diff options
Diffstat (limited to 'lib')
352 files changed, 53455 insertions, 44370 deletions
diff --git a/lib/core/hotcodereloading.nim b/lib/core/hotcodereloading.nim index 73f38402d..3a876885c 100644 --- a/lib/core/hotcodereloading.nim +++ b/lib/core/hotcodereloading.nim @@ -11,7 +11,7 @@ when defined(hotcodereloading): import - macros + std/macros template beforeCodeReload*(body: untyped) = hcrAddEventHandler(true, proc = body) {.executeOnReload.} diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 42d67388f..523727479 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,13 +9,16 @@ ## This module contains Nim's support for locks and condition vars. +#[ +for js, for now we treat locks as noop's to avoid pushing `when defined(js)` +in client code that uses locks. +]# when not compileOption("threads") and not defined(nimdoc): when false: # fix #12330 {.error: "Locks requires --threads:on option.".} -const insideRLocksModule = false -include "system/syslocks" +import std/private/syslocks type Lock* = SysLock ## Nim lock; whether this is re-entrant @@ -24,47 +27,59 @@ type {.push stackTrace: off.} + +proc `$`*(lock: Lock): string = + # workaround bug #14873 + result = "()" + proc initLock*(lock: var Lock) {.inline.} = ## Initializes the given lock. - initSysLock(lock) + when not defined(js): + initSysLock(lock) -proc deinitLock*(lock: var Lock) {.inline.} = +proc deinitLock*(lock: Lock) {.inline.} = ## Frees the resources associated with the lock. deinitSys(lock) -proc tryAcquire*(lock: var Lock): bool = +proc tryAcquire*(lock: var Lock): bool {.inline.} = ## Tries to acquire the given lock. Returns `true` on success. result = tryAcquireSys(lock) -proc acquire*(lock: var Lock) = +proc acquire*(lock: var Lock) {.inline.} = ## Acquires the given lock. - acquireSys(lock) + when not defined(js): + acquireSys(lock) -proc release*(lock: var Lock) = +proc release*(lock: var Lock) {.inline.} = ## Releases the given lock. - releaseSys(lock) + when not defined(js): + releaseSys(lock) proc initCond*(cond: var Cond) {.inline.} = ## Initializes the given condition variable. initSysCond(cond) -proc deinitCond*(cond: var Cond) {.inline.} = - ## Frees the resources associated with the lock. +proc deinitCond*(cond: Cond) {.inline.} = + ## Frees the resources associated with the condition variable. deinitSysCond(cond) proc wait*(cond: var Cond, lock: var Lock) {.inline.} = - ## waits on the condition variable `cond`. + ## Waits on the condition variable `cond`. waitSysCond(cond, lock) proc signal*(cond: var Cond) {.inline.} = - ## sends a signal to the condition variable `cond`. + ## Sends a signal to the condition variable `cond`. signalSysCond(cond) +proc broadcast*(cond: var Cond) {.inline.} = + ## Unblocks all threads currently blocked on the + ## specified condition variable `cond`. + broadcastSysCond(cond) + template withLock*(a: Lock, body: untyped) = ## Acquires the given lock, executes the statements in body and ## releases the lock after the statements finish executing. - mixin acquire, release acquire(a) {.locks: [a].}: try: diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim index bd48b5bd4..39999fa11 100644 --- a/lib/core/macrocache.nim +++ b/lib/core/macrocache.nim @@ -7,39 +7,238 @@ # distribution, for details about the copyright. # -## This module provides an API for macros that need to collect compile -## time information across module boundaries in global variables. -## Starting with version 0.19 of Nim this is not directly supported anymore -## as it breaks incremental compilations. -## Instead the API here needs to be used. See XXX (wikipedia page) for a -## theoretical foundation behind this. +## This module provides an API for macros to collect compile-time information +## across module boundaries. It should be used instead of global `{.compileTime.}` +## variables as those break incremental compilation. +## +## The main feature of this module is that if you create `CacheTable`s or +## any other `Cache` types with the same name in different modules, their +## content will be shared, meaning that you can fill a `CacheTable` in +## one module, and iterate over its contents in another. + +runnableExamples: + import std/macros + + const mcTable = CacheTable"myTable" + const mcSeq = CacheSeq"mySeq" + const mcCounter = CacheCounter"myCounter" + + static: + # add new key "val" with the value `myval` + let myval = newLit("hello ic") + mcTable["val"] = myval + assert mcTable["val"].kind == nnkStrLit + + # Can access the same cache from different static contexts + # All the information is retained + static: + # get value from `mcTable` and add it to `mcSeq` + mcSeq.add(mcTable["val"]) + assert mcSeq.len == 1 + + static: + assert mcSeq[0].strVal == "hello ic" + + # increase `mcCounter` by 3 + mcCounter.inc(3) + assert mcCounter.value == 3 + type CacheSeq* = distinct string + ## Compile-time sequence of `NimNode`s. CacheTable* = distinct string + ## Compile-time table of key-value pairs. + ## + ## Keys are `string`s and values are `NimNode`s. CacheCounter* = distinct string + ## Compile-time counter, uses `int` for storing the count. + +proc value*(c: CacheCounter): int {.magic: "NccValue".} = + ## Returns the value of a counter `c`. + runnableExamples: + static: + let counter = CacheCounter"valTest" + # default value is 0 + assert counter.value == 0 + + inc counter + assert counter.value == 1 + +proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".} = + ## Increments the counter `c` with the value `by`. + runnableExamples: + static: + let counter = CacheCounter"incTest" + inc counter + inc counter, 5 + + assert counter.value == 6 + +proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} = + ## Adds `value` to `s`. + runnableExamples: + import std/macros + const mySeq = CacheSeq"addTest" + + static: + mySeq.add(newLit(5)) + mySeq.add(newLit("hello ic")) + + assert mySeq.len == 2 + assert mySeq[1].strVal == "hello ic" + +proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} = + ## Adds `value` to `s`. + ## + ## .. hint:: This doesn't do anything if `value` is already in `s`. + runnableExamples: + import std/macros + const mySeq = CacheSeq"inclTest" + + static: + mySeq.incl(newLit(5)) + mySeq.incl(newLit(5)) + + # still one element + assert mySeq.len == 1 + +proc len*(s: CacheSeq): int {.magic: "NcsLen".} = + ## Returns the length of `s`. + runnableExamples: + import std/macros -proc value*(c: CacheCounter): int {.magic: "NccValue".} -proc inc*(c: CacheCounter; by = 1) {.magic: "NccInc".} + const mySeq = CacheSeq"lenTest" + static: + let val = newLit("helper") + mySeq.add(val) + assert mySeq.len == 1 -proc add*(s: CacheSeq; value: NimNode) {.magic: "NcsAdd".} -proc incl*(s: CacheSeq; value: NimNode) {.magic: "NcsIncl".} -proc len*(s: CacheSeq): int {.magic: "NcsLen".} -proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".} + mySeq.add(val) + assert mySeq.len == 2 + +proc `[]`*(s: CacheSeq; i: int): NimNode {.magic: "NcsAt".} = + ## Returns the `i`th value from `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"subTest" + static: + mySeq.add(newLit(42)) + assert mySeq[0].intVal == 42 + +proc `[]`*(s: CacheSeq; i: BackwardsIndex): NimNode = + ## Returns the `i`th last value from `s`. + runnableExamples: + import std/macros + + const mySeq = CacheSeq"backTest" + static: + mySeq &= newLit(42) + mySeq &= newLit(7) + assert mySeq[^1].intVal == 7 # Last item + assert mySeq[^2].intVal == 42 # Second last item + s[s.len - int(i)] iterator items*(s: CacheSeq): NimNode = + ## Iterates over each item in `s`. + runnableExamples: + import std/macros + const myseq = CacheSeq"itemsTest" + + static: + myseq.add(newLit(5)) + myseq.add(newLit(42)) + + for val in myseq: + # check that all values in `myseq` are int literals + assert val.kind == nnkIntLit + for i in 0 ..< len(s): yield s[i] -proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} - ## 'key' has to be unique! +proc `[]=`*(t: CacheTable; key: string, value: NimNode) {.magic: "NctPut".} = + ## Inserts a `(key, value)` pair into `t`. + ## + ## .. warning:: `key` has to be unique! Assigning `value` to a `key` that is already + ## in the table will result in a compiler error. + runnableExamples: + import std/macros + + const mcTable = CacheTable"subTest" + static: + # assign newLit(5) to the key "value" + mcTable["value"] = newLit(5) + + # check that we can get the value back + assert mcTable["value"].kind == nnkIntLit -proc len*(t: CacheTable): int {.magic: "NctLen".} -proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} +proc len*(t: CacheTable): int {.magic: "NctLen".} = + ## Returns the number of elements in `t`. + runnableExamples: + import std/macros + + const dataTable = CacheTable"lenTest" + static: + dataTable["key"] = newLit(5) + assert dataTable.len == 1 + +proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} = + ## Retrieves the `NimNode` value at `t[key]`. + runnableExamples: + import std/macros + + const mcTable = CacheTable"subTest" + static: + mcTable["toAdd"] = newStmtList() + + # get the NimNode back + assert mcTable["toAdd"].kind == nnkStmtList + +proc hasKey*(t: CacheTable; key: string): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * [contains proc][contains(CacheTable, string)] for use with the `in` operator + runnableExamples: + import std/macros + const mcTable = CacheTable"hasKeyEx" + static: + assert not mcTable.hasKey("foo") + mcTable["foo"] = newEmptyNode() + # Will now be true since we inserted a value + assert mcTable.hasKey("foo") + discard "Implemented in vmops" + +proc contains*(t: CacheTable; key: string): bool {.inline.} = + ## Alias of [hasKey][hasKey(CacheTable, string)] for use with the `in` operator. + runnableExamples: + import std/macros + const mcTable = CacheTable"containsEx" + static: + mcTable["foo"] = newEmptyNode() + # Will be true since we gave it a value before + assert "foo" in mcTable + t.hasKey(key) proc hasNext(t: CacheTable; iter: int): bool {.magic: "NctHasNext".} proc next(t: CacheTable; iter: int): (string, NimNode, int) {.magic: "NctNext".} iterator pairs*(t: CacheTable): (string, NimNode) = + ## Iterates over all `(key, value)` pairs in `t`. + runnableExamples: + import std/macros + const mytabl = CacheTable"values" + + static: + mytabl["intVal"] = newLit(5) + mytabl["otherVal"] = newLit(6) + for key, val in mytabl: + # make sure that we actually get the same keys + assert key in ["intVal", "otherVal"] + + # all vals are int literals + assert val.kind == nnkIntLit + var h = 0 while hasNext(t, h): let (a, b, h2) = next(t, h) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index f97b4a15f..7646b165c 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -8,6 +8,11 @@ # include "system/inclrtl" +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] + ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. @@ -18,6 +23,8 @@ include "system/inclrtl" ## .. include:: ../../doc/astspec.txt +## .. importdoc:: system.nim + # If you look for the implementation of the magic symbol # ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`. @@ -70,21 +77,24 @@ type nnkTupleTy, nnkTupleClassTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkRefTy, nnkPtrTy, nnkVarTy, - nnkConstTy, nnkMutableTy, + nnkConstTy, nnkOutTy, nnkDistinctTy, nnkProcTy, nnkIteratorTy, # iterator type - nnkSharedTy, # 'shared T' + nnkSinkAsgn, nnkEnumTy, nnkEnumFieldDef, - nnkArglist, nnkPattern + nnkArgList, nnkPattern nnkHiddenTryStmt, nnkClosure, nnkGotoState, nnkState, nnkBreakState, nnkFuncDef, - nnkTupleConstr + nnkTupleConstr, + nnkError, ## erroneous AST node + nnkModuleRef, nnkReplayAction, nnkNilRodNode ## internal IC nodes + nnkOpenSym NimNodeKinds* = set[NimNodeKind] NimTypeKind* = enum # some types are no longer used, see ast.nim @@ -105,7 +115,7 @@ type ntyError, ntyBuiltinTypeClass, ntyUserTypeClass, ntyUserTypeClassInst, ntyCompositeTypeClass, ntyInferred, ntyAnd, ntyOr, ntyNot, - ntyAnything, ntyStatic, ntyFromExpr, ntyOpt, ntyVoid + ntyAnything, ntyStatic, ntyFromExpr, ntyOptDeprecated, ntyVoid TNimTypeKinds* {.deprecated.} = set[NimTypeKind] NimSymKind* = enum @@ -119,11 +129,15 @@ type TNimSymKinds* {.deprecated.} = set[NimSymKind] +const + nnkMutableTy* {.deprecated.} = nnkOutTy + nnkSharedTy* {.deprecated.} = nnkSinkAsgn + type NimIdent* {.deprecated.} = object of RootObj ## Represents a Nim identifier in the AST. **Note**: This is only ## rarely useful, for identifier construction from a string - ## use ``ident"abc"``. + ## use `ident"abc"`. NimSymObj = object # hidden NimSym* {.deprecated.} = ref NimSymObj @@ -133,16 +147,13 @@ type const nnkLiterals* = {nnkCharLit..nnkNilLit} + # see matching set CallNodes below nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, - nnkCallStrLit} + nnkCallStrLit, nnkHiddenCallConv} nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit} {.push warnings: off.} -proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated: - "Deprecated since version 0.18.0: Use 'ident' or 'newIdentNode' instead.".} - ## Constructs an identifier from the string `s`. - proc toNimIdent*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated: "Deprecated since version 0.18.0: Use 'ident' or 'newIdentNode' instead.".} ## Constructs an identifier from the string `s`. @@ -163,7 +174,7 @@ proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect, deprecated proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} = ## Compares two Nim nodes' types. Return true if the types are the same, - ## eg. true when comparing alias with original type. + ## e.g. true when comparing alias with original type. discard proc len*(n: NimNode): int {.magic: "NLen", noSideEffect.} @@ -178,9 +189,9 @@ proc `[]`*(n: NimNode, i: BackwardsIndex): NimNode = n[n.len - i.int] template `^^`(n: NimNode, i: untyped): untyped = (when i is BackwardsIndex: n.len - int(i) else: int(i)) -proc `[]`*[T, U](n: NimNode, x: HSlice[T, U]): seq[NimNode] = +proc `[]`*[T, U: Ordinal](n: NimNode, x: HSlice[T, U]): seq[NimNode] = ## Slice operation for NimNode. - ## Returns a seq of child of `n` who inclusive range [n[x.a], n[x.b]]. + ## Returns a seq of child of `n` who inclusive range `[n[x.a], n[x.b]]`. let xa = n ^^ x.a let L = (n ^^ x.b) - xa + 1 result = newSeq[NimNode](L) @@ -196,13 +207,12 @@ proc `[]=`*(n: NimNode, i: BackwardsIndex, child: NimNode) = n[n.len - i.int] = child template `or`*(x, y: NimNode): NimNode = - ## Evaluate ``x`` and when it is not an empty node, return - ## it. Otherwise evaluate to ``y``. Can be used to chain several + ## Evaluate `x` and when it is not an empty node, return + ## it. Otherwise evaluate to `y`. Can be used to chain several ## expressions to get the first expression that is not empty. - ## - ## .. code-block:: nim - ## + ## ```nim ## let node = mightBeEmpty() or mightAlsoBeEmpty() or fallbackNode + ## ``` let arg = x if arg != nil and arg.kind != nnkEmpty: @@ -211,12 +221,12 @@ template `or`*(x, y: NimNode): NimNode = y proc add*(father, child: NimNode): NimNode {.magic: "NAdd", discardable, - noSideEffect, locks: 0.} + noSideEffect.} ## Adds the `child` to the `father` node. Returns the ## father node so that calls can be nested. proc add*(father: NimNode, children: varargs[NimNode]): NimNode {. - magic: "NAddMultiple", discardable, noSideEffect, locks: 0.} + magic: "NAddMultiple", discardable, noSideEffect.} ## Adds each child of `children` to the `father` node. ## Returns the `father` node so that calls can be nested. @@ -232,7 +242,17 @@ proc intVal*(n: NimNode): BiggestInt {.magic: "NIntVal", noSideEffect.} proc floatVal*(n: NimNode): BiggestFloat {.magic: "NFloatVal", noSideEffect.} ## Returns a float from any floating point literal. -{.push warnings: off.} + +proc symKind*(symbol: NimNode): NimSymKind {.magic: "NSymKind", noSideEffect.} +proc getImpl*(symbol: NimNode): NimNode {.magic: "GetImpl", noSideEffect.} + ## Returns a copy of the declaration of a symbol or `nil`. +proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} + ## Returns the string value of an identifier, symbol, comment, or string literal. + ## + ## See also: + ## * `strVal= proc<#strVal=,NimNode,string>`_ for setting the string value. + +{.push warnings: off.} # silence `deprecated` proc ident*(n: NimNode): NimIdent {.magic: "NIdent", noSideEffect, deprecated: "Deprecated since version 0.18.1; All functionality is defined on 'NimNode'.".} @@ -242,65 +262,38 @@ proc symbol*(n: NimNode): NimSym {.magic: "NSymbol", noSideEffect, deprecated: proc getImpl*(s: NimSym): NimNode {.magic: "GetImpl", noSideEffect, deprecated: "use `getImpl: NimNode -> NimNode` instead".} -when defined(nimSymKind): - proc symKind*(symbol: NimNode): NimSymKind {.magic: "NSymKind", noSideEffect.} - proc getImpl*(symbol: NimNode): NimNode {.magic: "GetImpl", noSideEffect.} - ## Returns a copy of the declaration of a symbol or `nil`. - proc strVal*(n: NimNode): string {.magic: "NStrVal", noSideEffect.} - ## Returns the string value of an identifier, symbol, comment, or string literal. - ## - ## See also: - ## * `strVal= proc<#strVal=,NimNode,string>`_ for setting the string value. - - proc `$`*(i: NimIdent): string {.magic: "NStrVal", noSideEffect, deprecated: - "Deprecated since version 0.18.1; Use 'strVal' instead.".} - ## Converts a Nim identifier to a string. - - proc `$`*(s: NimSym): string {.magic: "NStrVal", noSideEffect, deprecated: - "Deprecated since version 0.18.1; Use 'strVal' instead.".} - ## Converts a Nim symbol to a string. +proc `$`*(i: NimIdent): string {.magic: "NStrVal", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Use 'strVal' instead.".} + ## Converts a Nim identifier to a string. -else: # bootstrapping substitute - proc getImpl*(symbol: NimNode): NimNode = - symbol.symbol.getImpl - - proc strValOld(n: NimNode): string {.magic: "NStrVal", noSideEffect.} - - proc `$`*(s: NimSym): string {.magic: "IdentToStr", noSideEffect.} - - proc `$`*(i: NimIdent): string {.magic: "IdentToStr", noSideEffect.} - - proc strVal*(n: NimNode): string = - if n.kind == nnkIdent: - $n.ident - elif n.kind == nnkSym: - $n.symbol - else: - n.strValOld +proc `$`*(s: NimSym): string {.magic: "NStrVal", noSideEffect, deprecated: + "Deprecated since version 0.18.1; Use 'strVal' instead.".} + ## Converts a Nim symbol to a string. {.pop.} -when defined(nimSymImplTransform): +when (NimMajor, NimMinor, NimPatch) >= (1, 3, 5) or defined(nimSymImplTransform): proc getImplTransformed*(symbol: NimNode): NimNode {.magic: "GetImplTransf", noSideEffect.} - ## For a typed proc returns the AST after transformation pass. - -when defined(nimHasSymOwnerInMacro): - proc owner*(sym: NimNode): NimNode {.magic: "SymOwner", noSideEffect.} - ## Accepts a node of kind `nnkSym` and returns its owner's symbol. - ## The meaning of 'owner' depends on `sym`'s `NimSymKind` and declaration - ## context. For top level declarations this is an `nskModule` symbol, - ## for proc local variables an `nskProc` symbol, for enum/object fields an - ## `nskType` symbol, etc. For symbols without an owner, `nil` is returned. - ## - ## See also: - ## * `symKind proc<#symKind,NimNode>`_ to get the kind of a symbol - ## * `getImpl proc<#getImpl,NimNode>`_ to get the declaration of a symbol - -when defined(nimHasInstantiationOfInMacro): - proc isInstantiationOf*(instanceProcSym, genProcSym: NimNode): bool {.magic: "SymIsInstantiationOf", noSideEffect.} - ## Checks if a proc symbol is an instance of the generic proc symbol. - ## Useful to check proc symbols against generic symbols - ## returned by `bindSym`. + ## For a typed proc returns the AST after transformation pass; this is useful + ## for debugging how the compiler transforms code (e.g.: `defer`, `for`) but + ## note that code transformations are implementation dependent and subject to change. + ## See an example in `tests/macros/tmacros_various.nim`. + +proc owner*(sym: NimNode): NimNode {.magic: "SymOwner", noSideEffect, deprecated.} + ## Accepts a node of kind `nnkSym` and returns its owner's symbol. + ## The meaning of 'owner' depends on `sym`'s `NimSymKind` and declaration + ## context. For top level declarations this is an `nskModule` symbol, + ## for proc local variables an `nskProc` symbol, for enum/object fields an + ## `nskType` symbol, etc. For symbols without an owner, `nil` is returned. + ## + ## See also: + ## * `symKind proc<#symKind,NimNode>`_ to get the kind of a symbol + ## * `getImpl proc<#getImpl,NimNode>`_ to get the declaration of a symbol + +proc isInstantiationOf*(instanceProcSym, genProcSym: NimNode): bool {.magic: "SymIsInstantiationOf", noSideEffect.} + ## Checks if a proc symbol is an instance of the generic proc symbol. + ## Useful to check proc symbols against generic symbols + ## returned by `bindSym`. proc getType*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} ## With 'getType' you can access the node's `type`:idx:. A Nim type is @@ -311,11 +304,11 @@ proc getType*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} ## kind of type it is, call `typeKind` on getType's result. proc getType*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} - ## Version of ``getType`` which takes a ``typedesc``. + ## Version of `getType` which takes a `typedesc`. proc typeKind*(n: NimNode): NimTypeKind {.magic: "NGetType", noSideEffect.} ## Returns the type kind of the node 'n' that should represent a type, that - ## means the node should have been obtained via ``getType``. + ## means the node should have been obtained via `getType`. proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = ## Returns the `type`:idx: of a node in a form matching the way the @@ -336,12 +329,12 @@ proc getTypeInst*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = doAssert(dumpTypeInst(c) == "Vec[4, float32]") proc getTypeInst*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} - ## Version of ``getTypeInst`` which takes a ``typedesc``. + ## Version of `getTypeInst` which takes a `typedesc`. proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = ## Returns the `type`:idx: of a node in a form matching the implementation ## of the type. Any intermediate aliases are expanded to arrive at the final - ## type implementation. You can instead use ``getImpl`` on a symbol if you + ## type implementation. You can instead use `getImpl` on a symbol if you ## want to find the intermediate aliases. runnableExamples: type @@ -356,18 +349,16 @@ proc getTypeImpl*(n: NimNode): NimNode {.magic: "NGetType", noSideEffect.} = newLit(x.getTypeImpl.repr) let t = """ object - arr: array[0 .. 3, float32] -""" + arr: array[0 .. 3, float32]""" doAssert(dumpTypeImpl(a) == t) doAssert(dumpTypeImpl(b) == t) doAssert(dumpTypeImpl(c) == t) -when defined(nimHasSignatureHashInMacro): - proc signatureHash*(n: NimNode): string {.magic: "NSigHash", noSideEffect.} - ## Returns a stable identifier derived from the signature of a symbol. - ## The signature combines many factors such as the type of the symbol, - ## the owning module of the symbol and others. The same identifier is - ## used in the back-end to produce the mangled symbol name. +proc signatureHash*(n: NimNode): string {.magic: "NSigHash", noSideEffect.} + ## Returns a stable identifier derived from the signature of a symbol. + ## The signature combines many factors such as the type of the symbol, + ## the owning module of the symbol and others. The same identifier is + ## used in the back-end to produce the mangled symbol name. proc symBodyHash*(s: NimNode): string {.noSideEffect.} = ## Returns a stable digest for symbols derived not only from type signature @@ -377,7 +368,7 @@ proc symBodyHash*(s: NimNode): string {.noSideEffect.} = discard proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} - ## Version of ``getTypeImpl`` which takes a ``typedesc``. + ## Version of `getTypeImpl` which takes a `typedesc`. proc `intVal=`*(n: NimNode, val: BiggestInt) {.magic: "NSetIntVal", noSideEffect.} proc `floatVal=`*(n: NimNode, val: BiggestFloat) {.magic: "NSetFloatVal", noSideEffect.} @@ -392,13 +383,6 @@ proc `ident=`*(n: NimNode, val: NimIdent) {.magic: "NSetIdent", noSideEffect, de {.pop.} -#proc `typ=`*(n: NimNode, typ: typedesc) {.magic: "NSetType".} -# this is not sound! Unfortunately forbidding 'typ=' is not enough, as you -# can easily do: -# let bracket = semCheck([1, 2]) -# let fake = semCheck(2.0) -# bracket[0] = fake # constructs a mixed array with ints and floats! - proc `strVal=`*(n: NimNode, val: string) {.magic: "NSetStrVal", noSideEffect.} ## Sets the string value of a string literal or comment. ## Setting `strVal` is disallowed for `nnkIdent` and `nnkSym` nodes; a new node @@ -414,15 +398,43 @@ proc newNimNode*(kind: NimNodeKind, {.magic: "NNewNimNode", noSideEffect.} ## Creates a new AST node of the specified kind. ## - ## The ``lineInfoFrom`` parameter is used for line information when the + ## The `lineInfoFrom` parameter is used for line information when the ## produced code crashes. You should ensure that it is set to a node that ## you are transforming. -proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} -proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} +proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.} = + ## Creates a new AST node by copying the node `n`. Note that unlike `copyNimTree`, + ## child nodes of `n` are not copied. + runnableExamples: + macro foo(x: typed) = + var s = copyNimNode(x) + doAssert s.len == 0 + doAssert s.kind == nnkStmtList + + foo: + let x = 12 + echo x + +proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.} = + ## Creates a new AST node by recursively copying the node `n`. Note that + ## unlike `copyNimNode`, this copies `n`, the children of `n`, etc. + runnableExamples: + macro foo(x: typed) = + var s = copyNimTree(x) + doAssert s.len == 2 + doAssert s.kind == nnkStmtList + + foo: + let x = 12 + echo x + +when defined(nimHasNoReturnError): + {.pragma: errorNoReturn, noreturn.} +else: + {.pragma: errorNoReturn.} -proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign.} - ## Writes an error message at compile time. The optional ``n: NimNode`` +proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign, errorNoReturn.} + ## Writes an error message at compile time. The optional `n: NimNode` ## parameter is used as the source for file and line number information in ## the compilation error message. @@ -432,44 +444,48 @@ proc warning*(msg: string, n: NimNode = nil) {.magic: "NWarning", benign.} proc hint*(msg: string, n: NimNode = nil) {.magic: "NHint", benign.} ## Writes a hint message at compile time. -proc newStrLitNode*(s: string): NimNode {.compileTime, noSideEffect.} = +proc newStrLitNode*(s: string): NimNode {.noSideEffect.} = ## Creates a string literal node from `s`. result = newNimNode(nnkStrLit) result.strVal = s -proc newCommentStmtNode*(s: string): NimNode {.compileTime, noSideEffect.} = +proc newCommentStmtNode*(s: string): NimNode {.noSideEffect.} = ## Creates a comment statement node. result = newNimNode(nnkCommentStmt) result.strVal = s -proc newIntLitNode*(i: BiggestInt): NimNode {.compileTime.} = +proc newIntLitNode*(i: BiggestInt): NimNode = ## Creates an int literal node from `i`. result = newNimNode(nnkIntLit) result.intVal = i -proc newFloatLitNode*(f: BiggestFloat): NimNode {.compileTime.} = +proc newFloatLitNode*(f: BiggestFloat): NimNode = ## Creates a float literal node from `f`. result = newNimNode(nnkFloatLit) result.floatVal = f {.push warnings: off.} -proc newIdentNode*(i: NimIdent): NimNode {.compileTime, deprecated.} = +proc newIdentNode*(i: NimIdent): NimNode {.deprecated: "use ident(string)".} = ## Creates an identifier node from `i`. result = newNimNode(nnkIdent) result.ident = i {.pop.} -proc newIdentNode*(i: string): NimNode {.magic: "StrToIdent", noSideEffect, compilerproc.} +proc newIdentNode*(i: string): NimNode {.magic: "StrToIdent", noSideEffect.} ## Creates an identifier node from `i`. It is simply an alias for - ## ``ident(string)``. Use that, it's shorter. + ## `ident(string)`. Use that, it's shorter. +proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.} + ## Create a new ident node from a string. type - BindSymRule* = enum ## specifies how ``bindSym`` behaves + BindSymRule* = enum ## Specifies how `bindSym` behaves. The difference + ## between open and closed symbols can be found in + ## `<manual.html#symbol-lookup-in-generics-open-and-closed-symbols>`_ brClosed, ## only the symbols in current scope are bound - brOpen, ## open wrt overloaded symbols, but may be a single + brOpen, ## open for overloaded symbols, but may be a single ## symbol if not ambiguous (the rules match that of ## binding in generics) brForceOpen ## same as brOpen, but it will always be open even @@ -478,22 +494,17 @@ type proc bindSym*(ident: string | NimNode, rule: BindSymRule = brClosed): NimNode {. magic: "NBindSym", noSideEffect.} - ## Ceates a node that binds `ident` to a symbol node. The bound symbol + ## Creates a node that binds `ident` to a symbol node. The bound symbol ## may be an overloaded symbol. - ## if `ident` is a NimNode, it must have ``nnkIdent`` kind. - ## If ``rule == brClosed`` either an ``nnkClosedSymChoice`` tree is - ## returned or ``nnkSym`` if the symbol is not ambiguous. - ## If ``rule == brOpen`` either an ``nnkOpenSymChoice`` tree is - ## returned or ``nnkSym`` if the symbol is not ambiguous. - ## If ``rule == brForceOpen`` always an ``nnkOpenSymChoice`` tree is + ## if `ident` is a NimNode, it must have `nnkIdent` kind. + ## If `rule == brClosed` either an `nnkClosedSymChoice` tree is + ## returned or `nnkSym` if the symbol is not ambiguous. + ## If `rule == brOpen` either an `nnkOpenSymChoice` tree is + ## returned or `nnkSym` if the symbol is not ambiguous. + ## If `rule == brForceOpen` always an `nnkOpenSymChoice` tree is ## returned even if the symbol is not ambiguous. ## - ## Experimental feature: - ## use {.experimental: "dynamicBindSym".} to activate it. - ## If called from template / regular code, `ident` and `rule` must be - ## constant expression / literal value. - ## If called from macros / compile time procs / static blocks, - ## `ident` and `rule` can be VM computed value. + ## See the `manual <manual.html#macros-bindsym>`_ for more details. proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. magic: "NGenSym", noSideEffect.} @@ -501,10 +512,11 @@ proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. ## needs to occur in a declaration context. proc callsite*(): NimNode {.magic: "NCallSite", benign, deprecated: - "Deprecated since v0.18.1; use varargs[untyped] in the macro prototype instead".} + "Deprecated since v0.18.1; use `varargs[untyped]` in the macro prototype instead".} ## Returns the AST of the invocation expression that invoked this macro. + # see https://github.com/nim-lang/RFCs/issues/387 as candidate replacement. -proc toStrLit*(n: NimNode): NimNode {.compileTime.} = +proc toStrLit*(n: NimNode): NimNode = ## Converts the AST `n` to the concrete Nim code and wraps that ## in a string literal node. return newStrLitNode(repr(n)) @@ -528,125 +540,194 @@ proc getColumn(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.} proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.} proc copyLineInfo*(arg: NimNode, info: NimNode) {.magic: "NLineInfo", noSideEffect.} - ## Copy lineinfo from ``info``. + ## Copy lineinfo from `info`. + +proc setLine(arg: NimNode, line: uint16) {.magic: "NLineInfo", noSideEffect.} +proc setColumn(arg: NimNode, column: int16) {.magic: "NLineInfo", noSideEffect.} +proc setFile(arg: NimNode, file: string) {.magic: "NLineInfo", noSideEffect.} -proc lineInfoObj*(n: NimNode): LineInfo {.compileTime.} = - ## Returns ``LineInfo`` of ``n``, using absolute path for ``filename``. - result.filename = n.getFile - result.line = n.getLine - result.column = n.getColumn +proc setLineInfo*(arg: NimNode, file: string, line: int, column: int) = + ## Sets the line info on the NimNode. The file needs to exists, but can be a + ## relative path. If you want to attach line info to a block using `quote` + ## you'll need to add the line information after the quote block. + arg.setFile(file) + arg.setLine(line.uint16) + arg.setColumn(column.int16) -proc lineInfo*(arg: NimNode): string {.compileTime.} = +proc setLineInfo*(arg: NimNode, lineInfo: LineInfo) = + ## See `setLineInfo proc<#setLineInfo,NimNode,string,int,int>`_ + setLineInfo(arg, lineInfo.filename, lineInfo.line, lineInfo.column) + +proc lineInfoObj*(n: NimNode): LineInfo = + ## Returns `LineInfo` of `n`, using absolute path for `filename`. + result = LineInfo(filename: n.getFile, line: n.getLine, column: n.getColumn) + +proc lineInfo*(arg: NimNode): string = ## Return line info in the form `filepath(line, column)`. $arg.lineInfoObj -proc internalParseExpr(s: string): NimNode {. +proc internalParseExpr(s, filename: string): NimNode {. magic: "ParseExprToAst", noSideEffect.} -proc internalParseStmt(s: string): NimNode {. +proc internalParseStmt(s, filename: string): NimNode {. magic: "ParseStmtToAst", noSideEffect.} proc internalErrorFlag*(): string {.magic: "NError", noSideEffect.} ## Some builtins set an error flag. This is then turned into a proper ## exception. **Note**: Ordinary application code should not call this. -proc parseExpr*(s: string): NimNode {.noSideEffect, compileTime.} = +proc parseExpr*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. - ## Expects a single expression. Raises ``ValueError`` for parsing errors. - result = internalParseExpr(s) + ## Expects a single expression. Raises `ValueError` for parsing errors. + ## A filename can be given for more informative errors. + result = internalParseExpr(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc parseStmt*(s: string): NimNode {.noSideEffect, compileTime.} = +proc parseStmt*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. - ## Expects one or more statements. Raises ``ValueError`` for parsing errors. - result = internalParseStmt(s) + ## Expects one or more statements. Raises `ValueError` for parsing errors. + ## A filename can be given for more informative errors. + result = internalParseStmt(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.} ## Obtains the AST nodes returned from a macro or template invocation. + ## See also `genasts.genAst`. ## Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## macro FooMacro() = ## var ast = getAst(BarTemplate()) + ## ``` -proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} +proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} = ## Quasi-quoting operator. ## Accepts an expression or a block and returns the AST that represents it. ## Within the quoted AST, you are able to interpolate NimNode expressions ## from the surrounding scope. If no operator is given, quoting is done using ## backticks. Otherwise, the given operator must be used as a prefix operator - ## for any interpolated expression. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## macro check(ex: untyped) = - ## # this is a simplified version of the check macro from the - ## # unittest module. - ## - ## # If there is a failed check, we want to make it easy for - ## # the user to jump to the faulty line in the code, so we - ## # get the line info here: - ## var info = ex.lineinfo + ## for any interpolated expression. The original meaning of the interpolation + ## operator may be obtained by escaping it (by prefixing it with itself) when used + ## as a unary operator: + ## e.g. `@` is escaped as `@@`, `&%` is escaped as `&%&%` and so on; see examples. ## - ## # We will also display the code string of the failed check: - ## var expString = ex.toStrLit + ## A custom operator interpolation needs accent quoted (``) whenever it resolves + ## to a symbol. ## - ## # Finally we compose the code to implement the check: - ## result = quote do: - ## if not `ex`: - ## echo `info` & ": Check failed: " & `expString` + ## See also `genasts <genasts.html>`_ which avoids some issues with `quote`. + runnableExamples: + macro check(ex: untyped) = + # this is a simplified version of the check macro from the + # unittest module. + + # If there is a failed check, we want to make it easy for + # the user to jump to the faulty line in the code, so we + # get the line info here: + var info = ex.lineinfo + + # We will also display the code string of the failed check: + var expString = ex.toStrLit -proc expectKind*(n: NimNode, k: NimNodeKind) {.compileTime.} = + # Finally we compose the code to implement the check: + result = quote do: + if not `ex`: + echo `info` & ": Check failed: " & `expString` + check 1 + 1 == 2 + + runnableExamples: + # example showing how to define a symbol that requires backtick without + # quoting it. + var destroyCalled = false + macro bar() = + let s = newTree(nnkAccQuoted, ident"=destroy") + # let s = ident"`=destroy`" # this would not work + result = quote do: + type Foo = object + # proc `=destroy`(a: var Foo) = destroyCalled = true # this would not work + proc `s`(a: var Foo) = destroyCalled = true + block: + let a = Foo() + bar() + doAssert destroyCalled + + runnableExamples: + # custom `op` + var destroyCalled = false + macro bar(ident) = + var x = 1.5 + result = quote("@") do: + type Foo = object + let `@ident` = 0 # custom op interpolated symbols need quoted (``) + proc `=destroy`(a: var Foo) = + doAssert @x == 1.5 + doAssert compiles(@x == 1.5) + let b1 = @[1,2] + let b2 = @@[1,2] + doAssert $b1 == "[1, 2]" + doAssert $b2 == "@[1, 2]" + destroyCalled = true + block: + let a = Foo() + bar(someident) + doAssert destroyCalled + + proc `&%`(x: int): int = 1 + proc `&%`(x, y: int): int = 2 + + macro bar2() = + var x = 3 + result = quote("&%") do: + var y = &%x # quoting operator + doAssert &%&%y == 1 # unary operator => need to escape + doAssert y &% y == 2 # binary operator => no need to escape + doAssert y == 3 + bar2() + +proc expectKind*(n: NimNode, k: NimNodeKind) = ## Checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check the AST that is passed to them. if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind, n) -proc expectMinLen*(n: NimNode, min: int) {.compileTime.} = +proc expectMinLen*(n: NimNode, min: int) = ## Checks that `n` has at least `min` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. - if n.len < min: error("macro expects a node with " & $min & " children", n) + if n.len < min: error("Expected a node with at least " & $min & " children, got " & $n.len, n) -proc expectLen*(n: NimNode, len: int) {.compileTime.} = +proc expectLen*(n: NimNode, len: int) = ## Checks that `n` has exactly `len` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. - if n.len != len: error("macro expects a node with " & $len & " children", n) + if n.len != len: error("Expected a node with " & $len & " children, got " & $n.len, n) -proc expectLen*(n: NimNode, min, max: int) {.compileTime.} = - ## Checks that `n` has a number of children in the range ``min..max``. +proc expectLen*(n: NimNode, min, max: int) = + ## Checks that `n` has a number of children in the range `min..max`. ## If this is not the case, compilation aborts with an error message. ## This is useful for writing macros that check its number of arguments. if n.len < min or n.len > max: - error("macro expects a node with " & $min & ".." & $max & " children", n) + error("Expected a node with " & $min & ".." & $max & " children, got " & $n.len, n) proc newTree*(kind: NimNodeKind, - children: varargs[NimNode]): NimNode {.compileTime.} = + children: varargs[NimNode]): NimNode = ## Produces a new node with children. result = newNimNode(kind) result.add(children) -proc newCall*(theProc: NimNode, - args: varargs[NimNode]): NimNode {.compileTime.} = +proc newCall*(theProc: NimNode, args: varargs[NimNode]): NimNode = ## Produces a new call node. `theProc` is the proc that is called with - ## the arguments ``args[0..]``. + ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(theProc) result.add(args) {.push warnings: off.} -proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.compileTime, deprecated: +proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.deprecated: "Deprecated since v0.18.1; use 'newCall(string, ...)' or 'newCall(NimNode, ...)' instead".} = ## Produces a new call node. `theProc` is the proc that is called with - ## the arguments ``args[0..]``. + ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(newIdentNode(theProc)) result.add(args) @@ -654,128 +735,128 @@ proc newCall*(theProc: NimIdent, args: varargs[NimNode]): NimNode {.compileTime, {.pop.} proc newCall*(theProc: string, - args: varargs[NimNode]): NimNode {.compileTime.} = + args: varargs[NimNode]): NimNode = ## Produces a new call node. `theProc` is the proc that is called with - ## the arguments ``args[0..]``. + ## the arguments `args[0..]`. result = newNimNode(nnkCall) result.add(newIdentNode(theProc)) result.add(args) -proc newLit*(c: char): NimNode {.compileTime.} = +proc newLit*(c: char): NimNode = ## Produces a new character literal node. result = newNimNode(nnkCharLit) result.intVal = ord(c) -proc newLit*(i: int): NimNode {.compileTime.} = +proc newLit*(i: int): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkIntLit) result.intVal = i -proc newLit*(i: int8): NimNode {.compileTime.} = +proc newLit*(i: int8): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt8Lit) result.intVal = i -proc newLit*(i: int16): NimNode {.compileTime.} = +proc newLit*(i: int16): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt16Lit) result.intVal = i -proc newLit*(i: int32): NimNode {.compileTime.} = +proc newLit*(i: int32): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt32Lit) result.intVal = i -proc newLit*(i: int64): NimNode {.compileTime.} = +proc newLit*(i: int64): NimNode = ## Produces a new integer literal node. result = newNimNode(nnkInt64Lit) result.intVal = i -proc newLit*(i: uint): NimNode {.compileTime.} = +proc newLit*(i: uint): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUIntLit) result.intVal = BiggestInt(i) -proc newLit*(i: uint8): NimNode {.compileTime.} = +proc newLit*(i: uint8): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt8Lit) result.intVal = BiggestInt(i) -proc newLit*(i: uint16): NimNode {.compileTime.} = +proc newLit*(i: uint16): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt16Lit) result.intVal = BiggestInt(i) -proc newLit*(i: uint32): NimNode {.compileTime.} = +proc newLit*(i: uint32): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt32Lit) result.intVal = BiggestInt(i) -proc newLit*(i: uint64): NimNode {.compileTime.} = +proc newLit*(i: uint64): NimNode = ## Produces a new unsigned integer literal node. result = newNimNode(nnkUInt64Lit) result.intVal = BiggestInt(i) -proc newLit*(b: bool): NimNode {.compileTime.} = +proc newLit*(b: bool): NimNode = ## Produces a new boolean literal node. result = if b: bindSym"true" else: bindSym"false" -proc newLit*(s: string): NimNode {.compileTime.} = +proc newLit*(s: string): NimNode = ## Produces a new string literal node. result = newNimNode(nnkStrLit) result.strVal = s when false: # the float type is not really a distinct type as described in https://github.com/nim-lang/Nim/issues/5875 - proc newLit*(f: float): NimNode {.compileTime.} = + proc newLit*(f: float): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloatLit) result.floatVal = f -proc newLit*(f: float32): NimNode {.compileTime.} = +proc newLit*(f: float32): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloat32Lit) result.floatVal = f -proc newLit*(f: float64): NimNode {.compileTime.} = +proc newLit*(f: float64): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloat64Lit) result.floatVal = f when declared(float128): - proc newLit*(f: float128): NimNode {.compileTime.} = + proc newLit*(f: float128): NimNode = ## Produces a new float literal node. result = newNimNode(nnkFloat128Lit) result.floatVal = f -proc newLit*(arg: enum): NimNode {.compileTime.} = +proc newLit*(arg: enum): NimNode = result = newCall( - arg.type.getTypeInst[1], + arg.typeof.getTypeInst[1], newLit(int(arg)) ) -proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} -proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} -proc newLit*[T](s: set[T]): NimNode {.compileTime.} -proc newLit*(arg: tuple): NimNode {.compileTime.} +proc newLit*[N,T](arg: array[N,T]): NimNode +proc newLit*[T](arg: seq[T]): NimNode +proc newLit*[T](s: set[T]): NimNode +proc newLit*[T: tuple](arg: T): NimNode -proc newLit*(arg: object): NimNode {.compileTime.} = - result = nnkObjConstr.newTree(arg.type.getTypeInst[1]) +proc newLit*(arg: object): NimNode = + result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1]) for a, b in arg.fieldPairs: result.add nnkExprColonExpr.newTree( newIdentNode(a), newLit(b) ) -proc newLit*(arg: ref object): NimNode {.compileTime.} = +proc newLit*(arg: ref object): NimNode = ## produces a new ref type literal node. - result = nnkObjConstr.newTree(arg.type.getTypeInst[1]) + result = nnkObjConstr.newTree(arg.typeof.getTypeInst[1]) for a, b in fieldPairs(arg[]): result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b)) -proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} = +proc newLit*[N,T](arg: array[N,T]): NimNode = result = nnkBracket.newTree for x in arg: result.add newLit(x) -proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} = +proc newLit*[T](arg: seq[T]): NimNode = let bracket = nnkBracket.newTree for x in arg: bracket.add newLit(x) @@ -788,19 +869,32 @@ proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} = var typ = getTypeInst(typeof(arg))[1] result = newCall(typ,result) -proc newLit*[T](s: set[T]): NimNode {.compileTime.} = +proc newLit*[T](s: set[T]): NimNode = result = nnkCurly.newTree for x in s: result.add newLit(x) + if result.len == 0: + # add type cast for empty set + var typ = getTypeInst(typeof(s))[1] + result = newCall(typ,result) -proc newLit*(arg: tuple): NimNode {.compileTime.} = - result = nnkPar.newTree - for a,b in arg.fieldPairs: - result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b)) +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + ## See `typetraits.isNamedTuple` -proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} = +proc newLit*[T: tuple](arg: T): NimNode = + ## use -d:nimHasWorkaround14720 to restore behavior prior to PR, forcing + ## a named tuple even when `arg` is unnamed. + result = nnkTupleConstr.newTree + when defined(nimHasWorkaround14720) or isNamedTuple(T): + for a, b in arg.fieldPairs: + result.add nnkExprColonExpr.newTree(newIdentNode(a), newLit(b)) + else: + for b in arg.fields: + result.add newLit(b) + +proc nestList*(op: NimNode; pack: NimNode): NimNode = ## Nests the list `pack` into a tree of call expressions: - ## ``[a, b, c]`` is transformed into ``op(a, op(c, d))``. + ## `[a, b, c]` is transformed into `op(a, op(c, d))`. ## This is also known as fold expression. if pack.len < 1: error("`nestList` expects a node with at least 1 child") @@ -808,21 +902,36 @@ proc nestList*(op: NimNode; pack: NimNode): NimNode {.compileTime.} = for i in countdown(pack.len - 2, 0): result = newCall(op, pack[i], result) -proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode {.compileTime.} = +proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode = ## Nests the list `pack` into a tree of call expressions: - ## ``[a, b, c]`` is transformed into ``op(a, op(c, d))``. + ## `[a, b, c]` is transformed into `op(a, op(c, d))`. ## This is also known as fold expression. result = init for i in countdown(pack.len - 1, 0): result = newCall(op, pack[i], result) -{.push warnings: off.} +proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. -proc nestList*(theProc: NimIdent, x: NimNode): NimNode {.compileTime, deprecated: - "Deprecated since v0.18.1; use one of 'nestList(NimNode, ...)' instead.".} = - nestList(newIdentNode(theProc), x) +proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` can be an identifier or a + ## symbol. `a` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. -{.pop.} +proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `b` can be an identifier or a + ## symbol. `b` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` and `b` can be an + ## identifier or a symbol. Both may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +const collapseSymChoice = not defined(nimLegacyMacrosCollapseSymChoice) proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indented = false) {.benign.} = if level > 0: @@ -845,35 +954,52 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent discard # same as nil node in this representation of nnkCharLit .. nnkInt64Lit: res.add(" " & $n.intVal) + of nnkUIntLit .. nnkUInt64Lit: + res.add(" " & $cast[uint64](n.intVal)) of nnkFloatLit .. nnkFloat64Lit: res.add(" " & $n.floatVal) of nnkStrLit .. nnkTripleStrLit, nnkCommentStmt, nnkIdent, nnkSym: res.add(" " & $n.strVal.newLit.repr) of nnkNone: assert false + elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice: + res.add(" " & $n.len) + if n.len > 0: + var allSameSymName = true + for i in 0..<n.len: + if n[i].kind != nnkSym or not eqIdent(n[i], n[0]): + allSameSymName = false + break + if allSameSymName: + res.add(" " & $n[0].strVal.newLit.repr) + else: + for j in 0 ..< n.len: + n[j].treeTraverse(res, level+1, isLisp, indented) else: - for j in 0 .. n.len-1: + for j in 0 ..< n.len: n[j].treeTraverse(res, level+1, isLisp, indented) if isLisp: res.add(")") -proc treeRepr*(n: NimNode): string {.compileTime, benign.} = +proc treeRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to a human-readable tree-like string. ## - ## See also `repr`, `lispRepr`, and `astGenRepr`. + ## See also `repr`, `lispRepr`_, and `astGenRepr`_. + result = "" n.treeTraverse(result, isLisp = false, indented = true) -proc lispRepr*(n: NimNode; indented = false): string {.compileTime, benign.} = - ## Convert the AST ``n`` to a human-readable lisp-like string. +proc lispRepr*(n: NimNode; indented = false): string {.benign.} = + ## Convert the AST `n` to a human-readable lisp-like string. ## - ## See also ``repr``, ``treeRepr``, and ``astGenRepr``. + ## See also `repr`, `treeRepr`_, and `astGenRepr`_. + result = "" n.treeTraverse(result, isLisp = true, indented = indented) -proc astGenRepr*(n: NimNode): string {.compileTime, benign.} = - ## Convert the AST ``n`` to the code required to generate that AST. +proc astGenRepr*(n: NimNode): string {.benign.} = + ## Convert the AST `n` to the code required to generate that AST. ## - ## See also ``repr``, ``treeRepr``, and ``lispRepr``. + ## See also `repr`_, `treeRepr`_, and `lispRepr`_. const NodeKinds = {nnkEmpty, nnkIdent, nnkSym, nnkNone, nnkCommentStmt} @@ -898,6 +1024,10 @@ proc astGenRepr*(n: NimNode): string {.compileTime, benign.} = of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkIdent, nnkSym: res.add(n.strVal.newLit.repr) of nnkNone: assert false + elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice: + res.add(", # unrepresentable symbols: " & $n.len) + if n.len > 0: + res.add(" " & n[0].strVal.newLit.repr) else: res.add(".newTree(") for j in 0..<n.len: @@ -918,179 +1048,177 @@ proc astGenRepr*(n: NimNode): string {.compileTime, benign.} = macro dumpTree*(s: untyped): untyped = echo s.treeRepr ## Accepts a block of nim code and prints the parsed abstract syntax - ## tree using the ``treeRepr`` proc. Printing is done *at compile time*. + ## tree using the `treeRepr` proc. Printing is done *at compile time*. ## ## You can use this as a tool to explore the Nim's abstract syntax ## tree and to discover what kind of nodes must be created to represent ## a certain expression/statement. ## ## For example: - ## - ## .. code-block:: nim - ## dumpTree: - ## echo "Hello, World!" + ## ```nim + ## dumpTree: + ## echo "Hello, World!" + ## ``` ## ## Outputs: + ## ``` + ## StmtList + ## Command + ## Ident "echo" + ## StrLit "Hello, World!" + ## ``` ## - ## .. code-block:: - ## StmtList - ## Command - ## Ident "echo" - ## StrLit "Hello, World!" - ## - ## Also see ``dumpAstGen`` and ``dumpLisp``. + ## Also see `dumpAstGen` and `dumpLisp`. macro dumpLisp*(s: untyped): untyped = echo s.lispRepr(indented = true) ## Accepts a block of nim code and prints the parsed abstract syntax - ## tree using the ``lispRepr`` proc. Printing is done *at compile time*. + ## tree using the `lispRepr` proc. Printing is done *at compile time*. ## ## You can use this as a tool to explore the Nim's abstract syntax ## tree and to discover what kind of nodes must be created to represent ## a certain expression/statement. ## ## For example: - ## - ## .. code-block:: nim - ## dumpLisp: - ## echo "Hello, World!" + ## ```nim + ## dumpLisp: + ## echo "Hello, World!" + ## ``` ## ## Outputs: + ## ``` + ## (StmtList + ## (Command + ## (Ident "echo") + ## (StrLit "Hello, World!"))) + ## ``` ## - ## .. code-block:: - ## (StmtList - ## (Command - ## (Ident "echo") - ## (StrLit "Hello, World!"))) - ## - ## Also see ``dumpAstGen`` and ``dumpTree``. + ## Also see `dumpAstGen` and `dumpTree`. macro dumpAstGen*(s: untyped): untyped = echo s.astGenRepr ## Accepts a block of nim code and prints the parsed abstract syntax - ## tree using the ``astGenRepr`` proc. Printing is done *at compile time*. + ## tree using the `astGenRepr` proc. Printing is done *at compile time*. ## ## You can use this as a tool to write macros quicker by writing example ## outputs and then copying the snippets into the macro for modification. ## ## For example: - ## - ## .. code-block:: nim - ## dumpAstGen: - ## echo "Hello, World!" + ## ```nim + ## dumpAstGen: + ## echo "Hello, World!" + ## ``` ## ## Outputs: - ## - ## .. code-block:: nim - ## nnkStmtList.newTree( - ## nnkCommand.newTree( - ## newIdentNode("echo"), - ## newLit("Hello, World!") - ## ) - ## ) - ## - ## Also see ``dumpTree`` and ``dumpLisp``. - -macro dumpTreeImm*(s: untyped): untyped {.deprecated.} = echo s.treeRepr - ## Deprecated. Use `dumpTree` instead. - -macro dumpLispImm*(s: untyped): untyped {.deprecated.} = echo s.lispRepr - ## Deprecated. Use `dumpLisp` instead. - -proc newEmptyNode*(): NimNode {.compileTime, noSideEffect.} = + ## ``` + ## nnkStmtList.newTree( + ## nnkCommand.newTree( + ## newIdentNode("echo"), + ## newLit("Hello, World!") + ## ) + ## ) + ## ``` + ## + ## Also see `dumpTree` and `dumpLisp`. + +proc newEmptyNode*(): NimNode {.noSideEffect.} = ## Create a new empty node. result = newNimNode(nnkEmpty) -proc newStmtList*(stmts: varargs[NimNode]): NimNode {.compileTime.}= +proc newStmtList*(stmts: varargs[NimNode]): NimNode = ## Create a new statement list. result = newNimNode(nnkStmtList).add(stmts) -proc newPar*(exprs: varargs[NimNode]): NimNode {.compileTime.}= +proc newPar*(exprs: NimNode): NimNode = ## Create a new parentheses-enclosed expression. newNimNode(nnkPar).add(exprs) -proc newBlockStmt*(label, body: NimNode): NimNode {.compileTime.} = +proc newPar*(exprs: varargs[NimNode]): NimNode {.deprecated: + "don't use newPar/nnkPar to construct tuple expressions; use nnkTupleConstr instead".} = + ## Create a new parentheses-enclosed expression. + newNimNode(nnkPar).add(exprs) + +proc newBlockStmt*(label, body: NimNode): NimNode = ## Create a new block statement with label. return newNimNode(nnkBlockStmt).add(label, body) -proc newBlockStmt*(body: NimNode): NimNode {.compileTime.} = +proc newBlockStmt*(body: NimNode): NimNode = ## Create a new block: stmt. return newNimNode(nnkBlockStmt).add(newEmptyNode(), body) -proc newVarStmt*(name, value: NimNode): NimNode {.compileTime.} = +proc newVarStmt*(name, value: NimNode): NimNode = ## Create a new var stmt. return newNimNode(nnkVarSection).add( newNimNode(nnkIdentDefs).add(name, newNimNode(nnkEmpty), value)) -proc newLetStmt*(name, value: NimNode): NimNode {.compileTime.} = +proc newLetStmt*(name, value: NimNode): NimNode = ## Create a new let stmt. return newNimNode(nnkLetSection).add( newNimNode(nnkIdentDefs).add(name, newNimNode(nnkEmpty), value)) -proc newConstStmt*(name, value: NimNode): NimNode {.compileTime.} = +proc newConstStmt*(name, value: NimNode): NimNode = ## Create a new const stmt. newNimNode(nnkConstSection).add( newNimNode(nnkConstDef).add(name, newNimNode(nnkEmpty), value)) -proc newAssignment*(lhs, rhs: NimNode): NimNode {.compileTime.} = +proc newAssignment*(lhs, rhs: NimNode): NimNode = return newNimNode(nnkAsgn).add(lhs, rhs) -proc newDotExpr*(a, b: NimNode): NimNode {.compileTime.} = +proc newDotExpr*(a, b: NimNode): NimNode = ## Create new dot expression. ## a.dot(b) -> `a.b` return newNimNode(nnkDotExpr).add(a, b) -proc newColonExpr*(a, b: NimNode): NimNode {.compileTime.} = +proc newColonExpr*(a, b: NimNode): NimNode = ## Create new colon expression. ## newColonExpr(a, b) -> `a: b` newNimNode(nnkExprColonExpr).add(a, b) proc newIdentDefs*(name, kind: NimNode; - default = newEmptyNode()): NimNode {.compileTime.} = - ## Creates a new ``nnkIdentDefs`` node of a specific kind and value. + default = newEmptyNode()): NimNode = + ## Creates a new `nnkIdentDefs` node of a specific kind and value. ## - ## ``nnkIdentDefs`` need to have at least three children, but they can have + ## `nnkIdentDefs` need to have at least three children, but they can have ## more: first comes a list of identifiers followed by a type and value ## nodes. This helper proc creates a three node subtree, the first subnode - ## being a single identifier name. Both the ``kind`` node and ``default`` - ## (value) nodes may be empty depending on where the ``nnkIdentDefs`` - ## appears: tuple or object definitions will have an empty ``default`` node, - ## ``let`` or ``var`` blocks may have an empty ``kind`` node if the + ## being a single identifier name. Both the `kind` node and `default` + ## (value) nodes may be empty depending on where the `nnkIdentDefs` + ## appears: tuple or object definitions will have an empty `default` node, + ## `let` or `var` blocks may have an empty `kind` node if the ## identifier is being assigned a value. Example: ## - ## .. code-block:: nim - ## + ## ```nim ## var varSection = newNimNode(nnkVarSection).add( ## newIdentDefs(ident("a"), ident("string")), ## newIdentDefs(ident("b"), newEmptyNode(), newLit(3))) ## # --> var ## # a: string ## # b = 3 + ## ``` ## ## If you need to create multiple identifiers you need to use the lower level - ## ``newNimNode``: - ## - ## .. code-block:: nim - ## + ## `newNimNode`: + ## ```nim ## result = newNimNode(nnkIdentDefs).add( ## ident("a"), ident("b"), ident("c"), ident("string"), ## newStrLitNode("Hello")) + ## ``` newNimNode(nnkIdentDefs).add(name, kind, default) -proc newNilLit*(): NimNode {.compileTime.} = +proc newNilLit*(): NimNode = ## New nil literal shortcut. result = newNimNode(nnkNilLit) -proc last*(node: NimNode): NimNode {.compileTime.} = node[node.len-1] +proc last*(node: NimNode): NimNode = node[node.len-1] ## Return the last item in nodes children. Same as `node[^1]`. const RoutineNodes* = {nnkProcDef, nnkFuncDef, nnkMethodDef, nnkDo, nnkLambda, - nnkIteratorDef, nnkTemplateDef, nnkConverterDef} + nnkIteratorDef, nnkTemplateDef, nnkConverterDef, nnkMacroDef} AtomicNodes* = {nnkNone..nnkNilLit} - CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, - nnkCallStrLit, nnkHiddenCallConv} + # see matching set nnkCallKinds above + CallNodes* = nnkCallKinds -proc expectKind*(n: NimNode; k: set[NimNodeKind]) {.compileTime.} = +proc expectKind*(n: NimNode; k: set[NimNodeKind]) = ## Checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check the AST that is passed to them. @@ -1100,10 +1228,10 @@ proc newProc*(name = newEmptyNode(); params: openArray[NimNode] = [newEmptyNode()]; body: NimNode = newStmtList(); procType = nnkProcDef; - pragmas: NimNode = newEmptyNode()): NimNode {.compileTime.} = + pragmas: NimNode = newEmptyNode()): NimNode = ## Shortcut for creating a new proc. ## - ## The ``params`` array must start with the return type of the proc, + ## The `params` array must start with the return type of the proc, ## followed by a list of IdentDefs which specify the params. if procType notin RoutineNodes: error("Expected one of " & $RoutineNodes & ", got " & $procType) @@ -1117,17 +1245,14 @@ proc newProc*(name = newEmptyNode(); newEmptyNode(), body) -proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): - NimNode {.compileTime.} = - ## Constructor for ``if`` statements. - ## - ## .. code-block:: nim - ## - ## newIfStmt( - ## (Ident, StmtList), - ## ... - ## ) - ## +proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): NimNode = + ## Constructor for `if` statements. + ## ```nim + ## newIfStmt( + ## (Ident, StmtList), + ## ... + ## ) + ## ``` result = newNimNode(nnkIfStmt) if len(branches) < 1: error("If statement must have at least one branch") @@ -1135,20 +1260,18 @@ proc newIfStmt*(branches: varargs[tuple[cond, body: NimNode]]): result.add(newTree(nnkElifBranch, i.cond, i.body)) proc newEnum*(name: NimNode, fields: openArray[NimNode], - public, pure: bool): NimNode {.compileTime.} = + public, pure: bool): NimNode = ## Creates a new enum. `name` must be an ident. Fields are allowed to be - ## either idents or EnumFieldDef - ## - ## .. code-block:: nim - ## - ## newEnum( - ## name = ident("Colors"), - ## fields = [ident("Blue"), ident("Red")], - ## public = true, pure = false) - ## - ## # type Colors* = Blue Red + ## either idents or EnumFieldDef: + ## ```nim + ## newEnum( + ## name = ident("Colors"), + ## fields = [ident("Blue"), ident("Red")], + ## public = true, pure = false) ## + ## # type Colors* = Blue Red + ## ``` expectKind name, nnkIdent if len(fields) < 1: @@ -1178,7 +1301,7 @@ proc newEnum*(name: NimNode, fields: openArray[NimNode], return typeSect -proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}= +proc copyChildrenTo*(src, dest: NimNode) = ## Copy all children from `src` to `dest`. for i in 0 ..< src.len: dest.add src[i].copyNimTree @@ -1186,7 +1309,7 @@ proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}= template expectRoutine(node: NimNode) = expectKind(node, RoutineNodes) -proc name*(someProc: NimNode): NimNode {.compileTime.} = +proc name*(someProc: NimNode): NimNode = someProc.expectRoutine result = someProc[0] if result.kind == nnkPostfix: @@ -1197,21 +1320,28 @@ proc name*(someProc: NimNode): NimNode {.compileTime.} = elif result.kind == nnkAccQuoted: result = result[0] -proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} = +proc `name=`*(someProc: NimNode; val: NimNode) = someProc.expectRoutine if someProc[0].kind == nnkPostfix: someProc[0][1] = val else: someProc[0] = val -proc params*(someProc: NimNode): NimNode {.compileTime.} = - someProc.expectRoutine - result = someProc[3] -proc `params=`* (someProc: NimNode; params: NimNode) {.compileTime.}= - someProc.expectRoutine +proc params*(someProc: NimNode): NimNode = + if someProc.kind == nnkProcTy: + someProc[0] + else: + someProc.expectRoutine + someProc[3] + +proc `params=`* (someProc: NimNode; params: NimNode) = expectKind(params, nnkFormalParams) - someProc[3] = params + if someProc.kind == nnkProcTy: + someProc[0] = params + else: + someProc.expectRoutine + someProc[3] = params -proc pragma*(someProc: NimNode): NimNode {.compileTime.} = +proc pragma*(someProc: NimNode): NimNode = ## Get the pragma of a proc type. ## These will be expanded. if someProc.kind == nnkProcTy: @@ -1219,7 +1349,7 @@ proc pragma*(someProc: NimNode): NimNode {.compileTime.} = else: someProc.expectRoutine result = someProc[4] -proc `pragma=`*(someProc: NimNode; val: NimNode) {.compileTime.}= +proc `pragma=`*(someProc: NimNode; val: NimNode) = ## Set the pragma of a proc type. expectKind(val, {nnkEmpty, nnkPragma}) if someProc.kind == nnkProcTy: @@ -1228,7 +1358,7 @@ proc `pragma=`*(someProc: NimNode; val: NimNode) {.compileTime.}= someProc.expectRoutine someProc[4] = val -proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = +proc addPragma*(someProc, pragma: NimNode) = ## Adds pragma to routine definition. someProc.expectKind(RoutineNodes + {nnkProcTy}) var pragmaNode = someProc.pragma @@ -1240,7 +1370,7 @@ proc addPragma*(someProc, pragma: NimNode) {.compileTime.} = template badNodeKind(n, f) = error("Invalid node kind " & $n.kind & " for macros.`" & $f & "`", n) -proc body*(someProc: NimNode): NimNode {.compileTime.} = +proc body*(someProc: NimNode): NimNode = case someProc.kind: of RoutineNodes: return someProc[6] @@ -1251,7 +1381,7 @@ proc body*(someProc: NimNode): NimNode {.compileTime.} = else: badNodeKind someProc, "body" -proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = +proc `body=`*(someProc: NimNode, val: NimNode) = case someProc.kind of RoutineNodes: someProc[6] = val @@ -1262,46 +1392,53 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} = else: badNodeKind someProc, "body=" -proc basename*(a: NimNode): NimNode {.compileTime, benign.} +proc basename*(a: NimNode): NimNode = + ## Pull an identifier from prefix/postfix expressions. + case a.kind + of nnkIdent: result = a + of nnkPostfix, nnkPrefix: result = a[1] + of nnkPragmaExpr: result = basename(a[0]) + else: + error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & + repr(a), a) -proc `$`*(node: NimNode): string {.compileTime.} = +proc `$`*(node: NimNode): string = ## Get the string of an identifier node. case node.kind of nnkPostfix: result = node.basename.strVal & "*" of nnkStrLit..nnkTripleStrLit, nnkCommentStmt, nnkSym, nnkIdent: result = node.strVal - of nnkOpenSymChoice, nnkClosedSymChoice: + of nnkOpenSymChoice, nnkClosedSymChoice, nnkOpenSym: result = $node[0] of nnkAccQuoted: - result = $node[0] + result = "" + for i in 0 ..< node.len: + result.add(repr(node[i])) else: badNodeKind node, "$" -proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.} - ## Create a new ident node from a string. - iterator items*(n: NimNode): NimNode {.inline.} = - ## Iterates over the children of the NimNode ``n``. + ## Iterates over the children of the NimNode `n`. for i in 0 ..< n.len: yield n[i] iterator pairs*(n: NimNode): (int, NimNode) {.inline.} = - ## Iterates over the children of the NimNode ``n`` and its indices. + ## Iterates over the children of the NimNode `n` and its indices. for i in 0 ..< n.len: yield (i, n[i]) iterator children*(n: NimNode): NimNode {.inline.} = - ## Iterates over the children of the NimNode ``n``. + ## Iterates over the children of the NimNode `n`. for i in 0 ..< n.len: yield n[i] template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = ## Find the first child node matching condition (or nil). - ## - ## .. code-block:: nim + ## ```nim ## var res = findChild(n, it.kind == nnkPostfix and - ## it.basename.ident == toNimIdent"foo") + ## it.basename.ident == ident"foo") + ## ``` block: var res: NimNode for it in n.children: @@ -1310,8 +1447,8 @@ template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = break res -proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = - ## Insert node ``b`` into node ``a`` at ``pos``. +proc insert*(a: NimNode; pos: int; b: NimNode) = + ## Insert node `b` into node `a` at `pos`. if len(a)-1 < pos: # add some empty nodes first for i in len(a)-1..pos-2: @@ -1325,17 +1462,7 @@ proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} = a[i + 1] = a[i] a[pos] = b -proc basename*(a: NimNode): NimNode = - ## Pull an identifier from prefix/postfix expressions. - case a.kind - of nnkIdent: result = a - of nnkPostfix, nnkPrefix: result = a[1] - of nnkPragmaExpr: result = basename(a[0]) - else: - error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & - repr(a), a) - -proc `basename=`*(a: NimNode; val: string) {.compileTime.}= +proc `basename=`*(a: NimNode; val: string) = case a.kind of nnkIdent: a.strVal = val @@ -1346,103 +1473,49 @@ proc `basename=`*(a: NimNode; val: string) {.compileTime.}= error("Do not know how to get basename of (" & treeRepr(a) & ")\n" & repr(a), a) -proc postfix*(node: NimNode; op: string): NimNode {.compileTime.} = +proc postfix*(node: NimNode; op: string): NimNode = newNimNode(nnkPostfix).add(ident(op), node) -proc prefix*(node: NimNode; op: string): NimNode {.compileTime.} = +proc prefix*(node: NimNode; op: string): NimNode = newNimNode(nnkPrefix).add(ident(op), node) proc infix*(a: NimNode; op: string; - b: NimNode): NimNode {.compileTime.} = + b: NimNode): NimNode = newNimNode(nnkInfix).add(ident(op), a, b) -proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. - compileTime.} = +proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] = node.expectKind nnkPostfix result = (node[1], $node[0]) -proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. - compileTime.} = +proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] = node.expectKind nnkPrefix result = (node[1], $node[0]) -proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; - right: NimNode] {.compileTime.} = +proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] = expectKind(node, nnkInfix) result = (node[1], $node[0], node[2]) -proc copy*(node: NimNode): NimNode {.compileTime.} = +proc copy*(node: NimNode): NimNode = ## An alias for `copyNimTree<#copyNimTree,NimNode>`_. return node.copyNimTree() -when defined(nimVmEqIdent): - proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. - - proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. ``a`` can be an identifier or a - ## symbol. ``a`` may be wrapped in an export marker - ## (``nnkPostfix``) or quoted with backticks (``nnkAccQuoted``), - ## these nodes will be unwrapped. - - proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. ``b`` can be an identifier or a - ## symbol. ``b`` may be wrapped in an export marker - ## (``nnkPostfix``) or quoted with backticks (``nnkAccQuoted``), - ## these nodes will be unwrapped. - - proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. ``a`` and ``b`` can be an - ## identifier or a symbol. Both may be wrapped in an export marker - ## (``nnkPostfix``) or quoted with backticks (``nnkAccQuoted``), - ## these nodes will be unwrapped. +proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} = + ## Check that `eqIdent(n,name)` holds true. If this is not the + ## case, compilation aborts with an error message. This is useful + ## for writing macros that check the AST that is passed to them. + if not eqIdent(n, name): + error("Expected identifier to be `" & name & "` here", n) -else: - # this procedure is optimized for native code, it should not be compiled to nimVM bytecode. - proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = - proc toLower(c: char): char {.inline.} = - if c in {'A'..'Z'}: result = chr(ord(c) + (ord('a') - ord('A'))) - else: result = c - var i = 0 - var j = 0 - # first char is case sensitive - if a[0] != b[0]: return 1 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - - - proc eqIdent*(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 - ## Check if two idents are equal. - - proc eqIdent*(node: NimNode; s: string): bool {.compileTime.} = - ## Check if node is some identifier node (``nnkIdent``, ``nnkSym``, etc.) - ## is the same as ``s``. Note that this is the preferred way to check! Most - ## other ways like ``node.ident`` are much more error-prone, unfortunately. - case node.kind - of nnkSym, nnkIdent: - result = eqIdent(node.strVal, s) - of nnkOpenSymChoice, nnkClosedSymChoice: - result = eqIdent($node[0], s) - else: - result = false - -proc hasArgOfName*(params: NimNode; name: string): bool {.compileTime.}= - ## Search ``nnkFormalParams`` for an argument. +proc hasArgOfName*(params: NimNode; name: string): bool = + ## Search `nnkFormalParams` for an argument. expectKind(params, nnkFormalParams) - for i in 1 ..< params.len: - template node: untyped = params[i] - if name.eqIdent( $ node[0]): - return true + for i in 1..<params.len: + for j in 0..<params[i].len-2: + if name.eqIdent($params[i][j]): + return true -proc addIdentIfAbsent*(dest: NimNode, ident: string) {.compileTime.} = - ## Add ``ident`` to ``dest`` if it is not present. This is intended for use +proc addIdentIfAbsent*(dest: NimNode, ident: string) = + ## Add `ident` to `dest` if it is not present. This is intended for use ## with pragmas. for node in dest.children: case node.kind @@ -1453,15 +1526,14 @@ proc addIdentIfAbsent*(dest: NimNode, ident: string) {.compileTime.} = else: discard dest.add(ident(ident)) -proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = +proc boolVal*(n: NimNode): bool {.noSideEffect.} = if n.kind == nnkIntLit: n.intVal != 0 else: n == bindSym"true" # hacky solution for now -when defined(nimMacrosGetNodeId): - proc nodeID*(n: NimNode): int {.magic: NodeId.} - ## Returns the id of ``n``, when the compiler has been compiled - ## with the flag ``-d:useNodeids``, otherwise returns ``-1``. This - ## proc is for the purpose to debug the compiler only. +proc nodeID*(n: NimNode): int {.magic: "NodeId".} + ## Returns the id of `n`, when the compiler has been compiled + ## with the flag `-d:useNodeids`, otherwise returns `-1`. This + ## proc is for the purpose to debug the compiler only. macro expandMacros*(body: typed): untyped = ## Expands one level of macro - useful for debugging. @@ -1470,31 +1542,53 @@ macro expandMacros*(body: typed): untyped = ## ## For instance, ## - ## .. code-block:: nim - ## import future, macros + ## ```nim + ## import std/[sugar, macros] ## ## let ## x = 10 ## y = 20 ## expandMacros: ## dump(x + y) + ## ``` ## ## will actually dump `x + y`, but at the same time will print at - ## compile time the expansion of the ``dump`` macro, which in this - ## case is ``debugEcho ["x + y", " = ", x + y]``. + ## compile time the expansion of the `dump` macro, which in this + ## case is `debugEcho ["x + y", " = ", x + y]`. echo body.toStrLit result = body +proc extractTypeImpl(n: NimNode): NimNode = + ## attempts to extract the type definition of the given symbol + case n.kind + of nnkSym: # can extract an impl + result = n.getImpl.extractTypeImpl() + of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n + of nnkBracketExpr: + if n.typeKind == ntyTypeDesc: + result = n[1].extractTypeImpl() + else: + doAssert n.typeKind == ntyGenericInst + result = n[0].getImpl() + of nnkTypeDef: + result = n[2] + else: error("Invalid node to retrieve type implementation of: " & $n.kind) + proc customPragmaNode(n: NimNode): NimNode = - expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) + expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkType, nnkCheckedFieldExpr}) let typ = n.getTypeInst() if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy: return typ[1][1] elif typ.typeKind == ntyTypeDesc: - let impl = typ[1].getImpl() - if impl[0].kind == nnkPragmaExpr: + let impl = getImpl( + if kind(typ[1]) == nnkBracketExpr: typ[1][0] + else: typ[1] + ) + if impl.kind == nnkNilLit: + return impl + elif impl[0].kind == nnkPragmaExpr: return impl[0][1] else: return impl[0] # handle types which don't have macro at all @@ -1503,10 +1597,10 @@ proc customPragmaNode(n: NimNode): NimNode = let impl = n.getImpl() if impl.kind in RoutineNodes: return impl.pragma - elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr: + elif impl.kind in {nnkIdentDefs, nnkConstDef} and impl[0].kind == nnkPragmaExpr: return impl[0][1] else: - let timpl = typ.getImpl() + let timpl = getImpl(if typ.kind == nnkBracketExpr: typ[0] else: typ) if timpl.len>0 and timpl[0].len>1: return timpl[0][1] else: @@ -1514,12 +1608,13 @@ proc customPragmaNode(n: NimNode): NimNode = if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) - let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) - var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst) + var typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) + while typInst.kind in {nnkVarTy, nnkBracketExpr}: typInst = typInst[0] + var typDef = getImpl(typInst) while typDef != nil: typDef.expectKind(nnkTypeDef) - let typ = typDef[2] - typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) + let typ = typDef[2].extractTypeImpl() + if typ.kind notin {nnkRefTy, nnkPtrTy, nnkObjectTy}: break let isRef = typ.kind in {nnkRefTy, nnkPtrTy} if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) typDef = getImpl(typ[0]) @@ -1530,18 +1625,17 @@ proc customPragmaNode(n: NimNode): NimNode = for i in 0..<identDefsStack.len: identDefsStack[i] = obj[2][i] while identDefsStack.len > 0: var identDefs = identDefsStack.pop() - if identDefs.kind == nnkRecCase: - identDefsStack.add(identDefs[0]) - for i in 1..<identDefs.len: - let varNode = identDefs[i] - # if it is and empty branch, skip - if varNode[0].kind == nnkNilLit: continue - if varNode[1].kind == nnkIdentDefs: - identDefsStack.add(varNode[1]) - else: # nnkRecList - for j in 0 ..< varNode[1].len: - identDefsStack.add(varNode[1][j]) + case identDefs.kind + of nnkRecList: + for child in identDefs.children: + identDefsStack.add(child) + of nnkRecCase: + # Add condition definition + identDefsStack.add(identDefs[0]) + # Add branches + for i in 1 ..< identDefs.len: + identDefsStack.add(identDefs[i].last) else: for i in 0 .. identDefs.len - 3: let varNode = identDefs[i] @@ -1562,9 +1656,9 @@ macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr` ## (if checking a field), a proc or a type has custom pragma `cp`. ## - ## See also `getCustomPragmaVal`. + ## See also `getCustomPragmaVal`_. ## - ## .. code-block:: nim + ## ```nim ## template myAttr() {.pragma.} ## type ## MyObj = object @@ -1575,6 +1669,7 @@ macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = ## var o: MyObj ## assert(o.myField.hasCustomPragma(myAttr)) ## assert(myProc.hasCustomPragma(myAttr)) + ## ``` let pragmaNode = customPragmaNode(n) for p in pragmaNode: if (p.kind == nnkSym and p == cp) or @@ -1586,9 +1681,9 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = ## Expands to value of custom pragma `cp` of expression `n` which is expected ## to be `nnkDotExpr`, a proc or a type. ## - ## See also `hasCustomPragma` + ## See also `hasCustomPragma`_. ## - ## .. code-block:: nim + ## ```nim ## template serializationKey(key: string) {.pragma.} ## type ## MyObj {.serializationKey: "mo".} = object @@ -1597,10 +1692,12 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = ## assert(o.myField.getCustomPragmaVal(serializationKey) == "mf") ## assert(o.getCustomPragmaVal(serializationKey) == "mo") ## assert(MyObj.getCustomPragmaVal(serializationKey) == "mo") + ## ``` + result = nil let pragmaNode = customPragmaNode(n) for p in pragmaNode: if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: - if p.len == 2: + if p.len == 2 or (p.len == 3 and p[1].kind == nnkSym and p[1].symKind == nskType): result = p[1] else: let def = p[0].getImpl[3] @@ -1613,22 +1710,22 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = if result.kind == nnkEmpty: error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error, - -when not defined(booting): - template emit*(e: static[string]): untyped {.deprecated.} = - ## Accepts a single string argument and treats it as nim code - ## that should be inserted verbatim in the program - ## Example: - ## - ## .. code-block:: nim - ## emit("echo " & '"' & "hello world".toUpper & '"') - ## - ## Deprecated since version 0.15 since it's so rarely useful. - macro payload: untyped {.gensym.} = - result = parseStmt(e) - payload() - macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped = + ## Calls `callee` with `args` unpacked as individual arguments. + ## This is useful in 2 cases: + ## * when forwarding `varargs[T]` for some typed `T` + ## * when forwarding `varargs[untyped]` when `args` can potentially be empty, + ## due to a compiler limitation + runnableExamples: + template call1(fun: typed; args: varargs[untyped]): untyped = + unpackVarargs(fun, args) + # when varargsLen(args) > 0: fun(args) else: fun() # this would also work + template call2(fun: typed; args: varargs[typed]): untyped = + unpackVarargs(fun, args) + proc fn1(a = 0, b = 1) = discard (a, b) + call1(fn1, 10, 11) + call1(fn1) # `args` is empty in this case + if false: call2(echo, 10, 11) # would print 1011 result = newCall(callee) for i in 0 ..< args.len: result.add args[i] @@ -1640,17 +1737,17 @@ proc getProjectPath*(): string = discard ## which returns the path of the source file containing that template ## call. ## - ## For example, assume a ``dir1/foo.nim`` that imports a ``dir2/bar.nim``, - ## have the ``bar.nim`` print out both ``getProjectPath`` and - ## ``currentSourcePath`` outputs. + ## For example, assume a `dir1/foo.nim` that imports a `dir2/bar.nim`, + ## have the `bar.nim` print out both `getProjectPath` and + ## `currentSourcePath` outputs. ## - ## Now when ``foo.nim`` is compiled, the ``getProjectPath`` from - ## ``bar.nim`` will return the ``dir1/`` path, while the ``currentSourcePath`` - ## will return the path to the ``bar.nim`` source file. + ## Now when `foo.nim` is compiled, the `getProjectPath` from + ## `bar.nim` will return the `dir1/` path, while the `currentSourcePath` + ## will return the path to the `bar.nim` source file. ## - ## Now when ``bar.nim`` is compiled directly, the ``getProjectPath`` - ## will now return the ``dir2/`` path, and the ``currentSourcePath`` - ## will still return the same path, the path to the ``bar.nim`` source + ## Now when `bar.nim` is compiled directly, the `getProjectPath` + ## will now return the `dir2/` path, and the `currentSourcePath` + ## will still return the same path, the path to the `bar.nim` source ## file. ## ## The path returned by this proc is set at compile time. @@ -1658,22 +1755,58 @@ proc getProjectPath*(): string = discard ## See also: ## * `getCurrentDir proc <os.html#getCurrentDir>`_ -when defined(nimMacrosSizealignof): - proc getSize*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = - ## Returns the same result as ``system.sizeof`` if the size is - ## known by the Nim compiler. Returns a negative value if the Nim - ## compiler does not know the size. - proc getAlign*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = - ## Returns the same result as ``system.alignof`` if the alignment - ## is known by the Nim compiler. It works on ``NimNode`` for use - ## in macro context. Returns a negative value if the Nim compiler - ## does not know the alignment. - proc getOffset*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = - ## Returns the same result as ``system.offsetof`` if the offset is - ## known by the Nim compiler. It expects a resolved symbol node - ## from a field of a type. Therefore it only requires one argument - ## instead of two. Returns a negative value if the Nim compiler - ## does not know the offset. +proc getSize*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = + ## Returns the same result as `system.sizeof` if the size is + ## known by the Nim compiler. Returns a negative value if the Nim + ## compiler does not know the size. +proc getAlign*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = + ## Returns the same result as `system.alignof` if the alignment + ## is known by the Nim compiler. It works on `NimNode` for use + ## in macro context. Returns a negative value if the Nim compiler + ## does not know the alignment. +proc getOffset*(arg: NimNode): int {.magic: "NSizeOf", noSideEffect.} = + ## Returns the same result as `system.offsetof` if the offset is + ## known by the Nim compiler. It expects a resolved symbol node + ## from a field of a type. Therefore it only requires one argument + ## instead of two. Returns a negative value if the Nim compiler + ## does not know the offset. proc isExported*(n: NimNode): bool {.noSideEffect.} = ## Returns whether the symbol is exported or not. + +proc extractDocCommentsAndRunnables*(n: NimNode): NimNode = + ## returns a `nnkStmtList` containing the top-level doc comments and + ## runnableExamples in `a`, stopping at the first child that is neither. + ## Example: + ## + ## ```nim + ## import std/macros + ## macro transf(a): untyped = + ## result = quote do: + ## proc fun2*() = discard + ## let header = extractDocCommentsAndRunnables(a.body) + ## # correct usage: rest is appended + ## result.body = header + ## result.body.add quote do: discard # just an example + ## # incorrect usage: nesting inside a nnkStmtList: + ## # result.body = quote do: (`header`; discard) + ## + ## proc fun*() {.transf.} = + ## ## first comment + ## runnableExamples: discard + ## runnableExamples: discard + ## ## last comment + ## discard # first statement after doc comments + runnableExamples + ## ## not docgen'd + ## ``` + + result = newStmtList() + for ni in n: + case ni.kind + of nnkCommentStmt: + result.add ni + of nnkCall, nnkCommand: + if ni[0].kind == nnkIdent and ni[0].eqIdent "runnableExamples": + result.add ni + else: break + else: break diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 34d7687f9..8cb0cef05 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -11,10 +11,12 @@ when not compileOption("threads") and not defined(nimdoc): - {.error: "Rlocks requires --threads:on option.".} + when false: + # make rlocks modlue consistent with locks module, + # so they can replace each other seamlessly. + {.error: "Rlocks requires --threads:on option.".} -const insideRLocksModule = true -include "system/syslocks" +import std/private/syslocks type RLock* = SysLock ## Nim lock, re-entrant @@ -24,32 +26,32 @@ proc initRLock*(lock: var RLock) {.inline.} = when defined(posix): var a: SysLockAttr initSysLockAttr(a) - setSysLockType(a, SysLockType_Reentrant()) + setSysLockType(a, SysLockType_Reentrant) initSysLock(lock, a.addr) else: initSysLock(lock) -proc deinitRLock*(lock: var RLock) {.inline.} = +proc deinitRLock*(lock: RLock) {.inline.} = ## Frees the resources associated with the lock. deinitSys(lock) -proc tryAcquire*(lock: var RLock): bool = +proc tryAcquire*(lock: var RLock): bool {.inline.} = ## Tries to acquire the given lock. Returns `true` on success. result = tryAcquireSys(lock) -proc acquire*(lock: var RLock) = +proc acquire*(lock: var RLock) {.inline.} = ## Acquires the given lock. acquireSys(lock) -proc release*(lock: var RLock) = +proc release*(lock: var RLock) {.inline.} = ## Releases the given lock. releaseSys(lock) -template withRLock*(lock: var RLock, code: untyped): untyped = +template withRLock*(lock: RLock, code: untyped) = ## Acquires the given lock and then executes the code. - block: - acquire(lock) - defer: - release(lock) - {.locks: [lock].}: + acquire(lock) + {.locks: [lock].}: + try: code + finally: + release(lock) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index d8bbfb642..f2fee91c4 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -9,18 +9,29 @@ ## This module implements an interface to Nim's `runtime type information`:idx: ## (`RTTI`:idx:). See the `marshal <marshal.html>`_ module for an example of -## what this module allows you to do. +## what this allows you to do. ## -## Note that even though ``Any`` and its operations hide the nasty low level -## details from its clients, it remains inherently unsafe! Also, Nim's -## runtime type information will evolve and may eventually be deprecated. -## As an alternative approach to programmatically understanding and -## manipulating types, consider using the `macros <macros.html>`_ package to -## work with the types' AST representation at compile time. See, for example, -## the `getTypeImpl proc<macros.html#getTypeImpl,NimNode>`_. As an alternative -## approach to storing arbitrary types at runtime, consider using generics. -## -## +## .. note:: Even though `Any` and its operations hide the nasty low level +## details from its users, it remains inherently unsafe! Also, Nim's +## runtime type information will evolve and may eventually be deprecated. +## As an alternative approach to programmatically understanding and +## manipulating types, consider using the `macros <macros.html>`_ module to +## work with the types' AST representation at compile time. See for example +## the `getTypeImpl proc <macros.html#getTypeImpl,NimNode>`_. As an alternative +## approach to storing arbitrary types at runtime, consider using generics. + +runnableExamples: + var x: Any + + var i = 42 + x = i.toAny + assert x.kind == akInt + assert x.getInt == 42 + + var s = @[1, 2, 3] + x = s.toAny + assert x.kind == akSequence + assert x.len == 3 {.push hints: off.} @@ -29,45 +40,50 @@ include "system/hti.nim" {.pop.} +when defined(nimPreviewSlimSystem): + import std/assertions + + type - AnyKind* = enum ## what kind of ``any`` it is - akNone = 0, ## invalid any - akBool = 1, ## any represents a ``bool`` - akChar = 2, ## any represents a ``char`` - akEnum = 14, ## any represents an enum - akArray = 16, ## any represents an array - akObject = 17, ## any represents an object - akTuple = 18, ## any represents a tuple - akSet = 19, ## any represents a set - akRange = 20, ## any represents a range - akPtr = 21, ## any represents a ptr - akRef = 22, ## any represents a ref - akSequence = 24, ## any represents a sequence - akProc = 25, ## any represents a proc - akPointer = 26, ## any represents a pointer - akString = 28, ## any represents a string - akCString = 29, ## any represents a cstring - akInt = 31, ## any represents an int - akInt8 = 32, ## any represents an int8 - akInt16 = 33, ## any represents an int16 - akInt32 = 34, ## any represents an int32 - akInt64 = 35, ## any represents an int64 - akFloat = 36, ## any represents a float - akFloat32 = 37, ## any represents a float32 - akFloat64 = 38, ## any represents a float64 - akFloat128 = 39, ## any represents a float128 - akUInt = 40, ## any represents an unsigned int - akUInt8 = 41, ## any represents an unsigned int8 - akUInt16 = 42, ## any represents an unsigned in16 - akUInt32 = 43, ## any represents an unsigned int32 - akUInt64 = 44, ## any represents an unsigned int64 + AnyKind* = enum ## The kind of `Any`. + akNone = 0, ## invalid + akBool = 1, ## bool + akChar = 2, ## char + akEnum = 14, ## enum + akArray = 16, ## array + akObject = 17, ## object + akTuple = 18, ## tuple + akSet = 19, ## set + akRange = 20, ## range + akPtr = 21, ## ptr + akRef = 22, ## ref + akSequence = 24, ## sequence + akProc = 25, ## proc + akPointer = 26, ## pointer + akString = 28, ## string + akCString = 29, ## cstring + akInt = 31, ## int + akInt8 = 32, ## int8 + akInt16 = 33, ## int16 + akInt32 = 34, ## int32 + akInt64 = 35, ## int64 + akFloat = 36, ## float + akFloat32 = 37, ## float32 + akFloat64 = 38, ## float64 + akFloat128 = 39, ## float128 + akUInt = 40, ## uint + akUInt8 = 41, ## uint8 + akUInt16 = 42, ## uin16 + akUInt32 = 43, ## uint32 + akUInt64 = 44, ## uint64 # akOpt = 44+18 ## the builtin 'opt' type. - Any* = object ## can represent any nim value; NOTE: the wrapped - ## value can be modified with its wrapper! This means - ## that ``Any`` keeps a non-traced pointer to its - ## wrapped value and **must not** live longer than - ## its wrapped value. + Any* = object + ## A type that can represent any nim value. + ## + ## .. danger:: The wrapped value can be modified with its wrapper! This means + ## that `Any` keeps a non-traced pointer to its wrapped value and + ## **must not** live longer than its wrapped value. value: pointer when defined(js): rawType: PNimType @@ -75,13 +91,25 @@ type rawTypePtr: pointer ppointer = ptr pointer - pbyteArray = ptr array[0xffff, int8] + pbyteArray = ptr array[0xffff, uint8] - TGenericSeq {.importc.} = object - len, space: int - when defined(gogc): - elemSize: int - PGenSeq = ptr TGenericSeq +when not defined(gcDestructors): + type + TGenericSeq {.importc.} = object + len, space: int + when defined(gogc): + elemSize: int + PGenSeq = ptr TGenericSeq + + when defined(gogc): + const GenericSeqSize = 3 * sizeof(int) + else: + const GenericSeqSize = 2 * sizeof(int) + +else: + include system/seqs_v2_reimpl + +from std/private/strimpl import cmpNimIdentifier when not defined(js): template rawType(x: Any): PNimType = @@ -90,39 +118,42 @@ when not defined(js): template `rawType=`(x: var Any, p: PNimType) = x.rawTypePtr = cast[pointer](p) -when defined(gogc): - const GenericSeqSize = (3 * sizeof(int)) -else: - const GenericSeqSize = (2 * sizeof(int)) - proc genericAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} -proc genericShallowAssign(dest, src: pointer, mt: PNimType) {. - importCompilerProc.} -proc incrSeq(seq: PGenSeq, elemSize: int): PGenSeq {.importCompilerProc.} -proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.} -proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.} -proc objectInit(dest: pointer, typ: PNimType) {.importCompilerProc.} -template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b) +when not defined(gcDestructors): + proc genericShallowAssign(dest, src: pointer, mt: PNimType) {.importCompilerProc.} + proc incrSeq(seq: PGenSeq, elemSize, elemAlign: int): PGenSeq {.importCompilerProc.} + proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.} + proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.} + proc objectInit(dest: pointer, typ: PNimType) {.importCompilerProc.} +else: + proc nimNewObj(size, align: int): pointer {.importCompilerProc.} + proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.importCompilerProc.} + proc prepareSeqAddUninit(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + importCompilerProc.} + proc zeroNewElements(len: int; p: pointer; addlen, elemSize, elemAlign: int) {. + importCompilerProc.} + +template `+!!`(a, b): untyped = cast[pointer](cast[int](a) + b) proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = assert(n.kind == nkCase) var d: int - var a = cast[ByteAddress](aa) + let a = cast[int](aa) case n.typ.size - of 1: d = ze(cast[ptr int8](a +% n.offset)[]) - of 2: d = ze(cast[ptr int16](a +% n.offset)[]) - of 4: d = int(cast[ptr int32](a +% n.offset)[]) - of 8: d = int(cast[ptr int64](a +% n.offset)[]) + of 1: d = int(cast[ptr uint8](a +% n.offset)[]) + of 2: d = int(cast[ptr uint16](a +% n.offset)[]) + of 4: d = int(cast[ptr uint32](a +% n.offset)[]) + of 8: d = int(cast[ptr uint64](a +% n.offset)[]) else: assert(false) return d proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = - var discr = getDiscriminant(aa, n) + let discr = getDiscriminant(aa, n) if discr <% n.len: result = n.sons[discr] if result == nil: result = n.sons[n.len] - # n.sons[n.len] contains the ``else`` part (but may be nil) + # n.sons[n.len] contains the `else` part (but may be nil) else: result = n.sons[n.len] @@ -130,148 +161,188 @@ proc newAny(value: pointer, rawType: PNimType): Any {.inline.} = result.value = value result.rawType = rawType -when declared(system.VarSlot): - proc toAny*(x: VarSlot): Any {.inline.} = - ## constructs a ``Any`` object from a variable slot ``x``. - ## This captures `x`'s address, so `x` can be modified with its - ## ``Any`` wrapper! The client needs to ensure that the wrapper - ## **does not** live longer than `x`! - ## This is provided for easier reflection capabilities of a debugger. - result.value = x.address - result.rawType = x.typ - proc toAny*[T](x: var T): Any {.inline.} = - ## constructs a ``Any`` object from `x`. This captures `x`'s address, so - ## `x` can be modified with its ``Any`` wrapper! The client needs to ensure + ## Constructs an `Any` object from `x`. This captures `x`'s address, so + ## `x` can be modified with its `Any` wrapper! The caller needs to ensure ## that the wrapper **does not** live longer than `x`! newAny(addr(x), cast[PNimType](getTypeInfo(x))) proc kind*(x: Any): AnyKind {.inline.} = - ## get the type kind + ## Gets the type kind. result = AnyKind(ord(x.rawType.kind)) proc size*(x: Any): int {.inline.} = - ## returns the size of `x`'s type. + ## Returns the size of `x`'s type. result = x.rawType.size proc baseTypeKind*(x: Any): AnyKind {.inline.} = - ## get the base type's kind; ``akNone`` is returned if `x` has no base type. + ## Gets the base type's kind. If `x` has no base type, `akNone` is returned. if x.rawType.base != nil: result = AnyKind(ord(x.rawType.base.kind)) proc baseTypeSize*(x: Any): int {.inline.} = - ## returns the size of `x`'s basetype. + ## Returns the size of `x`'s base type. If `x` has no base type, 0 is returned. if x.rawType.base != nil: result = x.rawType.base.size proc invokeNew*(x: Any) = - ## performs ``new(x)``. `x` needs to represent a ``ref``. + ## Performs `new(x)`. `x` needs to represent a `ref`. assert x.rawType.kind == tyRef - var z = newObj(x.rawType, x.rawType.base.size) - genericAssign(x.value, addr(z), x.rawType) + when defined(gcDestructors): + cast[ppointer](x.value)[] = nimNewObj(x.rawType.base.size, x.rawType.base.align) + else: + var z = newObj(x.rawType, x.rawType.base.size) + genericAssign(x.value, addr(z), x.rawType) proc invokeNewSeq*(x: Any, len: int) = - ## performs ``newSeq(x, len)``. `x` needs to represent a ``seq``. + ## Performs `newSeq(x, len)`. `x` needs to represent a `seq`. assert x.rawType.kind == tySequence - var z = newSeq(x.rawType, len) - genericShallowAssign(x.value, addr(z), x.rawType) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + s.len = len + let elem = x.rawType.base + s.p = cast[ptr NimSeqPayloadReimpl](newSeqPayload(len, elem.size, elem.align)) + else: + var z = newSeq(x.rawType, len) + genericShallowAssign(x.value, addr(z), x.rawType) proc extendSeq*(x: Any) = - ## performs ``setLen(x, x.len+1)``. `x` needs to represent a ``seq``. + ## Performs `setLen(x, x.len+1)`. `x` needs to represent a `seq`. assert x.rawType.kind == tySequence - var y = cast[ptr PGenSeq](x.value)[] - var z = incrSeq(y, x.rawType.base.size) - # 'incrSeq' already freed the memory for us and copied over the RC! - # So we simply copy the raw pointer into 'x.value': - cast[ppointer](x.value)[] = z - #genericShallowAssign(x.value, addr(z), x.rawType) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + let elem = x.rawType.base + if s.p == nil or s.p.cap < s.len+1: + s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAddUninit(s.len, s.p, 1, elem.size, elem.align)) + zeroNewElements(s.len, s.p, 1, elem.size, elem.align) + inc s.len + else: + var y = cast[ptr PGenSeq](x.value)[] + var z = incrSeq(y, x.rawType.base.size, x.rawType.base.align) + # 'incrSeq' already freed the memory for us and copied over the RC! + # So we simply copy the raw pointer into 'x.value': + cast[ppointer](x.value)[] = z + #genericShallowAssign(x.value, addr(z), x.rawType) proc setObjectRuntimeType*(x: Any) = - ## this needs to be called to set `x`'s runtime object type field. + ## This needs to be called to set `x`'s runtime object type field. assert x.rawType.kind == tyObject - objectInit(x.value, x.rawType) + when defined(gcDestructors): + cast[ppointer](x.value)[] = x.rawType.typeInfoV2 + else: + objectInit(x.value, x.rawType) proc skipRange(x: PNimType): PNimType {.inline.} = result = x if result.kind == tyRange: result = result.base +proc align(address, alignment: int): int = + result = (address + (alignment - 1)) and not (alignment - 1) + proc `[]`*(x: Any, i: int): Any = - ## accessor for an any `x` that represents an array or a sequence. + ## Accessor for an any `x` that represents an array or a sequence. case x.rawType.kind of tyArray: - var bs = x.rawType.base.size + let bs = x.rawType.base.size if i >=% x.rawType.size div bs: - raise newException(IndexError, formatErrorIndexBound(i, x.rawType.size div bs)) + raise newException(IndexDefect, formatErrorIndexBound(i, x.rawType.size div bs)) return newAny(x.value +!! i*bs, x.rawType.base) of tySequence: - var s = cast[ppointer](x.value)[] - if s == nil: raise newException(ValueError, "sequence is nil") - var bs = x.rawType.base.size - if i >=% cast[PGenSeq](s).len: - raise newException(IndexError, formatErrorIndexBound(i, cast[PGenSeq](s).len-1)) - return newAny(s +!! (GenericSeqSize+i*bs), x.rawType.base) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + if i >=% s.len: + raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) + let bs = x.rawType.base.size + let ba = x.rawType.base.align + let headerSize = align(sizeof(int), ba) + return newAny(s.p +!! (headerSize+i*bs), x.rawType.base) + else: + var s = cast[ppointer](x.value)[] + if s == nil: raise newException(ValueError, "sequence is nil") + let bs = x.rawType.base.size + if i >=% cast[PGenSeq](s).len: + raise newException(IndexDefect, formatErrorIndexBound(i, cast[PGenSeq](s).len-1)) + return newAny(s +!! (align(GenericSeqSize, x.rawType.base.align)+i*bs), x.rawType.base) else: assert false proc `[]=`*(x: Any, i: int, y: Any) = - ## accessor for an any `x` that represents an array or a sequence. + ## Accessor for an any `x` that represents an array or a sequence. case x.rawType.kind of tyArray: var bs = x.rawType.base.size if i >=% x.rawType.size div bs: - raise newException(IndexError, formatErrorIndexBound(i, x.rawType.size div bs)) + raise newException(IndexDefect, formatErrorIndexBound(i, x.rawType.size div bs)) assert y.rawType == x.rawType.base genericAssign(x.value +!! i*bs, y.value, y.rawType) of tySequence: - var s = cast[ppointer](x.value)[] - if s == nil: raise newException(ValueError, "sequence is nil") - var bs = x.rawType.base.size - if i >=% cast[PGenSeq](s).len: - raise newException(IndexError, formatErrorIndexBound(i, cast[PGenSeq](s).len-1)) - assert y.rawType == x.rawType.base - genericAssign(s +!! (GenericSeqSize+i*bs), y.value, y.rawType) + when defined(gcDestructors): + var s = cast[ptr NimSeqV2Reimpl](x.value) + if i >=% s.len: + raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) + let bs = x.rawType.base.size + let ba = x.rawType.base.align + let headerSize = align(sizeof(int), ba) + assert y.rawType == x.rawType.base + genericAssign(s.p +!! (headerSize+i*bs), y.value, y.rawType) + else: + var s = cast[ppointer](x.value)[] + if s == nil: raise newException(ValueError, "sequence is nil") + var bs = x.rawType.base.size + if i >=% cast[PGenSeq](s).len: + raise newException(IndexDefect, formatErrorIndexBound(i, cast[PGenSeq](s).len-1)) + assert y.rawType == x.rawType.base + genericAssign(s +!! (align(GenericSeqSize, x.rawType.base.align)+i*bs), y.value, y.rawType) else: assert false proc len*(x: Any): int = - ## len for an any `x` that represents an array or a sequence. + ## `len` for an any `x` that represents an array or a sequence. case x.rawType.kind of tyArray: result = x.rawType.size div x.rawType.base.size of tySequence: - let pgenSeq = cast[PGenSeq](cast[ppointer](x.value)[]) - if isNil(pgenSeq): - result = 0 + when defined(gcDestructors): + result = cast[ptr NimSeqV2Reimpl](x.value).len else: - result = pgenSeq.len + let pgenSeq = cast[PGenSeq](cast[ppointer](x.value)[]) + if isNil(pgenSeq): + result = 0 + else: + result = pgenSeq.len else: assert false proc base*(x: Any): Any = - ## returns base Any (useful for inherited object types). + ## Returns the base type of `x` (useful for inherited object types). result.rawType = x.rawType.base result.value = x.value proc isNil*(x: Any): bool = - ## `isNil` for an any `x` that represents a cstring, proc or + ## `isNil` for an `x` that represents a cstring, proc or ## some pointer type. - assert x.rawType.kind in {tyCString, tyRef, tyPtr, tyPointer, tyProc} + assert x.rawType.kind in {tyCstring, tyRef, tyPtr, tyPointer, tyProc} result = isNil(cast[ppointer](x.value)[]) +const pointerLike = + when defined(gcDestructors): {tyCstring, tyRef, tyPtr, tyPointer, tyProc} + else: {tyString, tyCstring, tyRef, tyPtr, tyPointer, tySequence, tyProc} + proc getPointer*(x: Any): pointer = - ## retrieve the pointer value out of `x`. ``x`` needs to be of kind - ## ``akString``, ``akCString``, ``akProc``, ``akRef``, ``akPtr``, - ## ``akPointer``, ``akSequence``. - assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer, - tySequence, tyProc} + ## Retrieves the pointer value out of `x`. `x` needs to be of kind + ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`, + ## `akPointer` or `akSequence`. + assert x.rawType.kind in pointerLike result = cast[ppointer](x.value)[] proc setPointer*(x: Any, y: pointer) = - ## sets the pointer value of `x`. ``x`` needs to be of kind - ## ``akString``, ``akCString``, ``akProc``, ``akRef``, ``akPtr``, - ## ``akPointer``, ``akSequence``. - assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer, - tySequence, tyProc} - cast[ppointer](x.value)[] = y + ## Sets the pointer value of `x`. `x` needs to be of kind + ## `akString`, `akCString`, `akProc`, `akRef`, `akPtr`, + ## `akPointer` or `akSequence`. + assert x.rawType.kind in pointerLike + if y != nil and x.rawType.kind != tyPointer: + genericAssign(x.value, y, x.rawType) + else: + cast[ppointer](x.value)[] = y proc fieldsAux(p: pointer, n: ptr TNimNode, ret: var seq[tuple[name: cstring, any: Any]]) = @@ -288,15 +359,15 @@ proc fieldsAux(p: pointer, n: ptr TNimNode, if m != nil: fieldsAux(p, m, ret) iterator fields*(x: Any): tuple[name: string, any: Any] = - ## iterates over every active field of the any `x` that represents an object + ## Iterates over every active field of `x`. `x` needs to represent an object ## or a tuple. assert x.rawType.kind in {tyTuple, tyObject} - var p = x.value + let p = x.value var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] - var ret: seq[tuple[name: cstring, any: Any]] = @[] + var ret: seq[tuple[name: cstring, any: Any]] if t.kind == tyObject: while true: fieldsAux(p, t.node, ret) @@ -307,48 +378,31 @@ iterator fields*(x: Any): tuple[name: string, any: Any] = for name, any in items(ret): yield ($name, any) -proc cmpIgnoreStyle(a, b: cstring): int {.noSideEffect.} = - proc toLower(c: char): char {.inline.} = - if c in {'A'..'Z'}: result = chr(ord(c) + (ord('a') - ord('A'))) - else: result = c - var i = 0 - var j = 0 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLower(a[i]) - var bb = toLower(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) - -proc getFieldNode(p: pointer, n: ptr TNimNode, - name: cstring): ptr TNimNode = +proc getFieldNode(p: pointer, n: ptr TNimNode, name: cstring): ptr TNimNode = case n.kind of nkNone: assert(false) of nkSlot: - if cmpIgnoreStyle(n.name, name) == 0: + if cmpNimIdentifier(n.name, name) == 0: result = n of nkList: for i in 0..n.len-1: result = getFieldNode(p, n.sons[i], name) if result != nil: break of nkCase: - if cmpIgnoreStyle(n.name, name) == 0: + if cmpNimIdentifier(n.name, name) == 0: result = n else: - var m = selectBranch(p, n) + let m = selectBranch(p, n) if m != nil: result = getFieldNode(p, m, name) proc `[]=`*(x: Any, fieldName: string, value: Any) = - ## sets a field of `x`; `x` represents an object or a tuple. + ## Sets a field of `x`. `x` needs to represent an object or a tuple. var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] assert x.rawType.kind in {tyTuple, tyObject} - var n = getFieldNode(x.value, t.node, fieldName) + let n = getFieldNode(x.value, t.node, fieldName) if n != nil: assert n.typ == value.rawType genericAssign(x.value +!! n.offset, value.value, value.rawType) @@ -356,13 +410,13 @@ proc `[]=`*(x: Any, fieldName: string, value: Any) = raise newException(ValueError, "invalid field name: " & fieldName) proc `[]`*(x: Any, fieldName: string): Any = - ## gets a field of `x`; `x` represents an object or a tuple. + ## Gets a field of `x`. `x` needs to represent an object or a tuple. var t = x.rawType # XXX BUG: does not work yet, however is questionable anyway when false: if x.rawType.kind == tyObject: t = cast[ptr PNimType](x.value)[] assert x.rawType.kind in {tyTuple, tyObject} - var n = getFieldNode(x.value, t.node, fieldName) + let n = getFieldNode(x.value, t.node, fieldName) if n != nil: result.value = x.value +!! n.offset result.rawType = n.typ @@ -372,47 +426,47 @@ proc `[]`*(x: Any, fieldName: string): Any = raise newException(ValueError, "invalid field name: " & fieldName) proc `[]`*(x: Any): Any = - ## dereference operation for the any `x` that represents a ptr or a ref. + ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref. assert x.rawType.kind in {tyRef, tyPtr} result.value = cast[ppointer](x.value)[] result.rawType = x.rawType.base proc `[]=`*(x, y: Any) = - ## dereference operation for the any `x` that represents a ptr or a ref. + ## Dereference operator for `Any`. `x` needs to represent a ptr or a ref. assert x.rawType.kind in {tyRef, tyPtr} assert y.rawType == x.rawType.base genericAssign(cast[ppointer](x.value)[], y.value, y.rawType) proc getInt*(x: Any): int = - ## retrieve the int value out of `x`. `x` needs to represent an int. + ## Retrieves the `int` value out of `x`. `x` needs to represent an `int`. assert skipRange(x.rawType).kind == tyInt result = cast[ptr int](x.value)[] proc getInt8*(x: Any): int8 = - ## retrieve the int8 value out of `x`. `x` needs to represent an int8. + ## Retrieves the `int8` value out of `x`. `x` needs to represent an `int8`. assert skipRange(x.rawType).kind == tyInt8 result = cast[ptr int8](x.value)[] proc getInt16*(x: Any): int16 = - ## retrieve the int16 value out of `x`. `x` needs to represent an int16. + ## Retrieves the `int16` value out of `x`. `x` needs to represent an `int16`. assert skipRange(x.rawType).kind == tyInt16 result = cast[ptr int16](x.value)[] proc getInt32*(x: Any): int32 = - ## retrieve the int32 value out of `x`. `x` needs to represent an int32. + ## Retrieves the `int32` value out of `x`. `x` needs to represent an `int32`. assert skipRange(x.rawType).kind == tyInt32 result = cast[ptr int32](x.value)[] proc getInt64*(x: Any): int64 = - ## retrieve the int64 value out of `x`. `x` needs to represent an int64. + ## Retrieves the `int64` value out of `x`. `x` needs to represent an `int64`. assert skipRange(x.rawType).kind == tyInt64 result = cast[ptr int64](x.value)[] proc getBiggestInt*(x: Any): BiggestInt = - ## retrieve the integer value out of `x`. `x` needs to represent + ## Retrieves the integer value out of `x`. `x` needs to represent ## some integer, a bool, a char, an enum or a small enough bit set. - ## The value might be sign-extended to ``BiggestInt``. - var t = skipRange(x.rawType) + ## The value might be sign-extended to `BiggestInt`. + let t = skipRange(x.rawType) case t.kind of tyInt: result = BiggestInt(cast[ptr int](x.value)[]) of tyInt8: result = BiggestInt(cast[ptr int8](x.value)[]) @@ -423,8 +477,8 @@ proc getBiggestInt*(x: Any): BiggestInt = of tyChar: result = BiggestInt(cast[ptr char](x.value)[]) of tyEnum, tySet: case t.size - of 1: result = ze64(cast[ptr int8](x.value)[]) - of 2: result = ze64(cast[ptr int16](x.value)[]) + of 1: result = int64(cast[ptr uint8](x.value)[]) + of 2: result = int64(cast[ptr uint16](x.value)[]) of 4: result = BiggestInt(cast[ptr int32](x.value)[]) of 8: result = BiggestInt(cast[ptr int64](x.value)[]) else: assert false @@ -435,9 +489,9 @@ proc getBiggestInt*(x: Any): BiggestInt = else: assert false proc setBiggestInt*(x: Any, y: BiggestInt) = - ## sets the integer value of `x`. `x` needs to represent + ## Sets the integer value of `x`. `x` needs to represent ## some integer, a bool, a char, an enum or a small enough bit set. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyInt: cast[ptr int](x.value)[] = int(y) of tyInt8: cast[ptr int8](x.value)[] = int8(y) @@ -448,8 +502,8 @@ proc setBiggestInt*(x: Any, y: BiggestInt) = of tyChar: cast[ptr char](x.value)[] = chr(y.int) of tyEnum, tySet: case t.size - of 1: cast[ptr int8](x.value)[] = toU8(y.int) - of 2: cast[ptr int16](x.value)[] = toU16(y.int) + of 1: cast[ptr uint8](x.value)[] = uint8(y.int) + of 2: cast[ptr uint16](x.value)[] = uint16(y.int) of 4: cast[ptr int32](x.value)[] = int32(y) of 8: cast[ptr int64](x.value)[] = y else: assert false @@ -460,38 +514,34 @@ proc setBiggestInt*(x: Any, y: BiggestInt) = else: assert false proc getUInt*(x: Any): uint = - ## retrieve the uint value out of `x`, `x` needs to represent an uint. + ## Retrieves the `uint` value out of `x`. `x` needs to represent a `uint`. assert skipRange(x.rawType).kind == tyUInt result = cast[ptr uint](x.value)[] proc getUInt8*(x: Any): uint8 = - ## retrieve the uint8 value out of `x`, `x` needs to represent an - ## uint8. + ## Retrieves the `uint8` value out of `x`. `x` needs to represent a `uint8`. assert skipRange(x.rawType).kind == tyUInt8 result = cast[ptr uint8](x.value)[] proc getUInt16*(x: Any): uint16 = - ## retrieve the uint16 value out of `x`, `x` needs to represent an - ## uint16. + ## Retrieves the `uint16` value out of `x`. `x` needs to represent a `uint16`. assert skipRange(x.rawType).kind == tyUInt16 result = cast[ptr uint16](x.value)[] proc getUInt32*(x: Any): uint32 = - ## retrieve the uint32 value out of `x`, `x` needs to represent an - ## uint32. + ## Retrieves the `uint32` value out of `x`. `x` needs to represent a `uint32`. assert skipRange(x.rawType).kind == tyUInt32 result = cast[ptr uint32](x.value)[] proc getUInt64*(x: Any): uint64 = - ## retrieve the uint64 value out of `x`, `x` needs to represent an - ## uint64. + ## Retrieves the `uint64` value out of `x`. `x` needs to represent a `uint64`. assert skipRange(x.rawType).kind == tyUInt64 result = cast[ptr uint64](x.value)[] proc getBiggestUint*(x: Any): uint64 = - ## retrieve the unsigned integer value out of `x`. `x` needs to + ## Retrieves the unsigned integer value out of `x`. `x` needs to ## represent an unsigned integer. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind of tyUInt: result = uint64(cast[ptr uint](x.value)[]) of tyUInt8: result = uint64(cast[ptr uint8](x.value)[]) @@ -501,9 +551,9 @@ proc getBiggestUint*(x: Any): uint64 = else: assert false proc setBiggestUint*(x: Any; y: uint64) = - ## sets the unsigned integer value of `c`. `c` needs to represent an + ## Sets the unsigned integer value of `x`. `x` needs to represent an ## unsigned integer. - var t = skipRange(x.rawType) + let t = skipRange(x.rawType) case t.kind: of tyUInt: cast[ptr uint](x.value)[] = uint(y) of tyUInt8: cast[ptr uint8](x.value)[] = uint8(y) @@ -513,33 +563,33 @@ proc setBiggestUint*(x: Any; y: uint64) = else: assert false proc getChar*(x: Any): char = - ## retrieve the char value out of `x`. `x` needs to represent a char. - var t = skipRange(x.rawType) + ## Retrieves the `char` value out of `x`. `x` needs to represent a `char`. + let t = skipRange(x.rawType) assert t.kind == tyChar result = cast[ptr char](x.value)[] proc getBool*(x: Any): bool = - ## retrieve the bool value out of `x`. `x` needs to represent a bool. - var t = skipRange(x.rawType) + ## Retrieves the `bool` value out of `x`. `x` needs to represent a `bool`. + let t = skipRange(x.rawType) assert t.kind == tyBool result = cast[ptr bool](x.value)[] proc skipRange*(x: Any): Any = - ## skips the range information of `x`. + ## Skips the range information of `x`. assert x.rawType.kind == tyRange result.rawType = x.rawType.base result.value = x.value proc getEnumOrdinal*(x: Any, name: string): int = - ## gets the enum field ordinal from `name`. `x` needs to represent an enum + ## Gets the enum field ordinal from `name`. `x` needs to represent an enum ## but is only used to access the type information. In case of an error - ## ``low(int)`` is returned. - var typ = skipRange(x.rawType) + ## `low(int)` is returned. + let typ = skipRange(x.rawType) assert typ.kind == tyEnum - var n = typ.node - var s = n.sons + let n = typ.node + let s = n.sons for i in 0 .. n.len-1: - if cmpIgnoreStyle($s[i].name, name) == 0: + if cmpNimIdentifier($s[i].name, name) == 0: if ntfEnumHole notin typ.flags: return i else: @@ -547,45 +597,45 @@ proc getEnumOrdinal*(x: Any, name: string): int = result = low(int) proc getEnumField*(x: Any, ordinalValue: int): string = - ## gets the enum field name as a string. `x` needs to represent an enum + ## Gets the enum field name as a string. `x` needs to represent an enum ## but is only used to access the type information. The field name of ## `ordinalValue` is returned. - var typ = skipRange(x.rawType) + let typ = skipRange(x.rawType) assert typ.kind == tyEnum - var e = ordinalValue + let e = ordinalValue if ntfEnumHole notin typ.flags: if e <% typ.node.len: return $typ.node.sons[e].name else: # ugh we need a slow linear search: - var n = typ.node - var s = n.sons + let n = typ.node + let s = n.sons for i in 0 .. n.len-1: if s[i].offset == e: return $s[i].name result = $e proc getEnumField*(x: Any): string = - ## gets the enum field name as a string. `x` needs to represent an enum. + ## Gets the enum field name as a string. `x` needs to represent an enum. result = getEnumField(x, getBiggestInt(x).int) proc getFloat*(x: Any): float = - ## retrieve the float value out of `x`. `x` needs to represent an float. + ## Retrieves the `float` value out of `x`. `x` needs to represent a `float`. assert skipRange(x.rawType).kind == tyFloat result = cast[ptr float](x.value)[] proc getFloat32*(x: Any): float32 = - ## retrieve the float32 value out of `x`. `x` needs to represent an float32. + ## Retrieves the `float32` value out of `x`. `x` needs to represent a `float32`. assert skipRange(x.rawType).kind == tyFloat32 result = cast[ptr float32](x.value)[] proc getFloat64*(x: Any): float64 = - ## retrieve the float64 value out of `x`. `x` needs to represent an float64. + ## Retrieves the `float64` value out of `x`. `x` needs to represent a `float64`. assert skipRange(x.rawType).kind == tyFloat64 result = cast[ptr float64](x.value)[] proc getBiggestFloat*(x: Any): BiggestFloat = - ## retrieve the float value out of `x`. `x` needs to represent - ## some float. The value is extended to ``BiggestFloat``. + ## Retrieves the float value out of `x`. `x` needs to represent + ## some float. The value is extended to `BiggestFloat`. case skipRange(x.rawType).kind of tyFloat: result = BiggestFloat(cast[ptr float](x.value)[]) of tyFloat32: result = BiggestFloat(cast[ptr float32](x.value)[]) @@ -593,7 +643,7 @@ proc getBiggestFloat*(x: Any): BiggestFloat = else: assert false proc setBiggestFloat*(x: Any, y: BiggestFloat) = - ## sets the float value of `x`. `x` needs to represent + ## Sets the float value of `x`. `x` needs to represent ## some float. case skipRange(x.rawType).kind of tyFloat: cast[ptr float](x.value)[] = y @@ -602,56 +652,59 @@ proc setBiggestFloat*(x: Any, y: BiggestFloat) = else: assert false proc getString*(x: Any): string = - ## retrieve the string value out of `x`. `x` needs to represent a string. + ## Retrieves the `string` value out of `x`. `x` needs to represent a `string`. assert x.rawType.kind == tyString - if not isNil(cast[ptr pointer](x.value)[]): + when defined(gcDestructors): result = cast[ptr string](x.value)[] + else: + if not isNil(cast[ptr pointer](x.value)[]): + result = cast[ptr string](x.value)[] proc setString*(x: Any, y: string) = - ## sets the string value of `x`. `x` needs to represent a string. + ## Sets the `string` value of `x`. `x` needs to represent a `string`. assert x.rawType.kind == tyString - cast[ptr string](x.value)[] = y + cast[ptr string](x.value)[] = y # also correct for gcDestructors proc getCString*(x: Any): cstring = - ## retrieve the cstring value out of `x`. `x` needs to represent a cstring. - assert x.rawType.kind == tyCString + ## Retrieves the `cstring` value out of `x`. `x` needs to represent a `cstring`. + assert x.rawType.kind == tyCstring result = cast[ptr cstring](x.value)[] proc assign*(x, y: Any) = - ## copies the value of `y` to `x`. The assignment operator for ``Any`` + ## Copies the value of `y` to `x`. The assignment operator for `Any` ## does NOT do this; it performs a shallow copy instead! assert y.rawType == x.rawType genericAssign(x.value, y.value, y.rawType) iterator elements*(x: Any): int = - ## iterates over every element of `x` that represents a Nim bitset. + ## Iterates over every element of `x`. `x` needs to represent a `set`. assert x.rawType.kind == tySet - var typ = x.rawType - var p = x.value + let typ = x.rawType + let p = x.value # "typ.slots.len" field is for sets the "first" field var u: int64 case typ.size - of 1: u = ze64(cast[ptr int8](p)[]) - of 2: u = ze64(cast[ptr int16](p)[]) - of 4: u = ze64(cast[ptr int32](p)[]) + of 1: u = int64(cast[ptr uint8](p)[]) + of 2: u = int64(cast[ptr uint16](p)[]) + of 4: u = int64(cast[ptr uint32](p)[]) of 8: u = cast[ptr int64](p)[] else: - var a = cast[pbyteArray](p) + let a = cast[pbyteArray](p) for i in 0 .. typ.size*8-1: - if (ze(a[i div 8]) and (1 shl (i mod 8))) != 0: - yield i+typ.node.len + if (int(a[i div 8]) and (1 shl (i mod 8))) != 0: + yield i + typ.node.len if typ.size <= 8: for i in 0..sizeof(int64)*8-1: if (u and (1'i64 shl int64(i))) != 0'i64: - yield i+typ.node.len + yield i + typ.node.len proc inclSetElement*(x: Any, elem: int) = - ## includes an element `elem` in `x`. `x` needs to represent a Nim bitset. + ## Includes an element `elem` in `x`. `x` needs to represent a Nim bitset. assert x.rawType.kind == tySet - var typ = x.rawType - var p = x.value + let typ = x.rawType + let p = x.value # "typ.slots.len" field is for sets the "first" field - var e = elem - typ.node.len + let e = elem - typ.node.len case typ.size of 1: var a = cast[ptr int8](p) @@ -667,63 +720,4 @@ proc inclSetElement*(x: Any, elem: int) = a[] = a[] or (1'i64 shl e) else: var a = cast[pbyteArray](p) - a[e shr 3] = toU8(a[e shr 3] or (1 shl (e and 7))) - -when isMainModule: - type - TE = enum - blah, blah2 - - TestObj = object - test, asd: int - case test2: TE - of blah: - help: string - else: - nil - - var test = @[0,1,2,3,4] - var x = toAny(test) - var y = 78 - x[4] = toAny(y) - assert cast[ptr int](x[2].value)[] == 2 - - var test2: tuple[name: string, s: int] = ("test", 56) - var x2 = toAny(test2) - var i = 0 - for n, a in fields(x2): - case i - of 0: assert n == "name" and $a.kind == "akString" - of 1: assert n == "s" and $a.kind == "akInt" - else: assert false - inc i - - var test3: TestObj - test3.test = 42 - test3 = TestObj(test2: blah2) - var x3 = toAny(test3) - i = 0 - for n, a in fields(x3): - case i - of 0: assert n == "test" and $a.kind == "akInt" - of 1: assert n == "asd" and $a.kind == "akInt" - of 2: assert n == "test2" and $a.kind == "akEnum" - else: assert false - inc i - - var test4: ref string - new(test4) - test4[] = "test" - var x4 = toAny(test4) - assert($x4[].kind() == "akString") - - block: - # gimme a new scope dammit - var myarr: array[0..4, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"]] - var m = toAny(myArr) - for i in 0 .. m.len-1: - for j in 0 .. m[i].len-1: - echo getString(m[i][j]) + a[e shr 3] = a[e shr 3] or uint8(1 shl (e and 7)) diff --git a/lib/deprecated/pure/LockFreeHash.nim b/lib/deprecated/pure/LockFreeHash.nim deleted file mode 100644 index 97e1ef6ad..000000000 --- a/lib/deprecated/pure/LockFreeHash.nim +++ /dev/null @@ -1,609 +0,0 @@ -#nim c -t:-march=i686 --cpu:amd64 --threads:on -d:release lockfreehash.nim - -import math, hashes - -#------------------------------------------------------------------------------ -## Memory Utility Functions - -proc newHeap*[T](): ptr T = - result = cast[ptr T](alloc0(sizeof(T))) - -proc copyNew*[T](x: var T): ptr T = - var - size = sizeof(T) - mem = alloc(size) - copyMem(mem, x.addr, size) - return cast[ptr T](mem) - -proc copyTo*[T](val: var T, dest: int) = - copyMem(pointer(dest), val.addr, sizeof(T)) - -proc allocType*[T](): pointer = alloc(sizeof(T)) - -proc newShared*[T](): ptr T = - result = cast[ptr T](allocShared0(sizeof(T))) - -proc copyShared*[T](x: var T): ptr T = - var - size = sizeof(T) - mem = allocShared(size) - copyMem(mem, x.addr, size) - return cast[ptr T](mem) - -#------------------------------------------------------------------------------ -## Pointer arithmetic - -proc `+`*(p: pointer, i: int): pointer {.inline.} = - cast[pointer](cast[int](p) + i) - -const - minTableSize = 8 - reProbeLimit = 12 - minCopyWork = 4096 - intSize = sizeof(int) - - - -when sizeof(int) == 4: # 32bit - type - Raw = range[0..1073741823] - ## The range of uint values that can be stored directly in a value slot - ## when on a 32 bit platform -elif sizeof(int) == 8: # 64bit - type - Raw = range[0'i64..4611686018427387903'i64] - ## The range of uint values that can be stored directly in a value slot - ## when on a 64 bit platform -else: - {.error: "unsupported platform".} - -type - Entry = tuple - key: int - value: int - - EntryArr = ptr array[0..10_000_000, Entry] - - PConcTable[K, V] = ptr object {.pure.} - len: int - used: int - active: int - copyIdx: int - copyDone: int - next: PConcTable[K, V] - data: EntryArr - -proc setVal[K, V](table: var PConcTable[K, V], key: int, val: int, - expVal: int, match: bool): int - -#------------------------------------------------------------------------------ - -# Create a new table -proc newLFTable*[K, V](size: int = minTableSize): PConcTable[K, V] = - let - dataLen = max(nextPowerOfTwo(size), minTableSize) - dataSize = dataLen*sizeof(Entry) - dataMem = allocShared0(dataSize) - tableSize = 7 * intSize - tableMem = allocShared0(tableSize) - table = cast[PConcTable[K, V]](tableMem) - table.len = dataLen - table.used = 0 - table.active = 0 - table.copyIdx = 0 - table.copyDone = 0 - table.next = nil - table.data = cast[EntryArr](dataMem) - result = table - -#------------------------------------------------------------------------------ - -# Delete a table -proc deleteConcTable[K, V](tbl: PConcTable[K, V]) = - deallocShared(tbl.data) - deallocShared(tbl) - -#------------------------------------------------------------------------------ - -proc `[]`[K, V](table: var PConcTable[K, V], i: int): var Entry {.inline.} = - table.data[i] - -#------------------------------------------------------------------------------ -# State flags stored in ptr - - -proc pack[T](x: T): int {.inline.} = - result = (cast[int](x) shl 2) - #echo("packKey ",cast[int](x) , " -> ", result) - -# Pop the flags off returning a 4 byte aligned ptr to our Key or Val -proc pop(x: int): int {.inline.} = - result = x and 0xFFFFFFFC'i32 - -# Pop the raw value off of our Key or Val -proc popRaw(x: int): int {.inline.} = - result = x shr 2 - -# Pop the flags off returning a 4 byte aligned ptr to our Key or Val -proc popPtr[V](x: int): ptr V {.inline.} = - result = cast[ptr V](pop(x)) - #echo("popPtr " & $x & " -> " & $cast[int](result)) - -# Ghost (sentinel) -# K or V is no longer valid use new table -const Ghost = 0xFFFFFFFC -proc isGhost(x: int): bool {.inline.} = - result = x == 0xFFFFFFFC - -# Tombstone -# applied to V = K is dead -proc isTomb(x: int): bool {.inline.} = - result = (x and 0x00000002) != 0 - -proc setTomb(x: int): int {.inline.} = - result = x or 0x00000002 - -# Prime -# K or V is in new table copied from old -proc isPrime(x: int): bool {.inline.} = - result = (x and 0x00000001) != 0 - -proc setPrime(x: int): int {.inline.} = - result = x or 0x00000001 - -#------------------------------------------------------------------------------ - -##This is for i32 only need to override for i64 -proc hashInt(x: int): int {.inline.} = - var h = uint32(x) #shr 2'u32 - h = h xor (h shr 16'u32) - h *= 0x85ebca6b'u32 - h = h xor (h shr 13'u32) - h *= 0xc2b2ae35'u32 - h = h xor (h shr 16'u32) - result = int(h) - -#------------------------------------------------------------------------------ - -proc resize[K, V](self: PConcTable[K, V]): PConcTable[K, V] = - var next = atomic_load_n(self.next.addr, ATOMIC_RELAXED) - #echo("next = " & $cast[int](next)) - if next != nil: - #echo("A new table already exists, copy in progress") - return next - var - oldLen = atomic_load_n(self.len.addr, ATOMIC_RELAXED) - newTable = newLFTable[K, V](oldLen*2) - success = atomic_compare_exchange_n(self.next.addr, next.addr, newTable, - false, ATOMIC_RELAXED, ATOMIC_RELAXED) - if not success: - echo("someone beat us to it! delete table we just created and return his " & - $cast[int](next)) - deleteConcTable(newTable) - return next - else: - echo("Created New Table! " & $cast[int](newTable) & " Size = " & $newTable.len) - return newTable - - -#------------------------------------------------------------------------------ -#proc keyEQ[K](key1: ptr K, key2: ptr K): bool {.inline.} = -proc keyEQ[K](key1: int, key2: int): bool {.inline.} = - result = false - when K is Raw: - if key1 == key2: - result = true - else: - var - p1 = popPtr[K](key1) - p2 = popPtr[K](key2) - if p1 != nil and p2 != nil: - if cast[int](p1) == cast[int](p2): - return true - if p1[] == p2[]: - return true - -#------------------------------------------------------------------------------ - -#proc tableFull(self: var PConcTable[K,V]) : bool {.inline.} = - - -#------------------------------------------------------------------------------ - -proc copySlot[K, V](idx: int, oldTbl: var PConcTable[K, V], - newTbl: var PConcTable[K, V]): bool = - #echo("Copy idx " & $idx) - var - oldVal = 0 - oldkey = 0 - ok = false - result = false - #Block the key so no other threads waste time here - while not ok: - ok = atomic_compare_exchange_n(oldTbl[idx].key.addr, oldKey.addr, - setTomb(oldKey), false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #echo("oldKey was = " & $oldKey & " set it to tomb " & $setTomb(oldKey)) - #Prevent new values from appearing in the old table by priming - oldVal = atomic_load_n(oldTbl[idx].value.addr, ATOMIC_RELAXED) - while not isPrime(oldVal): - var box = if oldVal == 0 or isTomb(oldVal): oldVal.setTomb.setPrime - else: oldVal.setPrime - if atomic_compare_exchange_n(oldTbl[idx].value.addr, oldVal.addr, - box, false, ATOMIC_RELAXED, ATOMIC_RELAXED): - if isPrime(box) and isTomb(box): - return true - oldVal = box - break - #echo("oldVal was = ", oldVal, " set it to prime ", box) - if isPrime(oldVal) and isTomb(oldVal): - #when not (K is Raw): - # deallocShared(popPtr[K](oldKey)) - return false - if isTomb(oldVal): - echo("oldVal is Tomb!!!, should not happen") - if pop(oldVal) != 0: - result = setVal(newTbl, pop(oldKey), pop(oldVal), 0, true) == 0 - #if result: - #echo("Copied a Slot! idx= " & $idx & " key= " & $oldKey & " val= " & $oldVal) - #else: - #echo("copy slot failed") - # Our copy is done so we disable the old slot - while not ok: - ok = atomic_compare_exchange_n(oldTbl[idx].value.addr, oldVal.addr, - oldVal.setTomb.setPrime, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #echo("disabled old slot") - #echo"---------------------" - -#------------------------------------------------------------------------------ - -proc promote[K, V](table: var PConcTable[K, V]) = - var - newData = atomic_load_n(table.next.data.addr, ATOMIC_RELAXED) - newLen = atomic_load_n(table.next.len.addr, ATOMIC_RELAXED) - newUsed = atomic_load_n(table.next.used.addr, ATOMIC_RELAXED) - - deallocShared(table.data) - atomic_store_n(table.data.addr, newData, ATOMIC_RELAXED) - atomic_store_n(table.len.addr, newLen, ATOMIC_RELAXED) - atomic_store_n(table.used.addr, newUsed, ATOMIC_RELAXED) - atomic_store_n(table.copyIdx.addr, 0, ATOMIC_RELAXED) - atomic_store_n(table.copyDone.addr, 0, ATOMIC_RELAXED) - deallocShared(table.next) - atomic_store_n(table.next.addr, nil, ATOMIC_RELAXED) - echo("new table swapped!") - -#------------------------------------------------------------------------------ - -proc checkAndPromote[K, V](table: var PConcTable[K, V], workDone: int): bool = - var - oldLen = atomic_load_n(table.len.addr, ATOMIC_RELAXED) - copyDone = atomic_load_n(table.copyDone.addr, ATOMIC_RELAXED) - ok: bool - result = false - if workDone > 0: - #echo("len to copy =" & $oldLen) - #echo("copyDone + workDone = " & $copyDone & " + " & $workDone) - while not ok: - ok = atomic_compare_exchange_n(table.copyDone.addr, copyDone.addr, - copyDone + workDone, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #if ok: echo("set copyDone") - # If the copy is done we can promote this table - if copyDone + workDone >= oldLen: - # Swap new data - #echo("work is done!") - table.promote - result = true - -#------------------------------------------------------------------------------ - -proc copySlotAndCheck[K, V](table: var PConcTable[K, V], idx: int): - PConcTable[K, V] = - var - newTable = cast[PConcTable[K, V]](atomic_load_n(table.next.addr, - ATOMIC_RELAXED)) - result = newTable - if newTable != nil and copySlot(idx, table, newTable): - #echo("copied a single slot, idx = " & $idx) - if checkAndPromote(table, 1): return table - - -#------------------------------------------------------------------------------ - -proc helpCopy[K, V](table: var PConcTable[K, V]): PConcTable[K, V] = - var - newTable = cast[PConcTable[K, V]](atomic_load_n(table.next.addr, - ATOMIC_RELAXED)) - result = newTable - if newTable != nil: - var - oldLen = atomic_load_n(table.len.addr, ATOMIC_RELAXED) - copyDone = atomic_load_n(table.copyDone.addr, ATOMIC_RELAXED) - copyIdx = 0 - work = min(oldLen, minCopyWork) - #panicStart = -1 - workDone = 0 - if copyDone < oldLen: - var ok: bool - while not ok: - ok = atomic_compare_exchange_n(table.copyIdx.addr, copyIdx.addr, - copyIdx + work, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - #echo("copy idx = ", copyIdx) - for i in 0..work-1: - var idx = (copyIdx + i) and (oldLen - 1) - if copySlot(idx, table, newTable): - workDone += 1 - if workDone > 0: - #echo("did work ", workDone, " on thread ", cast[int](myThreadID[pointer]())) - if checkAndPromote(table, workDone): return table - # In case a thread finished all the work then got stalled before promotion - if checkAndPromote(table, 0): return table - - - -#------------------------------------------------------------------------------ - -proc setVal[K, V](table: var PConcTable[K, V], key: int, val: int, - expVal: int, match: bool): int = - #echo("-try set- in table ", " key = ", (popPtr[K](key)[]), " val = ", val) - when K is Raw: - var idx = hashInt(key) - else: - var idx = popPtr[K](key)[].hash - var - nextTable: PConcTable[K, V] - probes = 1 - # spin until we find a key slot or build and jump to next table - while true: - idx = idx and (table.len - 1) - #echo("try set idx = " & $idx & "for" & $key) - var - probedKey = 0 - openKey = atomic_compare_exchange_n(table[idx].key.addr, probedKey.addr, - key, false, ATOMIC_RELAXED, ATOMIC_RELAXED) - if openKey: - if val.isTomb: - #echo("val was tomb, bail, no reason to set an open slot to tomb") - return val - #increment used slots - #echo("found an open slot, total used = " & - #$atomic_add_fetch(table.used.addr, 1, ATOMIC_RELAXED)) - discard atomic_add_fetch(table.used.addr, 1, ATOMIC_RELAXED) - break # We found an open slot - #echo("set idx ", idx, " key = ", key, " probed = ", probedKey) - if keyEQ[K](probedKey, key): - #echo("we found the matching slot") - break # We found a matching slot - if (not(expVal != 0 and match)) and (probes >= reProbeLimit or key.isTomb): - if key.isTomb: echo("Key is Tombstone") - #if probes >= reProbeLimit: echo("Too much probing " & $probes) - #echo("try to resize") - #create next bigger table - nextTable = resize(table) - #help do some copying - #echo("help copy old table to new") - nextTable = helpCopy(table) - #now setVal in the new table instead - #echo("jumping to next table to set val") - return setVal(nextTable, key, val, expVal, match) - else: - idx += 1 - probes += 1 - # Done spinning for a new slot - var oldVal = atomic_load_n(table[idx].value.addr, ATOMIC_RELAXED) - if val == oldVal: - #echo("this val is already in the slot") - return oldVal - nextTable = atomic_load_n(table.next.addr, ATOMIC_SEQ_CST) - if nextTable == nil and - ((oldVal == 0 and - (probes >= reProbeLimit or table.used / table.len > 0.8)) or - (isPrime(oldVal))): - if table.used / table.len > 0.8: echo("resize because usage ratio = " & - $(table.used / table.len)) - if isPrime(oldVal): echo("old val isPrime, should be a rare mem ordering event") - nextTable = resize(table) - if nextTable != nil: - #echo("tomb old slot then set in new table") - nextTable = copySlotAndCheck(table, idx) - return setVal(nextTable, key, val, expVal, match) - # Finally ready to add new val to table - while true: - if match and oldVal != expVal: - #echo("set failed, no match oldVal= " & $oldVal & " expVal= " & $expVal) - return oldVal - if atomic_compare_exchange_n(table[idx].value.addr, oldVal.addr, - val, false, ATOMIC_RELEASE, ATOMIC_RELAXED): - #echo("val set at table " & $cast[int](table)) - if expVal != 0: - if (oldVal == 0 or isTomb(oldVal)) and not isTomb(val): - discard atomic_add_fetch(table.active.addr, 1, ATOMIC_RELAXED) - elif not (oldVal == 0 or isTomb(oldVal)) and isTomb(val): - discard atomic_add_fetch(table.active.addr, -1, ATOMIC_RELAXED) - if oldVal == 0 and expVal != 0: - return setTomb(oldVal) - else: return oldVal - if isPrime(oldVal): - nextTable = copySlotAndCheck(table, idx) - return setVal(nextTable, key, val, expVal, match) - -#------------------------------------------------------------------------------ - -proc getVal[K, V](table: var PConcTable[K, V], key: int): int = - #echo("-try get- key = " & $key) - when K is Raw: - var idx = hashInt(key) - else: - var idx = popPtr[K](key)[].hash - #echo("get idx ", idx) - var - probes = 0 - val: int - while true: - idx = idx and (table.len - 1) - var - newTable: PConcTable[K, V] # = atomic_load_n(table.next.addr, ATOMIC_ACQUIRE) - probedKey = atomic_load_n(table[idx].key.addr, ATOMIC_SEQ_CST) - if keyEQ[K](probedKey, key): - #echo("found key after ", probes+1) - val = atomic_load_n(table[idx].value.addr, ATOMIC_ACQUIRE) - if not isPrime(val): - if isTomb(val): - #echo("val was tomb but not prime") - return 0 - else: - #echo("-GotIt- idx = ", idx, " key = ", key, " val ", val ) - return val - else: - newTable = copySlotAndCheck(table, idx) - return getVal(newTable, key) - else: - #echo("probe ", probes, " idx = ", idx, " key = ", key, " found ", probedKey ) - if probes >= reProbeLimit*4 or key.isTomb: - if newTable == nil: - #echo("too many probes and no new table ", key, " ", idx ) - return 0 - else: - newTable = helpCopy(table) - return getVal(newTable, key) - idx += 1 - probes += 1 - -#------------------------------------------------------------------------------ - -#proc set*(table: var PConcTable[Raw,Raw], key: Raw, val: Raw) = -# discard setVal(table, pack(key), pack(key), 0, false) - -#proc set*[V](table: var PConcTable[Raw,V], key: Raw, val: ptr V) = -# discard setVal(table, pack(key), cast[int](val), 0, false) - -proc set*[K, V](table: var PConcTable[K, V], key: var K, val: var V) = - when not (K is Raw): - var newKey = cast[int](copyShared(key)) - else: - var newKey = pack(key) - when not (V is Raw): - var newVal = cast[int](copyShared(val)) - else: - var newVal = pack(val) - var oldPtr = pop(setVal(table, newKey, newVal, 0, false)) - #echo("oldPtr = ", cast[int](oldPtr), " newPtr = ", cast[int](newPtr)) - when not (V is Raw): - if newVal != oldPtr and oldPtr != 0: - deallocShared(cast[ptr V](oldPtr)) - - - -proc get*[K, V](table: var PConcTable[K, V], key: var K): V = - when not (V is Raw): - when not (K is Raw): - return popPtr[V](getVal(table, cast[int](key.addr)))[] - else: - return popPtr[V](getVal(table, pack(key)))[] - else: - when not (K is Raw): - return popRaw(getVal(table, cast[int](key.addr))) - else: - return popRaw(getVal(table, pack(key))) - - - - - - - - - - - -#proc `[]`[K,V](table: var PConcTable[K,V], key: K): PEntry[K,V] {.inline.} = -# getVal(table, key) - -#proc `[]=`[K,V](table: var PConcTable[K,V], key: K, val: V): PEntry[K,V] {.inline.} = -# setVal(table, key, val) - - - - - - -#Tests ---------------------------- -when not defined(testing) and isMainModule: - import locks, times, mersenne - - const - numTests = 100000 - numThreads = 10 - - - - type - TestObj = tuple - thr: int - f0: int - f1: int - - Data = tuple[k: string, v: TestObj] - PDataArr = array[0..numTests-1, Data] - Dict = PConcTable[string, TestObj] - - var - thr: array[0..numThreads-1, Thread[Dict]] - - table = newLFTable[string, TestObj](8) - rand = newMersenneTwister(2525) - - proc createSampleData(len: int): PDataArr = - #result = cast[PDataArr](allocShared0(sizeof(Data)*numTests)) - for i in 0..len-1: - result[i].k = "mark" & $(i+1) - #echo("mark" & $(i+1), " ", hash("mark" & $(i+1))) - result[i].v.thr = 0 - result[i].v.f0 = i+1 - result[i].v.f1 = 0 - #echo("key = " & $(i+1) & " Val ptr = " & $cast[int](result[i].v.addr)) - - - - proc threadProc(tp: Dict) {.thread.} = - var t = cpuTime(); - for i in 1..numTests: - var key = "mark" & $(i) - var got = table.get(key) - got.thr = cast[int](myThreadID[pointer]()) - got.f1 = got.f1 + 1 - table.set(key, got) - t = cpuTime() - t - echo t - - - var testData = createSampleData(numTests) - - for i in 0..numTests-1: - table.set(testData[i].k, testData[i].v) - - var i = 0 - while i < numThreads: - createThread(thr[i], threadProc, table) - i += 1 - - joinThreads(thr) - - - - - - var fails = 0 - - for i in 0..numTests-1: - var got = table.get(testData[i].k) - if got.f0 != i+1 or got.f1 != numThreads: - fails += 1 - echo(got) - - echo("Failed read or write = ", fails) - - - #for i in 1..numTests: - # echo(i, " = ", hashInt(i) and 8191) - - deleteConcTable(table) diff --git a/lib/deprecated/pure/events.nim b/lib/deprecated/pure/events.nim deleted file mode 100644 index d2a8dd085..000000000 --- a/lib/deprecated/pure/events.nim +++ /dev/null @@ -1,102 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2011 Alexander Mitchell-Robinson -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## :Author: Alexander Mitchell-Robinson -## -## Unstable API. -## -## This module implements an event system that is not dependent on external -## graphical toolkits. It was originally called ``NimEE`` because -## it was inspired by Python's PyEE module. There are two ways you can use -## events: one is a python-inspired way; the other is more of a C-style way. -## -## .. code-block:: Nim -## var ee = initEventEmitter() -## var genericargs: EventArgs -## proc handleevent(e: EventArgs) = -## echo("Handled!") -## -## # Python way -## ee.on("EventName", handleevent) -## ee.emit("EventName", genericargs) -## -## # C/Java way -## # Declare a type -## type -## SomeObject = object of RootObj -## SomeEvent: EventHandler -## var myobj: SomeObject -## myobj.SomeEvent = initEventHandler("SomeEvent") -## myobj.SomeEvent.addHandler(handleevent) -## ee.emit(myobj.SomeEvent, genericargs) - -type - EventArgs* = object of RootObj ## Base object for event arguments that are passed to callback functions. - EventHandler* = tuple[name: string, handlers: seq[proc(e: EventArgs) {.closure.}]] ## An eventhandler for an event. - -type - EventEmitter* = object ## An object that fires events and holds event handlers for an object. - s: seq[EventHandler] - EventError* = object of ValueError - -proc initEventHandler*(name: string): EventHandler = - ## Initializes an EventHandler with the specified name and returns it. - result.handlers = @[] - result.name = name - -proc addHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) = - ## Adds the callback to the specified event handler. - handler.handlers.add(fn) - -proc removeHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) = - ## Removes the callback from the specified event handler. - for i in countup(0, len(handler.handlers)-1): - if fn == handler.handlers[i]: - handler.handlers.del(i) - break - -proc containsHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}): bool = - ## Checks if a callback is registered to this event handler. - return handler.handlers.contains(fn) - - -proc clearHandlers*(handler: var EventHandler) = - ## Clears all of the callbacks from the event handler. - setLen(handler.handlers, 0) - -proc getEventHandler(emitter: var EventEmitter, event: string): int = - for k in 0..high(emitter.s): - if emitter.s[k].name == event: return k - return -1 - -proc on*(emitter: var EventEmitter, event: string, fn: proc(e: EventArgs) {.closure.}) = - ## Assigns a event handler with the specified callback. If the event - ## doesn't exist, it will be created. - var i = getEventHandler(emitter, event) - if i < 0: - var eh = initEventHandler(event) - addHandler(eh, fn) - emitter.s.add(eh) - else: - addHandler(emitter.s[i], fn) - -proc emit*(emitter: var EventEmitter, eventhandler: var EventHandler, - args: EventArgs) = - ## Fires an event handler with specified event arguments. - for fn in items(eventhandler.handlers): fn(args) - -proc emit*(emitter: var EventEmitter, event: string, args: EventArgs) = - ## Fires an event handler with specified event arguments. - var i = getEventHandler(emitter, event) - if i >= 0: - emit(emitter, emitter.s[i], args) - -proc initEventEmitter*(): EventEmitter = - ## Creates and returns a new EventEmitter. - result.s = @[] diff --git a/lib/deprecated/pure/future.nim b/lib/deprecated/pure/future.nim new file mode 100644 index 000000000..0e06161f2 --- /dev/null +++ b/lib/deprecated/pure/future.nim @@ -0,0 +1,6 @@ +## This module is a deprecated alias for the `sugar` module. Deprecated since 0.19.0. + +{.deprecated: "Use the new 'sugar' module instead".} + +import std/sugar +export sugar diff --git a/lib/pure/mersenne.nim b/lib/deprecated/pure/mersenne.nim index a2b8914d9..37c5085b1 100644 --- a/lib/pure/mersenne.nim +++ b/lib/deprecated/pure/mersenne.nim @@ -2,17 +2,28 @@ # # Nim's Runtime Library # (c) Copyright 2015 Nim Contributors -# # See the file "copying.txt", included in this # distribution, for details about the copyright. -# +## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister) +## random number generator. +## .. note:: The procs in this module work at compile-time. +{.deprecated: "use `std/random` instead".} + +runnableExamples: + var rand = newMersenneTwister(uint32.high) ## must be "var" + doAssert rand.getNum() != rand.getNum() ## pseudorandom number +## See also +## ======== +## * `random module<random.html>`_ for Nim's standard random number generator type MersenneTwister* = object + ## The Mersenne Twister. mt: array[0..623, uint32] index: int proc newMersenneTwister*(seed: uint32): MersenneTwister = + ## Creates a new `MersenneTwister` with seed `seed`. result.index = 0 result.mt[0] = seed for i in 1'u32 .. 623'u32: @@ -20,6 +31,7 @@ proc newMersenneTwister*(seed: uint32): MersenneTwister = (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = + for i in 0..623: var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) @@ -28,20 +40,12 @@ proc generateNumbers(m: var MersenneTwister) = m.mt[i] = m.mt[i] xor 0x9908b0df'u32 proc getNum*(m: var MersenneTwister): uint32 = - ## Returns the next pseudo random number ranging from 0 to high(uint32) + ## Returns the next pseudorandom `uint32`. if m.index == 0: generateNumbers(m) result = m.mt[m.index] m.index = (m.index + 1) mod m.mt.len - result = result xor (result shr 11'u32) result = result xor ((result shl 7'u32) and 0x9d2c5680'u32) result = result xor ((result shl 15'u32) and 0xefc60000'u32) result = result xor (result shr 18'u32) - -# Test -when not defined(testing) and isMainModule: - var mt = newMersenneTwister(2525) - - for i in 0..99: - echo mt.getNum diff --git a/lib/deprecated/pure/ospaths.nim b/lib/deprecated/pure/ospaths.nim index 34d452fd2..43fcb17cc 100644 --- a/lib/deprecated/pure/ospaths.nim +++ b/lib/deprecated/pure/ospaths.nim @@ -7,10 +7,11 @@ # distribution, for details about the copyright. # -## This module is deprecated, ``import os`` instead. -{.deprecated: "import os.nim instead".} +## This module is deprecated since 0.20.0, `import std/os` instead. -import os +{.deprecated: "use `std/os` instead".} + +import std/os export ReadEnvEffect, WriteEnvEffect, ReadDirEffect, WriteDirEffect, OSErrorCode, doslikeFileSystem, CurDir, ParDir, DirSep, AltSep, PathSep, FileSystemCaseSensitive, ExeExt, ScriptExt, DynlibFormat, ExtSep, joinPath, `/`, splitPath, parentDir, diff --git a/lib/deprecated/pure/oswalkdir.nim b/lib/deprecated/pure/oswalkdir.nim new file mode 100644 index 000000000..57a2cb81d --- /dev/null +++ b/lib/deprecated/pure/oswalkdir.nim @@ -0,0 +1,13 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module is deprecated, `import std/os` instead. +{.deprecated: "import 'std/os' instead".} +import std/os +export PathComponent, walkDir, walkDirRec diff --git a/lib/deprecated/pure/parseopt2.nim b/lib/deprecated/pure/parseopt2.nim deleted file mode 100644 index e57b7e2c9..000000000 --- a/lib/deprecated/pure/parseopt2.nim +++ /dev/null @@ -1,155 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module provides the standard Nim command line parser. -## It supports one convenience iterator over all command line options and some -## lower-level features. -## -## Supported syntax: -## -## 1. short options - ``-abcd``, where a, b, c, d are names -## 2. long option - ``--foo:bar``, ``--foo=bar`` or ``--foo`` -## 3. argument - everything else - -{.deprecated: "Use the 'parseopt' module instead".} -{.push debugger: off.} - -include "system/inclrtl" - -import - os, strutils - -type - CmdLineKind* = enum ## the detected command line token - cmdEnd, ## end of command line reached - cmdArgument, ## argument detected - cmdLongOption, ## a long option ``--option`` detected - cmdShortOption ## a short option ``-c`` detected - OptParser* = - object of RootObj ## this object implements the command line parser - cmd: seq[string] - pos: int - remainingShortOptions: string - kind*: CmdLineKind ## the detected command line token - key*, val*: TaintedString ## key and value pair; ``key`` is the option - ## or the argument, ``value`` is not "" if - ## the option was given a value - -proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} = - ## Initializes option parses with cmdline. cmdline should not contain - ## argument 0 - program name. - ## If cmdline.len == 0 default to current command line arguments. - result.remainingShortOptions = "" - when not defined(createNimRtl): - if cmdline.len == 0: - result.cmd = commandLineParams() - return - else: - assert cmdline != nil, "Cannot determine command line arguments." - - result.cmd = @cmdline - -when not defined(createNimRtl): - proc initOptParser*(): OptParser = - ## Initializes option parser from current command line arguments. - return initOptParser(commandLineParams()) - -proc next*(p: var OptParser) {.rtl, extern: "npo2$1".} - -proc nextOption(p: var OptParser, token: string, allowEmpty: bool) = - for splitchar in [':', '=']: - if splitchar in token: - let pos = token.find(splitchar) - p.key = token[0..pos-1] - p.val = token[pos+1..token.len-1] - return - - p.key = token - if allowEmpty: - p.val = "" - else: - p.remainingShortOptions = token[0..token.len-1] - p.next() - -proc next(p: var OptParser) = - if p.remainingShortOptions.len != 0: - p.kind = cmdShortOption - p.key = TaintedString(p.remainingShortOptions[0..0]) - p.val = "" - p.remainingShortOptions = p.remainingShortOptions[1..p.remainingShortOptions.len-1] - return - - if p.pos >= p.cmd.len: - p.kind = cmdEnd - return - - let token = p.cmd[p.pos] - p.pos += 1 - - if token.startsWith("--"): - p.kind = cmdLongOption - nextOption(p, token[2..token.len-1], allowEmpty=true) - elif token.startsWith("-"): - p.kind = cmdShortOption - nextOption(p, token[1..token.len-1], allowEmpty=true) - else: - p.kind = cmdArgument - p.key = token - p.val = "" - -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1".} = - ## Returns the part of command line string that has not been parsed yet, - ## properly quoted. - return p.cmd[p.pos..p.cmd.len-1].quoteShellCommand - -type - GetoptResult* = tuple[kind: CmdLineKind, key, val: TaintedString] - -iterator getopt*(p: var OptParser): GetoptResult = - ## This is an convenience iterator for iterating over the given OptParser object. - ## Example: - ## - ## .. code-block:: nim - ## var p = initOptParser("--left --debug:3 -l=4 -r:2") - ## for kind, key, val in p.getopt(): - ## case kind - ## of cmdArgument: - ## filename = key - ## of cmdLongOption, cmdShortOption: - ## case key - ## of "help", "h": writeHelp() - ## of "version", "v": writeVersion() - ## of cmdEnd: assert(false) # cannot happen - ## if filename == "": - ## # no filename has been given, so we show the help: - ## writeHelp() - p.pos = 0 - while true: - next(p) - if p.kind == cmdEnd: break - yield (p.kind, p.key, p.val) - -when declared(paramCount): - iterator getopt*(): GetoptResult = - ## This is an convenience iterator for iterating over the command line arguments. - ## This create a new OptParser object. - ## See above for a more detailed example - ## - ## .. code-block:: nim - ## for kind, key, val in getopt(): - ## # this will iterate over all arguments passed to the cmdline. - ## continue - ## - var p = initOptParser() - while true: - next(p) - if p.kind == cmdEnd: break - yield (p.kind, p.key, p.val) - -{.pop.} diff --git a/lib/deprecated/pure/securehash.nim b/lib/deprecated/pure/securehash.nim deleted file mode 100644 index c6cde599a..000000000 --- a/lib/deprecated/pure/securehash.nim +++ /dev/null @@ -1,6 +0,0 @@ - - -## This module is a deprecated alias for the ``sha1`` module. -{.deprecated.} - -include "../std/sha1" diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim deleted file mode 100644 index 17e2e5888..000000000 --- a/lib/deprecated/pure/sharedstrings.nim +++ /dev/null @@ -1,152 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Shared string support for Nim. - -type - UncheckedCharArray = UncheckedArray[char] - -type - Buffer = ptr object - refcount: int - capacity, realLen: int - data: UncheckedCharArray - - SharedString* = object ## A string that can be shared. Slicing is O(1). - buffer: Buffer - first, len: int - -proc decRef(b: Buffer) {.inline.} = - if atomicDec(b.refcount) <= 0: - deallocShared(b) - -proc incRef(b: Buffer) {.inline.} = - atomicInc(b.refcount) - -{.experimental.} - -proc `=destroy`*(s: SharedString) = - #echo "destroyed" - if not s.buffer.isNil: - decRef(s.buffer) - -when false: - proc `=`*(dest: var SharedString; src: SharedString) = - incRef(src.buffer) - if not dest.buffer.isNil: - decRef(dest.buffer) - dest.buffer = src.buffer - dest.first = src.first - dest.len = src.len - -proc len*(s: SharedString): int = s.len - -proc `[]`*(s: SharedString; i: Natural): char = - if i < s.len: result = s.buffer.data[i+s.first] - else: raise newException(IndexError, formatErrorIndexBound(i, s.len-1)) - -proc `[]=`*(s: var SharedString; i: Natural; value: char) = - if i < s.len: s.buffer.data[i+s.first] = value - else: raise newException(IndexError, formatErrorIndexBound(i, s.len-1)) - -proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString = - #incRef(src.buffer) - if ab.a < s.len: - result.buffer = s.buffer - result.first = ab.a - result.len = min(s.len, ab.b - ab.a + 1) - # else: produce empty string ;-) - -proc newBuffer(cap, len: int): Buffer = - assert cap >= len - result = cast[Buffer](allocShared0(sizeof(int)*3 + cap)) - result.refcount = 0 - result.capacity = cap - result.realLen = len - -proc newSharedString*(len: Natural): SharedString = - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - result.first = 0 - result.len = len - -proc newSharedString*(s: string): SharedString = - let len = s.len - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - copyMem(addr result.buffer.data[0], cstring(s), s.len) - result.first = 0 - result.len = len - -when declared(atomicLoadN): - template load(x): untyped = atomicLoadN(addr x, ATOMIC_SEQ_CST) -else: - # XXX Fixme - template load(x): untyped = x - -proc add*(s: var SharedString; t: cstring; len: Natural) = - if len == 0: return - let newLen = s.len + len - if s.buffer.isNil: - s.buffer = newBuffer(len, len) - copyMem(addr s.buffer.data[0], t, len) - s.len = len - elif newLen >= s.buffer.capacity or s.first != 0 or - s.len != s.buffer.realLen or load(s.buffer.refcount) > 1: - let oldBuf = s.buffer - s.buffer = newBuffer(max(s.buffer.capacity * 3 div 2, newLen), newLen) - copyMem(addr s.buffer.data[0], addr oldBuf.data[s.first], s.len) - copyMem(addr s.buffer.data[s.len], t, len) - decRef(oldBuf) - else: - copyMem(addr s.buffer.data[s.len], t, len) - s.buffer.realLen += len - s.len += len - -proc add*(s: var SharedString; t: string) = - s.add(t.cstring, t.len) - -proc rawData*(s: var SharedString): pointer = - if s.buffer.isNil: result = nil - else: result = addr s.buffer.data[s.first] - -proc add*(s: var SharedString; t: SharedString) = - if t.buffer.isNil: return - s.add(cast[cstring](addr s.buffer.data[s.first]), t.len) - -proc `$`*(s: SharedString): string = - result = newString(s.len) - if s.len > 0: - copyMem(addr result[0], addr s.buffer.data[s.first], s.len) - -proc `==`*(s: SharedString; t: string): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - cstring(t), t.len) - -proc `==`*(s, t: SharedString): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - addr t.buffer.data[t.first], t.len) - -iterator items*(s: SharedString): char = - let buf = s.buffer.data - let x = s.first - if buf != nil: - for i in 0..<s.len: - yield buf[i+x] - -import hashes - -proc hash*(s: SharedString): THash = - var h: THash = 0 - for x in s: h = h !& x.hash - result = !$h diff --git a/lib/std/sums.nim b/lib/deprecated/pure/sums.nim index dc9d8d507..a6ce1b85d 100644 --- a/lib/std/sums.nim +++ b/lib/deprecated/pure/sums.nim @@ -6,12 +6,39 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -## Fast sumation functions. +## Accurate summation functions. +{.deprecated: "use the nimble package `sums` instead.".} + +runnableExamples: + import std/math + + template `~=`(x, y: float): bool = abs(x - y) < 1e-4 + + let + n = 1_000_000 + first = 1e10 + small = 0.1 + var data = @[first] + for _ in 1 .. n: + data.add(small) + + let result = first + small * n.float + + doAssert abs(sum(data) - result) > 0.3 + doAssert sumKbn(data) ~= result + doAssert sumPairs(data) ~= result + +## See also +## ======== +## * `math module <math.html>`_ for a standard `sum proc <math.html#sum,openArray[T]>`_ func sumKbn*[T](x: openArray[T]): T = - ## Kahan (compensated) summation: O(1) error growth, at the expense - ## of a considerable increase in computational expense. + ## Kahan-Babuška-Neumaier summation: O(1) error growth, at the expense + ## of a considerable increase in computational cost. + ## + ## See: + ## * https://en.wikipedia.org/wiki/Kahan_summation_algorithm#Further_enhancements if len(x) == 0: return var sum = x[0] var c = T(0) @@ -35,13 +62,13 @@ func sumPairwise[T](x: openArray[T], i0, n: int): T = result = sumPairwise(x, i0, n2) + sumPairwise(x, i0 + n2, n - n2) func sumPairs*[T](x: openArray[T]): T = - ## Pairwise (cascade) summation of ``x[i0:i0+n-1]``, with O(log n) error growth + ## Pairwise (cascade) summation of `x[i0:i0+n-1]`, with O(log n) error growth ## (vs O(n) for a simple loop) with negligible performance cost if ## the base case is large enough. ## ## See, e.g.: - ## * http://en.wikipedia.org/wiki/Pairwise_summation - ## Higham, Nicholas J. (1993), "The accuracy of floating point + ## * https://en.wikipedia.org/wiki/Pairwise_summation + ## * Higham, Nicholas J. (1993), "The accuracy of floating point ## summation", SIAM Journal on Scientific Computing 14 (4): 783–799. ## ## In fact, the root-mean-square error growth, assuming random roundoff @@ -49,30 +76,5 @@ func sumPairs*[T](x: openArray[T]): T = ## in practice. See: ## * Manfred Tasche and Hansmartin Zeuner, Handbook of ## Analytic-Computational Methods in Applied Mathematics (2000). - ## let n = len(x) if n == 0: T(0) else: sumPairwise(x, 0, n) - - -when isMainModule: - from math import pow - - var epsilon = 1.0 - while 1.0 + epsilon != 1.0: - epsilon /= 2.0 - let data = @[1.0, epsilon, -epsilon] - assert sumKbn(data) == 1.0 - assert sumPairs(data) != 1.0 # known to fail - assert (1.0 + epsilon) - epsilon != 1.0 - - var tc1: seq[float] - for n in 1 .. 1000: - tc1.add 1.0 / n.float - assert sumKbn(tc1) == 7.485470860550345 - assert sumPairs(tc1) == 7.485470860550345 - - var tc2: seq[float] - for n in 1 .. 1000: - tc2.add pow(-1.0, n.float) / n.float - assert sumKbn(tc2) == -0.6926474305598203 - assert sumPairs(tc2) == -0.6926474305598204 diff --git a/lib/deps.txt b/lib/deps.txt new file mode 100644 index 000000000..2a44b5fe2 --- /dev/null +++ b/lib/deps.txt @@ -0,0 +1,14 @@ +# Dragonbox.nim and Schubfach.nim + +"Dragonbox" is Nim's "float64 to string" algorithm. +"Schubfach" is Nim's "float32 to string" algorithm. These are based on + +https://github.com/abolz/Drachennest/blob/master/src/dragonbox.cc +https://github.com/abolz/Drachennest/blob/master/src/schubfach_32.cc + +commit e6714a39ad331b4489d0b6aaf3968635bff4eb5e + +The `.cc` files were translated by c2nim via `--cpp --keepBodies --nep1` +and then modified to remove the unsafe code. + +We used c2nim as of commit f0469c909d9e2e28d59687e394bf5ac862f561b6. diff --git a/lib/experimental/diff.nim b/lib/experimental/diff.nim index 5a69687a3..669e9f613 100644 --- a/lib/experimental/diff.nim +++ b/lib/experimental/diff.nim @@ -10,28 +10,31 @@ ## This module implements an algorithm to compute the ## `diff`:idx: between two sequences of lines. ## -## A basic example of ``diffInt`` on 2 arrays of integers: -## -## .. code::nim -## -## import experimental/diff -## echo diffInt([0, 1, 2, 3, 4, 5, 6, 7, 8], [-1, 1, 2, 3, 4, 5, 666, 7, 42]) -## -## Another short example of ``diffText`` to diff strings: -## -## .. code::nim -## -## import experimental/diff -## # 2 samples of text for testing (from "The Call of Cthulhu" by Lovecraft) -## let txt0 = """I have looked upon all the universe has to hold of horror, -## even skies of spring and flowers of summer must ever be poison to me.""" -## let txt1 = """I have looked upon all your code has to hold of bugs, -## even skies of spring and flowers of summer must ever be poison to me.""" -## -## echo diffText(txt0, txt1) -## ## - To learn more see `Diff on Wikipedia. <http://wikipedia.org/wiki/Diff>`_ +runnableExamples: + assert diffInt( + [0, 1, 2, 3, 4, 5, 6, 7, 8], + [-1, 1, 2, 3, 4, 5, 666, 7, 42]) == + @[Item(startA: 0, startB: 0, deletedA: 1, insertedB: 1), + Item(startA: 6, startB: 6, deletedA: 1, insertedB: 1), + Item(startA: 8, startB: 8, deletedA: 1, insertedB: 1)] + +runnableExamples: + # 2 samples of text (from "The Call of Cthulhu" by Lovecraft) + let txt0 = """ +abc +def ghi +jkl2""" + let txt1 = """ +bacx +abc +def ghi +jkl""" + assert diffText(txt0, txt1) == + @[Item(startA: 0, startB: 0, deletedA: 0, insertedB: 1), + Item(startA: 2, startB: 3, deletedA: 1, insertedB: 1)] + # code owner: Arne Döring # # This is based on C# code written by Matthias Hertel, http://www.mathertel.de @@ -40,7 +43,10 @@ # "An O(ND) Difference Algorithm and its Variations" by Eugene Myers # Algorithmica Vol. 1 No. 2, 1986, p 251. -import tables, strutils +import std/[tables, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions type Item* = object ## An Item in the list of differences. @@ -59,7 +65,7 @@ type Smsrd = object x, y: int -# template to avoid a seq copy. Required until ``sink`` parameters are ready. +# template to avoid a seq copy. Required until `sink` parameters are ready. template newDiffData(initData: seq[int]; L: int): DiffData = DiffData( data: initData, @@ -71,9 +77,9 @@ proc len(d: DiffData): int {.inline.} = d.data.len proc diffCodes(aText: string; h: var Table[string, int]): DiffData = ## This function converts all textlines of the text into unique numbers for every unique textline ## so further work can work only with simple numbers. - ## ``aText`` the input text - ## ``h`` This extern initialized hashtable is used for storing all ever used textlines. - ## ``trimSpace`` ignore leading and trailing space characters + ## `aText` the input text + ## `h` This extern initialized hashtable is used for storing all ever used textlines. + ## `trimSpace` ignore leading and trailing space characters ## Returns a array of integers. var lastUsedCode = h.len result.data = newSeq[int]() @@ -108,14 +114,14 @@ proc optimize(data: var DiffData) = proc sms(dataA: var DiffData; lowerA, upperA: int; dataB: DiffData; lowerB, upperB: int; downVector, upVector: var openArray[int]): Smsrd = ## This is the algorithm to find the Shortest Middle Snake (sms). - ## ``dataA`` sequence A - ## ``lowerA`` lower bound of the actual range in dataA - ## ``upperA`` upper bound of the actual range in dataA (exclusive) - ## ``dataB`` sequence B - ## ``lowerB`` lower bound of the actual range in dataB - ## ``upperB`` upper bound of the actual range in dataB (exclusive) - ## ``downVector`` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons. - ## ``upVector`` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons. + ## `dataA` sequence A + ## `lowerA` lower bound of the actual range in dataA + ## `upperA` upper bound of the actual range in dataA (exclusive) + ## `dataB` sequence B + ## `lowerB` lower bound of the actual range in dataB + ## `upperB` upper bound of the actual range in dataB (exclusive) + ## `downVector` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons. + ## `upVector` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons. ## Returns a MiddleSnakeData record containing x,y and u,v. let max = dataA.len + dataB.len + 1 @@ -138,7 +144,7 @@ proc sms(dataA: var DiffData; lowerA, upperA: int; dataB: DiffData; lowerB, uppe for D in 0 .. maxD: # Extend the forward path. - for k in countUp(downK - D, downK + D, 2): + for k in countup(downK - D, downK + D, 2): # find the only or better starting point var x: int if k == downK - D: @@ -164,7 +170,7 @@ proc sms(dataA: var DiffData; lowerA, upperA: int; dataB: DiffData; lowerB, uppe y: downVector[downOffset + k] - k) # Extend the reverse path. - for k in countUp(upK - D, upK + D, 2): + for k in countup(upK - D, upK + D, 2): # find the only or better starting point var x: int if k == upK + D: @@ -195,14 +201,14 @@ proc lcs(dataA: var DiffData; lowerA, upperA: int; dataB: var DiffData; lowerB, ## algorithm. ## The published algorithm passes recursively parts of the A and B sequences. ## To avoid copying these arrays the lower and upper bounds are passed while the sequences stay constant. - ## ``dataA`` sequence A - ## ``lowerA`` lower bound of the actual range in dataA - ## ``upperA`` upper bound of the actual range in dataA (exclusive) - ## ``dataB`` sequence B - ## ``lowerB`` lower bound of the actual range in dataB - ## ``upperB`` upper bound of the actual range in dataB (exclusive) - ## ``downVector`` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons. - ## ``upVector`` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons. + ## `dataA` sequence A + ## `lowerA` lower bound of the actual range in dataA + ## `upperA` upper bound of the actual range in dataA (exclusive) + ## `dataB` sequence B + ## `lowerB` lower bound of the actual range in dataB + ## `upperB` upper bound of the actual range in dataB (exclusive) + ## `downVector` a vector for the (0,0) to (x,y) search. Passed as a parameter for speed reasons. + ## `upVector` a vector for the (u,v) to (N,M) search. Passed as a parameter for speed reasons. # make mutable copy: var lowerA = lowerA @@ -275,9 +281,9 @@ proc createDiffs(dataA, dataB: DiffData): seq[Item] = proc diffInt*(arrayA, arrayB: openArray[int]): seq[Item] = ## Find the difference in 2 arrays of integers. ## - ## ``arrayA`` A-version of the numbers (usually the old one) + ## `arrayA` A-version of the numbers (usually the old one) ## - ## ``arrayB`` B-version of the numbers (usually the new one) + ## `arrayB` B-version of the numbers (usually the new one) ## ## Returns a sequence of Items that describe the differences. @@ -288,9 +294,9 @@ proc diffInt*(arrayA, arrayB: openArray[int]): seq[Item] = var dataB = newDiffData(@arrayB, arrayB.len) let max = dataA.len + dataB.len + 1 - ## vector for the (0,0) to (x,y) search + # vector for the (0,0) to (x,y) search var downVector = newSeq[int](2 * max + 2) - ## vector for the (u,v) to (N,M) search + # vector for the (u,v) to (N,M) search var upVector = newSeq[int](2 * max + 2) lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector) @@ -304,12 +310,12 @@ proc diffText*(textA, textB: string): seq[Item] = ## textlines into a common hashtable so i can find duplicates in there, and generating a ## new number each time a new textline is inserted. ## - ## ``textA`` A-version of the text (usually the old one) + ## `textA` A-version of the text (usually the old one) ## - ## ``textB`` B-version of the text (usually the new one) + ## `textB` B-version of the text (usually the new one) ## ## Returns a seq of Items that describe the differences. - + # See also `gitutils.diffStrings`. # prepare the input-text and convert to comparable numbers. var h = initTable[string, int]() # TextA.len + TextB.len <- probably wrong initial size # The A-Version of the data (original data) to be compared. @@ -321,9 +327,9 @@ proc diffText*(textA, textB: string): seq[Item] = h.clear # free up hashtable memory (maybe) let max = dataA.len + dataB.len + 1 - ## vector for the (0,0) to (x,y) search + # vector for the (0,0) to (x,y) search var downVector = newSeq[int](2 * max + 2) - ## vector for the (u,v) to (N,M) search + # vector for the (u,v) to (N,M) search var upVector = newSeq[int](2 * max + 2) lcs(dataA, 0, dataA.len, dataB, 0, dataB.len, downVector, upVector) @@ -331,83 +337,3 @@ proc diffText*(textA, textB: string): seq[Item] = optimize(dataA) optimize(dataB) result = createDiffs(dataA, dataB) - -when isMainModule: - - proc testHelper(f: seq[Item]): string = - for it in f: - result.add( - $it.deletedA & "." & $it.insertedB & "." & $it.startA & "." & $it.startB & "*" - ) - - proc main() = - var a, b: string - - stdout.writeLine("Diff Self Test...") - - # test all changes - a = "a,b,c,d,e,f,g,h,i,j,k,l".replace(',', '\n') - b = "0,1,2,3,4,5,6,7,8,9".replace(',', '\n') - assert(testHelper(diffText(a, b)) == - "12.10.0.0*", - "all-changes test failed.") - stdout.writeLine("all-changes test passed.") - # test all same - a = "a,b,c,d,e,f,g,h,i,j,k,l".replace(',', '\n') - b = a - assert(testHelper(diffText(a, b)) == - "", - "all-same test failed.") - stdout.writeLine("all-same test passed.") - - # test snake - a = "a,b,c,d,e,f".replace(',', '\n') - b = "b,c,d,e,f,x".replace(',', '\n') - assert(testHelper(diffText(a, b)) == - "1.0.0.0*0.1.6.5*", - "snake test failed.") - stdout.writeLine("snake test passed.") - - # 2002.09.20 - repro - a = "c1,a,c2,b,c,d,e,g,h,i,j,c3,k,l".replace(',', '\n') - b = "C1,a,C2,b,c,d,e,I1,e,g,h,i,j,C3,k,I2,l".replace(',', '\n') - assert(testHelper(diffText(a, b)) == - "1.1.0.0*1.1.2.2*0.2.7.7*1.1.11.13*0.1.13.15*", - "repro20020920 test failed.") - stdout.writeLine("repro20020920 test passed.") - - # 2003.02.07 - repro - a = "F".replace(',', '\n') - b = "0,F,1,2,3,4,5,6,7".replace(',', '\n') - assert(testHelper(diffText(a, b)) == - "0.1.0.0*0.7.1.2*", - "repro20030207 test failed.") - stdout.writeLine("repro20030207 test passed.") - - # Muegel - repro - a = "HELLO\nWORLD" - b = "\n\nhello\n\n\n\nworld\n" - assert(testHelper(diffText(a, b)) == - "2.8.0.0*", - "repro20030409 test failed.") - stdout.writeLine("repro20030409 test passed.") - - # test some differences - a = "a,b,-,c,d,e,f,f".replace(',', '\n') - b = "a,b,x,c,e,f".replace(',', '\n') - assert(testHelper(diffText(a, b)) == - "1.1.2.2*1.0.4.4*1.0.7.6*", - "some-changes test failed.") - stdout.writeLine("some-changes test passed.") - - # test one change within long chain of repeats - a = "a,a,a,a,a,a,a,a,a,a".replace(',', '\n') - b = "a,a,a,a,-,a,a,a,a,a".replace(',', '\n') - assert(testHelper(diffText(a, b)) == - "0.1.4.4*1.0.9.10*", - "long chain of repeats test failed.") - - stdout.writeLine("End.") - stdout.flushFile - - main() diff --git a/lib/genode/alloc.nim b/lib/genode/alloc.nim index a21a3ad7b..24fb9954e 100644 --- a/lib/genode/alloc.nim +++ b/lib/genode/alloc.nim @@ -15,20 +15,20 @@ when not defined(genode): {.error: "Genode only module".} when not declared(GenodeEnv): - include genode/env + import genode/env -type DataspaceCapability {. - importcpp: "Genode::Dataspace_capability", pure.} = object +type RamDataspaceCapability {. + importcpp: "Genode::Ram_dataspace_capability", pure.} = object type Map = object attachment: pointer size: int - ds: DataspaceCapability + ds: RamDataspaceCapability SlabMeta = object next: ptr MapSlab - ds: DataspaceCapability + ds: RamDataspaceCapability MapSlab = object meta: SlabMeta @@ -45,11 +45,11 @@ proc capsAvail(env: GenodeEnv): int {. ## Return the number of available capabilities. ## Each dataspace allocation consumes a capability. -proc allocDataspace(env: GenodeEnv; size: int): DataspaceCapability {. +proc allocDataspace(env: GenodeEnv; size: int): RamDataspaceCapability {. importcpp: "#->pd().alloc(@)".} ## Allocate a dataspace and its capability. -proc attachDataspace(env: GenodeEnv; ds: DataspaceCapability): pointer {. +proc attachDataspace(env: GenodeEnv; ds: RamDataspaceCapability): pointer {. importcpp: "#->rm().attach(@)".} ## Attach a dataspace into the component address-space. @@ -57,7 +57,7 @@ proc detachAddress(env: GenodeEnv; p: pointer) {. importcpp: "#->rm().detach(@)".} ## Detach a dataspace from the component address-space. -proc freeDataspace(env: GenodeEnv; ds: DataspaceCapability) {. +proc freeDataspace(env: GenodeEnv; ds: RamDataspaceCapability) {. importcpp: "#->pd().free(@)".} ## Free a dataspace. @@ -111,7 +111,7 @@ proc osDeallocPages(p: pointer; size: int) = if m.attachment == p: if m.size != size: echo "cannot partially detach dataspace" - quit -1 + rawQuit -1 runtimeEnv.detachAddress m.attachment runtimeEnv.freeDataspace m.ds m[] = Map() diff --git a/lib/genode/constructibles.nim b/lib/genode/constructibles.nim new file mode 100644 index 000000000..3a4a646e0 --- /dev/null +++ b/lib/genode/constructibles.nim @@ -0,0 +1,21 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +type Constructible*[T] {. + importcpp: "Genode::Constructible", + header: "<util/reconstructible.h>", byref, pure.} = object + +proc construct*[T](x: Constructible[T]) {.importcpp.} + ## Construct a constructible C++ object. + +proc destruct*[T](x: Constructible[T]) {.importcpp.} + ## Destruct a constructible C++ object. + +proc constructed*[T](x: Constructible[T]): bool {.importcpp.} + ## Test if an object is constructed. diff --git a/lib/genode/entrypoints.nim b/lib/genode/entrypoints.nim new file mode 100644 index 000000000..0bf5e0e0e --- /dev/null +++ b/lib/genode/entrypoints.nim @@ -0,0 +1,22 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## See `Genode Foundations - Entrypoint <https://genode.org/documentation/genode-foundations/21.05/functional_specification/Entrypoint.html>` +## for a description of Entrypoints. + +type + EntrypointObj {. + importcpp: "Genode::Entrypoint", + header: "<base/entrypoint.h>", + pure.} = object + Entrypoint* = ptr EntrypointObj + ## Opaque Entrypoint object. + +proc ep*(env: GenodeEnv): Entrypoint {.importcpp: "(&#->ep())".} + ## Access the entrypoint associated with `env`. diff --git a/lib/genode/env.nim b/lib/genode/env.nim index 2b180d1b3..babe2a8a0 100644 --- a/lib/genode/env.nim +++ b/lib/genode/env.nim @@ -11,19 +11,19 @@ # This file contains the minimum required definitions # for interacting with the initial Genode environment. # It is reserved for use only within the standard -# library. See ``componentConstructHook`` in the system +# library. See `componentConstructHook` in the system # module for accessing the Genode environment after the # standard library has finished initializating. # when not defined(genode): - {.error: "Genode only include".} + {.error: "Genode only module".} type - GenodeEnvObj {.importcpp: "Genode::Env", header: "<base/env.h>", pure.} = object - GenodeEnvPtr = ptr GenodeEnvObj + GenodeEnvObj* {.importcpp: "Genode::Env", header: "<base/env.h>", pure.} = object + GenodeEnvPtr* = ptr GenodeEnvObj -const runtimeEnvSym = "nim_runtime_env" +const runtimeEnvSym* = "nim_runtime_env" when not defined(nimscript): - var runtimeEnv {.importcpp: runtimeEnvSym.}: GenodeEnvPtr + var runtimeEnv* {.importcpp: runtimeEnvSym.}: GenodeEnvPtr diff --git a/lib/genode/signals.nim b/lib/genode/signals.nim new file mode 100644 index 000000000..7d1875730 --- /dev/null +++ b/lib/genode/signals.nim @@ -0,0 +1,77 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Emery Hemingway +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## See `Genode Foundations - Asynchronous notifications <https://genode.org/documentation/genode-foundations/21.05/architecture/Inter-component_communication.html#Asynchronous_notifications>` +## for a description of Genode signals. + +when not defined(genode) or defined(nimdoc): + {.error: "Genode only module".} + +import ./entrypoints, ./constructibles + +export ep # Entrypoint accessor on GenodeEnv + +type + SignalContextCapability* {. + importcpp: "Genode::Signal_context_capability", + header: "<base/signal.h>", pure.} = object + ## Capability to an asynchronous signal context. + +proc isValid*(cap: SignalContextCapability): bool {.importcpp: "#.valid()".} + ## Call the Genode core to check if this `SignalContextCapability` is valid. + # TODO: RpcEffect + +type + HandlerProc = proc () {.closure, gcsafe.} + + SignalHandlerBase {. + importcpp: "Nim::SignalHandler", + header: "genode_cpp/signals.h", + pure.} = object + + SignalHandlerCpp = Constructible[SignalHandlerBase] + + SignalHandlerObj = object + cpp: SignalHandlerCpp + cb: HandlerProc + ## Signal handling procedure called during dispatch. + + SignalHandler* = ref SignalHandlerObj + ## Nim object enclosing a Genode signal handler. + +proc construct(cpp: SignalHandlerCpp; ep: Entrypoint; sh: SignalHandler) {.importcpp.} + +proc cap(cpp: SignalHandlerCpp): SignalContextCapability {.importcpp: "#->cap()".} + +proc newSignalHandler*(ep: Entrypoint; cb: HandlerProc): SignalHandler = + ## Create a new signal handler. A label is recommended for + ## debugging purposes. A signal handler will not be garbage + ## collected until after it has been dissolved. + result = SignalHandler(cb: cb) + result.cpp.construct(ep, result) + GCref result + +proc dissolve*(sig: SignalHandler) = + ## Dissolve signal dispatcher from entrypoint. + # TODO: =destroy? + destruct sig.cpp + sig.cb = nil # lose the callback + GCunref sig + +proc cap*(sig: SignalHandler): SignalContextCapability = + ## Signal context capability. Can be delegated to external components. + sig.cpp.cap + +proc submit*(cap: SignalContextCapability) {. + importcpp: "Genode::Signal_transmitter(#).submit()".} + ## Submit a signal to a context capability. + +proc nimHandleSignal(p: pointer) {.exportc.} = + ## C symbol invoked by entrypoint during signal dispatch. + cast[SignalHandler](p).cb() diff --git a/lib/genode_cpp/signals.h b/lib/genode_cpp/signals.h new file mode 100644 index 000000000..fa3975d38 --- /dev/null +++ b/lib/genode_cpp/signals.h @@ -0,0 +1,39 @@ +/* + * + * Nim's Runtime Library + * (c) Copyright 2022 Emery Hemingway + * + * See the file "copying.txt", included in this + * distribution, for details about the copyright. + * + */ + +#ifndef _NIM_SIGNALS_H_ +#define _NIM_SIGNALS_H_ + +#include <libc/component.h> +#include <base/signal.h> +#include <util/reconstructible.h> + +// Symbol for calling back into Nim +extern "C" void nimHandleSignal(void *arg); + +namespace Nim { struct SignalHandler; } + +struct Nim::SignalHandler +{ + // Pointer to the Nim handler object. + void *arg; + + void handle_signal() { + Libc::with_libc([this] () { nimHandleSignal(arg); }); } + + Genode::Signal_handler<SignalHandler> handler; + + SignalHandler(Genode::Entrypoint *ep, void *arg) + : arg(arg), handler(*ep, *this, &SignalHandler::handle_signal) { } + + Genode::Signal_context_capability cap() { return handler; } +}; + +#endif diff --git a/lib/genode_cpp/syslocks.h b/lib/genode_cpp/syslocks.h index 8ba39abc2..b5d5ae694 100644 --- a/lib/genode_cpp/syslocks.h +++ b/lib/genode_cpp/syslocks.h @@ -13,7 +13,7 @@ /* Genode includes */ #include <base/semaphore.h> -#include <base/lock.h> +#include <base/mutex.h> namespace Nim { struct SysLock; @@ -22,15 +22,14 @@ namespace Nim { struct Nim::SysLock { - Genode::Lock _lock_a, _lock_b; + Genode::Mutex _mutex_a, _mutex_b; bool _locked; void acquireSys() { - _lock_a.lock(); + Genode::Mutex::Guard guard(_mutex_a); _locked = true; - _lock_a.unlock(); - _lock_b.lock(); + _mutex_b.acquire(); } bool tryAcquireSys() @@ -38,23 +37,22 @@ struct Nim::SysLock if (_locked) return false; - _lock_a.lock(); + Genode::Mutex::Guard guard(_mutex_a); + if (_locked) { - _lock_a.unlock(); return false; } else { _locked = true; - _lock_b.lock(); - _lock_a.unlock(); + _mutex_b.acquire(); return true; } } void releaseSys() { + Genode::Mutex::Guard guard(_mutex_a); _locked = false; - _lock_a.unlock(); - _lock_b.unlock(); + _mutex_b.release(); } }; @@ -73,6 +71,11 @@ struct Nim::SysCond { _semaphore.up(); } + + void broadcastSysCond() + { + _semaphore.up(); + } }; #endif diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim deleted file mode 100644 index 510503a63..000000000 --- a/lib/impure/db_mysql.nim +++ /dev/null @@ -1,397 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `mySQL`:idx: database wrapper. The same interface is -## implemented for other databases too. -## -## See also: `db_odbc <db_odbc.html>`_, `db_sqlite <db_sqlite.html>`_, -## `db_postgres <db_postgres.html>`_. -## -## Parameter substitution -## ====================== -## -## All ``db_*`` modules support the same form of parameter substitution. -## That is, using the ``?`` (question mark) to signify the place where a -## value should be placed. For example: -## -## .. code-block:: Nim -## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" -## -## -## Examples -## ======== -## -## Opening a connection to a database -## ---------------------------------- -## -## .. code-block:: Nim -## import db_mysql -## let db = open("localhost", "user", "password", "dbname") -## db.close() -## -## Creating a table -## ---------------- -## -## .. code-block:: Nim -## db.exec(sql"DROP TABLE IF EXISTS myTable") -## db.exec(sql("""CREATE TABLE myTable ( -## id integer, -## name varchar(50) not null)""")) -## -## Inserting data -## -------------- -## -## .. code-block:: Nim -## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", -## "Dominik") -## -## Larger example -## -------------- -## -## .. code-block:: Nim -## -## import db_mysql, math -## -## let theDb = open("localhost", "nim", "nim", "test") -## -## theDb.exec(sql"Drop table if exists myTestTbl") -## theDb.exec(sql("create table myTestTbl (" & -## " Id INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY, " & -## " Name VARCHAR(50) NOT NULL, " & -## " i INT(11), " & -## " f DECIMAL(18,10))")) -## -## theDb.exec(sql"START TRANSACTION") -## for i in 1..1000: -## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#" & $i, i, sqrt(i.float)) -## theDb.exec(sql"COMMIT") -## -## for x in theDb.fastRows(sql"select * from myTestTbl"): -## echo x -## -## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) -## -## theDb.close() - - -import strutils, mysql - -import db_common -export db_common - -type - DbConn* = distinct PMySQL ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = object ## a handle that can be used to get a row's - ## column text on demand - row: cstringArray - len: int - -proc dbError*(db: DbConn) {.noreturn.} = - ## raises a DbError exception. - var e: ref DbError - new(e) - e.msg = $mysql.error(PMySQL db) - raise e - -when false: - proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) = - var stmt = mysql_stmt_init(db) - if stmt == nil: dbError(db) - if mysql_stmt_prepare(stmt, query, len(query)) != 0: - dbError(db) - var - binding: seq[MYSQL_BIND] - discard mysql_stmt_close(stmt) - -proc dbQuote*(s: string): string = - ## DB quotes the string. - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var q = dbFormat(query, args) - return mysql.realQuery(PMySQL db, q, q.len) == 0'i32 - -proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = - var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) - -proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query and raises EDB if not successful. - var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) - -proc newRow(L: int): Row = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = - if row != nil: - while mysql.fetchRow(sqlres) != nil: discard - mysql.freeResult(sqlres) - -iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the query and iterates over the result dataset. - ## - ## This is very fast, but potentially dangerous. Use this iterator only - ## if you require **ALL** the rows. - ## - ## Breaking the fastRows() iterator during a loop will cause the next - ## database query to raise an [EDb] exception ``Commands out of sync``. - rawExec(db, query, args) - var sqlres = mysql.useResult(PMySQL db) - if sqlres != nil: - var - L = int(mysql.numFields(sqlres)) - row: cstringArray - result: Row - backup: Row - newSeq(result, L) - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - for i in 0..L-1: - setLen(result[i], 0) - result[i].add row[i] - yield result - properFreeResult(sqlres, row) - -iterator instantRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## Same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - rawExec(db, query, args) - var sqlres = mysql.useResult(PMySQL db) - if sqlres != nil: - let L = int(mysql.numFields(sqlres)) - var row: cstringArray - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - yield InstantRow(row: row, len: L) - properFreeResult(sqlres, row) - -proc setTypeName(t: var DbType; f: PFIELD) = - t.name = $f.name - t.maxReprLen = Natural(f.max_length) - if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true - case f.ftype - of TYPE_DECIMAL: - t.kind = dbDecimal - of TYPE_TINY: - t.kind = dbInt - t.size = 1 - of TYPE_SHORT: - t.kind = dbInt - t.size = 2 - of TYPE_LONG: - t.kind = dbInt - t.size = 4 - of TYPE_FLOAT: - t.kind = dbFloat - t.size = 4 - of TYPE_DOUBLE: - t.kind = dbFloat - t.size = 8 - of TYPE_NULL: - t.kind = dbNull - of TYPE_TIMESTAMP: - t.kind = dbTimestamp - of TYPE_LONGLONG: - t.kind = dbInt - t.size = 8 - of TYPE_INT24: - t.kind = dbInt - t.size = 3 - of TYPE_DATE: - t.kind = dbDate - of TYPE_TIME: - t.kind = dbTime - of TYPE_DATETIME: - t.kind = dbDatetime - of TYPE_YEAR: - t.kind = dbDate - of TYPE_NEWDATE: - t.kind = dbDate - of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING: - t.kind = dbVarchar - of TYPE_BIT: - t.kind = dbBit - of TYPE_NEWDECIMAL: - t.kind = dbDecimal - of TYPE_ENUM: t.kind = dbEnum - of TYPE_SET: t.kind = dbSet - of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB, - TYPE_BLOB: t.kind = dbBlob - of TYPE_GEOMETRY: - t.kind = dbGeometry - -proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) = - setLen(columns, L) - for i in 0..<L: - let fp = mysql.fetch_field_direct(res, cint(i)) - setTypeName(columns[i].typ, fp) - columns[i].name = $fp.name - columns[i].tableName = $fp.table - columns[i].primaryKey = (fp.flags and PRI_KEY_FLAG) != 0 - #columns[i].foreignKey = there is no such thing in mysql - -iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; - args: varargs[string, `$`]): InstantRow = - ## Same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - rawExec(db, query, args) - var sqlres = mysql.useResult(PMySQL db) - if sqlres != nil: - let L = int(mysql.numFields(sqlres)) - setColumnInfo(columns, sqlres, L) - var row: cstringArray - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - yield InstantRow(row: row, len: L) - properFreeResult(sqlres, row) - - -proc `[]`*(row: InstantRow, col: int): string {.inline.} = - ## Returns text for given column of the row. - $row.row[col] - -proc unsafeColumnAt*(row: InstantRow, index: int): cstring {.inline.} = - ## Return cstring of given column of the row - row.row[index] - -proc len*(row: InstantRow): int {.inline.} = - ## Returns number of columns in the row. - row.len - -proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## Retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - rawExec(db, query, args) - var sqlres = mysql.useResult(PMySQL db) - if sqlres != nil: - var L = int(mysql.numFields(sqlres)) - result = newRow(L) - var row = mysql.fetchRow(sqlres) - if row != nil: - for i in 0..L-1: - setLen(result[i], 0) - add(result[i], row[i]) - properFreeResult(sqlres, row) - -proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = - ## executes the query and returns the whole result dataset. - result = @[] - rawExec(db, query, args) - var sqlres = mysql.useResult(PMySQL db) - if sqlres != nil: - var L = int(mysql.numFields(sqlres)) - var row: cstringArray - var j = 0 - while true: - row = mysql.fetchRow(sqlres) - if row == nil: break - setLen(result, j+1) - newSeq(result[j], L) - for i in 0..L-1: - result[j][i] = $row[i] - inc(j) - mysql.freeResult(sqlres) - -iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `fastRows`, but slower and safe. - for r in items(getAllRows(db, query, args)): yield r - -proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - result = getRow(db, query, args)[0] - -proc tryInsertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. - var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: - result = -1'i64 - else: - result = mysql.insertId(PMySQL db) - -proc insertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## runs the query (typically "UPDATE") and returns the - ## number of affected rows - rawExec(db, query, args) - result = mysql.affectedRows(PMySQL db) - -proc close*(db: DbConn) {.tags: [DbEffect].} = - ## closes the database connection. - if PMySQL(db) != nil: mysql.close(PMySQL db) - -proc open*(connection, user, password, database: string): DbConn {. - tags: [DbEffect].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. - var res = mysql.init(nil) - if res == nil: dbError("could not open database connection") - let - colonPos = connection.find(':') - host = if colonPos < 0: connection - else: substr(connection, 0, colonPos-1) - port: int32 = if colonPos < 0: 0'i32 - else: substr(connection, colonPos+1).parseInt.int32 - if mysql.realConnect(res, host, user, password, database, - port, nil, 0) == nil: - var errmsg = $mysql.error(res) - mysql.close(res) - dbError(errmsg) - result = DbConn(res) - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [DbEffect].} = - ## sets the encoding of a database connection, returns true for - ## success, false for failure. - result = mysql.set_character_set(PMySQL connection, encoding) == 0 diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim deleted file mode 100644 index 7370adbf3..000000000 --- a/lib/impure/db_odbc.nim +++ /dev/null @@ -1,521 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `ODBC` database wrapper. -## -## This is the same interface that is implemented for other databases. -## -## This has NOT yet been (extensively) tested against ODBC drivers for -## Teradata, Oracle, Sybase, MSSqlvSvr, et. al. databases. -## -## Currently all queries are ANSI calls, not Unicode. -## -## See also: `db_postgres <db_postgres.html>`_, `db_sqlite <db_sqlite.html>`_, -## `db_mysql <db_mysql.html>`_. -## -## Parameter substitution -## ====================== -## -## All ``db_*`` modules support the same form of parameter substitution. -## That is, using the ``?`` (question mark) to signify the place where a -## value should be placed. For example: -## -## .. code-block:: Nim -## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" -## -## -## Examples -## ======== -## -## Opening a connection to a database -## ---------------------------------- -## -## .. code-block:: Nim -## import db_odbc -## var db = open("localhost", "user", "password", "dbname") -## db.close() -## -## Creating a table -## ---------------- -## -## .. code-block:: Nim -## db.exec(sql"DROP TABLE IF EXISTS myTable") -## db.exec(sql("""CREATE TABLE myTable ( -## id integer, -## name varchar(50) not null)""")) -## -## Inserting data -## -------------- -## -## .. code-block:: Nim -## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", -## "Andreas") -## -## Large example -## ------------- -## -## .. code-block:: Nim -## -## import db_odbc, math -## -## var theDb = open("localhost", "nim", "nim", "test") -## -## theDb.exec(sql"Drop table if exists myTestTbl") -## theDb.exec(sql("create table myTestTbl (" & -## " Id INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY, " & -## " Name VARCHAR(50) NOT NULL, " & -## " i INT(11), " & -## " f DECIMAL(18,10))")) -## -## theDb.exec(sql"START TRANSACTION") -## for i in 1..1000: -## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#" & $i, i, sqrt(i.float)) -## theDb.exec(sql"COMMIT") -## -## for x in theDb.fastRows(sql"select * from myTestTbl"): -## echo x -## -## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) -## -## theDb.close() - -import strutils, odbcsql -import db_common -export db_common - -type - OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt] - DbConn* = OdbcConnTyp ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = tuple[row: seq[string], len: int] ## a handle that can be - ## used to get a row's - ## column text on demand - -var - buf: array[0..4096, char] - -proc properFreeResult(hType: int, sqlres: var SqlHandle) {. - tags: [WriteDbEffect], raises: [].} = - try: - discard SQLFreeHandle(hType.TSqlSmallInt, sqlres) - sqlres = nil - except: discard - -proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. - tags: [ReadDbEffect], raises: [].} = - ## Returns ODBC error information - var - sqlState: array[0..512, char] - nativeErr: array[0..512, char] - errMsg: array[0..512, char] - retSz: TSqlSmallInt = 0 - res: TSqlSmallInt = 0 - try: - sqlState[0] = '\0' - nativeErr[0] = '\0' - errMsg[0] = '\0' - res = SQLErr(db.env, db.hDb, db.stmt, - cast[PSQLCHAR](sqlState.addr), - cast[PSQLCHAR](nativeErr.addr), - cast[PSQLCHAR](errMsg.addr), - 511.TSqlSmallInt, retSz.addr) - except: - discard - return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg)) - -proc dbError*(db: var DbConn) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} = - ## Raises an `[DbError]` exception with ODBC error information - var - e: ref DbError - ss, ne, msg: string = "" - isAnError = false - res: int = 0 - prevSs = "" - while true: - prevSs = ss - (res, ss, ne, msg) = db.getErrInfo() - if prevSs == ss: - break - # sqlState of 00000 is not an error - elif ss == "00000": - break - elif ss == "01000": - echo "\nWarning: ", ss, " ", msg - continue - else: - isAnError = true - echo "\nError: ", ss, " ", msg - if isAnError: - new(e) - e.msg = "ODBC Error" - if db.stmt != nil: - properFreeResult(SQL_HANDLE_STMT, db.stmt) - properFreeResult(SQL_HANDLE_DBC, db.hDb) - properFreeResult(SQL_HANDLE_ENV, db.env) - raise e - -proc sqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} = - ## Wrapper that raises [EDb] if ``resVal`` is neither SQL_SUCCESS or SQL_NO_DATA - if resVal notIn [SQL_SUCCESS, SQL_NO_DATA]: dbError(db) - -proc sqlGetDBMS(db: var DbConn): string {. - tags: [ReadDbEffect, WriteDbEffect], raises: [] .} = - ## Returns the ODBC SQL_DBMS_NAME string - const - SQL_DBMS_NAME = 17.SqlUSmallInt - var - sz: TSqlSmallInt = 0 - buf[0] = '\0' - try: - db.sqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr), - 4095.TSqlSmallInt, sz.addr)) - except: discard - return $(addr buf) - -proc dbQuote*(s: string): string {.noSideEffect.} = - ## DB quotes the string. - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. - noSideEffect.} = - ## Replace any ``?`` placeholders with `args`, - ## and quotes the arguments - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc prepareFetch(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): TSqlSmallInt {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - # Prepare a statement, execute it and fetch the data to the driver - # ready for retrieval of the data - # Used internally by iterators and retrieval procs - # requires calling - # properFreeResult(SQL_HANDLE_STMT, db.stmt) - # when finished - db.sqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) - var q = dbFormat(query, args) - db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) - db.sqlCheck(SQLExecute(db.stmt)) - result = SQLFetch(db.stmt) - db.sqlCheck(result) - -proc prepareFetchDirect(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - # Prepare a statement, execute it and fetch the data to the driver - # ready for retrieval of the data - # Used internally by iterators and retrieval procs - # requires calling - # properFreeResult(SQL_HANDLE_STMT, db.stmt) - # when finished - db.sqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) - var q = dbFormat(query, args) - db.sqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) - db.sqlCheck(SQLFetch(db.stmt)) - -proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [ReadDbEffect, WriteDbEffect], raises: [].} = - ## Tries to execute the query and returns true if successful, false otherwise. - var - res:TSqlSmallInt = -1 - try: - db.prepareFetchDirect(query, args) - var - rCnt = -1 - res = SQLRowCount(db.stmt, rCnt) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - if res != SQL_SUCCESS: dbError(db) - except: discard - return res == SQL_SUCCESS - -proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - db.prepareFetchDirect(query, args) - -proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query and raises EDB if not successful. - db.prepareFetchDirect(query, args) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - -proc newRow(L: int): Row {.noSideEFfect.} = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -iterator fastRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query and iterates over the result dataset. - ## - ## This is very fast, but potentially dangerous. Use this iterator only - ## if you require **ALL** the rows. - ## - ## Breaking the fastRows() iterator during a loop may cause a driver error - ## for subsequent queries - ## - ## Rows are retrieved from the server at each iteration. - var - rowRes: Row - sz: TSqlInteger = 0 - cCnt: TSqlSmallInt = 0 - res: TSqlSmallInt = 0 - res = db.prepareFetch(query, args) - if res == SQL_NO_DATA: - discard - elif res == SQL_SUCCESS: - res = SQLNumResultCols(db.stmt, cCnt) - rowRes = newRow(cCnt) - rowRes.setLen(max(cCnt,0)) - while res == SQL_SUCCESS: - for colId in 1..cCnt: - buf[0] = '\0' - db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr)) - rowRes[colId-1] = $(addr buf) - yield rowRes - res = SQLFetch(db.stmt) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - db.sqlCheck(res) - -iterator instantRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect, WriteDbEffect].} = - ## Same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. - var - rowRes: Row = @[] - sz: TSqlInteger = 0 - cCnt: TSqlSmallInt = 0 - res: TSqlSmallInt = 0 - res = db.prepareFetch(query, args) - if res == SQL_NO_DATA: - discard - elif res == SQL_SUCCESS: - res = SQLNumResultCols(db.stmt, cCnt) - rowRes = newRow(cCnt) - rowRes.setLen(max(cCnt,0)) - while res == SQL_SUCCESS: - for colId in 1..cCnt: - buf[0] = '\0' - db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr)) - rowRes[colId-1] = $(addr buf) - yield (row: rowRes, len: cCnt.int) - res = SQLFetch(db.stmt) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - db.sqlCheck(res) - -proc `[]`*(row: InstantRow, col: int): string {.inline.} = - ## Returns text for given column of the row - $row.row[col] - -proc unsafeColumnAt*(row: InstantRow, index: int): cstring {.inline.} = - ## Return cstring of given column of the row - row.row[index] - -proc len*(row: InstantRow): int {.inline.} = - ## Returns number of columns in the row - row.len - -proc getRow*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - var - rowRes: Row - sz: TSqlInteger = 0 - cCnt: TSqlSmallInt = 0 - res: TSqlSmallInt = 0 - res = db.prepareFetch(query, args) - if res == SQL_NO_DATA: - result = @[] - elif res == SQL_SUCCESS: - res = SQLNumResultCols(db.stmt, cCnt) - rowRes = newRow(cCnt) - rowRes.setLen(max(cCnt,0)) - for colId in 1..cCnt: - buf[0] = '\0' - db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr)) - rowRes[colId-1] = $(addr buf) - res = SQLFetch(db.stmt) - result = rowRes - properFreeResult(SQL_HANDLE_STMT, db.stmt) - db.sqlCheck(res) - -proc getAllRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} = - ## Executes the query and returns the whole result dataset. - var - rows: seq[Row] = @[] - rowRes: Row - sz: TSqlInteger = 0 - cCnt: TSqlSmallInt = 0 - res: TSqlSmallInt = 0 - res = db.prepareFetch(query, args) - if res == SQL_NO_DATA: - result = @[] - elif res == SQL_SUCCESS: - res = SQLNumResultCols(db.stmt, cCnt) - rowRes = newRow(cCnt) - rowRes.setLen(max(cCnt,0)) - while res == SQL_SUCCESS: - for colId in 1..cCnt: - buf[0] = '\0' - db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, - cast[cstring](buf.addr), 4095.TSqlSmallInt, - sz.addr)) - rowRes[colId-1] = $(addr buf) - rows.add(rowRes) - res = SQLFetch(db.stmt) - result = rows - properFreeResult(SQL_HANDLE_STMT, db.stmt) - db.sqlCheck(res) - -iterator rows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Same as `fastRows`, but slower and safe. - ## - ## This retrieves ALL rows into memory before - ## iterating through the rows. - ## Large dataset queries will impact on memory usage. - for r in items(getAllRows(db, query, args)): yield r - -proc getValue*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {. - tags: [ReadDbEffect, WriteDbEffect], raises: [].} = - ## Executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - result = "" - try: - result = getRow(db, query, args)[0] - except: discard - -proc tryInsertId*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect], raises: [].} = - ## Executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. - if not tryExec(db, query, args): - result = -1'i64 - else: - result = -1'i64 - try: - case sqlGetDBMS(db).toLower(): - of "postgresql": - result = getValue(db, sql"SELECT LASTVAL();", []).parseInt - of "mysql": - result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt - of "sqlite": - result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt - of "microsoft sql server": - result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt - of "oracle": - result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt - else: result = -1'i64 - except: discard - -proc insertId*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Executes the query (typically "INSERT") and returns the - ## generated ID for the row. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: var DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Runs the query (typically "UPDATE") and returns the - ## number of affected rows - result = -1 - db.sqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle)) - var q = dbFormat(query, args) - db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) - rawExec(db, query, args) - var rCnt = -1 - db.sqlCheck(SQLRowCount(db.hDb, rCnt)) - properFreeResult(SQL_HANDLE_STMT, db.stmt) - result = rCnt - -proc close*(db: var DbConn) {. - tags: [WriteDbEffect], raises: [].} = - ## Closes the database connection. - if db.hDb != nil: - try: - var res = SQLDisconnect(db.hDb) - if db.stmt != nil: - res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt) - res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb) - res = SQLFreeHandle(SQL_HANDLE_ENV, db.env) - db = (hDb: nil, env: nil, stmt: nil) - except: - discard - -proc open*(connection, user, password, database: string): DbConn {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Opens a database connection. - ## - ## Raises `EDb` if the connection could not be established. - ## - ## Currently the database parameter is ignored, - ## but included to match ``open()`` in the other db_xxxxx library modules. - var - val: TSqlInteger = SQL_OV_ODBC3 - resLen = 0 - result = (hDb: nil, env: nil, stmt: nil) - # allocate environment handle - var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env) - if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.") - res = SQLSetEnvAttr(result.env, - SQL_ATTR_ODBC_VERSION.TSqlInteger, - val, resLen.TSqlInteger) - if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.") - # allocate hDb handle - res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb) - if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.") - - # Connect: connection = dsn str, - res = SQLConnect(result.hDb, - connection.PSQLCHAR , connection.len.TSqlSmallInt, - user.PSQLCHAR, user.len.TSqlSmallInt, - password.PSQLCHAR, password.len.TSqlSmallInt) - if res != SQL_SUCCESS: - result.dbError() - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = - ## Currently not implemented for ODBC. - ## - ## Sets the encoding of a database connection, returns true for - ## success, false for failure. - ##result = set_character_set(connection, encoding) == 0 - dbError("setEncoding() is currently not implemented by the db_odbc module") diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim deleted file mode 100644 index 57c61fa23..000000000 --- a/lib/impure/db_postgres.nim +++ /dev/null @@ -1,548 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `PostgreSQL`:idx: database wrapper. This interface -## is implemented for other databases also. -## -## See also: `db_odbc <db_odbc.html>`_, `db_sqlite <db_sqlite.html>`_, -## `db_mysql <db_mysql.html>`_. -## -## Parameter substitution -## ====================== -## -## All ``db_*`` modules support the same form of parameter substitution. -## That is, using the ``?`` (question mark) to signify the place where a -## value should be placed. For example: -## -## .. code-block:: Nim -## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" -## -## **Note**: There are two approaches to parameter substitution support by -## this module. -## -## 1. ``SqlQuery`` using ``?, ?, ?, ...`` (same as all the ``db_*`` modules) -## -## 2. ``SqlPrepared`` using ``$1, $2, $3, ...`` -## -## .. code-block:: Nim -## prepare(db, "myExampleInsert", -## sql"""INSERT INTO myTable -## (colA, colB, colC) -## VALUES ($1, $2, $3)""", -## 3) -## -## Examples -## ======== -## -## Opening a connection to a database -## ---------------------------------- -## -## .. code-block:: Nim -## import db_postgres -## let db = open("localhost", "user", "password", "dbname") -## db.close() -## -## Creating a table -## ---------------- -## -## .. code-block:: Nim -## db.exec(sql"DROP TABLE IF EXISTS myTable") -## db.exec(sql("""CREATE TABLE myTable ( -## id integer, -## name varchar(50) not null)""")) -## -## Inserting data -## -------------- -## -## .. code-block:: Nim -## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", -## "Dominik") -import strutils, postgres - -import db_common -export db_common - -type - DbConn* = PPGconn ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = object ## a handle that can be - res: PPGresult ## used to get a row's - line: int ## column text on demand - SqlPrepared* = distinct string ## a identifier for the prepared queries - -proc dbError*(db: DbConn) {.noreturn.} = - ## raises a DbError exception. - var e: ref DbError - new(e) - e.msg = $pqErrorMessage(db) - raise e - -proc dbQuote*(s: string): string = - ## DB quotes the string. - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - if args.len > 0 and not string(formatstr).contains("?"): - dbError("""parameter substitution expects "?" """) - if args.len == 0: - return string(formatstr) - else: - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, - nil, nil, 0) - result = pqresultStatus(res) == PGRES_COMMAND_OK - pqclear(res) - -proc tryExec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): bool {.tags: [ - ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. - var arr = allocCStringArray(args) - var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - result = pqresultStatus(res) == PGRES_COMMAND_OK - pqclear(res) - -proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query and raises EDB if not successful. - var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, - nil, nil, 0) - if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db) - pqclear(res) - -proc exec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} = - var arr = allocCStringArray(args) - var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db) - pqclear(res) - -proc newRow(L: int): Row = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -proc setupQuery(db: DbConn, query: SqlQuery, - args: varargs[string]): PPGresult = - result = pqexec(db, dbFormat(query, args)) - if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) - -proc setupQuery(db: DbConn, stmtName: SqlPrepared, - args: varargs[string]): PPGresult = - var arr = allocCStringArray(args) - result = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) - -proc prepare*(db: DbConn; stmtName: string, query: SqlQuery; - nParams: int): SqlPrepared = - ## Creates a new ``SqlPrepared`` statement. Parameter substitution is done - ## via ``$1``, ``$2``, ``$3``, etc. - if nParams > 0 and not string(query).contains("$1"): - dbError("parameter substitution expects \"$1\"") - var res = pqprepare(db, stmtName, query.string, int32(nParams), nil) - if pqResultStatus(res) != PGRES_COMMAND_OK: dbError(db) - return SqlPrepared(stmtName) - -proc setRow(res: PPGresult, r: var Row, line, cols: int32) = - for col in 0'i32..cols-1: - setLen(r[col], 0) - let x = pqgetvalue(res, line, col) - if x.isNil: - r[col] = "" - else: - add(r[col], x) - -iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the query and iterates over the result dataset. This is very - ## fast, but potentially dangerous: If the for-loop-body executes another - ## query, the results can be undefined. For Postgres it is safe though. - var res = setupQuery(db, query, args) - var L = pqnfields(res) - var result = newRow(L) - for i in 0'i32..pqntuples(res)-1: - setRow(res, result, i, L) - yield result - pqclear(res) - -iterator fastRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the prepared query and iterates over the result dataset. - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - var result = newRow(L) - for i in 0'i32..pqNtuples(res)-1: - setRow(res, result, i, L) - yield result - pqClear(res) - -iterator instantRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, query, args) - for i in 0'i32..pqNtuples(res)-1: - yield InstantRow(res: res, line: i) - pqClear(res) - -iterator instantRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, stmtName, args) - for i in 0'i32..pqNtuples(res)-1: - yield InstantRow(res: res, line: i) - pqClear(res) - -proc getColumnType(res: PPGresult, col: int) : DbType = - ## returns DbType for given column in the row - ## defined in pg_type.h file in the postgres source code - ## Wire representation for types: http://www.npgsql.org/dev/types.html - var oid = pqftype(res, int32(col)) - ## The integer returned is the internal OID number of the type - case oid - of 16: return DbType(kind: DbTypeKind.dbBool, name: "bool") - of 17: return DbType(kind: DbTypeKind.dbBlob, name: "bytea") - - of 21: return DbType(kind: DbTypeKind.dbInt, name: "int2", size: 2) - of 23: return DbType(kind: DbTypeKind.dbInt, name: "int4", size: 4) - of 20: return DbType(kind: DbTypeKind.dbInt, name: "int8", size: 8) - of 1560: return DbType(kind: DbTypeKind.dbBit, name: "bit") - of 1562: return DbType(kind: DbTypeKind.dbInt, name: "varbit") - - of 18: return DbType(kind: DbTypeKind.dbFixedChar, name: "char") - of 19: return DbType(kind: DbTypeKind.dbFixedChar, name: "name") - of 1042: return DbType(kind: DbTypeKind.dbFixedChar, name: "bpchar") - - of 25: return DbType(kind: DbTypeKind.dbVarchar, name: "text") - of 1043: return DbType(kind: DbTypeKind.dbVarChar, name: "varchar") - of 2275: return DbType(kind: DbTypeKind.dbVarchar, name: "cstring") - - of 700: return DbType(kind: DbTypeKind.dbFloat, name: "float4") - of 701: return DbType(kind: DbTypeKind.dbFloat, name: "float8") - - of 790: return DbType(kind: DbTypeKind.dbDecimal, name: "money") - of 1700: return DbType(kind: DbTypeKind.dbDecimal, name: "numeric") - - of 704: return DbType(kind: DbTypeKind.dbTimeInterval, name: "tinterval") - of 702: return DbType(kind: DbTypeKind.dbTimestamp, name: "abstime") - of 703: return DbType(kind: DbTypeKind.dbTimeInterval, name: "reltime") - of 1082: return DbType(kind: DbTypeKind.dbDate, name: "date") - of 1083: return DbType(kind: DbTypeKind.dbTime, name: "time") - of 1114: return DbType(kind: DbTypeKind.dbTimestamp, name: "timestamp") - of 1184: return DbType(kind: DbTypeKind.dbTimestamp, name: "timestamptz") - of 1186: return DbType(kind: DbTypeKind.dbTimeInterval, name: "interval") - of 1266: return DbType(kind: DbTypeKind.dbTime, name: "timetz") - - of 114: return DbType(kind: DbTypeKind.dbJson, name: "json") - of 142: return DbType(kind: DbTypeKind.dbXml, name: "xml") - of 3802: return DbType(kind: DbTypeKind.dbJson, name: "jsonb") - - of 600: return DbType(kind: DbTypeKind.dbPoint, name: "point") - of 601: return DbType(kind: DbTypeKind.dbLseg, name: "lseg") - of 602: return DbType(kind: DbTypeKind.dbPath, name: "path") - of 603: return DbType(kind: DbTypeKind.dbBox, name: "box") - of 604: return DbType(kind: DbTypeKind.dbPolygon, name: "polygon") - of 628: return DbType(kind: DbTypeKind.dbLine, name: "line") - of 718: return DbType(kind: DbTypeKind.dbCircle, name: "circle") - - of 650: return DbType(kind: DbTypeKind.dbInet, name: "cidr") - of 829: return DbType(kind: DbTypeKind.dbMacAddress, name: "macaddr") - of 869: return DbType(kind: DbTypeKind.dbInet, name: "inet") - - of 2950: return DbType(kind: DbTypeKind.dbVarchar, name: "uuid") - of 3614: return DbType(kind: DbTypeKind.dbVarchar, name: "tsvector") - of 3615: return DbType(kind: DbTypeKind.dbVarchar, name: "tsquery") - of 2970: return DbType(kind: DbTypeKind.dbVarchar, name: "txid_snapshot") - - of 27: return DbType(kind: DbTypeKind.dbComposite, name: "tid") - of 1790: return DbType(kind: DbTypeKind.dbComposite, name: "refcursor") - of 2249: return DbType(kind: DbTypeKind.dbComposite, name: "record") - of 3904: return DbType(kind: DbTypeKind.dbComposite, name: "int4range") - of 3906: return DbType(kind: DbTypeKind.dbComposite, name: "numrange") - of 3908: return DbType(kind: DbTypeKind.dbComposite, name: "tsrange") - of 3910: return DbType(kind: DbTypeKind.dbComposite, name: "tstzrange") - of 3912: return DbType(kind: DbTypeKind.dbComposite, name: "daterange") - of 3926: return DbType(kind: DbTypeKind.dbComposite, name: "int8range") - - of 22: return DbType(kind: DbTypeKind.dbArray, name: "int2vector") - of 30: return DbType(kind: DbTypeKind.dbArray, name: "oidvector") - of 143: return DbType(kind: DbTypeKind.dbArray, name: "xml[]") - of 199: return DbType(kind: DbTypeKind.dbArray, name: "json[]") - of 629: return DbType(kind: DbTypeKind.dbArray, name: "line[]") - of 651: return DbType(kind: DbTypeKind.dbArray, name: "cidr[]") - of 719: return DbType(kind: DbTypeKind.dbArray, name: "circle[]") - of 791: return DbType(kind: DbTypeKind.dbArray, name: "money[]") - of 1000: return DbType(kind: DbTypeKind.dbArray, name: "bool[]") - of 1001: return DbType(kind: DbTypeKind.dbArray, name: "bytea[]") - of 1002: return DbType(kind: DbTypeKind.dbArray, name: "char[]") - of 1003: return DbType(kind: DbTypeKind.dbArray, name: "name[]") - of 1005: return DbType(kind: DbTypeKind.dbArray, name: "int2[]") - of 1006: return DbType(kind: DbTypeKind.dbArray, name: "int2vector[]") - of 1007: return DbType(kind: DbTypeKind.dbArray, name: "int4[]") - of 1008: return DbType(kind: DbTypeKind.dbArray, name: "regproc[]") - of 1009: return DbType(kind: DbTypeKind.dbArray, name: "text[]") - of 1028: return DbType(kind: DbTypeKind.dbArray, name: "oid[]") - of 1010: return DbType(kind: DbTypeKind.dbArray, name: "tid[]") - of 1011: return DbType(kind: DbTypeKind.dbArray, name: "xid[]") - of 1012: return DbType(kind: DbTypeKind.dbArray, name: "cid[]") - of 1013: return DbType(kind: DbTypeKind.dbArray, name: "oidvector[]") - of 1014: return DbType(kind: DbTypeKind.dbArray, name: "bpchar[]") - of 1015: return DbType(kind: DbTypeKind.dbArray, name: "varchar[]") - of 1016: return DbType(kind: DbTypeKind.dbArray, name: "int8[]") - of 1017: return DbType(kind: DbTypeKind.dbArray, name: "point[]") - of 1018: return DbType(kind: DbTypeKind.dbArray, name: "lseg[]") - of 1019: return DbType(kind: DbTypeKind.dbArray, name: "path[]") - of 1020: return DbType(kind: DbTypeKind.dbArray, name: "box[]") - of 1021: return DbType(kind: DbTypeKind.dbArray, name: "float4[]") - of 1022: return DbType(kind: DbTypeKind.dbArray, name: "float8[]") - of 1023: return DbType(kind: DbTypeKind.dbArray, name: "abstime[]") - of 1024: return DbType(kind: DbTypeKind.dbArray, name: "reltime[]") - of 1025: return DbType(kind: DbTypeKind.dbArray, name: "tinterval[]") - of 1027: return DbType(kind: DbTypeKind.dbArray, name: "polygon[]") - of 1040: return DbType(kind: DbTypeKind.dbArray, name: "macaddr[]") - of 1041: return DbType(kind: DbTypeKind.dbArray, name: "inet[]") - of 1263: return DbType(kind: DbTypeKind.dbArray, name: "cstring[]") - of 1115: return DbType(kind: DbTypeKind.dbArray, name: "timestamp[]") - of 1182: return DbType(kind: DbTypeKind.dbArray, name: "date[]") - of 1183: return DbType(kind: DbTypeKind.dbArray, name: "time[]") - of 1185: return DbType(kind: DbTypeKind.dbArray, name: "timestamptz[]") - of 1187: return DbType(kind: DbTypeKind.dbArray, name: "interval[]") - of 1231: return DbType(kind: DbTypeKind.dbArray, name: "numeric[]") - of 1270: return DbType(kind: DbTypeKind.dbArray, name: "timetz[]") - of 1561: return DbType(kind: DbTypeKind.dbArray, name: "bit[]") - of 1563: return DbType(kind: DbTypeKind.dbArray, name: "varbit[]") - of 2201: return DbType(kind: DbTypeKind.dbArray, name: "refcursor[]") - of 2951: return DbType(kind: DbTypeKind.dbArray, name: "uuid[]") - of 3643: return DbType(kind: DbTypeKind.dbArray, name: "tsvector[]") - of 3645: return DbType(kind: DbTypeKind.dbArray, name: "tsquery[]") - of 3807: return DbType(kind: DbTypeKind.dbArray, name: "jsonb[]") - of 2949: return DbType(kind: DbTypeKind.dbArray, name: "txid_snapshot[]") - of 3905: return DbType(kind: DbTypeKind.dbArray, name: "int4range[]") - of 3907: return DbType(kind: DbTypeKind.dbArray, name: "numrange[]") - of 3909: return DbType(kind: DbTypeKind.dbArray, name: "tsrange[]") - of 3911: return DbType(kind: DbTypeKind.dbArray, name: "tstzrange[]") - of 3913: return DbType(kind: DbTypeKind.dbArray, name: "daterange[]") - of 3927: return DbType(kind: DbTypeKind.dbArray, name: "int8range[]") - of 2287: return DbType(kind: DbTypeKind.dbArray, name: "record[]") - - of 705: return DbType(kind: DbTypeKind.dbUnknown, name: "unknown") - else: return DbType(kind: DbTypeKind.dbUnknown, name: $oid) ## Query the system table pg_type to determine exactly which type is referenced. - -proc setColumnInfo(columns: var DbColumns; res: PPGresult; L: int32) = - setLen(columns, L) - for i in 0'i32..<L: - columns[i].name = $pqfname(res, i) - columns[i].typ = getColumnType(res, i) - columns[i].tableName = $(pqftable(res, i)) ## Returns the OID of the table from which the given column was fetched. - ## Query the system table pg_class to determine exactly which table is referenced. - #columns[i].primaryKey = libpq does not have a function for that - #columns[i].foreignKey = libpq does not have a function for that - -iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - var res = setupQuery(db, query, args) - setColumnInfo(columns, res, pqnfields(res)) - for i in 0'i32..<pqntuples(res): - yield InstantRow(res: res, line: i) - pqClear(res) - -proc `[]`*(row: InstantRow; col: int): string {.inline.} = - ## returns text for given column of the row - $pqgetvalue(row.res, int32(row.line), int32(col)) - -proc unsafeColumnAt*(row: InstantRow, index: int): cstring {.inline.} = - ## Return cstring of given column of the row - pqgetvalue(row.res, int32(row.line), int32(index)) - -proc len*(row: InstantRow): int {.inline.} = - ## returns number of columns in the row - int(pqNfields(row.res)) - -proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. - var res = setupQuery(db, query, args) - var L = pqnfields(res) - result = newRow(L) - if pqntuples(res) > 0: - setRow(res, result, 0, L) - pqclear(res) - -proc getRow*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - result = newRow(L) - if pqntuples(res) > 0: - setRow(res, result, 0, L) - pqClear(res) - -proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {. - tags: [ReadDbEffect].} = - ## executes the query and returns the whole result dataset. - result = @[] - for r in fastRows(db, query, args): - result.add(r) - -proc getAllRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): seq[Row] {.tags: - [ReadDbEffect].} = - ## executes the prepared query and returns the whole result dataset. - result = @[] - for r in fastRows(db, stmtName, args): - result.add(r) - -iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `fastRows`, but slower and safe. - for r in items(getAllRows(db, query, args)): yield r - -iterator rows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `fastRows`, but slower and safe. - for r in items(getAllRows(db, stmtName, args)): yield r - -proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {. - tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - var res = setupQuery(db, query, args) - if pqntuples(res) > 0: - var x = pqgetvalue(res, 0, 0) - result = if isNil(x): "" else: $x - else: - result = "" - -proc getValue*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): string {. - tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. - var res = setupQuery(db, stmtName, args) - if pqntuples(res) > 0: - var x = pqgetvalue(res, 0, 0) - result = if isNil(x): "" else: $x - else: - result = "" - -proc tryInsertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [WriteDbEffect].}= - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. For Postgre this adds - ## ``RETURNING id`` to the query, so it only works if your primary key is - ## named ``id``. - var x = pqgetvalue(setupQuery(db, SqlQuery(string(query) & " RETURNING id"), - args), 0, 0) - if not isNil(x): - result = parseBiggestInt($x) - else: - result = -1 - -proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row. For Postgre this adds - ## ``RETURNING id`` to the query, so it only works if your primary key is - ## named ``id``. - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [ - ReadDbEffect, WriteDbEffect].} = - ## executes the query (typically "UPDATE") and returns the - ## number of affected rows. - var q = dbFormat(query, args) - var res = pqExec(db, q) - if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db) - result = parseBiggestInt($pqcmdTuples(res)) - pqclear(res) - -proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): int64 {.tags: [ - ReadDbEffect, WriteDbEffect].} = - ## executes the query (typically "UPDATE") and returns the - ## number of affected rows. - var arr = allocCStringArray(args) - var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, - nil, nil, 0) - deallocCStringArray(arr) - if pqresultStatus(res) != PGRES_COMMAND_OK: dbError(db) - result = parseBiggestInt($pqcmdTuples(res)) - pqclear(res) - -proc close*(db: DbConn) {.tags: [DbEffect].} = - ## closes the database connection. - if db != nil: pqfinish(db) - -proc open*(connection, user, password, database: string): DbConn {. - tags: [DbEffect].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. - ## - ## Clients can also use Postgres keyword/value connection strings to - ## connect. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## con = open("", "", "", "host=localhost port=5432 dbname=mydb") - ## - ## See http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING - ## for more information. - let - colonPos = connection.find(':') - host = if colonPos < 0: connection - else: substr(connection, 0, colonPos-1) - port = if colonPos < 0: "" - else: substr(connection, colonPos+1) - result = pqsetdbLogin(host, port, nil, nil, database, user, password) - if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [DbEffect].} = - ## sets the encoding of a database connection, returns true for - ## success, false for failure. - return pqsetClientEncoding(connection, encoding) == 0 - - -# Tests are in ../../tests/untestable/tpostgres. diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim deleted file mode 100644 index f182ae65a..000000000 --- a/lib/impure/db_sqlite.nim +++ /dev/null @@ -1,650 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## A higher level `SQLite`:idx: database wrapper. This interface -## is implemented for other databases too. -## -## Basic usage -## =========== -## -## The basic flow of using this module is: -## -## 1. Open database connection -## 2. Execute SQL query -## 3. Close database connection -## -## Parameter substitution -## ---------------------- -## -## All ``db_*`` modules support the same form of parameter substitution. -## That is, using the ``?`` (question mark) to signify the place where a -## value should be placed. For example: -## -## .. code-block:: Nim -## -## sql"INSERT INTO my_table (colA, colB, colC) VALUES (?, ?, ?)" -## -## Opening a connection to a database -## ---------------------------------- -## -## .. code-block:: Nim -## -## import db_sqlite -## -## # user, password, database name can be empty. -## # These params are not used on db_sqlite module. -## let db = open("mytest.db", "", "", "") -## db.close() -## -## Creating a table -## ---------------- -## -## .. code-block:: Nim -## -## db.exec(sql"DROP TABLE IF EXISTS my_table") -## db.exec(sql"""CREATE TABLE my_table ( -## id INTEGER, -## name VARCHAR(50) NOT NULL -## )""") -## -## Inserting data -## -------------- -## -## .. code-block:: Nim -## -## db.exec(sql"INSERT INTO my_table (id, name) VALUES (0, ?)", -## "Jack") -## -## Larger example -## -------------- -## -## .. code-block:: nim -## -## import db_sqlite, math -## -## let db = open("mytest.db", "", "", "") -## -## db.exec(sql"DROP TABLE IF EXISTS my_table") -## db.exec(sql"""CREATE TABLE my_table ( -## id INTEGER PRIMARY KEY, -## name VARCHAR(50) NOT NULL, -## i INT(11), -## f DECIMAL(18, 10) -## )""") -## -## db.exec(sql"BEGIN") -## for i in 1..1000: -## db.exec(sql"INSERT INTO my_table (name, i, f) VALUES (?, ?, ?)", -## "Item#" & $i, i, sqrt(i.float)) -## db.exec(sql"COMMIT") -## -## for x in db.fastRows(sql"SELECT * FROM my_table"): -## echo x -## -## let id = db.tryInsertId(sql"""INSERT INTO my_table (name, i, f) -## VALUES (?, ?, ?)""", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", db.getValue(sql"SELECT name FROM my_table WHERE id=?", id) -## -## db.close() -## -## -## Note -## ==== -## This module does not implement any ORM features such as mapping the types from the schema. -## Instead, a ``seq[string]`` is returned for each row. -## -## The reasoning is as follows: -## 1. it's close to what many DBs offer natively (char**) -## 2. it hides the number of types that the DB supports -## (int? int64? decimal up to 10 places? geo coords?) -## 3. it's convenient when all you do is to forward the data to somewhere else (echo, log, put the data into a new query) -## -## See also -## ======== -## -## * `db_odbc module <db_odbc.html>`_ for ODBC database wrapper -## * `db_mysql module <db_mysql.html>`_ for MySQL database wrapper -## * `db_postgres module <db_postgres.html>`_ for PostgreSQL database wrapper - -{.deadCodeElim: on.} # dce option deprecated - -import sqlite3 - -import db_common -export db_common - -type - DbConn* = PSqlite3 ## Encapsulates a database connection. - Row* = seq[string] ## A row of a dataset. `NULL` database values will be - ## converted to an empty string. - InstantRow* = PStmt ## A handle that can be used to get a row's column - ## text on demand. - -proc dbError*(db: DbConn) {.noreturn.} = - ## Raises a `DbError` exception. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## if not db.tryExec(sql"SELECT * FROM not_exist_table"): - ## dbError(db) - ## db.close() - var e: ref DbError - new(e) - e.msg = $sqlite3.errmsg(db) - raise e - -proc dbQuote*(s: string): string = - ## Escapes the `'` (single quote) char to `''`. - ## Because single quote is used for defining `VARCHAR` in SQL. - runnableExamples: - doAssert dbQuote("'") == "''''" - doAssert dbQuote("A Foobar's pen.") == "'A Foobar''s pen.'" - - result = "'" - for c in items(s): - if c == '\'': add(result, "''") - else: add(result, c) - add(result, '\'') - -proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) - -proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## Tries to execute the query and returns `true` if successful, `false` otherwise. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## if not db.tryExec(sql"SELECT * FROM my_table"): - ## dbError(db) - ## db.close() - assert(not db.isNil, "Database not connected.") - var q = dbFormat(query, args) - var stmt: sqlite3.PStmt - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - let x = step(stmt) - if x in {SQLITE_DONE, SQLITE_ROW}: - result = finalize(stmt) == SQLITE_OK - -proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## Executes the query and raises a `DbError` exception if not successful. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## try: - ## db.exec(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", - ## 1, "item#1") - ## except: - ## stderr.writeLine(getCurrentExceptionMsg()) - ## finally: - ## db.close() - if not tryExec(db, query, args): dbError(db) - -proc newRow(L: int): Row = - newSeq(result, L) - for i in 0..L-1: result[i] = "" - -proc setupQuery(db: DbConn, query: SqlQuery, - args: varargs[string]): PStmt = - assert(not db.isNil, "Database not connected.") - var q = dbFormat(query, args) - if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db) - -proc setRow(stmt: PStmt, r: var Row, cols: cint) = - for col in 0'i32..cols-1: - setLen(r[col], column_bytes(stmt, col)) # set capacity - setLen(r[col], 0) - let x = column_text(stmt, col) - if not isNil(x): add(r[col], x) - -iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## Executes the query and iterates over the result dataset. - ## - ## This is very fast, but potentially dangerous. Use this iterator only - ## if you require **ALL** the rows. - ## - ## **Note:** Breaking the `fastRows()` iterator during a loop will cause the - ## next database query to raise a `DbError` exception ``unable to close due - ## to ...``. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## for row in db.fastRows(sql"SELECT id, name FROM my_table"): - ## echo row - ## - ## # Output: - ## # @["1", "item#1"] - ## # @["2", "item#2"] - ## - ## db.close() - var stmt = setupQuery(db, query, args) - var L = (column_count(stmt)) - var result = newRow(L) - try: - while step(stmt) == SQLITE_ROW: - setRow(stmt, result, L) - yield result - finally: - if finalize(stmt) != SQLITE_OK: dbError(db) - -iterator instantRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## Similar to `fastRows iterator <#fastRows.i,DbConn,SqlQuery,varargs[string,]>`_ - ## but returns a handle that can be used to get column text - ## on demand using `[]`. Returned handle is valid only within the iterator body. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## for row in db.instantRows(sql"SELECT * FROM my_table"): - ## echo "id:" & row[0] - ## echo "name:" & row[1] - ## echo "length:" & $len(row) - ## - ## # Output: - ## # id:1 - ## # name:item#1 - ## # length:2 - ## # id:2 - ## # name:item#2 - ## # length:2 - ## - ## db.close() - var stmt = setupQuery(db, query, args) - try: - while step(stmt) == SQLITE_ROW: - yield stmt - finally: - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc toTypeKind(t: var DbType; x: int32) = - case x - of SQLITE_INTEGER: - t.kind = dbInt - t.size = 8 - of SQLITE_FLOAT: - t.kind = dbFloat - t.size = 8 - of SQLITE_BLOB: t.kind = dbBlob - of SQLITE_NULL: t.kind = dbNull - of SQLITE_TEXT: t.kind = dbVarchar - else: t.kind = dbUnknown - -proc setColumns(columns: var DbColumns; x: PStmt) = - let L = column_count(x) - setLen(columns, L) - for i in 0'i32 ..< L: - columns[i].name = $column_name(x, i) - columns[i].typ.name = $column_decltype(x, i) - toTypeKind(columns[i].typ, column_type(x, i)) - columns[i].tableName = $column_table_name(x, i) - -iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery, - args: varargs[string, `$`]): InstantRow - {.tags: [ReadDbEffect].} = - ## Similar to `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_, - ## but sets information about columns to `columns`. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## var columns: DbColumns - ## for row in db.instantRows(columns, sql"SELECT * FROM my_table"): - ## discard - ## echo columns[0] - ## - ## # Output: - ## # (name: "id", tableName: "my_table", typ: (kind: dbNull, - ## # notNull: false, name: "INTEGER", size: 0, maxReprLen: 0, precision: 0, - ## # scale: 0, min: 0, max: 0, validValues: @[]), primaryKey: false, - ## # foreignKey: false) - ## - ## db.close() - var stmt = setupQuery(db, query, args) - setColumns(columns, stmt) - try: - while step(stmt) == SQLITE_ROW: - yield stmt - finally: - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc `[]`*(row: InstantRow, col: int32): string {.inline.} = - ## Returns text for given column of the row. - ## - ## See also: - ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_ - ## example code - $column_text(row, col) - -proc unsafeColumnAt*(row: InstantRow, index: int32): cstring {.inline.} = - ## Returns cstring for given column of the row. - ## - ## See also: - ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_ - ## example code - column_text(row, index) - -proc len*(row: InstantRow): int32 {.inline.} = - ## Returns number of columns in a row. - ## - ## See also: - ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_ - ## example code - column_count(row) - -proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## Retrieves a single row. If the query doesn't return any rows, this proc - ## will return a `Row` with empty strings for each column. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## doAssert db.getRow(sql"SELECT id, name FROM my_table" - ## ) == Row(@["1", "item#1"]) - ## doAssert db.getRow(sql"SELECT id, name FROM my_table WHERE id = ?", - ## 2) == Row(@["2", "item#2"]) - ## - ## # Returns empty. - ## doAssert db.getRow(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", - ## 3, "item#3") == @[] - ## doAssert db.getRow(sql"DELETE FROM my_table WHERE id = ?", 3) == @[] - ## doAssert db.getRow(sql"UPDATE my_table SET name = 'ITEM#1' WHERE id = ?", - ## 1) == @[] - ## db.close() - var stmt = setupQuery(db, query, args) - var L = (column_count(stmt)) - result = newRow(L) - if step(stmt) == SQLITE_ROW: - setRow(stmt, result, L) - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = - ## Executes the query and returns the whole result dataset. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## doAssert db.getAllRows(sql"SELECT id, name FROM my_table") == @[Row(@["1", "item#1"]), Row(@["2", "item#2"])] - ## db.close() - result = @[] - for r in fastRows(db, query, args): - result.add(r) - -iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## Similar to `fastRows iterator <#fastRows.i,DbConn,SqlQuery,varargs[string,]>`_, - ## but slower and safe. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## for row in db.rows(sql"SELECT id, name FROM my_table"): - ## echo row - ## - ## ## Output: - ## ## @["1", "item#1"] - ## ## @["2", "item#2"] - ## - ## db.close() - for r in fastRows(db, query, args): yield r - -proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = - ## Executes the query and returns the first column of the first row of the - ## result dataset. Returns `""` if the dataset contains no rows or the database - ## value is `NULL`. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## doAssert db.getValue(sql"SELECT name FROM my_table WHERE id = ?", - ## 2) == "item#2" - ## doAssert db.getValue(sql"SELECT id, name FROM my_table") == "1" - ## doAssert db.getValue(sql"SELECT name, id FROM my_table") == "item#1" - ## - ## db.close() - var stmt = setupQuery(db, query, args) - if step(stmt) == SQLITE_ROW: - let cb = column_bytes(stmt, 0) - if cb == 0: - result = "" - else: - result = newStringOfCap(cb) - add(result, column_text(stmt, 0)) - else: - result = "" - if finalize(stmt) != SQLITE_OK: dbError(db) - -proc tryInsertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [].} = - ## Executes the query (typically "INSERT") and returns the - ## generated ID for the row or -1 in case of an error. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## db.exec(sql"CREATE TABLE my_table (id INTEGER, name VARCHAR(50) NOT NULL)") - ## - ## doAssert db.tryInsertID(sql"INSERT INTO not_exist_table (id, name) VALUES (?, ?)", - ## 1, "item#1") == -1 - ## db.close() - assert(not db.isNil, "Database not connected.") - var q = dbFormat(query, args) - var stmt: sqlite3.PStmt - result = -1 - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: - result = -1 - -proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## Executes the query (typically "INSERT") and returns the - ## generated ID for the row. - ## - ## Raises a `DbError` exception when failed to insert row. - ## For Postgre this adds ``RETURNING id`` to the query, so it only works - ## if your primary key is named ``id``. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## db.exec(sql"CREATE TABLE my_table (id INTEGER, name VARCHAR(50) NOT NULL)") - ## - ## for i in 0..2: - ## let id = db.insertID(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", i, "item#" & $i) - ## echo "LoopIndex = ", i, ", InsertID = ", id - ## - ## # Output: - ## # LoopIndex = 0, InsertID = 1 - ## # LoopIndex = 1, InsertID = 2 - ## # LoopIndex = 2, InsertID = 3 - ## - ## db.close() - result = tryInsertID(db, query, args) - if result < 0: dbError(db) - -proc execAffectedRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {. - tags: [ReadDbEffect, WriteDbEffect].} = - ## Executes the query (typically "UPDATE") and returns the - ## number of affected rows. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## - ## # Records of my_table: - ## # | id | name | - ## # |----|----------| - ## # | 1 | item#1 | - ## # | 2 | item#2 | - ## - ## doAssert db.execAffectedRows(sql"UPDATE my_table SET name = 'TEST'") == 2 - ## - ## db.close() - exec(db, query, args) - result = changes(db) - -proc close*(db: DbConn) {.tags: [DbEffect].} = - ## Closes the database connection. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## let db = open("mytest.db", "", "", "") - ## db.close() - if sqlite3.close(db) != SQLITE_OK: dbError(db) - -proc open*(connection, user, password, database: string): DbConn {. - tags: [DbEffect].} = - ## Opens a database connection. Raises a `DbError` exception if the connection - ## could not be established. - ## - ## **Note:** Only the ``connection`` parameter is used for ``sqlite``. - ## - ## **Examples:** - ## - ## .. code-block:: Nim - ## - ## try: - ## let db = open("mytest.db", "", "", "") - ## ## do something... - ## ## db.getAllRows(sql"SELECT * FROM my_table") - ## db.close() - ## except: - ## stderr.writeLine(getCurrentExceptionMsg()) - var db: DbConn - if sqlite3.open(connection, db) == SQLITE_OK: - result = db - else: - dbError(db) - -proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [DbEffect].} = - ## Sets the encoding of a database connection, returns `true` for - ## success, `false` for failure. - ## - ## **Note:** The encoding cannot be changed once it's been set. - ## According to SQLite3 documentation, any attempt to change - ## the encoding after the database is created will be silently - ## ignored. - exec(connection, sql"PRAGMA encoding = ?", [encoding]) - result = connection.getValue(sql"PRAGMA encoding") == encoding - -when not defined(testing) and isMainModule: - var db = open("db.sql", "", "", "") - exec(db, sql"create table tbl1(one varchar(10), two smallint)", []) - exec(db, sql"insert into tbl1 values('hello!',10)", []) - exec(db, sql"insert into tbl1 values('goodbye', 20)", []) - #db.query("create table tbl1(one varchar(10), two smallint)") - #db.query("insert into tbl1 values('hello!',10)") - #db.query("insert into tbl1 values('goodbye', 20)") - for r in db.rows(sql"select * from tbl1", []): - echo(r[0], r[1]) - for r in db.instantRows(sql"select * from tbl1", []): - echo(r[0], r[1]) - - db_sqlite.close(db) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 1d0952274..39d238055 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -6,6 +6,9 @@ # distribution, for details about the copyright. # +when defined(js): + {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE; See jsre for JS backend.".} + ## What is NRE? ## ============ ## @@ -17,15 +20,17 @@ ## search the internet for a wide variety of third-party documentation and ## tools. ## -## **Note**: If you love ``sequtils.toSeq`` we have bad news for you. This -## library doesn't work with it due to documented compiler limitations. As -## a workaround, use this: -## -## .. code-block:: nim -## -## import nre except toSeq -## -## +## .. warning:: If you love `sequtils.toSeq` we have bad news for you. This +## library doesn't work with it due to documented compiler limitations. As +## a workaround, use this: +runnableExamples: + # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`: + import std/sequtils + iterator iota(n: int): int = + for i in 0..<n: yield i + assert sequtils.toSeq(iota(3)) == @[0, 1, 2] +## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre) +## and [regex](https://github.com/nitely/nim-regex). ## Licencing ## --------- ## @@ -33,55 +38,58 @@ ## this module. ## ## .. _`some additional terms`: http://pcre.sourceforge.net/license.txt -## runnableExamples: + import std/sugar let vowels = re"[aeoui]" - - let expectedResults = [ - 1 .. 1, - 2 .. 2, - 4 .. 4, - 6 .. 6, - 7 .. 7, - ] - var i = 0 - for match in "moigagoo".findIter(vowels): - doAssert match.matchBounds == expectedResults[i] - inc i + let bounds = collect: + for match in "moiga".findIter(vowels): match.matchBounds + assert bounds == @[1 .. 1, 2 .. 2, 4 .. 4] + from std/sequtils import toSeq + let s = sequtils.toSeq("moiga".findIter(vowels)) + # fully qualified to avoid confusion with nre.toSeq + assert s.len == 3 let firstVowel = "foo".find(vowels) let hasVowel = firstVowel.isSome() - if hasVowel: - let matchBounds = firstVowel.get().captureBounds[-1] - doAssert matchBounds.a == 1 + assert hasVowel + let matchBounds = firstVowel.get().captureBounds[-1] + assert matchBounds.a == 1 + + # as with module `re`, unless specified otherwise, `start` parameter in each + # proc indicates where the scan starts, but outputs are relative to the start + # of the input string, not to `start`: + assert find("uxabc", re"(?<=x|y)ab", start = 1).get.captures[-1] == "ab" + assert find("uxabc", re"ab", start = 3).isNone -from pcre import nil +from std/pcre import nil import nre/private/util -import tables -from strutils import `%` -from math import ceil -import options -from unicode import runeLenAt +import std/tables +from std/strutils import `%` +import std/options +from std/unicode import runeLenAt + +when defined(nimPreviewSlimSystem): + import std/assertions export options type Regex* = ref object ## Represents the pattern that things are matched against, constructed with - ## ``re(string)``. Examples: ``re"foo"``, ``re(r"(*ANYCRLF)(?x)foo # - ## comment".`` + ## `re(string)`. Examples: `re"foo"`, `re(r"(*ANYCRLF)(?x)foo # + ## comment".` ## - ## ``pattern: string`` - ## the string that was used to create the pattern. For details on how + ## `pattern: string` + ## : the string that was used to create the pattern. For details on how ## to write a pattern, please see `the official PCRE pattern ## documentation. ## <https://www.pcre.org/original/doc/html/pcrepattern.html>`_ ## - ## ``captureCount: int`` - ## the number of captures that the pattern has. + ## `captureCount: int` + ## : the number of captures that the pattern has. ## - ## ``captureNameId: Table[string, int]`` - ## a table from the capture names to their numeric id. + ## `captureNameId: Table[string, int]` + ## : a table from the capture names to their numeric id. ## ## ## Options @@ -90,30 +98,30 @@ type ## The following options may appear anywhere in the pattern, and they affect ## the rest of it. ## - ## - ``(?i)`` - case insensitive - ## - ``(?m)`` - multi-line: ``^`` and ``$`` match the beginning and end of + ## - `(?i)` - case insensitive + ## - `(?m)` - multi-line: `^` and `$` match the beginning and end of ## lines, not of the subject string - ## - ``(?s)`` - ``.`` also matches newline (*dotall*) - ## - ``(?U)`` - expressions are not greedy by default. ``?`` can be added + ## - `(?s)` - `.` also matches newline (*dotall*) + ## - `(?U)` - expressions are not greedy by default. `?` can be added ## to a qualifier to make it greedy - ## - ``(?x)`` - whitespace and comments (``#``) are ignored (*extended*) - ## - ``(?X)`` - character escapes without special meaning (``\w`` vs. - ## ``\a``) are errors (*extra*) + ## - `(?x)` - whitespace and comments (`#`) are ignored (*extended*) + ## - `(?X)` - character escapes without special meaning (`\w` vs. + ## `\a`) are errors (*extra*) ## ## One or a combination of these options may appear only at the beginning ## of the pattern: ## - ## - ``(*UTF8)`` - treat both the pattern and subject as UTF-8 - ## - ``(*UCP)`` - Unicode character properties; ``\w`` matches ``я`` - ## - ``(*U)`` - a combination of the two options above - ## - ``(*FIRSTLINE*)`` - fails if there is not a match on the first line - ## - ``(*NO_AUTO_CAPTURE)`` - turn off auto-capture for groups; - ## ``(?<name>...)`` can be used to capture - ## - ``(*CR)`` - newlines are separated by ``\r`` - ## - ``(*LF)`` - newlines are separated by ``\n`` (UNIX default) - ## - ``(*CRLF)`` - newlines are separated by ``\r\n`` (Windows default) - ## - ``(*ANYCRLF)`` - newlines are separated by any of the above - ## - ``(*ANY)`` - newlines are separated by any of the above and Unicode + ## - `(*UTF8)` - treat both the pattern and subject as UTF-8 + ## - `(*UCP)` - Unicode character properties; `\w` matches `я` + ## - `(*U)` - a combination of the two options above + ## - `(*FIRSTLINE*)` - fails if there is not a match on the first line + ## - `(*NO_AUTO_CAPTURE)` - turn off auto-capture for groups; + ## `(?<name>...)` can be used to capture + ## - `(*CR)` - newlines are separated by `\r` + ## - `(*LF)` - newlines are separated by `\n` (UNIX default) + ## - `(*CRLF)` - newlines are separated by `\r\n` (Windows default) + ## - `(*ANYCRLF)` - newlines are separated by any of the above + ## - `(*ANY)` - newlines are separated by any of the above and Unicode ## newlines: ## ## single characters VT (vertical tab, U+000B), FF (form feed, U+000C), @@ -122,8 +130,8 @@ type ## are recognized only in UTF-8 mode. ## — man pcre ## - ## - ``(*JAVASCRIPT_COMPAT)`` - JavaScript compatibility - ## - ``(*NO_STUDY)`` - turn off studying; study is enabled by default + ## - `(*JAVASCRIPT_COMPAT)` - JavaScript compatibility + ## - `(*NO_STUDY)` - turn off studying; study is enabled by default ## ## For more details on the leading option groups, see the `Option ## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`_ @@ -133,11 +141,11 @@ type ## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`_. ## ## Some of these options are not part of PCRE and are converted by nre - ## into PCRE flags. These include ``NEVER_UTF``, ``ANCHORED``, - ## ``DOLLAR_ENDONLY``, ``FIRSTLINE``, ``NO_AUTO_CAPTURE``, - ## ``JAVASCRIPT_COMPAT``, ``U``, ``NO_STUDY``. In other PCRE wrappers, you + ## into PCRE flags. These include `NEVER_UTF`, `ANCHORED`, + ## `DOLLAR_ENDONLY`, `FIRSTLINE`, `NO_AUTO_CAPTURE`, + ## `JAVASCRIPT_COMPAT`, `U`, `NO_STUDY`. In other PCRE wrappers, you ## will need to pass these as separate flags to PCRE. - pattern*: string ## not nil + pattern*: string pcreObj: ptr pcre.Pcre ## not nil pcreExtra: ptr pcre.ExtraData ## nil @@ -147,50 +155,40 @@ type ## Usually seen as Option[RegexMatch], it represents the result of an ## execution. On failure, it is none, on success, it is some. ## - ## ``pattern: Regex`` - ## the pattern that is being matched + ## `pattern: Regex` + ## : the pattern that is being matched ## - ## ``str: string`` - ## the string that was matched against + ## `str: string` + ## : the string that was matched against ## - ## ``captures[]: string`` - ## the string value of whatever was captured at that id. If the value - ## is invalid, then behavior is undefined. If the id is ``-1``, then + ## `captures[]: string` + ## : the string value of whatever was captured at that id. If the value + ## is invalid, then behavior is undefined. If the id is `-1`, then ## the whole match is returned. If the given capture was not matched, - ## ``nil`` is returned. + ## `nil` is returned. See examples for `match`. ## - ## - ``"abc".match(re"(\w)").get.captures[0] == "a"`` - ## - ``"abc".match(re"(?<letter>\w)").get.captures["letter"] == "a"`` - ## - ``"abc".match(re"(\w)\w").get.captures[-1] == "ab"`` + ## `captureBounds[]: HSlice[int, int]` + ## : gets the bounds of the given capture according to the same rules as + ## the above. If the capture is not filled, then `None` is returned. + ## The bounds are both inclusive. See examples for `match`. ## - ## ``captureBounds[]: HSlice[int, int]`` - ## gets the bounds of the given capture according to the same rules as - ## the above. If the capture is not filled, then ``None`` is returned. - ## The bounds are both inclusive. + ## `match: string` + ## : the full text of the match. ## - ## - ``"abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0`` - ## - ``0 in "abc".match(re"(\w)").get.captureBounds == true`` - ## - ``"abc".match(re"").get.captureBounds[-1] == 0 .. -1`` - ## - ``"abc".match(re"abc").get.captureBounds[-1] == 0 .. 2`` + ## `matchBounds: HSlice[int, int]` + ## : the bounds of the match, as in `captureBounds[]` ## - ## ``match: string`` - ## the full text of the match. + ## `(captureBounds|captures).toTable` + ## : returns a table with each named capture as a key. ## - ## ``matchBounds: HSlice[int, int]`` - ## the bounds of the match, as in ``captureBounds[]`` + ## `(captureBounds|captures).toSeq` + ## : returns all the captures by their number. ## - ## ``(captureBounds|captures).toTable`` - ## returns a table with each named capture as a key. - ## - ## ``(captureBounds|captures).toSeq`` - ## returns all the captures by their number. - ## - ## ``$: string`` - ## same as ``match`` + ## `$: string` + ## : same as `match` pattern*: Regex ## The regex doing the matching. ## Not nil. str*: string ## The string that was matched against. - ## Not nil. pcreMatchBounds: seq[HSlice[cint, cint]] ## First item is the bounds of the match ## Other items are the captures ## `a` is inclusive start, `b` is exclusive end @@ -198,7 +196,7 @@ type Captures* = distinct RegexMatch CaptureBounds* = distinct RegexMatch - RegexError* = ref object of Exception + RegexError* = ref object of CatchableError RegexInternalError* = ref object of RegexError ## Internal error in the module, this probably means that there is a bug @@ -218,29 +216,19 @@ type ## for whatever reason. The message contains the error ## code. -runnableExamples: - # This MUST be kept in sync with the examples in RegexMatch - doAssert "abc".match(re"(\w)").get.captures[0] == "a" - doAssert "abc".match(re"(?<letter>\w)").get.captures["letter"] == "a" - doAssert "abc".match(re"(\w)\w").get.captures[-1] == "ab" - - doAssert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 - doAssert 0 in "abc".match(re"(\w)").get.captureBounds == true - doAssert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 - doAssert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 - - proc destroyRegex(pattern: Regex) = + `=destroy`(pattern.pattern) pcre.free_substring(cast[cstring](pattern.pcreObj)) if pattern.pcreExtra != nil: pcre.free_study(pattern.pcreExtra) + `=destroy`(pattern.captureNameToId) proc getinfo[T](pattern: Regex, opt: cint): T = let retcode = pcre.fullinfo(pattern.pcreObj, pattern.pcreExtra, opt, addr result) if retcode < 0: # XXX Error message that doesn't expose implementation details - raise newException(FieldError, "Invalid getinfo for $1, errno $2" % [$opt, $retcode]) + raise newException(FieldDefect, "Invalid getinfo for $1, errno $2" % [$opt, $retcode]) proc getNameToNumberTable(pattern: Regex): Table[string, int] = let entryCount = getinfo[cint](pattern, pcre.INFO_NAMECOUNT) @@ -300,7 +288,7 @@ proc matchesCrLf(pattern: Regex): bool = let newlineFlags = flags and (pcre.NEWLINE_CRLF or pcre.NEWLINE_ANY or pcre.NEWLINE_ANYCRLF) - if newLineFlags > 0u32: + if newlineFlags > 0u32: return true # get flags from build config @@ -331,7 +319,7 @@ func contains*(pattern: Captures, i: int): bool = func `[]`*(pattern: CaptureBounds, i: int): HSlice[int, int] = let pattern = RegexMatch(pattern) if not (i in pattern.captureBounds): - raise newException(IndexError, "Group '" & $i & "' was not captured") + raise newException(IndexDefect, "Group '" & $i & "' was not captured") let bounds = pattern.pcreMatchBounds[i + 1] int(bounds.a)..int(bounds.b-1) @@ -358,24 +346,26 @@ func contains*(pattern: CaptureBounds, name: string): bool = func contains*(pattern: Captures, name: string): bool = name in CaptureBounds(pattern) -func checkNamedCaptured(pattern: RegexMatch, name: string): void = +func checkNamedCaptured(pattern: RegexMatch, name: string) = if not (name in pattern.captureBounds): raise newException(KeyError, "Group '" & name & "' was not captured") func `[]`*(pattern: CaptureBounds, name: string): HSlice[int, int] = let pattern = RegexMatch(pattern) checkNamedCaptured(pattern, name) - pattern.captureBounds[pattern.pattern.captureNameToId[name]] + {.noSideEffect.}: + result = pattern.captureBounds[pattern.pattern.captureNameToId[name]] func `[]`*(pattern: Captures, name: string): string = let pattern = RegexMatch(pattern) checkNamedCaptured(pattern, name) - return pattern.captures[pattern.pattern.captureNameToId[name]] + {.noSideEffect.}: + result = pattern.captures[pattern.pattern.captureNameToId[name]] template toTableImpl() {.dirty.} = for key in RegexMatch(pattern).pattern.captureNameId.keys: if key in pattern: - result[key] = pattern[key] + result[key] = pattern[key] func toTable*(pattern: Captures): Table[string, string] = result = initTable[string, string]() @@ -489,7 +479,7 @@ proc re*(pattern: string): Regex = initRegex(pattern, flags, study) proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Option[RegexMatch] = - var myResult = RegexMatch(pattern : pattern, str : str) + var myResult = RegexMatch(pattern: pattern, str: str) # See PCRE man pages. # 2x capture count to make room for start-end pairs # 1x capture count as slack space for PCRE @@ -517,37 +507,46 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt of pcre.ERROR_NOMATCH: return none(RegexMatch) of pcre.ERROR_NULL: - raise newException(AccessViolationError, "Expected non-null parameters") + raise newException(AccessViolationDefect, "Expected non-null parameters") of pcre.ERROR_BADOPTION: - raise RegexInternalError(msg : "Unknown pattern flag. Either a bug or " & + raise RegexInternalError(msg: "Unknown pattern flag. Either a bug or " & "outdated PCRE.") of pcre.ERROR_BADUTF8, pcre.ERROR_SHORTUTF8, pcre.ERROR_BADUTF8_OFFSET: - raise InvalidUnicodeError(msg : "Invalid unicode byte sequence", - pos : myResult.pcreMatchBounds[0].a) + raise InvalidUnicodeError(msg: "Invalid unicode byte sequence", + pos: myResult.pcreMatchBounds[0].a) else: - raise RegexInternalError(msg : "Unknown internal error: " & $execRet) + raise RegexInternalError(msg: "Unknown internal error: " & $execRet) proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = - ## Like ` ``find(...)`` <#proc-find>`_, but anchored to the start of the + ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the ## string. - ## runnableExamples: - doAssert "foo".match(re"f").isSome - doAssert "foo".match(re"o").isNone + assert "foo".match(re"f").isSome + assert "foo".match(re"o").isNone + + assert "abc".match(re"(\w)").get.captures[0] == "a" + assert "abc".match(re"(?<letter>\w)").get.captures["letter"] == "a" + assert "abc".match(re"(\w)\w").get.captures[-1] == "ab" + assert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 + assert 0 in "abc".match(re"(\w)").get.captureBounds + assert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 + assert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = - ## Works the same as ` ``find(...)`` <#proc-find>`_, but finds every - ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not - ## ``"22", "22", "22"``. - ## - ## Arguments are the same as ` ``find(...)`` <#proc-find>`_ + ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every + ## non-overlapping match: + runnableExamples: + import std/sugar + assert collect(for a in "2222".findIter(re"22"): a.match) == @["22", "22"] + # not @["22", "22", "22"] + ## Arguments are the same as `find(...)<#find,string,Regex,int>`_ ## ## Variants: ## - ## - ``proc findAll(...)`` returns a ``seq[string]`` - # see pcredemo for explanation + ## - `proc findAll(...)` returns a `seq[string]` + # see pcredemo for explanation => https://www.pcre.org/original/doc/html/pcredemo.html let matchesCrLf = pattern.matchesCrLf() let unicode = uint32(getinfo[culong](pattern, pcre.INFO_OPTIONS) and pcre.UTF8) > 0u32 @@ -568,7 +567,7 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R # either the end of the input or the string # cannot be split here - we also need to bail # if we've never matched and we've already tried to... - if offset >= strlen or neverMatched: + if flags == 0 or offset >= strlen or neverMatched: # All matches found break if matchesCrLf and offset < (str.len - 1) and @@ -584,19 +583,18 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R else: neverMatched = false offset = match.get.matchBounds.b + 1 - yield match.get proc find*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = ## Finds the given pattern in the string between the end and start ## positions. ## - ## ``start`` - ## The start point at which to start matching. ``|abc`` is ``0``; - ## ``a|bc`` is ``1`` + ## `start` + ## : The start point at which to start matching. `|abc` is `0`; + ## `a|bc` is `1` ## - ## ``endpos`` - ## The maximum index for a match; ``int.high`` means the end of the + ## `endpos` + ## : The maximum index for a match; `int.high` means the end of the ## string, otherwise it’s an inclusive upper bound. return str.matchImpl(pattern, start, endpos, 0) @@ -608,12 +606,11 @@ proc findAll*(str: string, pattern: Regex, start = 0, endpos = int.high): seq[st proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool = ## Determine if the string contains the given pattern between the end and ## start positions: - ## This function is equivalent to ``isSome(str.find(pattern, start, endpos))``. - ## + ## This function is equivalent to `isSome(str.find(pattern, start, endpos))`. runnableExamples: - doAssert "abc".contains(re"bc") == true - doAssert "abc".contains(re"cd") == false - doAssert "abc".contains(re"a", start = 1) == false + assert "abc".contains(re"bc") + assert not "abc".contains(re"cd") + assert not "abc".contains(re"a", start = 1) return isSome(str.find(pattern, start, endpos)) @@ -621,20 +618,20 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] ## Splits the string with the given regex. This works according to the ## rules that Perl and Javascript use. ## - ## ``start`` behaves the same as in ` ``find(...)`` <#proc-find>`_. + ## `start` behaves the same as in `find(...)<#find,string,Regex,int>`_. ## runnableExamples: # - If the match is zero-width, then the string is still split: - doAssert "123".split(re"") == @["1", "2", "3"] + assert "123".split(re"") == @["1", "2", "3"] # - If the pattern has a capture in it, it is added after the string # split: - doAssert "12".split(re"(\d)") == @["", "1", "", "2", ""] + assert "12".split(re"(\d)") == @["", "1", "", "2", ""] - # - If ``maxsplit != -1``, then the string will only be split - # ``maxsplit - 1`` times. This means that there will be ``maxsplit`` + # - If `maxsplit != -1`, then the string will only be split + # `maxsplit - 1` times. This means that there will be `maxsplit` # strings in the output seq. - doAssert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] + assert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] result = @[] var lastIdx = start @@ -698,28 +695,27 @@ template replaceImpl(str: string, pattern: Regex, proc replace*(str: string, pattern: Regex, subproc: proc (match: RegexMatch): string): string = - ## Replaces each match of Regex in the string with ``subproc``, which should - ## never be or return ``nil``. + ## Replaces each match of Regex in the string with `subproc`, which should + ## never be or return `nil`. ## - ## If ``subproc`` is a ``proc (RegexMatch): string``, then it is executed with + ## If `subproc` is a `proc (RegexMatch): string`, then it is executed with ## each match and the return value is the replacement value. ## - ## If ``subproc`` is a ``proc (string): string``, then it is executed with the - ## full text of the match and and the return value is the replacement - ## value. + ## If `subproc` is a `proc (string): string`, then it is executed with the + ## full text of the match and the return value is the replacement value. ## - ## If ``subproc`` is a string, the syntax is as follows: + ## If `subproc` is a string, the syntax is as follows: ## - ## - ``$$`` - literal ``$`` - ## - ``$123`` - capture number ``123`` - ## - ``$foo`` - named capture ``foo`` - ## - ``${foo}`` - same as above - ## - ``$1$#`` - first and second captures - ## - ``$#`` - first capture - ## - ``$0`` - full match + ## - `$$` - literal `$` + ## - `$123` - capture number `123` + ## - `$foo` - named capture `foo` + ## - `${foo}` - same as above + ## - `$1$#` - first and second captures + ## - `$#` - first capture + ## - `$0` - full match ## - ## If a given capture is missing, ``IndexError`` thrown for un-named captures - ## and ``KeyError`` for named captures. + ## If a given capture is missing, `IndexDefect` thrown for un-named captures + ## and `KeyError` for named captures. replaceImpl(str, pattern, subproc(match)) proc replace*(str: string, pattern: Regex, @@ -731,8 +727,25 @@ proc replace*(str: string, pattern: Regex, sub: string): string = replaceImpl(str, pattern, formatStr(sub, match.captures[name], match.captures[id - 1])) -let SpecialCharMatcher = re"([\\+*?[^\]$(){}=!<>|:-])" -proc escapeRe*(str: string): string = - ## Escapes the string so it doesn’t match any special characters. - ## Incompatible with the Extra flag (``X``). - str.replace(SpecialCharMatcher, "\\$1") +proc escapeRe*(str: string): string {.gcsafe.} = + ## Escapes the string so it doesn't match any special characters. + ## Incompatible with the Extra flag (`X`). + ## + ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -` + runnableExamples: + assert escapeRe("fly+wind") == "fly\\+wind" + assert escapeRe("!") == "\\!" + assert escapeRe("nim*") == "nim\\*" + + #([\\+*?[^\]$(){}=!<>|:-]) + const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(', + ')', '{', '}', '=', '!', '<', '>', '|', ':', + '-'} + + for c in items(str): + case c + of SpecialCharMatcher: + result.add("\\") + result.add(c) + else: + result.add(c) diff --git a/lib/impure/nre/private/util.nim b/lib/impure/nre/private/util.nim index f7d8b1d60..ed8420776 100644 --- a/lib/impure/nre/private/util.nim +++ b/lib/impure/nre/private/util.nim @@ -1,5 +1,5 @@ ## INTERNAL FILE FOR USE ONLY BY nre.nim. -import tables +import std/tables const Ident = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} const StartIdent = Ident - {'0'..'9'} @@ -47,5 +47,5 @@ template formatStr*(howExpr, namegetter, idgetter): untyped = i += 1 val.add(namegetter) else: - raise newException(Exception, "Syntax error in format string at " & $i) + raise newException(ValueError, "Syntax error in format string at " & $i) val diff --git a/lib/impure/osinfo_posix.nim b/lib/impure/osinfo_posix.nim deleted file mode 100644 index 0362fca12..000000000 --- a/lib/impure/osinfo_posix.nim +++ /dev/null @@ -1,10 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.error: "This module has been moved to the 'osinfo' nimble package.".} diff --git a/lib/impure/osinfo_win.nim b/lib/impure/osinfo_win.nim deleted file mode 100644 index 0362fca12..000000000 --- a/lib/impure/osinfo_win.nim +++ /dev/null @@ -1,10 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.error: "This module has been moved to the 'osinfo' nimble package.".} diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index b78c0d8cf..f4fc26380 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -9,61 +9,66 @@ ## This module contains code for reading from `stdin`:idx:. On UNIX the ## linenoise library is wrapped and set up to provide default key bindings -## (e.g. you can navigate with the arrow keys). On Windows ``system.readLine`` +## (e.g. you can navigate with the arrow keys). On Windows `system.readLine` ## is used. This suffices because Windows' console already provides the ## wanted functionality. -{.deadCodeElim: on.} # dce option deprecated +runnableExamples("-r:off"): + echo readLineFromStdin("Is Nim awesome? (Y/n): ") + var line: string + while true: + let ok = readLineFromStdin("How are you? ", line) + if not ok: break # ctrl-C or ctrl-D will cause a break + if line.len > 0: echo line + echo "exiting" -when defined(Windows): - proc readLineFromStdin*(prompt: string): TaintedString {. + +when defined(windows): + when defined(nimPreviewSlimSystem): + import std/syncio + + proc readLineFromStdin*(prompt: string): string {. tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a line from stdin. stdout.write(prompt) + stdout.flushFile() result = readLine(stdin) - proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. + proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a `line` from stdin. `line` must not be - ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ``CR``, ``LF`` or - ## ``CRLF``. The newline character(s) are not part of the returned string. - ## Returns ``false`` if the end of the file has been reached, ``true`` - ## otherwise. If ``false`` is returned `line` contains no new data. + ## `nil`! May throw an IO exception. + ## A line of text may be delimited by `CR`, `LF` or + ## `CRLF`. The newline character(s) are not part of the returned string. + ## Returns `false` if the end of the file has been reached, `true` + ## otherwise. If `false` is returned `line` contains no new data. stdout.write(prompt) result = readLine(stdin, line) elif defined(genode): - proc readLineFromStdin*(prompt: string): TaintedString {. + proc readLineFromStdin*(prompt: string): string {. tags: [ReadIOEffect, WriteIOEffect].} = stdin.readLine() - proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. + proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = stdin.readLine(line) else: - import linenoise, termios - - proc readLineFromStdin*(prompt: string): TaintedString {. - tags: [ReadIOEffect, WriteIOEffect].} = - var buffer = linenoise.readLine(prompt) - if isNil(buffer): - raise newException(IOError, "Linenoise returned nil") - result = TaintedString($buffer) - if result.string.len > 0: - historyAdd(buffer) - linenoise.free(buffer) + import std/linenoise - proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. + proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) if isNil(buffer): - line.string.setLen(0) + line.setLen(0) return false - line = TaintedString($buffer) - if line.string.len > 0: + line = $buffer + if line.len > 0: historyAdd(buffer) linenoise.free(buffer) result = true + proc readLineFromStdin*(prompt: string): string {.inline.} = + if not readLineFromStdin(prompt, result): + raise newException(IOError, "Linenoise returned nil") diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 809180774..053c6ab55 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -7,6 +7,9 @@ # distribution, for details about the copyright. # +when defined(js): + {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE; See jsre for JS backend.".} + ## Regular expression support for Nim. ## ## This module is implemented by providing a wrapper around the @@ -14,27 +17,43 @@ ## C library. This means that your application will depend on the PCRE ## library's licence when using this module, which should not be a problem ## though. +## +## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre) +## and [regex](https://github.com/nitely/nim-regex). +## ## PCRE's licence follows: ## ## .. include:: ../../doc/regexprs.txt ## +runnableExamples: + ## Unless specified otherwise, `start` parameter in each proc indicates + ## where the scan starts, but outputs are relative to the start of the input + ## string, not to `start`: + doAssert find("uxabc", re"(?<=x|y)ab", start = 1) == 2 # lookbehind assertion + doAssert find("uxabc", re"ab", start = 3) == -1 # we're past `start` => not found + doAssert not match("xabc", re"^abc$", start = 1) + # can't match start of string since we're starting at 1 + import - pcre, strutils, rtarrays + std/[pcre, strutils, rtarrays] + +when defined(nimPreviewSlimSystem): + import std/syncio const MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that can be captured. - ## This limit still exists for ``replacef`` and ``parallelReplace``. + ## This limit still exists for `replacef` and `parallelReplace`. type RegexFlag* = enum ## options for regular expressions - reIgnoreCase = 0, ## do caseless matching - reMultiLine = 1, ## ``^`` and ``$`` match newlines within data - reDotAll = 2, ## ``.`` matches anything including NL - reExtended = 3, ## ignore whitespace and ``#`` comments - reStudy = 4 ## study the expression (may be omitted if the - ## expression will be used only once) + reIgnoreCase = 0, ## do caseless matching + reMultiLine = 1, ## `^` and `$` match newlines within data + reDotAll = 2, ## `.` matches anything including NL + reExtended = 3, ## ignore whitespace and `#` comments + reStudy = 4 ## study the expression (may be omitted if the + ## expression will be used only once) RegexDesc = object h: ptr Pcre @@ -46,10 +65,16 @@ type ## is raised if the pattern is no valid regular expression. when defined(gcDestructors): - proc `=destroy`(x: var RegexDesc) = - pcre.free_substring(cast[cstring](x.h)) - if not isNil(x.e): - pcre.free_study(x.e) + when defined(nimAllowNonVarDestructor): + proc `=destroy`(x: RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) + else: + proc `=destroy`(x: var RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = var e: ref RegexError @@ -67,7 +92,7 @@ proc rawCompile(pattern: string, flags: cint): ptr Pcre = proc finalizeRegEx(x: Regex) = # XXX This is a hack, but PCRE does not export its "free" function properly. - # Sigh. The hack relies on PCRE's implementation (see ``pcre_get.c``). + # Sigh. The hack relies on PCRE's implementation (see `pcre_get.c`). # Fortunately the implementation is unlikely to change. pcre.free_substring(cast[cstring](x.h)) if not isNil(x.e): @@ -77,8 +102,8 @@ proc re*(s: string, flags = {reStudy}): Regex = ## Constructor of regular expressions. ## ## Note that Nim's - ## extended raw string literals support the syntax ``re"[abc]"`` as - ## a short form for ``re(r"[abc]")``. Also note that since this + ## extended raw string literals support the syntax `re"[abc]"` as + ## a short form for `re(r"[abc]")`. Also note that since this ## compiles the regular expression, which is expensive, you should ## avoid putting it directly in the arguments of the functions like ## the examples show below if you plan to use it a lot of times, as @@ -129,13 +154,20 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string], else: matches[i-1] = "" return rawMatches[1] - rawMatches[0] +const MaxReBufSize* = high(cint) + ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even + ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1. + proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): tuple[first, last: int] = - ## returns the starting position and end position of ``pattern`` in ``buf`` - ## (where ``buf`` has length ``bufSize`` and is not necessarily ``'\0'`` terminated), + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), ## and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and ``(-1,0)`` is returned. + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `(-1,0)` is returned. + ## + ## Note: The memory for `matches` needs to be allocated before this function is + ## called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -151,20 +183,31 @@ proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], start = 0): tuple[first, last: int] {.inline.} = - ## returns the starting position and end position of ``pattern`` in ``s`` - ## and the captured substrings in the array ``matches``. + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. ## If it does not match, nothing - ## is written into ``matches`` and ``(-1,0)`` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + ## is written into `matches` and `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[string](1) + let (first, last) = findBounds("Hello World", re"(W\w+)", matches) + doAssert first == 6 + doAssert last == 10 + doAssert matches[0] == "World" + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[tuple[first, last: int]], - start = 0, bufSize = 0): tuple[first, last: int] = - ## returns the starting position and end position of ``pattern`` in ``buf`` - ## (where ``buf`` has length ``bufSize`` and is not necessarily ``'\0'`` terminated), - ## and the captured substrings in the array ``matches``. - ## If it does not match, nothing is written into ``matches`` and - ## ``(-1,0)`` is returned. + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -181,17 +224,38 @@ proc findBounds*(buf: cstring, pattern: Regex, proc findBounds*(s: string, pattern: Regex, matches: var openArray[tuple[first, last: int]], start = 0): tuple[first, last: int] {.inline.} = - ## returns the starting position and end position of ``pattern`` in ``s`` - ## and the captured substrings in the array ``matches``. - ## If it does not match, nothing is written into ``matches`` and - ## ``(-1,0)`` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[tuple[first, last: int]](1) + let (first, last) = findBounds("Hello World", re"(\w+)", matches) + doAssert first == 0 + doAssert last == 4 + doAssert matches[0] == (0, 4) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc findBoundsImpl(buf: cstring, pattern: Regex, + start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = + var rtarray = initRtArray[cint](3) + let rawMatches = rtarray.getRawData + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32, + cast[ptr cint](rawMatches), 3) + + if res < 0'i32: + result = (-1, 0) + else: + result = (int(rawMatches[0]), int(rawMatches[1]-1)) proc findBounds*(buf: cstring, pattern: Regex, start = 0, bufSize: int): tuple[first, last: int] = - ## returns the ``first`` and ``last`` position of ``pattern`` in ``buf``, - ## where ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). - ## If it does not match, ``(-1,0)`` is returned. + ## returns the `first` and `last` position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `(-1,0)` is returned. var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData @@ -202,16 +266,14 @@ proc findBounds*(buf: cstring, pattern: Regex, proc findBounds*(s: string, pattern: Regex, start = 0): tuple[first, last: int] {.inline.} = - ## returns the ``first`` and ``last`` position of ``pattern`` in ``s``. - ## If it does not match, ``(-1,0)`` is returned. + ## returns the `first` and `last` position of `pattern` in `s`. + ## If it does not match, `(-1,0)` is returned. ## ## Note: there is a speed improvement if the matches do not need to be captured. - ## - ## Example: - ## - ## .. code-block:: nim - ## assert findBounds("01234abc89", re"abc") == (5,7) - result = findBounds(cstring(s), pattern, start, s.len) + runnableExamples: + assert findBounds("01234abc89", re"abc") == (5,7) + result = findBounds(cstring(s), pattern, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint = var @@ -224,72 +286,77 @@ proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint) proc matchLen*(s: string, pattern: Regex, matches: var openArray[string], start = 0): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED) proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED) proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. ## - ## Example: - ## - ## .. code-block:: nim - ## echo matchLen("abcdefg", re"cde", 2) # => 3 - ## echo matchLen("abcdefg", re"abcde") # => 5 - ## echo matchLen("abcdefg", re"cde") # => -1 + runnableExamples: + doAssert matchLen("abcdefg", re"cde", 2) == 3 + doAssert matchLen("abcdefg", re"abcde") == 5 + doAssert matchLen("abcdefg", re"cde") == -1 result = matchOrFind(cstring(s), pattern, start.cint, s.len.cint, pcre.ANCHORED) proc matchLen*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. result = matchOrFind(buf, pattern, start.cint, bufSize, pcre.ANCHORED) proc match*(s: string, pattern: Regex, start = 0): bool {.inline.} = - ## returns ``true`` if ``s[start..]`` matches the ``pattern``. + ## returns `true` if `s[start..]` matches the `pattern`. result = matchLen(cstring(s), pattern, start, s.len) != -1 proc match*(s: string, pattern: Regex, matches: var openArray[string], start = 0): bool {.inline.} = - ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is + ## returns `true` if `s[start..]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is ## returned. ## - ## Example: - ## - ## .. code-block:: nim - ## var matches: array[2, string] - ## if match("abcdefg", re"c(d)ef(g)", matches, 2): - ## for s in matches: - ## echo s # => d g + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + import std/sequtils + var matches: array[2, string] + if match("abcdefg", re"c(d)ef(g)", matches, 2): + doAssert toSeq(matches) == @["d", "g"] result = matchLen(cstring(s), pattern, matches, start, s.len) != -1 proc match*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): bool {.inline.} = - ## returns ``true`` if ``buf[start..<bufSize]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is + ## returns `true` if `buf[start..<bufSize]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is ## returned. - ## ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchLen(buf, pattern, matches, start, bufSize) != -1 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], - start = 0, bufSize = 0): int = - ## returns the starting position of ``pattern`` in ``buf`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and ``-1`` is returned. - ## ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). + start = 0, bufSize: int): int = + ## returns the starting position of `pattern` in `buf` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -305,15 +372,17 @@ proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], proc find*(s: string, pattern: Regex, matches: var openArray[string], start = 0): int {.inline.} = - ## returns the starting position of ``pattern`` in ``s`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and ``-1`` is returned. + ## returns the starting position of `pattern` in `s` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = find(cstring(s), pattern, matches, start, s.len) proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = - ## returns the starting position of ``pattern`` in ``buf``, - ## where ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). - ## If it does not match, ``-1`` is returned. + ## returns the starting position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `-1` is returned. var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData @@ -323,15 +392,16 @@ proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = return rawMatches[0] proc find*(s: string, pattern: Regex, start = 0): int {.inline.} = - ## returns the starting position of ``pattern`` in ``s``. If it does not - ## match, ``-1`` is returned. - ## - ## Example: - ## - ## .. code-block:: nim - ## echo find("abcdefg", re"cde") # => 2 - ## echo find("abcdefg", re"abc") # => 0 - ## echo find("abcdefg", re"zz") # => -1 + ## returns the starting position of `pattern` in `s`. If it does not + ## match, `-1` is returned. We start the scan at `start`. + runnableExamples: + doAssert find("abcdefg", re"cde") == 2 + doAssert find("abcdefg", re"abc") == 0 + doAssert find("abcdefg", re"zz") == -1 # not found + doAssert find("abcdefg", re"cde", start = 2) == 2 # still 2 + doAssert find("abcdefg", re"cde", start = 3) == -1 # we're past the start position + doAssert find("xabc", re"(?<=x|y)abc", start = 1) == 1 + # lookbehind assertion `(?<=x|y)` can look behind `start` result = find(cstring(s), pattern, start, s.len) iterator findAll*(s: string, pattern: Regex, start = 0): string = @@ -354,7 +424,7 @@ iterator findAll*(s: string, pattern: Regex, start = 0): string = i = b iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string = - ## Yields all matching `substrings` of ``s`` that match ``pattern``. + ## Yields all matching `substrings` of `s` that match `pattern`. ## ## Note that since this is an iterator you should not modify the string you ## are iterating over: bad things could happen. @@ -375,32 +445,25 @@ iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string i = b proc findAll*(s: string, pattern: Regex, start = 0): seq[string] {.inline.} = - ## returns all matching `substrings` of ``s`` that match ``pattern``. - ## If it does not match, @[] is returned. + ## returns all matching `substrings` of `s` that match `pattern`. + ## If it does not match, `@[]` is returned. result = @[] for x in findAll(s, pattern, start): result.add x -when not defined(nimhygiene): - {.pragma: inject.} - template `=~` *(s: string, pattern: Regex): untyped = - ## This calls ``match`` with an implicit declared ``matches`` array that - ## can be used in the scope of the ``=~`` call: - ## - ## .. code-block:: nim - ## - ## if line =~ re"\s*(\w+)\s*\=\s*(\w+)": - ## # matches a key=value pair: - ## echo("Key: ", matches[0]) - ## echo("Value: ", matches[1]) - ## elif line =~ re"\s*(\#.*)": - ## # matches a comment - ## # note that the implicit ``matches`` array is different from the - ## # ``matches`` array of the first branch - ## echo("comment: ", matches[0]) - ## else: - ## echo("syntax error") - ## + ## This calls `match` with an implicit declared `matches` array that + ## can be used in the scope of the `=~` call: + runnableExamples: + proc parse(line: string): string = + if line =~ re"\s*(\w+)\s*\=\s*(\w+)": # matches a key=value pair: + result = $(matches[0], matches[1]) + elif line =~ re"\s*(\#.*)": # matches a comment + # note that the implicit `matches` array is different from 1st branch + result = $(matches[0],) + else: raiseAssert "unreachable" + doAssert not declared(matches) + doAssert parse("NAME = LENA") == """("NAME", "LENA")""" + doAssert parse(" # comment ... ") == """("# comment ... ",)""" bind MaxSubpatterns when not declaredInScope(matches): var matches {.inject.}: array[MaxSubpatterns, string] @@ -409,12 +472,14 @@ template `=~` *(s: string, pattern: Regex): untyped = # ------------------------- more string handling ------------------------------ proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} = - ## same as ``find(s, pattern, start) >= 0`` + ## same as `find(s, pattern, start) >= 0` return find(s, pattern, start) >= 0 proc contains*(s: string, pattern: Regex, matches: var openArray[string], start = 0): bool {.inline.} = - ## same as ``find(s, pattern, matches, start) >= 0`` + ## same as `find(s, pattern, matches, start) >= 0` + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return find(s, pattern, matches, start) >= 0 proc startsWith*(s: string, prefix: Regex): bool {.inline.} = @@ -427,44 +492,32 @@ proc endsWith*(s: string, suffix: Regex): bool {.inline.} = if matchLen(s, suffix, i) == s.len - i: return true proc replace*(s: string, sub: Regex, by = ""): string = - ## Replaces ``sub`` in ``s`` by the string ``by``. Captures cannot be - ## accessed in ``by``. - ## - ## Example: - ## - ## .. code-block:: nim - ## "var1=key; var2=key2".replace(re"(\w+)=(\w+)") - ## - ## Results in: - ## - ## .. code-block:: nim - ## - ## "; " + ## Replaces `sub` in `s` by the string `by`. Captures cannot be + ## accessed in `by`. + runnableExamples: + doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)") == "; " + doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?" result = "" var prev = 0 + var flags = int32(0) while prev < s.len: - var match = findBounds(s, sub, prev) + var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags) + flags = 0 if match.first < 0: break add(result, substr(s, prev, match.first-1)) add(result, by) - if match.last + 1 == prev: break + if match.first > match.last: + # 0-len match + flags = pcre.NOTEMPTY_ATSTART prev = match.last + 1 add(result, substr(s, prev)) proc replacef*(s: string, sub: Regex, by: string): string = - ## Replaces ``sub`` in ``s`` by the string ``by``. Captures can be accessed in ``by`` - ## with the notation ``$i`` and ``$#`` (see strutils.\`%\`). - ## - ## Example: - ## - ## .. code-block:: nim - ## "var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") - ## - ## Results in: - ## - ## .. code-block:: nim - ## - ## "var1<-keykey; var2<-key2key2" + ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` + ## with the notation `$i` and `$#` (see strutils.\`%\`). + runnableExamples: + doAssert "var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") == + "var1<-keykey; var2<-key2key2" result = "" var caps: array[MaxSubpatterns, string] var prev = 0 @@ -479,7 +532,7 @@ proc replacef*(s: string, sub: Regex, by: string): string = proc multiReplace*(s: string, subs: openArray[ tuple[pattern: Regex, repl: string]]): string = - ## Returns a modified copy of ``s`` with the substitutions in ``subs`` + ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" var i = 0 @@ -497,58 +550,41 @@ proc multiReplace*(s: string, subs: openArray[ # copy the rest: add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: openArray[ - tuple[pattern: Regex, repl: string]]): string {.deprecated: - "Deprecated since v0.18.0: Use ``multiReplace`` instead.".} = - ## Returns a modified copy of ``s`` with the substitutions in ``subs`` - ## applied in parallel. - result = multiReplace(s, subs) - proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = - ## reads in the file ``infile``, performs a parallel replacement (calls - ## ``parallelReplace``) and writes back to ``outfile``. Raises ``IOError`` if an + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises `IOError` if an ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string + var x = readFile(infile) writeFile(outfile, x.multiReplace(subs)) iterator split*(s: string, sep: Regex; maxsplit = -1): string = - ## Splits the string ``s`` into substrings. - ## - ## Substrings are separated by the regular expression ``sep`` - ## (and the portion matched by ``sep`` is not returned). - ## - ## Example: - ## - ## .. code-block:: nim - ## for word in split("00232this02939is39an22example111", re"\d+"): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: nim - ## "" - ## "this" - ## "is" - ## "an" - ## "example" - ## "" - ## + ## Splits the string `s` into substrings. + ## + ## Substrings are separated by the regular expression `sep` + ## (and the portion matched by `sep` is not returned). + runnableExamples: + import std/sequtils + doAssert toSeq(split("00232this02939is39an22example111", re"\d+")) == + @["", "this", "is", "an", "example", ""] var last = 0 var splits = maxsplit - var x: int + var x = -1 + if len(s) == 0: + last = 1 + if matchLen(s, sep, 0) == 0: + x = 0 while last <= len(s): var first = last var sepLen = 1 + if x == 0: + inc(last) while last < len(s): x = matchLen(s, sep, last) if x >= 0: sepLen = x break inc(last) - if x == 0: - if last >= len(s): break - inc last if splits == 0: last = len(s) yield substr(s, first, last-1) if splits == 0: break @@ -556,14 +592,14 @@ iterator split*(s: string, sep: Regex; maxsplit = -1): string = inc(last, sepLen) proc split*(s: string, sep: Regex, maxsplit = -1): seq[string] {.inline.} = - ## Splits the string ``s`` into a seq of substrings. + ## Splits the string `s` into a seq of substrings. ## - ## The portion matched by ``sep`` is not returned. + ## The portion matched by `sep` is not returned. result = @[] for x in split(s, sep, maxsplit): result.add x proc escapeRe*(s: string): string = - ## escapes ``s`` so that it is matched verbatim when used as a regular + ## escapes `s` so that it is matched verbatim when used as a regular ## expression. result = "" for c in items(s): @@ -573,109 +609,3 @@ proc escapeRe*(s: string): string = else: result.add("\\x") result.add(toHex(ord(c), 2)) - -when isMainModule: - doAssert match("(a b c)", rex"\( .* \)") - doAssert match("WHiLe", re("while", {reIgnoreCase})) - - doAssert "0158787".match(re"\d+") - doAssert "ABC 0232".match(re"\w+\s+\d+") - doAssert "ABC".match(rex"\d+ | \w+") - - {.push warnings:off.} - doAssert matchLen("key", re"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b") == 3 - {.pop.} - - var pattern = re"[a-z0-9]+\s*=\s*[a-z0-9]+" - doAssert matchLen("key1= cal9", pattern) == 11 - - doAssert find("_____abc_______", re"abc") == 5 - doAssert findBounds("_____abc_______", re"abc") == (5,7) - - var matches: array[6, string] - if match("abcdefg", re"c(d)ef(g)", matches, 2): - doAssert matches[0] == "d" - doAssert matches[1] == "g" - else: - doAssert false - - if "abc" =~ re"(a)bcxyz|(\w+)": - doAssert matches[1] == "abc" - else: - doAssert false - - if "abc" =~ re"(cba)?.*": - doAssert matches[0] == "" - else: doAssert false - - if "abc" =~ re"().*": - doAssert matches[0] == "" - else: doAssert false - - doAssert "var1=key; var2=key2".endsWith(re"\w+=\w+") - doAssert("var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - doAssert("var1=key; var2=key2".replace(re"(\w+)=(\w+)", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", re"\d+"): - accum.add(word) - doAssert(accum == @["", "this", "is", "an", "example", ""]) - - accum = @[] - for word in split("00232this02939is39an22example111", re"\d+", maxsplit=2): - accum.add(word) - doAssert(accum == @["", "this", "is39an22example111"]) - - accum = @[] - for word in split("AAA : : BBB", re"\s*:\s*"): - accum.add(word) - doAssert(accum == @["AAA", "", "BBB"]) - - doAssert(split("abc", re"") == @["a", "b", "c"]) - doAssert(split("", re"") == @[]) - - doAssert(split("a;b;c", re";") == @["a", "b", "c"]) - doAssert(split(";a;b;c", re";") == @["", "a", "b", "c"]) - doAssert(split(";a;b;c;", re";") == @["", "a", "b", "c", ""]) - doAssert(split("a;b;c;", re";") == @["a", "b", "c", ""]) - doAssert(split("00232this02939is39an22example111", re"\d+", maxsplit=2) == @["", "this", "is39an22example111"]) - - - for x in findAll("abcdef", re"^{.}", 3): - doAssert x == "d" - accum = @[] - for x in findAll("abcdef", re".", 3): - accum.add(x) - doAssert(accum == @["d", "e", "f"]) - - doAssert("XYZ".find(re"^\d*") == 0) - doAssert("XYZ".match(re"^\d*") == true) - - block: - var matches: array[16, string] - if match("abcdefghijklmnop", re"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)(p)", matches): - for i in 0..matches.high: - doAssert matches[i] == $chr(i + 'a'.ord) - else: - doAssert false - - block: # Buffer based RE - var cs: cstring = "_____abc_______" - doAssert(cs.find(re"abc", bufSize=15) == 5) - doAssert(cs.matchLen(re"_*abc", bufSize=15) == 8) - doAssert(cs.matchLen(re"abc", start=5, bufSize=15) == 3) - doAssert(cs.matchLen(re"abc", start=5, bufSize=7) == -1) - doAssert(cs.matchLen(re"abc_*", start=5, bufSize=10) == 5) - var accum: seq[string] = @[] - for x in cs.findAll(re"[a-z]", start=3, bufSize=15): - accum.add($x) - doAssert(accum == @["a","b","c"]) - - block: - # bug #9306 - doAssert replace("bar", re"^", "foo") == "foobar" - doAssert replace("foo", re"", "-") == "-foo" - doAssert replace("foo", re"$", "bar") == "foobar" - diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 219b1bed5..9b043f3e5 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -11,43 +11,48 @@ ## and libraries, writing async procedures in Nim and converting callback-based code ## to promises. ## -## A Nim procedure is asynchronous when it includes the ``{.async.}`` pragma. It -## should always have a ``Future[T]`` return type or not have a return type at all. -## A ``Future[void]`` return type is assumed by default. +## A Nim procedure is asynchronous when it includes the `{.async.}` pragma. It +## should always have a `Future[T]` return type or not have a return type at all. +## A `Future[void]` return type is assumed by default. ## -## This is roughly equivalent to the ``async`` keyword in JavaScript code. +## This is roughly equivalent to the `async` keyword in JavaScript code. ## -## .. code-block:: nim -## proc loadGame(name: string): Future[Game] {.async.} = -## # code +## ```nim +## proc loadGame(name: string): Future[Game] {.async.} = +## # code +## ``` ## ## should be equivalent to ## -## .. code-block:: javascript +## ```javascript ## async function loadGame(name) { ## // code ## } +## ``` ## -## A call to an asynchronous procedure usually needs ``await`` to wait for -## the completion of the ``Future``. +## A call to an asynchronous procedure usually needs `await` to wait for +## the completion of the `Future`. ## -## .. code-block:: nim +## ```nim ## var game = await loadGame(name) +## ``` ## ## Often, you might work with callback-based API-s. You can wrap them with -## asynchronous procedures using promises and ``newPromise``: +## asynchronous procedures using promises and `newPromise`: ## -## .. code-block:: nim +## ```nim ## proc loadGame(name: string): Future[Game] = ## var promise = newPromise() do (resolve: proc(response: Game)): ## cbBasedLoadGame(name) do (game: Game): ## resolve(game) ## return promise +## ``` ## -## Forward definitions work properly, you just need to always add the ``{.async.}`` pragma: +## Forward definitions work properly, you just need to always add the `{.async.}` pragma: ## -## .. code-block:: nim +## ```nim ## proc loadGame(name: string): Future[Game] {.async.} +## ``` ## ## JavaScript compatibility ## ======================== @@ -57,19 +62,22 @@ ## If you need to use this module with older versions of JavaScript, you can ## use a tool that backports the resulting JavaScript code, as babel. -import jsffi -import macros +# xxx code: javascript above gives `LanguageXNotSupported` warning. -when not defined(js) and not defined(nimdoc) and not defined(nimsuggest): +when not defined(js) and not defined(nimsuggest): {.fatal: "Module asyncjs is designed to be used with the JavaScript backend.".} +import std/jsffi +import std/macros +import std/private/since + type Future*[T] = ref object future*: T ## Wraps the return type of an asynchronous procedure. - PromiseJs* {.importcpp: "Promise".} = ref object - ## A JavaScript Promise + PromiseJs* {.importjs: "Promise".} = ref object + ## A JavaScript Promise. proc replaceReturn(node: var NimNode) = @@ -82,6 +90,8 @@ proc replaceReturn(node: var NimNode) = node[z] = nnkReturnStmt.newTree(value) elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result": node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(jsResolve, son[1])) + elif son.kind in RoutineNodes: + discard else: replaceReturn(son) inc z @@ -92,10 +102,18 @@ proc isFutureVoid(node: NimNode): bool = node[1].kind == nnkIdent and $node[1] == "void" proc generateJsasync(arg: NimNode): NimNode = - if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: + if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo, nnkProcTy}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.") + # Transform type X = proc (): something {.async.} + # into type X = proc (): Future[something] + if arg.kind == nnkProcTy: + result = arg + if arg[0][0].kind == nnkEmpty: + result[0][0] = quote do: Future[void] + return result + result = arg var isVoid = false let jsResolve = ident("jsResolve") @@ -111,16 +129,17 @@ proc generateJsasync(arg: NimNode): NimNode = if len(code) > 0: var awaitFunction = quote: - proc await[T](f: Future[T]): T {.importcpp: "(await #)", used.} + proc await[T](f: Future[T]): T {.importjs: "(await #)", used.} result.body.add(awaitFunction) var resolve: NimNode if isVoid: resolve = quote: - var `jsResolve` {.importcpp: "undefined".}: Future[void] + var `jsResolve` {.importjs: "undefined".}: Future[void] else: resolve = quote: - proc jsResolve[T](a: T): Future[T] {.importcpp: "#", used.} + proc jsResolve[T](a: T): Future[T] {.importjs: "#", used.} + proc jsResolve[T](a: Future[T]): Future[T] {.importjs: "#", used.} result.body.add(resolve) else: result.body = newEmptyNode() @@ -139,7 +158,7 @@ proc generateJsasync(arg: NimNode): NimNode = macro async*(arg: untyped): untyped = ## Macro which converts normal procedures into - ## javascript-compatible async procedures + ## javascript-compatible async procedures. if arg.kind == nnkStmtList: result = newStmtList() for oneProc in arg: @@ -147,10 +166,104 @@ macro async*(arg: untyped): untyped = else: result = generateJsasync(arg) -proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".} +proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importjs: "(new Promise(#))".} ## A helper for wrapping callback-based functions - ## into promises and async procedures + ## into promises and async procedures. -proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importcpp: "(new Promise(#))".} +proc newPromise*(handler: proc(resolve: proc())): Future[void] {.importjs: "(new Promise(#))".} ## A helper for wrapping callback-based functions - ## into promises and async procedures + ## into promises and async procedures. + +template maybeFuture(T): untyped = + # avoids `Future[Future[T]]` + when T is Future: T + else: Future[T] + + +since (1, 5, 1): + #[ + TODO: + * map `Promise.all()` + * proc toString*(a: Error): cstring {.importjs: "#.toString()".} + + Note: + We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible + in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options + and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho + ]# + + type Error* {.importjs: "Error".} = ref object of JsRoot + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error + message*: cstring + name*: cstring + + type OnReject* = proc(reason: Error) + + proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then + ## Returns a `Future` from the return type of `onSuccess(T.default)`. + runnableExamples("-r:off"): + from std/sugar import `=>` + + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 + + proc asyncFact(n: int): Future[int] {.async.} = + if n > 0: result = n * await asyncFact(n-1) + else: result = 1 + + proc main() {.async.} = + block: # then + assert asyncFact(3).await == 3*2 + assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2 + let x1 = await fn(3) + assert x1 == 3 * 2 + let x2 = await fn(4) + .then((a: int) => a.float) + .then((a: float) => $a) + assert x2 == "8.0" + + block: # then with `onReject` callback + var witness = 1 + await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 2 + await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 3 + + template impl(call): untyped = + # see D20210421T014713 + when typeof(block: call) is void: + var ret: Future[void] + else: + var ret = default(maybeFuture(typeof(call))) + typeof(ret) + when T is void: + type A = impl(onSuccess()) + else: + type A = impl(onSuccess(default(T))) + var ret: A + {.emit: "`ret` = `future`.then(`onSuccess`, `onReject`);".} + return ret + + proc catch*[T](future: Future[T], onReject: OnReject): Future[void] = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch + runnableExamples("-r:off"): + from std/sugar import `=>` + from std/strutils import contains + + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 + + proc main() {.async.} = + var reason: Error + await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work + assert reason == nil + await fn(7).catch((r: Error) => (reason = r)) + assert reason != nil + assert "foobar: 7" in $reason.message + + discard main() + + {.emit: "`result` = `future`.catch(`onReject`);".} diff --git a/lib/js/dom.nim b/lib/js/dom.nim index cd78f5560..be2a34db1 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -9,39 +9,73 @@ ## Declaration of the Document Object Model for the `JavaScript backend ## <backends.html#backends-the-javascript-target>`_. - -when not defined(js) and not defined(Nimdoc): +## +## +## Document Ready +## -------------- +## +## * Basic example of a document ready: +runnableExamples"-b:js -r:off": + proc example(e: Event) = echo "Document is ready" + document.addEventListener("DOMContentLoaded", example) # You can also use "load" event. +## * This example runs 5 seconds after the document ready: +runnableExamples"-b:js -r:off": + proc example() = echo "5 seconds after document ready" + proc domReady(e: Event) = discard setTimeout(example, 5_000) # Document is ready. + document.addEventListener("DOMContentLoaded", domReady) +## Document onUnload +## ----------------- +## +## * Simple example of how to implement code that runs when the page unloads: +runnableExamples"-b:js -r:off": + proc example(e: Event) = echo "Document is unloaded" + document.addEventListener("unload", example) # You can also use "beforeunload". +## Document Autorefresh +## -------------------- +## +## * Minimal example of a document autorefresh: +runnableExamples"-b:js -r:off": + proc example() = window.location.reload() + discard setTimeout(example, 5_000) +## - For more examples, see https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener + + +import std/private/since +when not defined(js): {.error: "This module only works on the JavaScript platform".} const DomApiVersion* = 3 ## the version of DOM API we try to follow. No guarantees though. type - EventTarget* = ref EventTargetObj - EventTargetObj {.importc.} = object of RootObj - onabort*: proc (event: Event) {.nimcall.} - onblur*: proc (event: Event) {.nimcall.} - onchange*: proc (event: Event) {.nimcall.} - onclick*: proc (event: Event) {.nimcall.} - ondblclick*: proc (event: Event) {.nimcall.} - onerror*: proc (event: Event) {.nimcall.} - onfocus*: proc (event: Event) {.nimcall.} - onkeydown*: proc (event: Event) {.nimcall.} - onkeypress*: proc (event: Event) {.nimcall.} - onkeyup*: proc (event: Event) {.nimcall.} - onload*: proc (event: Event) {.nimcall.} - onmousedown*: proc (event: Event) {.nimcall.} - onmousemove*: proc (event: Event) {.nimcall.} - onmouseout*: proc (event: Event) {.nimcall.} - onmouseover*: proc (event: Event) {.nimcall.} - onmouseup*: proc (event: Event) {.nimcall.} - onreset*: proc (event: Event) {.nimcall.} - onselect*: proc (event: Event) {.nimcall.} - onsubmit*: proc (event: Event) {.nimcall.} - onunload*: proc (event: Event) {.nimcall.} - - # https://developer.mozilla.org/en-US/docs/Web/Events + EventTarget* {.importc.} = ref object of RootObj + onabort*: proc (event: Event) {.closure.} + onblur*: proc (event: Event) {.closure.} + onchange*: proc (event: Event) {.closure.} + onclick*: proc (event: Event) {.closure.} + ondblclick*: proc (event: Event) {.closure.} + onerror*: proc (event: Event) {.closure.} + onfocus*: proc (event: Event) {.closure.} + onkeydown*: proc (event: Event) {.closure.} + onkeypress*: proc (event: Event) {.closure.} + onkeyup*: proc (event: Event) {.closure.} + onload*: proc (event: Event) {.closure.} + onmousedown*: proc (event: Event) {.closure.} + onmousemove*: proc (event: Event) {.closure.} + onmouseout*: proc (event: Event) {.closure.} + onmouseover*: proc (event: Event) {.closure.} + onmouseup*: proc (event: Event) {.closure.} + onreset*: proc (event: Event) {.closure.} + onselect*: proc (event: Event) {.closure.} + onstorage*: proc (event: Event) {.closure.} + onsubmit*: proc (event: Event) {.closure.} + onunload*: proc (event: Event) {.closure.} + onloadstart*: proc (event: Event) {.closure.} + onprogress*: proc (event: Event) {.closure.} + onloadend*: proc (event: Event) {.closure.} + DomEvent* {.pure.} = enum + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/Events>`_ Abort = "abort", BeforeInput = "beforeinput", Blur = "blur", @@ -69,6 +103,7 @@ type Resize = "resize", Scroll = "scroll", Select = "select", + Storage = "storage", Unload = "unload", Wheel = "wheel" @@ -103,8 +138,28 @@ type memory*: PerformanceMemory timing*: PerformanceTiming - Window* = ref WindowObj - WindowObj {.importc.} = object of EventTargetObj + Range* {.importc.} = ref object + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Range>`_ + collapsed*: bool + commonAncestorContainer*: Node + endContainer*: Node + endOffset*: int + startContainer*: Node + startOffset*: int + + Selection* {.importc.} = ref object + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Selection>`_ + anchorNode*: Node + anchorOffset*: int + focusNode*: Node + focusOffset*: int + isCollapsed*: bool + rangeCount*: int + `type`*: cstring + + Storage* {.importc.} = ref object + + Window* {.importc.} = ref object of EventTarget document*: Document event*: Event history*: History @@ -129,12 +184,13 @@ type screen*: Screen performance*: Performance onpopstate*: proc (event: Event) + localStorage*: Storage + sessionStorage*: Storage + parent*: Window - Frame* = ref FrameObj - FrameObj {.importc.} = object of WindowObj + Frame* {.importc.} = ref object of Window - ClassList* = ref ClassListObj - ClassListObj {.importc.} = object of RootObj + ClassList* {.importc.} = ref object of RootObj NodeType* = enum ElementNode = 1, @@ -150,8 +206,7 @@ type DocumentFragmentNode, NotationNode - Node* = ref NodeObj - NodeObj {.importc.} = object of EventTargetObj + Node* {.importc.} = ref object of EventTarget attributes*: seq[Node] childNodes*: seq[Node] children*: seq[Node] @@ -163,12 +218,21 @@ type nodeType*: NodeType nodeValue*: cstring parentNode*: Node + content*: Node previousSibling*: Node + ownerDocument*: Document innerHTML*: cstring + outerHTML*: cstring + innerText*: cstring + textContent*: cstring style*: Style + baseURI*: cstring + parentElement*: Element + isConnected*: bool - Document* = ref DocumentObj - DocumentObj {.importc.} = object of NodeObj + Document* {.importc.} = ref object of Node + activeElement*: Element + documentElement*: Element alinkColor*: cstring bgColor*: cstring body*: Element @@ -177,11 +241,13 @@ type defaultCharset*: cstring fgColor*: cstring head*: Element + hidden*: bool lastModified*: cstring linkColor*: cstring referrer*: cstring title*: cstring URL*: cstring + visibilityState*: cstring vlinkColor*: cstring anchors*: seq[AnchorElement] forms*: seq[FormElement] @@ -189,9 +255,10 @@ type applets*: seq[Element] embeds*: seq[EmbedElement] links*: seq[LinkElement] + fonts*: FontFaceSet - Element* = ref ElementObj - ElementObj {.importc.} = object of NodeObj + Element* {.importc.} = ref object of Node + className*: cstring classList*: ClassList checked*: bool defaultChecked*: bool @@ -211,9 +278,7 @@ type offsetLeft*: int offsetTop*: int - # https://developer.mozilla.org/en-US/docs/Web/API/ValidityState - ValidityState* = ref ValidityStateObj - ValidityStateObj {.importc.} = object + ValidityState* {.importc.} = ref object ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/ValidityState>`_ badInput*: bool customError*: bool patternMismatch*: bool @@ -226,21 +291,21 @@ type valid*: bool valueMissing*: bool - # https://developer.mozilla.org/en-US/docs/Web/API/Blob - Blob* = ref BlobObj - BlobObj {.importc.} = object of RootObj + Blob* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Blob>`_ size*: int `type`*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/File - File* = ref FileObj - FileObj {.importc.} = object of Blob + File* {.importc.} = ref object of Blob ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/File>`_ lastModified*: int name*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement - InputElement* = ref InputElementObj - InputElementObj {.importc.} = object of Element + TextAreaElement* {.importc.} = ref object of Element ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/HTMLTextAreaElement>`_ + value*: cstring + selectionStart*, selectionEnd*: int + selectionDirection*: cstring + rows*, cols*: int + + InputElement* {.importc.} = ref object of Element ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement>`_ # Properties related to the parent form formAction*: cstring formEncType*: cstring @@ -292,15 +357,13 @@ type valueAsDate*: cstring valueAsNumber*: float - LinkElement* = ref LinkObj - LinkObj {.importc.} = object of ElementObj + LinkElement* {.importc.} = ref object of Element target*: cstring text*: cstring x*: int y*: int - EmbedElement* = ref EmbedObj - EmbedObj {.importc.} = object of ElementObj + EmbedElement* {.importc.} = ref object of Element height*: int hspace*: int src*: cstring @@ -308,22 +371,18 @@ type `type`*: cstring vspace*: int - AnchorElement* = ref AnchorObj - AnchorObj {.importc.} = object of ElementObj + AnchorElement* {.importc.} = ref object of Element text*: cstring x*, y*: int - OptionElement* = ref OptionObj - OptionObj {.importc.} = object of ElementObj + OptionElement* {.importc.} = ref object of Element defaultSelected*: bool selected*: bool selectedIndex*: int text*: cstring value*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/HTMLFormElement - FormElement* = ref FormObj - FormObj {.importc.} = object of ElementObj + FormElement* {.importc.} = ref object of Element ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/HTMLFormElement>`_ acceptCharset*: cstring action*: cstring autocomplete*: cstring @@ -335,8 +394,7 @@ type noValidate*: bool target*: cstring - ImageElement* = ref ImageObj - ImageObj {.importc.} = object of ElementObj + ImageElement* {.importc.} = ref object of Element border*: int complete*: bool height*: int @@ -346,100 +404,375 @@ type vspace*: int width*: int - Style* = ref StyleObj - StyleObj {.importc.} = object of RootObj + Style* {.importc.} = ref object of RootObj + alignContent*: cstring + alignItems*: cstring + alignSelf*: cstring + all*: cstring + animation*: cstring + animationDelay*: cstring + animationDirection*: cstring + animationDuration*: cstring + animationFillMode*: cstring + animationIterationCount*: cstring + animationName*: cstring + animationPlayState*: cstring + animationTimingFunction*: cstring + backdropFilter*: cstring + backfaceVisibility*: cstring background*: cstring backgroundAttachment*: cstring + backgroundBlendMode*: cstring + backgroundClip*: cstring backgroundColor*: cstring backgroundImage*: cstring + backgroundOrigin*: cstring backgroundPosition*: cstring backgroundRepeat*: cstring + backgroundSize*: cstring + blockSize*: cstring border*: cstring + borderBlock*: cstring + borderBlockColor*: cstring + borderBlockEnd*: cstring + borderBlockEndColor*: cstring + borderBlockEndStyle*: cstring + borderBlockEndWidth*: cstring + borderBlockStart*: cstring + borderBlockStartColor*: cstring + borderBlockStartStyle*: cstring + borderBlockStartWidth*: cstring + borderBlockStyle*: cstring + borderBlockWidth*: cstring borderBottom*: cstring borderBottomColor*: cstring + borderBottomLeftRadius*: cstring + borderBottomRightRadius*: cstring borderBottomStyle*: cstring borderBottomWidth*: cstring + borderCollapse*: cstring borderColor*: cstring + borderEndEndRadius*: cstring + borderEndStartRadius*: cstring + borderImage*: cstring + borderImageOutset*: cstring + borderImageRepeat*: cstring + borderImageSlice*: cstring + borderImageSource*: cstring + borderImageWidth*: cstring + borderInline*: cstring + borderInlineColor*: cstring + borderInlineEnd*: cstring + borderInlineEndColor*: cstring + borderInlineEndStyle*: cstring + borderInlineEndWidth*: cstring + borderInlineStart*: cstring + borderInlineStartColor*: cstring + borderInlineStartStyle*: cstring + borderInlineStartWidth*: cstring + borderInlineStyle*: cstring + borderInlineWidth*: cstring borderLeft*: cstring borderLeftColor*: cstring borderLeftStyle*: cstring borderLeftWidth*: cstring + borderRadius*: cstring borderRight*: cstring borderRightColor*: cstring borderRightStyle*: cstring borderRightWidth*: cstring + borderSpacing*: cstring + borderStartEndRadius*: cstring + borderStartStartRadius*: cstring borderStyle*: cstring borderTop*: cstring borderTopColor*: cstring + borderTopLeftRadius*: cstring + borderTopRightRadius*: cstring borderTopStyle*: cstring borderTopWidth*: cstring borderWidth*: cstring bottom*: cstring + boxDecorationBreak*: cstring + boxShadow*: cstring + boxSizing*: cstring + breakAfter*: cstring + breakBefore*: cstring + breakInside*: cstring captionSide*: cstring + caretColor*: cstring clear*: cstring clip*: cstring + clipPath*: cstring color*: cstring + colorAdjust*: cstring + columnCount*: cstring + columnFill*: cstring + columnGap*: cstring + columnRule*: cstring + columnRuleColor*: cstring + columnRuleStyle*: cstring + columnRuleWidth*: cstring + columnSpan*: cstring + columnWidth*: cstring + columns*: cstring + contain*: cstring + content*: cstring + counterIncrement*: cstring + counterReset*: cstring + counterSet*: cstring cursor*: cstring direction*: cstring display*: cstring emptyCells*: cstring + filter*: cstring + flex*: cstring + flexBasis*: cstring + flexDirection*: cstring + flexFlow*: cstring + flexGrow*: cstring + flexShrink*: cstring + flexWrap*: cstring cssFloat*: cstring font*: cstring fontFamily*: cstring + fontFeatureSettings*: cstring + fontKerning*: cstring + fontLanguageOverride*: cstring + fontOpticalSizing*: cstring fontSize*: cstring + fontSizeAdjust*: cstring fontStretch*: cstring fontStyle*: cstring + fontSynthesis*: cstring fontVariant*: cstring + fontVariantAlternates*: cstring + fontVariantCaps*: cstring + fontVariantEastAsian*: cstring + fontVariantLigatures*: cstring + fontVariantNumeric*: cstring + fontVariantPosition*: cstring + fontVariationSettings*: cstring fontWeight*: cstring + gap*: cstring + grid*: cstring + gridArea*: cstring + gridAutoColumns*: cstring + gridAutoFlow*: cstring + gridAutoRows*: cstring + gridColumn*: cstring + gridColumnEnd*: cstring + gridColumnStart*: cstring + gridRow*: cstring + gridRowEnd*: cstring + gridRowStart*: cstring + gridTemplate*: cstring + gridTemplateAreas*: cstring + gridTemplateColumns*: cstring + gridTemplateRows*: cstring + hangingPunctuation*: cstring height*: cstring + hyphens*: cstring + imageOrientation*: cstring + imageRendering*: cstring + inlineSize*: cstring + inset*: cstring + insetBlock*: cstring + insetBlockEnd*: cstring + insetBlockStart*: cstring + insetInline*: cstring + insetInlineEnd*: cstring + insetInlineStart*: cstring + isolation*: cstring + justifyContent*: cstring + justifyItems*: cstring + justifySelf*: cstring left*: cstring letterSpacing*: cstring + lineBreak*: cstring lineHeight*: cstring listStyle*: cstring listStyleImage*: cstring listStylePosition*: cstring listStyleType*: cstring margin*: cstring + marginBlock*: cstring + marginBlockEnd*: cstring + marginBlockStart*: cstring marginBottom*: cstring + marginInline*: cstring + marginInlineEnd*: cstring + marginInlineStart*: cstring marginLeft*: cstring marginRight*: cstring marginTop*: cstring + mask*: cstring + maskBorder*: cstring + maskBorderMode*: cstring + maskBorderOutset*: cstring + maskBorderRepeat*: cstring + maskBorderSlice*: cstring + maskBorderSource*: cstring + maskBorderWidth*: cstring + maskClip*: cstring + maskComposite*: cstring + maskImage*: cstring + maskMode*: cstring + maskOrigin*: cstring + maskPosition*: cstring + maskRepeat*: cstring + maskSize*: cstring + maskType*: cstring + maxBlockSize*: cstring maxHeight*: cstring + maxInlineSize*: cstring maxWidth*: cstring + minBlockSize*: cstring minHeight*: cstring + minInlineSize*: cstring minWidth*: cstring + mixBlendMode*: cstring + objectFit*: cstring + objectPosition*: cstring + offset*: cstring + offsetAnchor*: cstring + offsetDistance*: cstring + offsetPath*: cstring + offsetRotate*: cstring opacity*: cstring + order*: cstring + orphans*: cstring + outline*: cstring + outlineColor*: cstring + outlineOffset*: cstring + outlineStyle*: cstring + outlineWidth*: cstring overflow*: cstring + overflowAnchor*: cstring + overflowBlock*: cstring + overflowInline*: cstring + overflowWrap*: cstring + overflowX*: cstring + overflowY*: cstring + overscrollBehavior*: cstring + overscrollBehaviorBlock*: cstring + overscrollBehaviorInline*: cstring + overscrollBehaviorX*: cstring + overscrollBehaviorY*: cstring padding*: cstring + paddingBlock*: cstring + paddingBlockEnd*: cstring + paddingBlockStart*: cstring paddingBottom*: cstring + paddingInline*: cstring + paddingInlineEnd*: cstring + paddingInlineStart*: cstring paddingLeft*: cstring paddingRight*: cstring paddingTop*: cstring pageBreakAfter*: cstring pageBreakBefore*: cstring + pageBreakInside*: cstring + paintOrder*: cstring + perspective*: cstring + perspectiveOrigin*: cstring + placeContent*: cstring + placeItems*: cstring + placeSelf*: cstring pointerEvents*: cstring position*: cstring + quotes*: cstring + resize*: cstring right*: cstring + rotate*: cstring + rowGap*: cstring + scale*: cstring + scrollBehavior*: cstring + scrollMargin*: cstring + scrollMarginBlock*: cstring + scrollMarginBlockEnd*: cstring + scrollMarginBlockStart*: cstring + scrollMarginBottom*: cstring + scrollMarginInline*: cstring + scrollMarginInlineEnd*: cstring + scrollMarginInlineStart*: cstring + scrollMarginLeft*: cstring + scrollMarginRight*: cstring + scrollMarginTop*: cstring + scrollPadding*: cstring + scrollPaddingBlock*: cstring + scrollPaddingBlockEnd*: cstring + scrollPaddingBlockStart*: cstring + scrollPaddingBottom*: cstring + scrollPaddingInline*: cstring + scrollPaddingInlineEnd*: cstring + scrollPaddingInlineStart*: cstring + scrollPaddingLeft*: cstring + scrollPaddingRight*: cstring + scrollPaddingTop*: cstring + scrollSnapAlign*: cstring + scrollSnapStop*: cstring + scrollSnapType*: cstring scrollbar3dLightColor*: cstring scrollbarArrowColor*: cstring scrollbarBaseColor*: cstring + scrollbarColor*: cstring scrollbarDarkshadowColor*: cstring scrollbarFaceColor*: cstring scrollbarHighlightColor*: cstring scrollbarShadowColor*: cstring scrollbarTrackColor*: cstring + scrollbarWidth*: cstring + shapeImageThreshold*: cstring + shapeMargin*: cstring + shapeOutside*: cstring + tabSize*: cstring tableLayout*: cstring textAlign*: cstring + textAlignLast*: cstring + textCombineUpright*: cstring textDecoration*: cstring + textDecorationColor*: cstring + textDecorationLine*: cstring + textDecorationSkipInk*: cstring + textDecorationStyle*: cstring + textDecorationThickness*: cstring + textEmphasis*: cstring + textEmphasisColor*: cstring + textEmphasisPosition*: cstring + textEmphasisStyle*: cstring textIndent*: cstring + textJustify*: cstring + textOrientation*: cstring + textOverflow*: cstring + textRendering*: cstring + textShadow*: cstring textTransform*: cstring - transform*: cstring + textUnderlineOffset*: cstring + textUnderlinePosition*: cstring top*: cstring + touchAction*: cstring + transform*: cstring + transformBox*: cstring + transformOrigin*: cstring + transformStyle*: cstring + transition*: cstring + transitionDelay*: cstring + transitionDuration*: cstring + transitionProperty*: cstring + transitionTimingFunction*: cstring + translate*: cstring + unicodeBidi*: cstring verticalAlign*: cstring visibility*: cstring + whiteSpace*: cstring + widows*: cstring width*: cstring + willChange*: cstring + wordBreak*: cstring wordSpacing*: cstring - zIndex*: int + writingMode*: cstring + zIndex*: cstring EventPhase* = enum None = 0, @@ -447,9 +780,7 @@ type AtTarget, BubblingPhase - # https://developer.mozilla.org/en-US/docs/Web/API/Event - Event* = ref EventObj - EventObj {.importc.} = object of RootObj + Event* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/Event>`_ bubbles*: bool cancelBubble*: bool cancelable*: bool @@ -461,15 +792,11 @@ type `type`*: cstring isTrusted*: bool - # https://developer.mozilla.org/en-US/docs/Web/API/UIEvent - UIEvent* = ref UIEventObj - UIEventObj {.importc.} = object of Event + UIEvent* {.importc.} = ref object of Event ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/UIEvent>`_ detail*: int64 view*: Window - # https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent - KeyboardEvent* = ref KeyboardEventObj - KeyboardEventObj {.importc.} = object of UIEvent + KeyboardEvent* {.importc.} = ref object of UIEvent ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent>`_ altKey*, ctrlKey*, metaKey*, shiftKey*: bool code*: cstring isComposing*: bool @@ -477,8 +804,7 @@ type keyCode*: int location*: int - # https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key/Key_Values - KeyboardEventKey* {.pure.} = enum + KeyboardEventKey* {.pure.} = enum ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key/Key_Values>`_ # Modifier keys Alt, AltGraph, @@ -832,9 +1158,7 @@ type FourthButton = 8, FifthButton = 16 - # https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent - MouseEvent* = ref MouseEventObj - MouseEventObj {.importc.} = object of UIEvent + MouseEvent* {.importc.} = ref object of UIEvent ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent>`_ altKey*, ctrlKey*, metaKey*, shiftKey*: bool button*: int buttons*: int @@ -851,15 +1175,11 @@ type File = "file", String = "string" - # https://developer.mozilla.org/en-US/docs/Web/API/DataTransferItem - DataTransferItem* = ref DataTransferItemObj - DataTransferItemObj {.importc.} = object of RootObj + DataTransferItem* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/DataTransferItem>`_ kind*: cstring `type`*: cstring - # https://developer.mozilla.org/en-US/docs/Web/API/DataTransfer - DataTransfer* = ref DataTransferObj - DataTransferObj {.importc.} = object of RootObj + DataTransfer* {.importc.} = ref object of RootObj ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/DataTransfer>`_ dropEffect*: cstring effectAllowed*: cstring files*: seq[Element] @@ -893,15 +1213,24 @@ type DragStart = "dragstart", Drop = "drop" - # https://developer.mozilla.org/en-US/docs/Web/API/DragEvent DragEvent* {.importc.} = object of MouseEvent + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/DragEvent>`_ dataTransfer*: DataTransfer + ClipboardEvent* {.importc.} = object of Event + ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/ClipboardEvent>`_ + clipboardData*: DataTransfer + + StorageEvent* {.importc.} = ref object of Event ## see `docs<https://developer.mozilla.org/en-US/docs/Web/API/StorageEvent>`_ + key*: cstring + newValue*, oldValue*: cstring + storageArea*: Storage + url*: cstring + TouchList* {.importc.} = ref object of RootObj length*: int - Touch* = ref TouchObj - TouchObj {.importc.} = object of RootObj + Touch* {.importc.} = ref object of RootObj identifier*: int screenX*, screenY*, clientX*, clientY*, pageX*, pageY*: int target*: Element @@ -909,12 +1238,10 @@ type rotationAngle*: int force*: float - TouchEvent* = ref TouchEventObj - TouchEventObj {.importc.} = object of UIEvent + TouchEvent* {.importc.} = ref object of UIEvent changedTouches*, targetTouches*, touches*: seq[Touch] - Location* = ref LocationObj - LocationObj {.importc.} = object of RootObj + Location* {.importc.} = ref object of RootObj hash*: cstring host*: cstring hostname*: cstring @@ -925,19 +1252,26 @@ type search*: cstring origin*: cstring - History* = ref HistoryObj - HistoryObj {.importc.} = object of RootObj + History* {.importc.} = ref object of RootObj length*: int - Navigator* = ref NavigatorObj - NavigatorObj {.importc.} = object of RootObj + Navigator* {.importc.} = ref object of RootObj appCodeName*: cstring appName*: cstring appVersion*: cstring + buildID*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/buildID cookieEnabled*: bool + deviceMemory*: float ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory + doNotTrack*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/doNotTrack language*: cstring + languages*: seq[cstring] ## https://developer.mozilla.org/en-US/docs/Web/API/NavigatorLanguage/languages + maxTouchPoints*: cint ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/maxTouchPoints + onLine*: bool ## https://developer.mozilla.org/en-US/docs/Web/API/NavigatorOnLine/onLine + oscpu*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/oscpu platform*: cstring userAgent*: cstring + vendor*: cstring ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vendor + webdriver*: bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/webdriver mimeTypes*: seq[ref MimeType] Plugin* {.importc.} = object of RootObj @@ -959,8 +1293,7 @@ type ToolBar* = LocationBar StatusBar* = LocationBar - Screen = ref ScreenObj - ScreenObj {.importc.} = object of RootObj + Screen* {.importc.} = ref object of RootObj availHeight*: int availWidth*: int colorDepth*: int @@ -969,13 +1302,80 @@ type width*: int TimeOut* {.importc.} = ref object of RootObj - Interval* {.importc.} = object of RootObj + Interval* {.importc.} = ref object of RootObj AddEventListenerOptions* = object capture*: bool once*: bool passive*: bool + FontFaceSetReady* {.importc.} = ref object + ## see: `docs<https://developer.mozilla.org/en-US/docs/Web/API/FontFaceSet/ready>`_ + then*: proc(cb: proc()) + + FontFaceSet* {.importc.} = ref object + ## see: `docs<https://developer.mozilla.org/en-US/docs/Web/API/FontFaceSet>`_ + ready*: FontFaceSetReady + onloadingdone*: proc(event: Event) + + ScrollIntoViewOptions* = object + behavior*: cstring + `block`*: cstring + inline*: cstring + + MediaQueryList* {.importc.} = ref object of EventTarget + matches*: bool + media*: cstring + +since (1, 3): + type + DomParser* = ref object + ## DOM Parser object (defined on browser only, may not be on NodeJS). + ## * https://developer.mozilla.org/en-US/docs/Web/API/DOMParser + ## + ## ```nim + ## let prsr = newDomParser() + ## discard prsr.parseFromString("<html><marquee>Hello World</marquee></html>".cstring, "text/html".cstring) + ## ``` + + DomException* {.importc.} = ref object + ## The DOMException interface represents an abnormal event (called an exception) + ## which occurs as a result of calling a method or accessing a property of a web API. + ## Each exception has a name, which is a short "CamelCase" style string identifying + ## the error or abnormal condition. + ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException + + FileReader* {.importc.} = ref object of EventTarget + ## The FileReader object lets web applications asynchronously read the contents of files + ## (or raw data buffers) stored on the user's computer, using File or Blob objects to specify + ## the file or data to read. + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader + + FileReaderState* = distinct range[0'u16..2'u16] + RootNodeOptions* = object of RootObj + composed*: bool + DocumentOrShadowRoot* {.importc.} = object of RootObj + activeElement*: Element + # styleSheets*: StyleSheetList + ShadowRoot* {.importc.} = ref object of DocumentOrShadowRoot + delegatesFocus*: bool + host*: Element + innerHTML*: cstring + mode*: cstring # "open" or "closed" + ShadowRootInit* = object of RootObj + mode*: cstring + delegatesFocus*: bool + + HTMLSlotElement* {.importc.} = ref object of RootObj + name*: cstring + SlotOptions* = object of RootObj + flatten*: bool + + const + fileReaderEmpty* = 0.FileReaderState + fileReaderLoading* = 1.FileReaderState + fileReaderDone* = 2.FileReaderState + proc id*(n: Node): cstring {.importcpp: "#.id", nodecl.} proc `id=`*(n: Node; x: cstring) {.importcpp: "#.id = #", nodecl.} proc class*(n: Node): cstring {.importcpp: "#.className", nodecl.} @@ -984,6 +1384,9 @@ proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.} proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.} +proc checked*(n: Node): bool {.importcpp: "#.checked", nodecl.} +proc `checked=`*(n: Node; v: bool) {.importcpp: "#.checked = #", nodecl.} + proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.} when defined(nodejs): @@ -994,6 +1397,7 @@ when defined(nodejs): result = cast[Element](x.childNodes[idx]) var document* = Document(nodeType: DocumentNode) + document.ownerDocument = document proc getElem(x: Element; id: cstring): Element = if x.id == id: return x @@ -1007,6 +1411,7 @@ when defined(nodejs): proc appendChild*(parent, n: Node) = n.parentNode = parent + n.ownerDocument = parent.ownerDocument parent.childNodes.add n proc replaceChild*(parent, newNode, oldNode: Node) = @@ -1018,7 +1423,7 @@ when defined(nodejs): parent.childNodes[i] = newNode return inc i - doAssert false, "old node not in node list" + raiseAssert "old node not in node list" proc removeChild*(parent, child: Node) = child.parentNode = nil @@ -1028,7 +1433,7 @@ when defined(nodejs): parent.childNodes.delete(i) return inc i - doAssert false, "old node not in node list" + raiseAssert "old node not in node list" proc insertBefore*(parent, newNode, before: Node) = appendChild(parent, newNode) @@ -1040,7 +1445,7 @@ when defined(nodejs): parent.childNodes[i-1] = newNode return inc i - #doAssert false, "before not in node list" + #raiseAssert "before not in node list" proc createElement*(d: Document, identifier: cstring): Element = new(result) @@ -1053,20 +1458,31 @@ when defined(nodejs): result.nodeValue = identifier result.nodeType = NodeType.TextNode + proc createComment*(d: Document, data: cstring): Node = + new(result) + result.nodeName = "#comment" + result.nodeValue = data + result.nodeType = NodeType.CommentNode + else: proc len*(x: Node): int {.importcpp: "#.childNodes.length".} proc `[]`*(x: Node; idx: int): Element {.importcpp: "#.childNodes[#]".} proc getElementById*(id: cstring): Element {.importc: "document.getElementById", nodecl.} proc appendChild*(n, child: Node) {.importcpp.} proc removeChild*(n, child: Node) {.importcpp.} + proc remove*(child: Node) {.importcpp.} proc replaceChild*(n, newNode, oldNode: Node) {.importcpp.} proc insertBefore*(n, newNode, before: Node) {.importcpp.} proc getElementById*(d: Document, id: cstring): Element {.importcpp.} proc createElement*(d: Document, identifier: cstring): Element {.importcpp.} + proc createElementNS*(d: Document, namespaceURI, qualifiedIdentifier: cstring): Element {.importcpp.} proc createTextNode*(d: Document, identifier: cstring): Node {.importcpp.} + proc createComment*(d: Document, data: cstring): Node {.importcpp.} -proc setTimeout*(action: proc(); ms: int): Timeout {.importc, nodecl.} -proc clearTimeout*(t: Timeout) {.importc, nodecl.} +proc setTimeout*(action: proc(); ms: int): TimeOut {.importc, nodecl.} +proc clearTimeout*(t: TimeOut) {.importc, nodecl.} +proc setInterval*(action: proc(); ms: int): Interval {.importc, nodecl.} +proc clearInterval*(i: Interval) {.importc, nodecl.} {.push importcpp.} @@ -1080,18 +1496,19 @@ proc removeEventListener*(et: EventTarget; ev: cstring; cb: proc(ev: Event)) proc alert*(w: Window, msg: cstring) proc back*(w: Window) proc blur*(w: Window) -proc captureEvents*(w: Window, eventMask: int) {.deprecated.} -proc clearInterval*(w: Window, interval: ref Interval) -proc clearTimeout*(w: Window, timeout: ref TimeOut) +proc clearInterval*(w: Window, interval: Interval) +proc clearTimeout*(w: Window, timeout: TimeOut) proc close*(w: Window) proc confirm*(w: Window, msg: cstring): bool proc disableExternalCapture*(w: Window) proc enableExternalCapture*(w: Window) proc find*(w: Window, text: cstring, caseSensitive = false, - backwards = false) + backwards = false): bool proc focus*(w: Window) proc forward*(w: Window) -proc getComputedStyle*(w: Window, e: Node, pe:Node = nil): Style +proc getComputedStyle*(w: Window, e: Node, pe: Node = nil): Style + ## .. warning:: The returned Style may or may not be read-only at run-time in the browser. getComputedStyle is performance costly. + proc handleEvent*(w: Window, e: Event) proc home*(w: Window) proc moveBy*(w: Window, x, y: int) @@ -1100,19 +1517,19 @@ proc open*(w: Window, uri, windowname: cstring, properties: cstring = nil): Window proc print*(w: Window) proc prompt*(w: Window, text, default: cstring): cstring -proc releaseEvents*(w: Window, eventMask: int) {.deprecated.} proc resizeBy*(w: Window, x, y: int) proc resizeTo*(w: Window, x, y: int) proc routeEvent*(w: Window, event: Event) proc scrollBy*(w: Window, x, y: int) proc scrollTo*(w: Window, x, y: int) -proc setInterval*(w: Window, code: cstring, pause: int): ref Interval -proc setInterval*(w: Window, function: proc (), pause: int): ref Interval -proc setTimeout*(w: Window, code: cstring, pause: int): ref TimeOut -proc setTimeout*(w: Window, function: proc (), pause: int): ref Interval +proc setInterval*(w: Window, code: cstring, pause: int): Interval +proc setInterval*(w: Window, function: proc (), pause: int): Interval +proc setTimeout*(w: Window, code: cstring, pause: int): TimeOut +proc setTimeout*(w: Window, function: proc (), pause: int): Interval proc stop*(w: Window) proc requestAnimationFrame*(w: Window, function: proc (time: float)): int proc cancelAnimationFrame*(w: Window, id: int) +proc matchMedia*(w: Window, mediaQueryString: cstring): MediaQueryList # Node "methods" proc appendData*(n: Node, data: cstring) @@ -1121,25 +1538,48 @@ proc deleteData*(n: Node, start, len: int) proc focus*(e: Node) proc getAttribute*(n: Node, attr: cstring): cstring proc getAttributeNode*(n: Node, attr: cstring): Node +proc hasAttribute*(n: Node, attr: cstring): bool proc hasChildNodes*(n: Node): bool +proc normalize*(n: Node) proc insertData*(n: Node, position: int, data: cstring) proc removeAttribute*(n: Node, attr: cstring) proc removeAttributeNode*(n, attr: Node) proc replaceData*(n: Node, start, len: int, text: cstring) proc scrollIntoView*(n: Node) +proc scrollIntoView*(n: Node, options: ScrollIntoViewOptions) proc setAttribute*(n: Node, name, value: cstring) proc setAttributeNode*(n: Node, attr: Node) +proc querySelector*(n: Node, selectors: cstring): Element +proc querySelectorAll*(n: Node, selectors: cstring): seq[Element] +proc compareDocumentPosition*(n: Node, otherNode:Node): int +proc lookupPrefix*(n: Node): cstring +proc lookupNamespaceURI*(n: Node): cstring +proc isDefaultNamespace*(n: Node): bool +proc contains*(n: Node): bool +proc isEqualNode*(n: Node): bool +proc isSameNode*(n: Node): bool + +since (1, 3): + proc getRootNode*(n: Node,options: RootNodeOptions): Node + + # DocumentOrShadowRoot + proc getSelection*(n: DocumentOrShadowRoot): Selection + proc elementFromPoint*(n: DocumentOrShadowRoot; x, y: float): Element + + # shadow dom + proc attachShadow*(n: Element): ShadowRoot + proc assignedNodes*(n: HTMLSlotElement; options: SlotOptions): seq[Node] + proc assignedElements*(n: HTMLSlotElement; options: SlotOptions): seq[Element] # Document "methods" -proc captureEvents*(d: Document, eventMask: int) {.deprecated.} proc createAttribute*(d: Document, identifier: cstring): Node proc getElementsByName*(d: Document, name: cstring): seq[Element] proc getElementsByTagName*(d: Document, name: cstring): seq[Element] proc getElementsByClassName*(d: Document, name: cstring): seq[Element] -proc getSelection*(d: Document): cstring +proc insertNode*(range: Range, node: Node) +proc getSelection*(d: Document): Selection proc handleEvent*(d: Document, event: Event) proc open*(d: Document) -proc releaseEvents*(d: Document, eventMask: int) {.deprecated.} proc routeEvent*(d: Document, event: Event) proc write*(d: Document, text: cstring) proc writeln*(d: Document, text: cstring) @@ -1177,6 +1617,12 @@ proc pushState*[T](h: History, stateObject: T, title, url: cstring) # Navigator "methods" proc javaEnabled*(h: Navigator): bool +since (1, 3): + proc canShare*(self: Navigator; data: cstring): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/canShare + proc sendBeacon*(self: Navigator; url, data: cstring): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/sendBeacon + proc vibrate*(self: Navigator; pattern: cint): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vibrate + proc vibrate*(self: Navigator; pattern: openArray[cint]): bool ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/vibrate + proc registerProtocolHandler*(self: Navigator; scheme, url, title: cstring) ## https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler # ClassList "methods" proc add*(c: ClassList, class: cstring) @@ -1209,7 +1655,7 @@ proc item*(list: TouchList, i: int): Touch proc clearData*(dt: DataTransfer, format: cstring) proc getData*(dt: DataTransfer, format: cstring): cstring proc setData*(dt: DataTransfer, format: cstring, data: cstring) -proc setDragImage*(dt: DataTransfer, img: Element, xOffset: int64, yOffset: int64) +proc setDragImage*(dt: DataTransfer, img: Element, xOffset: int, yOffset: int) # DataTransferItem "methods" proc getAsFile*(dti: DataTransferItem): File @@ -1223,6 +1669,22 @@ proc checkValidity*(e: InputElement): bool # Blob "methods" proc slice*(e: Blob, startindex: int = 0, endindex: int = e.size, contentType: cstring = "") +# Performance "methods" +proc now*(p: Performance): float + +# Selection "methods" +proc removeAllRanges*(s: Selection) +proc deleteFromDocument*(s: Selection) +proc getRangeAt*(s: Selection, index: int): Range +converter toString*(s: Selection): cstring +proc `$`*(s: Selection): string = $(s.toString()) + +# Storage "methods" +proc getItem*(s: Storage, key: cstring): cstring +proc setItem*(s: Storage, key, value: cstring) +proc clear*(s: Storage) +proc removeItem*(s: Storage, key: cstring) + {.pop.} proc setAttr*(n: Node; key, val: cstring) {.importcpp: "#.setAttribute(@)".} @@ -1245,6 +1707,7 @@ proc decodeURIComponent*(uri: cstring): cstring {.importc, nodecl.} proc encodeURIComponent*(uri: cstring): cstring {.importc, nodecl.} proc isFinite*(x: BiggestFloat): bool {.importc, nodecl.} proc isNaN*(x: BiggestFloat): bool {.importc, nodecl.} + ## see also `math.isNaN`. proc newEvent*(name: cstring): Event {.importcpp: "new Event(@)", constructor.} @@ -1278,3 +1741,101 @@ proc offsetHeight*(e: Node): int {.importcpp: "#.offsetHeight", nodecl.} proc offsetWidth*(e: Node): int {.importcpp: "#.offsetWidth", nodecl.} proc offsetTop*(e: Node): int {.importcpp: "#.offsetTop", nodecl.} proc offsetLeft*(e: Node): int {.importcpp: "#.offsetLeft", nodecl.} + +since (1, 3): + func newDomParser*(): DomParser {.importcpp: "new DOMParser()".} + ## DOM Parser constructor. + func parseFromString*(this: DomParser; str: cstring; mimeType: cstring): Document {.importcpp.} + ## Parse from string to `Document`. + + proc newDomException*(): DomException {.importcpp: "new DomException()", constructor.} + ## DOM Exception constructor + proc message*(ex: DomException): cstring {.importcpp: "#.message", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException/message + proc name*(ex: DomException): cstring {.importcpp: "#.name", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/DOMException/name + + proc newFileReader*(): FileReader {.importcpp: "new FileReader()", constructor.} + ## File Reader constructor + proc error*(f: FileReader): DomException {.importcpp: "#.error", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/error + proc readyState*(f: FileReader): FileReaderState {.importcpp: "#.readyState", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readyState + proc resultAsString*(f: FileReader): cstring {.importcpp: "#.result", nodecl.} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/result + proc abort*(f: FileReader) {.importcpp: "#.abort()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/abort + proc readAsBinaryString*(f: FileReader, b: Blob) {.importcpp: "#.readAsBinaryString(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsBinaryString + proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL + proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText + +since (1, 5): + proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] {.importcpp.} + + +since (1, 7): + + proc insertAdjacentText*(self: Node; position, data: cstring) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentText + + proc insertAdjacentElement*(self: Node; position: cstring; element: Node) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentElement + + proc insertAdjacentHTML*(self: Node; position, html: cstring) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML + + proc after*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/after + + proc before*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/before + + proc append*(self: Node; element: Node): Node {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/append + + proc closest*(self: Node; cssSelector: cstring): Node {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/closest + + proc hasAttributeNS*(self: Node; namespace, localName: cstring): bool {.importjs: "(#.$1(#, #) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/hasAttributeNS + + proc removeAttributeNS*(self: Node; namespace, attributeName: cstring) {.importjs: "#.$1(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/removeAttributeNS + + proc hasPointerCapture*(self: Node; pointerId: SomeNumber): bool {.importjs: "(#.$1(#) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/hasPointerCapture + + proc releasePointerCapture*(self: Node; pointerId: SomeNumber) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/releasePointerCapture + + proc requestPointerLock*(self: Node) {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/requestPointerLock + + proc replaceChildren*(self: Node; replacements: Node) {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/replaceChildren + + proc replaceWith*(self: Node; replacements: Node) {.importjs: "#.$1(@)", varargs.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/replaceWith + + proc scrollIntoViewIfNeeded*(self: Node; centerIfNeeded: bool) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoViewIfNeeded + + proc setHTML*(self: Node; html: cstring) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/setHTML + + proc toggleAttribute*(self: Node; name: cstring; force = false): bool {.importjs: "(#.$1(#, #) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/toggleAttribute + + proc matches*(self: Node; cssSelector: cstring): bool {.importjs: "(#.$1(#) || false)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Element/matches + + +since (2, 1): + type VisualViewport* {.importc.} = ref object of EventTarget + offsetLeft*, offsetTop*, pageLeft*, pageTop*, width*, height*, scale*: float + onResize*, onScroll*: proc (event: Event) {.closure.} + + func visualViewport*(self: Window): VisualViewport {.importjs: "#.$1", nodecl.} diff --git a/lib/js/jsconsole.nim b/lib/js/jsconsole.nim index 199a5607e..e74127334 100644 --- a/lib/js/jsconsole.nim +++ b/lib/js/jsconsole.nim @@ -9,61 +9,117 @@ ## Wrapper for the `console` object for the `JavaScript backend ## <backends.html#backends-the-javascript-target>`_. - -when not defined(js) and not defined(Nimdoc): +## +## Styled Messages +## =============== +## +## CSS-styled messages in the browser are useful for debugging purposes. +## To use them, prefix the message with one or more `%c`, +## and provide the CSS style as the last argument. +## The amount of `%c`'s must match the amount of CSS-styled strings. +## +runnableExamples("-r:off"): + console.log "%c My Debug Message", "color: red" # Notice the "%c" + console.log "%c My Debug %c Message", "color: red", "font-size: 2em" + +import std/private/since, std/private/miscdollars # toLocation + +when not defined(js): {.error: "This module only works on the JavaScript platform".} -import macros +type Console* = ref object of JsRoot + +proc log*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/log + +proc debug*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/debug + +proc info*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/info + +proc error*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/error + +template exception*(console: Console, args: varargs[untyped]) = + ## Alias for `console.error()`. + error(console, args) -type Console* {.importc.} = ref object of RootObj +proc trace*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/trace -proc convertToConsoleLoggable*[T](v: T): RootRef {.importcpp: "#".} -template convertToConsoleLoggable*(v: string): RootRef = cast[RootRef](cstring(v)) +proc warn*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/warn -proc logImpl(console: Console) {.importcpp: "log", varargs.} -proc debugImpl(console: Console) {.importcpp: "debug", varargs.} -proc infoImpl(console: Console) {.importcpp: "info", varargs.} -proc errorImpl(console: Console) {.importcpp: "error", varargs.} -proc warnImpl(console: Console) {.importcpp: "warn", varargs.} +proc clear*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/clear -proc makeConsoleCall(console: NimNode, procName: NimNode, args: NimNode): NimNode = - result = newCall(procName, console) - for c in args: result.add(c) +proc count*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/count -macro log*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = - makeConsoleCall(console, bindSym "logImpl", args) +proc countReset*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/countReset -macro debug*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = - makeConsoleCall(console, bindSym "debugImpl", args) +proc group*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/group -macro info*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = - makeConsoleCall(console, bindSym "infoImpl", args) +proc groupCollapsed*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/groupCollapsed -macro error*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = - makeConsoleCall(console, bindSym "errorImpl", args) +proc groupEnd*(console: Console) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/groupEnd +proc time*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/time -macro warn*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = - ## https://developer.mozilla.org/en-US/docs/Web/API/Console/warn - makeConsoleCall(console, bindSym "warnImpl", args) +proc timeEnd*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/timeEnd -proc clear*(console: Console) {.importcpp: "clear".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/clear +proc timeLog*(console: Console, label = "".cstring) {.importcpp.} + ## https://developer.mozilla.org/docs/Web/API/Console/timeLog -proc count*(console: Console, label = "".cstring) {.importcpp: "count".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/count +proc table*(console: Console) {.importcpp, varargs.} + ## https://developer.mozilla.org/docs/Web/API/Console/table -proc countReset*(console: Console, label = "".cstring) {.importcpp: "countReset".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/countReset +since (1, 5): + type InstantiationInfo = tuple[filename: string, line: int, column: int] -proc group*(console: Console, label = "".cstring) {.importcpp: "group".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/group + func getMsg(info: InstantiationInfo; msg: string): string = + var temp = "" + temp.toLocation(info.filename, info.line, info.column + 1) + result.addQuoted("[jsAssert] " & temp) + result.add ',' + result.addQuoted(msg) -proc groupCollapsed*(console: Console, label = "".cstring) {.importcpp: "groupCollapsed".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/groupCollapsed + template jsAssert*(console: Console; assertion) = + ## JavaScript `console.assert`, for NodeJS this prints to stderr, + ## assert failure just prints to console and do not quit the program, + ## this is not meant to be better or even equal than normal assertions, + ## is just for when you need faster performance *and* assertions, + ## otherwise use the normal assertions for better user experience. + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/assert + runnableExamples: + console.jsAssert(42 == 42) # OK + console.jsAssert(42 != 42) # Fail, prints "Assertion failed" and continues + console.jsAssert('`' == '\n' and '\t' == '\0') # Message correctly formatted + assert 42 == 42 # Normal assertions keep working -proc groupEnd*(console: Console) {.importcpp: "groupEnd".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/groupEnd + const + loc = instantiationInfo(fullPaths = compileOption("excessiveStackTrace")) + msg = getMsg(loc, astToStr(assertion)).cstring + {.line: loc.}: + {.emit: ["console.assert(", assertion, ", ", msg, ");"].} -proc time*(console: Console, label = "".cstring) {.importcpp: "time".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/time + func dir*(console: Console; obj: auto) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/dir -proc timeEnd*(console: Console, label = "".cstring) {.importcpp: "timeEnd".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeEnd + func dirxml*(console: Console; obj: auto) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/dirxml -proc timeLog*(console: Console, label = "".cstring) {.importcpp: "timeLog".} ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeLog + func timeStamp*(console: Console; label: cstring) {.importcpp.} + ## https://developer.mozilla.org/en-US/docs/Web/API/Console/timeStamp + ## + ## ..warning:: non-standard var console* {.importc, nodecl.}: Console diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index 2e2bd2402..be353875c 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -11,10 +11,11 @@ ## ## Unless your application has very ## specific requirements and solely targets JavaScript, you should be using -## the relevant functions in the ``math``, ``json``, and ``times`` stdlib +## the relevant functions in the `math`, `json`, and `times` stdlib ## modules instead. +import std/private/[since, jsutils] -when not defined(js) and not defined(Nimdoc): +when not defined(js): {.error: "This module only works on the JavaScript platform".} type @@ -73,9 +74,16 @@ proc parse*(d: DateLib, s: cstring): int {.importcpp.} proc newDate*(): DateTime {. importcpp: "new Date()".} -proc newDate*(date: int|int64|string): DateTime {. +proc newDate*(date: int|string): DateTime {. importcpp: "new Date(#)".} +whenJsNoBigInt64: + proc newDate*(date: int64): DateTime {. + importcpp: "new Date(#)".} +do: + proc newDate*(date: int64): DateTime {. + importcpp: "new Date(Number(#))".} + proc newDate*(year, month, day, hours, minutes, seconds, milliseconds: int): DateTime {. importcpp: "new Date(#,#,#,#,#,#,#)".} @@ -87,20 +95,59 @@ proc getMilliseconds*(d: DateTime): int {.importcpp.} proc getMinutes*(d: DateTime): int {.importcpp.} proc getMonth*(d: DateTime): int {.importcpp.} proc getSeconds*(d: DateTime): int {.importcpp.} -proc getYear*(d: DateTime): int {.importcpp.} proc getTime*(d: DateTime): int {.importcpp.} -proc toString*(d: DateTime): cstring {.importcpp.} +proc getTimezoneOffset*(d: DateTime): int {.importcpp.} proc getUTCDate*(d: DateTime): int {.importcpp.} +proc getUTCDay*(d: DateTime): int {.importcpp.} proc getUTCFullYear*(d: DateTime): int {.importcpp.} proc getUTCHours*(d: DateTime): int {.importcpp.} proc getUTCMilliseconds*(d: DateTime): int {.importcpp.} proc getUTCMinutes*(d: DateTime): int {.importcpp.} proc getUTCMonth*(d: DateTime): int {.importcpp.} proc getUTCSeconds*(d: DateTime): int {.importcpp.} -proc getUTCDay*(d: DateTime): int {.importcpp.} -proc getTimezoneOffset*(d: DateTime): int {.importcpp.} +proc getYear*(d: DateTime): int {.importcpp.} + proc setFullYear*(d: DateTime, year: int) {.importcpp.} +func toDateString*(d: DateTime): cstring {.importcpp.} +func toISOString*(d: DateTime): cstring {.importcpp.} +func toJSON*(d: DateTime): cstring {.importcpp.} +proc toString*(d: DateTime): cstring {.importcpp.} +func toTimeString*(d: DateTime): cstring {.importcpp.} +func toUTCString*(d: DateTime): cstring {.importcpp.} + #JSON library proc stringify*(l: JsonLib, s: JsRoot): cstring {.importcpp.} proc parse*(l: JsonLib, s: cstring): JsRoot {.importcpp.} + + +since (1, 5): + func debugger*() {.importjs: "debugger@".} + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger + + func copyWithin*[T](self: openArray[T]; target: int): seq[T] {.importjs: "#.copyWithin(#)".} + func copyWithin*[T](self: openArray[T]; target, start: int): seq[T] {.importjs: "#.copyWithin(#, #)".} + func copyWithin*[T](self: openArray[T]; target, start, ends: int): seq[T] {.importjs: "#.copyWithin(#, #, #)".} = + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin + ## `copyWithin` uses shallow copy. + runnableExamples: + assert ['a', 'b', 'c', 'd', 'e'].copyWithin(0, 3, 4) == @['d', 'b', 'c', 'd', 'e'] + assert ['a', 'b', 'c', 'd', 'e'].copyWithin(1, 3) == @['a', 'd', 'e', 'd', 'e'] + assert [1, 2, 3, 4, 5].copyWithin(-2) == @[1, 2, 3, 1, 2] + assert [1, 2, 3, 4, 5].copyWithin(0, 3) == @[4, 5, 3, 4, 5] + assert [1, 2, 3, 4, 5].copyWithin(0, 3, 4) == @[4, 2, 3, 4, 5] + assert [1, 2, 3, 4, 5].copyWithin(-2, -3, -1) == @[1, 2, 3, 3, 4] + + +since (1, 7): + func shift*[T](self: seq[T]): T {.importjs: "#.$1()".} = + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift + runnableExamples: + var arrai = @[1, 2, 3] + assert arrai.shift() == 1 + assert arrai == @[2, 3] + + func queueMicrotask*(function: proc) {.importjs: "$1(#)".} = + ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask + ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide + runnableExamples"-r:off": queueMicrotask(proc() = echo "Microtask") diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 479c8c3a8..d50d58ae5 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -8,35 +8,34 @@ # ## This Module implements types and macros to facilitate the wrapping of, and -## interaction with JavaScript libraries. Using the provided types ``JsObject`` -## and ``JsAssoc`` together with the provided macros allows for smoother +## interaction with JavaScript libraries. Using the provided types `JsObject` +## and `JsAssoc` together with the provided macros allows for smoother ## interfacing with JavaScript, allowing for example quick and easy imports of ## JavaScript variables: -## -## .. code-block:: nim -## -## # Here, we are using jQuery for just a few calls and do not want to wrap the -## # whole library: -## -## # import the document object and the console -## var document {.importc, nodecl.}: JsObject -## var console {.importc, nodecl.}: JsObject -## # import the "$" function -## proc jq(selector: JsObject): JsObject {.importcpp: "$(#)".} -## -## # Use jQuery to make the following code run, after the document is ready. -## # This uses an experimental ``.()`` operator for ``JsObject``, to emit -## # JavaScript calls, when no corresponding proc exists for ``JsObject``. -## proc main = -## jq(document).ready(proc() = -## console.log("Hello JavaScript!") -## ) -## - -when not defined(js) and not defined(nimdoc) and not defined(nimsuggest): + +runnableExamples: + # Here, we are using jQuery for just a few calls and do not want to wrap the + # whole library: + + # import the document object and the console + var document {.importc, nodecl.}: JsObject + var console {.importc, nodecl.}: JsObject + # import the "$" function + proc jq(selector: JsObject): JsObject {.importjs: "$$(#)".} + + # Use jQuery to make the following code run, after the document is ready. + # This uses an experimental `.()` operator for `JsObject`, to emit + # JavaScript calls, when no corresponding proc exists for `JsObject`. + proc main = + jq(document).ready(proc() = + console.log("Hello JavaScript!") + ) + + +when not defined(js) and not defined(nimsuggest): {.fatal: "Module jsFFI is designed to be used with the JavaScript backend.".} -import macros, tables +import std/[macros, tables] const setImpl = "#[#] = #" @@ -65,13 +64,13 @@ proc validJsName(name: string): bool = if chr notin {'A'..'Z','a'..'z','_','$','0'..'9'}: return false -template mangleJsName(name: cstring): cstring = +template mangleJsName(name: string): string = inc nameCounter "mangledName" & $nameCounter # only values that can be mapped 1 to 1 with cstring should be keys: they have an injective function with cstring -proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importcpp: "parseInt(#)".} +proc toJsKey*[T: SomeInteger](text: cstring, t: type T): T {.importjs: "parseInt(#)".} proc toJsKey*[T: enum](text: cstring, t: type T): T = T(text.toJsKey(int)) @@ -79,11 +78,11 @@ proc toJsKey*[T: enum](text: cstring, t: type T): T = proc toJsKey*(text: cstring, t: type cstring): cstring = text -proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importcpp: "parseFloat(#)".} +proc toJsKey*[T: SomeFloat](text: cstring, t: type T): T {.importjs: "parseFloat(#)".} type JsKey* = concept a, type T - cstring.toJsKey(T) is type(a) + cstring.toJsKey(T) is T JsObject* = ref object of JsRoot ## Dynamically typed wrapper around a JavaScript object. @@ -94,21 +93,21 @@ type var jsArguments* {.importc: "arguments", nodecl}: JsObject - ## JavaScript's arguments pseudo-variable + ## JavaScript's arguments pseudo-variable. jsNull* {.importc: "null", nodecl.}: JsObject - ## JavaScript's null literal + ## JavaScript's null literal. jsUndefined* {.importc: "undefined", nodecl.}: JsObject - ## JavaScript's undefined literal + ## JavaScript's undefined literal. jsDirname* {.importc: "__dirname", nodecl.}: cstring - ## JavaScript's __dirname pseudo-variable + ## JavaScript's __dirname pseudo-variable. jsFilename* {.importc: "__filename", nodecl.}: cstring - ## JavaScript's __filename pseudo-variable + ## JavaScript's __filename pseudo-variable. -proc isNull*[T](x: T): bool {.noSideEffect, importcpp: "(# === null)".} - ## check if a value is exactly null +proc isNull*[T](x: T): bool {.noSideEffect, importjs: "(# === null)".} + ## Checks if a value is exactly null. -proc isUndefined*[T](x: T): bool {.noSideEffect, importcpp: "(# === undefined)".} - ## check if a value is exactly undefined +proc isUndefined*[T](x: T): bool {.noSideEffect, importjs: "(# === undefined)".} + ## Checks if a value is exactly undefined. # Exceptions type @@ -122,36 +121,36 @@ type JsURIError* {.importc: "URIError".} = object of JsError # New -proc newJsObject*: JsObject {.importcpp: "{@}".} - ## Creates a new empty JsObject +proc newJsObject*: JsObject {.importjs: "{@}".} + ## Creates a new empty JsObject. -proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importcpp: "{@}".} +proc newJsAssoc*[K: JsKey, V]: JsAssoc[K, V] {.importjs: "{@}".} ## Creates a new empty JsAssoc with key type `K` and value type `V`. # Checks proc hasOwnProperty*(x: JsObject, prop: cstring): bool - {.importcpp: "#.hasOwnProperty(#)".} + {.importjs: "#.hasOwnProperty(#)".} ## Checks, whether `x` has a property of name `prop`. -proc jsTypeOf*(x: JsObject): cstring {.importcpp: "typeof(#)".} +proc jsTypeOf*(x: JsObject): cstring {.importjs: "typeof(#)".} ## Returns the name of the JsObject's JavaScript type as a cstring. -proc jsNew*(x: auto): JsObject {.importcpp: "(new #)".} +proc jsNew*(x: auto): JsObject {.importjs: "(new #)".} ## Turns a regular function call into an invocation of the - ## JavaScript's `new` operator + ## JavaScript's `new` operator. -proc jsDelete*(x: auto): JsObject {.importcpp: "(delete #)".} - ## JavaScript's `delete` operator +proc jsDelete*(x: auto): JsObject {.importjs: "(delete #)".} + ## JavaScript's `delete` operator. proc require*(module: cstring): JsObject {.importc.} - ## JavaScript's `require` function + ## JavaScript's `require` function. # Conversion to and from JsObject -proc to*(x: JsObject, T: typedesc): T {.importcpp: "(#)".} +proc to*(x: JsObject, T: typedesc): T {.importjs: "(#)".} ## Converts a JsObject `x` to type `T`. -proc toJs*[T](val: T): JsObject {.importcpp: "(#)".} - ## Converts a value of any type to type JsObject +proc toJs*[T](val: T): JsObject {.importjs: "(#)".} + ## Converts a value of any type to type JsObject. template toJs*(s: string): JsObject = cstring(s).toJs @@ -161,49 +160,51 @@ macro jsFromAst*(n: untyped): untyped = result = newProc(procType = nnkDo, body = result) return quote: toJs(`result`) -proc `&`*(a, b: cstring): cstring {.importcpp: "(# + #)".} - ## Concatenation operator for JavaScript strings - -proc `+` *(x, y: JsObject): JsObject {.importcpp: "(# + #)".} -proc `-` *(x, y: JsObject): JsObject {.importcpp: "(# - #)".} -proc `*` *(x, y: JsObject): JsObject {.importcpp: "(# * #)".} -proc `/` *(x, y: JsObject): JsObject {.importcpp: "(# / #)".} -proc `%` *(x, y: JsObject): JsObject {.importcpp: "(# % #)".} -proc `+=` *(x, y: JsObject): JsObject {.importcpp: "(# += #)", discardable.} -proc `-=` *(x, y: JsObject): JsObject {.importcpp: "(# -= #)", discardable.} -proc `*=` *(x, y: JsObject): JsObject {.importcpp: "(# *= #)", discardable.} -proc `/=` *(x, y: JsObject): JsObject {.importcpp: "(# /= #)", discardable.} -proc `%=` *(x, y: JsObject): JsObject {.importcpp: "(# %= #)", discardable.} -proc `++` *(x: JsObject): JsObject {.importcpp: "(++#)".} -proc `--` *(x: JsObject): JsObject {.importcpp: "(--#)".} -proc `>` *(x, y: JsObject): JsObject {.importcpp: "(# > #)".} -proc `<` *(x, y: JsObject): JsObject {.importcpp: "(# < #)".} -proc `>=` *(x, y: JsObject): JsObject {.importcpp: "(# >= #)".} -proc `<=` *(x, y: JsObject): JsObject {.importcpp: "(# <= #)".} -proc `and`*(x, y: JsObject): JsObject {.importcpp: "(# && #)".} -proc `or` *(x, y: JsObject): JsObject {.importcpp: "(# || #)".} -proc `not`*(x: JsObject): JsObject {.importcpp: "(!#)".} -proc `in` *(x, y: JsObject): JsObject {.importcpp: "(# in #)".} - -proc `[]`*(obj: JsObject, field: cstring): JsObject {.importcpp: getImpl.} - ## Return the value of a property of name `field` from a JsObject `obj`. - -proc `[]`*(obj: JsObject, field: int): JsObject {.importcpp: getImpl.} - ## Return the value of a property of name `field` from a JsObject `obj`. - -proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importcpp: setImpl.} - ## Set the value of a property of name `field` in a JsObject `obj` to `v`. - -proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importcpp: setImpl.} - ## Set the value of a property of name `field` in a JsObject `obj` to `v`. +proc `&`*(a, b: cstring): cstring {.importjs: "(# + #)".} + ## Concatenation operator for JavaScript strings. + +proc `+`*(x, y: JsObject): JsObject {.importjs: "(# + #)".} +proc `-`*(x, y: JsObject): JsObject {.importjs: "(# - #)".} +proc `*`*(x, y: JsObject): JsObject {.importjs: "(# * #)".} +proc `/`*(x, y: JsObject): JsObject {.importjs: "(# / #)".} +proc `%`*(x, y: JsObject): JsObject {.importjs: "(# % #)".} +proc `+=`*(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} +proc `-=`*(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} +proc `*=`*(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} +proc `/=`*(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} +proc `%=`*(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} +proc `++`*(x: JsObject): JsObject {.importjs: "(++#)".} +proc `--`*(x: JsObject): JsObject {.importjs: "(--#)".} +proc `>`*(x, y: JsObject): JsObject {.importjs: "(# > #)".} +proc `<`*(x, y: JsObject): JsObject {.importjs: "(# < #)".} +proc `>=`*(x, y: JsObject): JsObject {.importjs: "(# >= #)".} +proc `<=`*(x, y: JsObject): JsObject {.importjs: "(# <= #)".} +proc `**`*(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} + # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 +proc `and`*(x, y: JsObject): JsObject {.importjs: "(# && #)".} +proc `or`*(x, y: JsObject): JsObject {.importjs: "(# || #)".} +proc `not`*(x: JsObject): JsObject {.importjs: "(!#)".} +proc `in`*(x, y: JsObject): JsObject {.importjs: "(# in #)".} + +proc `[]`*(obj: JsObject, field: cstring): JsObject {.importjs: getImpl.} + ## Returns the value of a property of name `field` from a JsObject `obj`. + +proc `[]`*(obj: JsObject, field: int): JsObject {.importjs: getImpl.} + ## Returns the value of a property of name `field` from a JsObject `obj`. + +proc `[]=`*[T](obj: JsObject, field: cstring, val: T) {.importjs: setImpl.} + ## Sets the value of a property of name `field` in a JsObject `obj` to `v`. + +proc `[]=`*[T](obj: JsObject, field: int, val: T) {.importjs: setImpl.} + ## Sets the value of a property of name `field` in a JsObject `obj` to `v`. proc `[]`*[K: JsKey, V](obj: JsAssoc[K, V], field: K): V - {.importcpp: getImpl.} - ## Return the value of a property of name `field` from a JsAssoc `obj`. + {.importjs: getImpl.} + ## Returns the value of a property of name `field` from a JsAssoc `obj`. proc `[]=`*[K: JsKey, V](obj: JsAssoc[K, V], field: K, val: V) - {.importcpp: setImpl.} - ## Set the value of a property of name `field` in a JsAssoc `obj` to `v`. + {.importjs: setImpl.} + ## Sets the value of a property of name `field` in a JsAssoc `obj` to `v`. proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V = obj[cstring(field)] @@ -211,8 +212,8 @@ proc `[]`*[V](obj: JsAssoc[cstring, V], field: string): V = proc `[]=`*[V](obj: JsAssoc[cstring, V], field: string, val: V) = obj[cstring(field)] = val -proc `==`*(x, y: JsRoot): bool {.importcpp: "(# === #)".} - ## Compare two JsObjects or JsAssocs. Be careful though, as this is comparison +proc `==`*(x, y: JsRoot): bool {.importjs: "(# === #)".} + ## Compares two JsObjects or JsAssocs. Be careful though, as this is comparison ## like in JavaScript, so if your JsObjects are in fact JavaScript Objects, ## and not strings or numbers, this is a *comparison of references*. @@ -220,46 +221,46 @@ proc `==`*(x, y: JsRoot): bool {.importcpp: "(# === #)".} macro `.`*(obj: JsObject, field: untyped): JsObject = ## Experimental dot accessor (get) for type JsObject. ## Returns the value of a property of name `field` from a JsObject `x`. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## let obj = newJsObject() - ## obj.a = 20 - ## console.log(obj.a) # puts 20 onto the console. + runnableExamples: + let obj = newJsObject() + obj.a = 20 + assert obj.a.to(int) == 20 if validJsName($field): let importString = "#." & $field + let helperName = genSym(nskProc, "helper") result = quote do: - proc helper(o: JsObject): JsObject - {.importcpp: `importString`, gensym.} - helper(`obj`) + proc `helperName`(o: JsObject): JsObject + {.importjs: `importString`.} + `helperName`(`obj`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) let importString = "#." & mangledNames[$field] + let helperName = genSym(nskProc, "helper") result = quote do: - proc helper(o: JsObject): JsObject - {.importcpp: `importString`, gensym.} - helper(`obj`) + proc `helperName`(o: JsObject): JsObject + {.importjs: `importString`.} + `helperName`(`obj`) macro `.=`*(obj: JsObject, field, value: untyped): untyped = ## Experimental dot accessor (set) for type JsObject. ## Sets the value of a property of name `field` in a JsObject `x` to `value`. if validJsName($field): let importString = "#." & $field & " = #" + let helperName = genSym(nskProc, "helper") result = quote do: - proc helper(o: JsObject, v: auto) - {.importcpp: `importString`, gensym.} - helper(`obj`, `value`) + proc `helperName`(o: JsObject, v: auto) + {.importjs: `importString`.} + `helperName`(`obj`, `value`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) let importString = "#." & mangledNames[$field] & " = #" + let helperName = genSym(nskProc, "helper") result = quote do: - proc helper(o: JsObject, v: auto) - {.importcpp: `importString`, gensym.} - helper(`obj`, `value`) + proc `helperName`(o: JsObject, v: auto) + {.importjs: `importString`.} + `helperName`(`obj`, `value`) macro `.()`*(obj: JsObject, field: untyped, @@ -271,26 +272,26 @@ macro `.()`*(obj: JsObject, ## so be careful when using this.) ## ## Example: - ## - ## .. code-block:: nim - ## - ## # Let's get back to the console example: - ## var console {.importc, nodecl.}: JsObject - ## let res = console.log("I return undefined!") - ## console.log(res) # This prints undefined, as console.log always returns - ## # undefined. Thus one has to be careful, when using - ## # JsObject calls. + ## ```nim + ## # Let's get back to the console example: + ## var console {.importc, nodecl.}: JsObject + ## let res = console.log("I return undefined!") + ## console.log(res) # This prints undefined, as console.log always returns + ## # undefined. Thus one has to be careful, when using + ## # JsObject calls. + ## ``` var importString: string if validJsName($field): importString = "#." & $field & "(@)" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] & "(@)" - result = quote: - proc helper(o: JsObject): JsObject - {.importcpp: `importString`, gensym, discardable.} - helper(`obj`) + let helperName = genSym(nskProc, "helper") + result = quote do: + proc `helperName`(o: JsObject): JsObject + {.importjs: `importString`, discardable.} + `helperName`(`obj`) for idx in 0 ..< args.len: let paramName = newIdentNode("param" & $idx) result[0][3].add newIdentDefs(paramName, newIdentNode("JsObject")) @@ -305,12 +306,13 @@ macro `.`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & $field else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] + let helperName = genSym(nskProc, "helper") result = quote do: - proc helper(o: type(`obj`)): `obj`.V - {.importcpp: `importString`, gensym.} - helper(`obj`) + proc `helperName`(o: type(`obj`)): `obj`.V + {.importjs: `importString`.} + `helperName`(`obj`) macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], field: untyped, @@ -322,12 +324,13 @@ macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & $field & " = #" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] & " = #" + let helperName = genSym(nskProc, "helper") result = quote do: - proc helper(o: type(`obj`), v: `obj`.V) - {.importcpp: `importString`, gensym.} - helper(`obj`, `value`) + proc `helperName`(o: type(`obj`), v: `obj`.V) + {.importjs: `importString`.} + `helperName`(`obj`, `value`) macro `.()`*[K: cstring, V: proc](obj: JsAssoc[K, V], field: untyped, @@ -345,14 +348,14 @@ macro `.()`*[K: cstring, V: proc](obj: JsAssoc[K, V], # Iterators: iterator pairs*(obj: JsObject): (cstring, JsObject) = - ## Yields tuples of type ``(cstring, JsObject)``, with the first entry + ## Yields tuples of type `(cstring, JsObject)`, with the first entry ## being the `name` of a fields in the JsObject and the second being its ## value wrapped into a JsObject. var k: cstring var v: JsObject {.emit: "for (var `k` in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(`k`)) continue;".} - {.emit: " `v`=`obj`[`k`];".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `obj`[`k`];".} yield (k, v) {.emit: "}".} @@ -360,8 +363,8 @@ iterator items*(obj: JsObject): JsObject = ## Yields the `values` of each field in a JsObject, wrapped into a JsObject. var v: JsObject {.emit: "for (var k in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(k)) continue;".} - {.emit: " `v`=`obj`[k];".} + {.emit: " if (!`obj`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `obj`[k];".} yield v {.emit: "}".} @@ -369,62 +372,61 @@ iterator keys*(obj: JsObject): cstring = ## Yields the `names` of each field in a JsObject. var k: cstring {.emit: "for (var `k` in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(`k`)) continue;".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} yield k {.emit: "}".} iterator pairs*[K: JsKey, V](assoc: JsAssoc[K, V]): (K,V) = - ## Yields tuples of type ``(K, V)``, with the first entry + ## Yields tuples of type `(K, V)`, with the first entry ## being a `key` in the JsAssoc and the second being its corresponding value. var k: cstring var v: V {.emit: "for (var `k` in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(`k`)) continue;".} - {.emit: " `v`=`assoc`[`k`];".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `assoc`[`k`];".} yield (k.toJsKey(K), v) {.emit: "}".} -iterator items*[K, V](assoc: JSAssoc[K, V]): V = +iterator items*[K, V](assoc: JsAssoc[K, V]): V = ## Yields the `values` in a JsAssoc. var v: V {.emit: "for (var k in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(k)) continue;".} - {.emit: " `v`=`assoc`[k];".} + {.emit: " if (!`assoc`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `assoc`[k];".} yield v {.emit: "}".} -iterator keys*[K: JsKey, V](assoc: JSAssoc[K, V]): K = +iterator keys*[K: JsKey, V](assoc: JsAssoc[K, V]): K = ## Yields the `keys` in a JsAssoc. var k: cstring {.emit: "for (var `k` in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(`k`)) continue;".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} yield k.toJsKey(K) {.emit: "}".} # Literal generation macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto = - ## Takes a ``typedesc`` as its first argument, and a series of expressions of - ## type ``key: value``, and returns a value of the specified type with each - ## field ``key`` set to ``value``, as specified in the arguments of ``{}``. + ## Takes a `typedesc` as its first argument, and a series of expressions of + ## type `key: value`, and returns a value of the specified type with each + ## field `key` set to `value`, as specified in the arguments of `{}`. ## ## Example: ## - ## .. code-block:: nim - ## - ## # Let's say we have a type with a ton of fields, where some fields do not - ## # need to be set, and we do not want those fields to be set to ``nil``: - ## type - ## ExtremelyHugeType = ref object - ## a, b, c, d, e, f, g: int - ## h, i, j, k, l: cstring - ## # And even more fields ... + ## ```nim + ## # Let's say we have a type with a ton of fields, where some fields do not + ## # need to be set, and we do not want those fields to be set to `nil`: + ## type + ## ExtremelyHugeType = ref object + ## a, b, c, d, e, f, g: int + ## h, i, j, k, l: cstring + ## # And even more fields ... ## - ## let obj = ExtremelyHugeType{ a: 1, k: "foo".cstring, d: 42 } - ## - ## # This generates roughly the same JavaScript as: - ## {.emit: "var obj = {a: 1, k: "foo", d: 42};".} + ## let obj = ExtremelyHugeType{ a: 1, k: "foo".cstring, d: 42 } ## + ## # This generates roughly the same JavaScript as: + ## {.emit: "var obj = {a: 1, k: "foo", d: 42};".} + ## ``` let a = ident"a" var body = quote do: var `a` {.noinit.}: `typ` @@ -458,32 +460,41 @@ macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto = # Macro to build a lambda using JavaScript's `this` # from a proc, `this` being the first argument. -macro bindMethod*(procedure: typed): auto = +proc replaceSyms(n: NimNode): NimNode = + if n.kind == nnkSym: + result = newIdentNode($n) + else: + result = n + for i in 0..<n.len: + result[i] = replaceSyms(n[i]) + +macro bindMethod*(procedure: typed): auto {.deprecated: "Don't use it with closures".} = ## Takes the name of a procedure and wraps it into a lambda missing the first - ## argument, which passes the JavaScript builtin ``this`` as the first + ## argument, which passes the JavaScript builtin `this` as the first ## argument to the procedure. Returns the resulting lambda. ## ## Example: ## ## We want to generate roughly this JavaScript: + ## ```js + ## var obj = {a: 10}; + ## obj.someMethod = function() { + ## return this.a + 42; + ## }; + ## ``` ## - ## .. code-block:: js - ## var obj = {a: 10}; - ## obj.someMethod = function() { - ## return this.a + 42; - ## }; - ## - ## We can achieve this using the ``bindMethod`` macro: + ## We can achieve this using the `bindMethod` macro: ## - ## .. code-block:: nim - ## let obj = JsObject{ a: 10 } - ## proc someMethodImpl(that: JsObject): int = - ## that.a.to(int) + 42 - ## obj.someMethod = bindMethod someMethodImpl + ## ```nim + ## let obj = JsObject{ a: 10 } + ## proc someMethodImpl(that: JsObject): int = + ## that.a.to(int) + 42 + ## obj.someMethod = bindMethod someMethodImpl ## - ## # Alternatively: - ## obj.someMethod = bindMethod - ## proc(that: JsObject): int = that.a.to(int) + 42 + ## # Alternatively: + ## obj.someMethod = bindMethod + ## proc(that: JsObject): int = that.a.to(int) + 42 + ## ``` if not (procedure.kind == nnkSym or procedure.kind == nnkLambda): error("Argument has to be a proc or a symbol corresponding to a proc.") var @@ -491,7 +502,7 @@ macro bindMethod*(procedure: typed): auto = getImpl(procedure) else: procedure - args = rawProc[3] + args = rawProc[3].copyNimTree.replaceSyms thisType = args[1][1] params = newNimNode(nnkFormalParams).add(args[0]) body = newNimNode(nnkLambda) diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim new file mode 100644 index 000000000..2d931eb20 --- /dev/null +++ b/lib/js/jsre.nim @@ -0,0 +1,97 @@ +## Regular Expressions for the JavaScript target. +## * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions +when not defined(js): + {.error: "This module only works on the JavaScript platform".} + +type RegExp* = ref object of JsRoot + ## Regular Expressions for JavaScript target. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp + flags*: cstring ## cstring that contains the flags of the RegExp object. + dotAll*: bool ## Whether `.` matches newlines or not. + global*: bool ## Whether to test against all possible matches in a string, or only against the first. + ignoreCase*: bool ## Whether to ignore case while attempting a match in a string. + multiline*: bool ## Whether to search in strings across multiple lines. + source*: cstring ## The text of the pattern. + sticky*: bool ## Whether the search is sticky. + unicode*: bool ## Whether Unicode features are enabled. + lastIndex*: cint ## Index at which to start the next match (read/write property). + input*: cstring ## Read-only and modified on successful match. + lastMatch*: cstring ## Ditto. + lastParen*: cstring ## Ditto. + leftContext*: cstring ## Ditto. + rightContext*: cstring ## Ditto. + hasIndices*: bool ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices + + +func newRegExp*(pattern: cstring; flags: cstring): RegExp {.importjs: "new RegExp(@)".} + ## Creates a new RegExp object. + +func newRegExp*(pattern: cstring): RegExp {.importjs: "new RegExp(@)".} + +func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.compile(@)".} + ## Recompiles a regular expression during execution of a script. + +func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} + ## Returns a new string with some or all matches of a pattern replaced by given replacement + +func replace*(pattern: cstring, self: RegExp, cb: proc (args: varargs[cstring]): cstring): cstring {.importcpp.} + ## Returns a new string with some or all matches of a pattern replaced by given callback function + +func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.split(#) || [])".} + ## Divides a string into an ordered list of substrings and returns the array + +func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.match(#) || [])".} + ## Returns an array of matches of a RegExp against given string + +func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "(#.exec(#) || [])".} + ## Executes a search for a match in its string parameter. + +func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} + ## Returns a string representing the RegExp object. + +func `$`*(self: RegExp): string = $toCstring(self) + +func contains*(pattern: cstring; self: RegExp): bool = + ## Tests for a substring match in its string parameter. + runnableExamples: + let jsregex: RegExp = newRegExp(r"bc$", r"i") + assert jsregex in r"abc" + assert jsregex notin r"abcd" + assert "xabc".contains jsregex + {.emit: "`result` = `self`.test(`pattern`);".} + +func startsWith*(pattern: cstring; self: RegExp): bool = + ## Tests if string starts with given RegExp + runnableExamples: + let jsregex: RegExp = newRegExp(r"abc", r"i") + assert "abcd".startsWith jsregex + pattern.contains(newRegExp(("^" & $(self.source)).cstring, self.flags)) + +func endsWith*(pattern: cstring; self: RegExp): bool = + ## Tests if string ends with given RegExp + runnableExamples: + let jsregex: RegExp = newRegExp(r"bcd", r"i") + assert "abcd".endsWith jsregex + pattern.contains(newRegExp(($(self.source) & "$").cstring, self.flags)) + + +runnableExamples: + let jsregex: RegExp = newRegExp(r"\s+", r"i") + jsregex.compile(r"\w+", r"i") + assert "nim javascript".contains jsregex + assert jsregex.exec(r"nim javascript") == @["nim".cstring] + assert jsregex.toCstring() == r"/\w+/i" + jsregex.compile(r"[0-9]", r"i") + assert "0123456789abcd".contains jsregex + assert $jsregex == "/[0-9]/i" + jsregex.compile(r"abc", r"i") + assert "abcd".startsWith jsregex + assert "dabc".endsWith jsregex + jsregex.compile(r"\d", r"i") + assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] + jsregex.compile(r"[lw]", r"i") + assert "hello world".replace(jsregex,"X") == "heXlo world" + jsregex.compile(r"([a-z])\1*", r"g") + assert "abbcccdddd".replace(jsregex, proc (m: varargs[cstring]): cstring = ($m[0] & $(m.len)).cstring) == "a1b2c3d4" + let digitsRegex: RegExp = newRegExp(r"\d") + assert "foo".match(digitsRegex) == @[] diff --git a/lib/nimbase.h b/lib/nimbase.h index 004ba170b..cf0c8002b 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -10,14 +10,12 @@ /* compiler symbols: __BORLANDC__ _MSC_VER -__WATCOMC__ -__LCC__ __GNUC__ -__DMC__ -__POCC__ __TINYC__ __clang__ __AVR__ +__arm__ +__EMSCRIPTEN__ */ @@ -75,7 +73,8 @@ __AVR__ #endif /* ------------------------------------------------------------------------- */ -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__ZEPHYR__) +/* Zephyr does some magic in it's headers that override the GCC stdlib. This breaks that. */ # define _GNU_SOURCE 1 #endif @@ -88,11 +87,8 @@ __AVR__ #endif /* calling convention mess ----------------------------------------------- */ -#if defined(__GNUC__) || defined(__LCC__) || defined(__POCC__) \ - || defined(__TINYC__) +#if defined(__GNUC__) || defined(__TINYC__) /* these should support C99's inline */ - /* the test for __POCC__ has to come before the test for _MSC_VER, - because PellesC defines _MSC_VER too. This is brain-dead. */ # define N_INLINE(rettype, name) inline rettype name #elif defined(__BORLANDC__) || defined(_MSC_VER) /* Borland's compiler is really STRANGE here; note that the __fastcall @@ -100,21 +96,13 @@ __AVR__ the return type, so we do not handle this mess in the code generator but rather here. */ # define N_INLINE(rettype, name) __inline rettype name -#elif defined(__DMC__) -# define N_INLINE(rettype, name) inline rettype name -#elif defined(__WATCOMC__) -# define N_INLINE(rettype, name) __inline rettype name #else /* others are less picky: */ # define N_INLINE(rettype, name) rettype __inline name #endif #define N_INLINE_PTR(rettype, name) rettype (*name) -#if defined(__POCC__) -# define NIM_CONST /* PCC is really picky with const modifiers */ -# undef _MSC_VER /* Yeah, right PCC defines _MSC_VER even if it is - not that compatible. Well done. */ -#elif defined(__cplusplus) +#if defined(__cplusplus) # define NIM_CONST /* C++ is picky with const modifiers */ #else # define NIM_CONST const @@ -124,12 +112,17 @@ __AVR__ NIM_THREADVAR declaration based on http://stackoverflow.com/questions/18298280/how-to-declare-a-variable-as-thread-local-portably */ -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__ +#if defined _WIN32 +# if defined _MSC_VER || defined __BORLANDC__ +# define NIM_THREADVAR __declspec(thread) +# else +# define NIM_THREADVAR __thread +# endif +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__ # define NIM_THREADVAR _Thread_local #elif defined _WIN32 && ( \ defined _MSC_VER || \ defined __ICL || \ - defined __DMC__ || \ defined __BORLANDC__ ) # define NIM_THREADVAR __declspec(thread) #elif defined(__TINYC__) || defined(__GENODE__) @@ -143,9 +136,12 @@ __AVR__ # error "Cannot define NIM_THREADVAR" #endif +#if defined(__cplusplus) + #define NIM_THREAD_LOCAL thread_local +#endif + /* --------------- how int64 constants should be declared: ----------- */ -#if defined(__GNUC__) || defined(__LCC__) || \ - defined(__POCC__) || defined(__DMC__) || defined(_MSC_VER) +#if defined(__GNUC__) || defined(_MSC_VER) # define IL64(x) x##LL #else /* works only without LL */ # define IL64(x) ((NI64)x) @@ -173,20 +169,23 @@ __AVR__ # define N_STDCALL(rettype, name) rettype __stdcall name # define N_SYSCALL(rettype, name) rettype __syscall name # define N_FASTCALL(rettype, name) rettype __fastcall name +# define N_THISCALL(rettype, name) rettype __thiscall name # define N_SAFECALL(rettype, name) rettype __stdcall name /* function pointers with calling convention: */ # define N_CDECL_PTR(rettype, name) rettype (__cdecl *name) # define N_STDCALL_PTR(rettype, name) rettype (__stdcall *name) # define N_SYSCALL_PTR(rettype, name) rettype (__syscall *name) # define N_FASTCALL_PTR(rettype, name) rettype (__fastcall *name) +# define N_THISCALL_PTR(rettype, name) rettype (__thiscall *name) # define N_SAFECALL_PTR(rettype, name) rettype (__stdcall *name) -# ifdef __cplusplus -# define N_LIB_EXPORT NIM_EXTERNC __declspec(dllexport) +# ifdef __EMSCRIPTEN__ +# define N_LIB_EXPORT NIM_EXTERNC __declspec(dllexport) __attribute__((used)) +# define N_LIB_EXPORT_VAR __declspec(dllexport) __attribute__((used)) # else # define N_LIB_EXPORT NIM_EXTERNC __declspec(dllexport) +# define N_LIB_EXPORT_VAR __declspec(dllexport) # endif -# define N_LIB_EXPORT_VAR __declspec(dllexport) # define N_LIB_IMPORT extern __declspec(dllimport) #else # define N_LIB_PRIVATE __attribute__((visibility("hidden"))) @@ -215,8 +214,13 @@ __AVR__ # define N_FASTCALL_PTR(rettype, name) rettype (*name) # define N_SAFECALL_PTR(rettype, name) rettype (*name) # endif -# define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default"))) -# define N_LIB_EXPORT_VAR __attribute__((visibility("default"))) +# ifdef __EMSCRIPTEN__ +# define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default"), used)) +# define N_LIB_EXPORT_VAR __attribute__((visibility("default"), used)) +# else +# define N_LIB_EXPORT NIM_EXTERNC __attribute__((visibility("default"))) +# define N_LIB_EXPORT_VAR __attribute__((visibility("default"))) +# endif # define N_LIB_IMPORT extern #endif @@ -234,8 +238,7 @@ __AVR__ #define N_NOINLINE_PTR(rettype, name) rettype (*name) -#if defined(__BORLANDC__) || defined(__WATCOMC__) || \ - defined(__POCC__) || defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) +#if defined(__BORLANDC__) || defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) /* these compilers have a fastcall so use it: */ # ifdef __TINYC__ # define N_NIMCALL(rettype, name) rettype __attribute((__fastcall)) name @@ -262,14 +265,32 @@ __AVR__ #include <limits.h> #include <stddef.h> +// define NIM_STATIC_ASSERT +// example use case: CT sizeof for importc types verification +// where we have {.completeStruct.} (or lack of {.incompleteStruct.}) +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) +#define NIM_STATIC_ASSERT(x, msg) _Static_assert((x), msg) +#elif defined(__cplusplus) +#define NIM_STATIC_ASSERT(x, msg) static_assert((x), msg) +#else +#define _NIM_STATIC_ASSERT_FINAL(x, append_name) typedef int NIM_STATIC_ASSERT_AUX ## append_name[(x) ? 1 : -1]; +#define _NIM_STATIC_ASSERT_STAGE_3(x, line) _NIM_STATIC_ASSERT_FINAL(x, _AT_LINE_##line) +#define _NIM_STATIC_ASSERT_STAGE_2(x, line) _NIM_STATIC_ASSERT_STAGE_3(x, line) +#define NIM_STATIC_ASSERT(x, msg) _NIM_STATIC_ASSERT_STAGE_2(x,__LINE__) +// On failure, your C compiler will say something like: +// "error: 'NIM_STATIC_ASSERT_AUX_AT_LINE_XXX' declared as an array with a negative size" +// Adding the line number helps to avoid redefinitions which are not allowed in +// old GCC versions, however the order of evaluation for __LINE__ is a little tricky, +// hence all the helper macros. See https://stackoverflow.com/a/3385694 for more info. +#endif + /* C99 compiler? */ #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901)) # define HAVE_STDINT_H #endif /* Known compiler with stdint.h that doesn't fit the general pattern? */ -#if defined(__LCC__) || defined(__DMC__) || defined(__POCC__) || \ - defined(__AVR__) || (defined(__cplusplus) && (__cplusplus < 201103)) +#if defined(__AVR__) || (defined(__cplusplus) && (__cplusplus < 201103)) # define HAVE_STDINT_H #endif @@ -288,40 +309,48 @@ __AVR__ namespace USE_NIM_NAMESPACE { #endif +// preexisting check, seems paranoid, maybe remove +#if defined(NIM_TRUE) || defined(NIM_FALSE) || defined(NIM_BOOL) +#error "nim reserved preprocessor macros clash" +#endif + /* bool types (C++ has it): */ #ifdef __cplusplus -# ifndef NIM_TRUE -# define NIM_TRUE true -# endif -# ifndef NIM_FALSE -# define NIM_FALSE false -# endif -# define NIM_BOOL bool +#define NIM_BOOL bool +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901) +// see #13798: to avoid conflicts for code emitting `#include <stdbool.h>` +#define NIM_BOOL _Bool +#else +typedef unsigned char NIM_BOOL; // best effort +#endif + +NIM_STATIC_ASSERT(sizeof(NIM_BOOL) == 1, ""); // check whether really needed +NIM_STATIC_ASSERT(CHAR_BIT == 8, ""); + // fail fast for (rare) environments where this doesn't hold, as some implicit + // assumptions would need revisiting (e.g. `uint8` or https://github.com/nim-lang/Nim/pull/18505) + +#define NIM_TRUE true +#define NIM_FALSE false + +#ifdef __cplusplus # if __cplusplus >= 201103L # /* nullptr is more type safe (less implicit conversions than 0) */ # define NIM_NIL nullptr # else -# /* consider using NULL if comment below for NIM_NIL doesn't apply to C++ */ +# // both `((void*)0)` and `NULL` would cause codegen to emit +# // error: assigning to 'Foo *' from incompatible type 'void *' +# // but codegen could be fixed if need. See also potential caveat regarding +# // NULL. +# // However, `0` causes other issues, see #13798 # define NIM_NIL 0 # endif #else -# ifdef bool -# define NIM_BOOL bool -# else - typedef unsigned char NIM_BOOL; -# endif -# ifndef NIM_TRUE -# define NIM_TRUE ((NIM_BOOL) 1) -# endif -# ifndef NIM_FALSE -# define NIM_FALSE ((NIM_BOOL) 0) -# endif +# include <stdbool.h> # define NIM_NIL ((void*)0) /* C's NULL is fucked up in some C compilers, so the generated code does not rely on it anymore */ #endif -#if defined(__BORLANDC__) || defined(__DMC__) \ - || defined(__WATCOMC__) || defined(_MSC_VER) +#if defined(__BORLANDC__) || defined(_MSC_VER) typedef signed char NI8; typedef signed short int NI16; typedef signed int NI32; @@ -447,7 +476,9 @@ typedef char* NCSTRING; } name = {{length, (NI) ((NU)length | NIM_STRLIT_FLAG)}, str} /* declared size of a sequence/variable length array: */ -#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) +#if defined(__cplusplus) && defined(__clang__) +# define SEQ_DECL_SIZE 1 +#elif defined(__GNUC__) || defined(_MSC_VER) # define SEQ_DECL_SIZE /* empty is correct! */ #else # define SEQ_DECL_SIZE 1000000 @@ -456,7 +487,6 @@ typedef char* NCSTRING; #define ALLOC_0(size) calloc(1, size) #define DL_ALLOC_0(size) dlcalloc(1, size) -#define GenericSeqSize sizeof(TGenericSeq) #define paramCount() cmdCount // NAN definition copied from math.h included in the Windows SDK version 10.0.14393.0 @@ -489,6 +519,7 @@ struct TFrame_ { NCSTRING filename; NI16 len; NI16 calldepth; + NI frameMsgLen; }; #define NIM_POSIX_INIT __attribute__((constructor)) @@ -515,10 +546,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } # define GC_GUARD #endif -/* Test to see if Nim and the C compiler agree on the size of a pointer. - On disagreement, your C compiler will say something like: - "error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */ -typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; +// Test to see if Nim and the C compiler agree on the size of a pointer. +NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, "Pointer size mismatch between Nim and C/C++ backend. You probably need to setup the backend compiler for target CPU."); #ifdef USE_NIM_NAMESPACE } @@ -526,8 +555,10 @@ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == siz #if defined(_MSC_VER) # define NIM_ALIGN(x) __declspec(align(x)) +# define NIM_ALIGNOF(x) __alignof(x) #else # define NIM_ALIGN(x) __attribute__((aligned(x))) +# define NIM_ALIGNOF(x) __alignof__(x) #endif /* ---------------- platform specific includes ----------------------- */ @@ -541,8 +572,38 @@ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == siz # include <sys/types.h> #endif -/* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */ -#define NIM_CHECK_SIZE(typ, sz) \ - _Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size") +/* these exist to make the codegen logic simpler */ +#define nimModInt(a, b, res) (((*res) = (a) % (b)), 0) +#define nimModInt64(a, b, res) (((*res) = (a) % (b)), 0) + +#if (!defined(_MSC_VER) || defined(__clang__)) && !defined(NIM_EmulateOverflowChecks) + /* these exist because we cannot have .compilerProcs that are importc'ed + by a different name */ + + #define nimAddInt64(a, b, res) __builtin_saddll_overflow(a, b, (long long int*)res) + #define nimSubInt64(a, b, res) __builtin_ssubll_overflow(a, b, (long long int*)res) + #define nimMulInt64(a, b, res) __builtin_smulll_overflow(a, b, (long long int*)res) + + #if NIM_INTBITS == 32 + #if defined(__arm__) && defined(__GNUC__) + /* arm-none-eabi-gcc targets defines int32_t as long int */ + #define nimAddInt(a, b, res) __builtin_saddl_overflow(a, b, res) + #define nimSubInt(a, b, res) __builtin_ssubl_overflow(a, b, res) + #define nimMulInt(a, b, res) __builtin_smull_overflow(a, b, res) + #else + #define nimAddInt(a, b, res) __builtin_sadd_overflow(a, b, res) + #define nimSubInt(a, b, res) __builtin_ssub_overflow(a, b, res) + #define nimMulInt(a, b, res) __builtin_smul_overflow(a, b, res) + #endif + #else + /* map it to the 'long long' variant */ + #define nimAddInt(a, b, res) __builtin_saddll_overflow(a, b, (long long int*)res) + #define nimSubInt(a, b, res) __builtin_ssubll_overflow(a, b, (long long int*)res) + #define nimMulInt(a, b, res) __builtin_smulll_overflow(a, b, (long long int*)res) + #endif +#endif + +#define NIM_NOALIAS __restrict +/* __restrict is said to work for all the C(++) compilers out there that we support */ #endif /* NIMBASE_H */ diff --git a/lib/nimhcr.nim b/lib/nimhcr.nim index 1bafa98cb..e87bb2413 100644 --- a/lib/nimhcr.nim +++ b/lib/nimhcr.nim @@ -1,3 +1,7 @@ +discard """ +batchable: false +""" + # # # Nim's Runtime Library @@ -16,7 +20,7 @@ # by storing them on the heap. For procs, we produce on the fly simple # trampolines that can be dynamically overwritten to jump to a different # target. In the host program, all globals and procs are first registered -# here with ``hcrRegisterGlobal`` and ``hcrRegisterProc`` and then the +# here with `hcrRegisterGlobal` and `hcrRegisterProc` and then the # returned permanent locations are used in every reference to these symbols # onwards. # @@ -196,6 +200,9 @@ # block. Perhaps something can be done about this - some way of re-allocating # the state and transferring the old... +when defined(nimPreviewSlimSystem): + import std/assertions + when not defined(js) and (defined(hotcodereloading) or defined(createNimHcr) or defined(testNimHcr)): @@ -205,14 +212,14 @@ when not defined(js) and (defined(hotcodereloading) or else: "so" type HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.} - HcrGcMarkerProc = proc () {.nimcall.} + HcrGcMarkerProc = proc () {.nimcall, raises: [].} HcrModuleInitializer* = proc () {.nimcall.} when defined(createNimHcr): when system.appType != "lib": {.error: "This file has to be compiled as a library!".} - import os, tables, sets, times, strutils, reservedmem, dynlib + import std/[os, tables, sets, times, strutils, reservedmem, dynlib] template trace(args: varargs[untyped]) = when defined(testNimHcr) or defined(traceHcr): @@ -226,41 +233,41 @@ when defined(createNimHcr): when defined(testNimHcr): return ($arg).splitFile.name.splitFile.name else: return $arg - {.pragma: nimhcr, compilerProc, exportc, dynlib.} - - when hostCPU in ["i386", "amd64"]: - type - ShortJumpInstruction {.packed.} = object - opcode: byte - offset: int32 - - LongJumpInstruction {.packed.} = object - opcode1: byte - opcode2: byte - offset: int32 - absoluteAddr: pointer - - proc writeJump(jumpTableEntry: ptr LongJumpInstruction, targetFn: pointer) = - let - jumpFrom = jumpTableEntry.shift(sizeof(ShortJumpInstruction)) - jumpDistance = distance(jumpFrom, targetFn) - - if abs(jumpDistance) < 0x7fff0000: - let shortJump = cast[ptr ShortJumpInstruction](jumpTableEntry) - shortJump.opcode = 0xE9 # relative jump - shortJump.offset = int32(jumpDistance) + {.pragma: nimhcr, compilerproc, exportc, dynlib.} + + # XXX these types are CPU specific and need ARM etc support + type + ShortJumpInstruction {.packed.} = object + opcode: byte + offset: int32 + + LongJumpInstruction {.packed.} = object + opcode1: byte + opcode2: byte + offset: int32 + absoluteAddr: pointer + + proc writeJump(jumpTableEntry: ptr LongJumpInstruction, targetFn: pointer) = + let + jumpFrom = jumpTableEntry.shift(sizeof(ShortJumpInstruction)) + jumpDistance = distance(jumpFrom, targetFn) + + if abs(jumpDistance) < 0x7fff0000: + let shortJump = cast[ptr ShortJumpInstruction](jumpTableEntry) + shortJump.opcode = 0xE9 # relative jump + shortJump.offset = int32(jumpDistance) + else: + jumpTableEntry.opcode1 = 0xff # indirect absolute jump + jumpTableEntry.opcode2 = 0x25 + when hostCPU == "i386": + # on x86 we write the absolute address of the following pointer + jumpTableEntry.offset = cast[int32](addr jumpTableEntry.absoluteAddr) else: - jumpTableEntry.opcode1 = 0xff # indirect absolute jump - jumpTableEntry.opcode2 = 0x25 - when hostCPU == "i386": - # on x86 we write the absolute address of the following pointer - jumpTableEntry.offset = cast[int32](addr jumpTableEntry.absoluteAddr) - else: - # on x64, we use a relative address for the same location - jumpTableEntry.offset = 0 - jumpTableEntry.absoluteAddr = targetFn - - elif hostCPU == "arm": + # on x64, we use a relative address for the same location + jumpTableEntry.offset = 0 + jumpTableEntry.absoluteAddr = targetFn + + if hostCPU == "arm": const jumpSize = 8 elif hostCPU == "arm64": const jumpSize = 16 @@ -298,7 +305,7 @@ when defined(createNimHcr): hash: string gen: int lastModification: Time - handlers: seq[tuple[isBefore: bool, cb: proc ()]] + handlers: seq[tuple[isBefore: bool, cb: proc () {.nimcall.}]] proc newModuleDesc(): ModuleDesc = result.procs = initTable[string, ProcSym]() @@ -419,7 +426,7 @@ when defined(createNimHcr): if modules.contains(name): unloadDll(name) else: - modules.add(name, newModuleDesc()) + modules[name] = newModuleDesc() let copiedName = name & ".copy." & dllExt copyFileWithPermissions(name, copiedName) @@ -483,7 +490,7 @@ when defined(createNimHcr): recursiveDiscovery(modules[curr].imports) allModulesOrderedByDFS.add(curr) continue - loadDll(curr) + loadDll(curr.cstring) # first load all dependencies of the current module and init it after that recursiveDiscovery(modules[curr].imports) @@ -493,20 +500,20 @@ when defined(createNimHcr): proc initModules() = # first init the pointers to hcr functions and also do the registering of typeinfo globals for curr in modulesToInit: - initHcrData(curr) - initTypeInfoGlobals(curr) + initHcrData(curr.cstring) + initTypeInfoGlobals(curr.cstring) # for now system always gets fully inited before any other module (including when reloading) - initPointerData(system) - initGlobalScope(system) + initPointerData(system.cstring) + initGlobalScope(system.cstring) # proceed with the DatInit calls - for all modules - including the main one! for curr in allModulesOrderedByDFS: if curr != system: - initPointerData(curr) + initPointerData(curr.cstring) mainDatInit() # execute top-level code (in global scope) for curr in modulesToInit: if curr != system: - initGlobalScope(curr) + initGlobalScope(curr.cstring) # cleanup old symbols which are gone now for curr in modulesToInit: cleanupSymbols(curr) @@ -550,13 +557,17 @@ when defined(createNimHcr): # Future versions of NIMHCR won't use the GC, because all globals and the # metadata needed to access them will be placed in shared memory, so they # can be manipulated from external programs without reloading. - GC_disable() - defer: GC_enable() + when declared(GC_disable): + GC_disable() + defer: GC_enable() + elif declared(GC_disableOrc): + GC_disableOrc() + defer: GC_enableOrc() inc(generation) trace "HCR RELOADING: ", generation - var traversedHandlerModules = initSet[string]() + var traversedHandlerModules = initHashSet[string]() proc recursiveExecuteHandlers(isBefore: bool, module: string) = # do not process an already traversed module @@ -591,18 +602,18 @@ when defined(createNimHcr): hashToModuleMap.del(modules[name].hash) modules.del(name) - proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} = + proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.} = modules[currentModule].handlers.add( (isBefore: isBefore, cb: cb)) proc hcrAddModule*(module: cstring) {.nimhcr.} = if not modules.contains($module): - modules.add($module, newModuleDesc()) + modules[$module] = newModuleDesc() proc hcrGeneration*(): int {.nimhcr.} = generation - proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} = + proc hcrMarkGlobals*() {.compilerproc, exportc, dynlib, nimcall, gcsafe.} = # This is gcsafe, because it will be registered # only in the GC of the main thread. {.gcsafe.}: @@ -618,7 +629,7 @@ elif defined(hotcodereloading) or defined(testNimHcr): elif defined(macosx): "libnimhcr." & dllExt else: "libnimhcr." & dllExt - {.pragma: nimhcr, compilerProc, importc, dynlib: nimhcrLibname.} + {.pragma: nimhcr, compilerproc, importc, dynlib: nimhcrLibname.} proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} @@ -642,19 +653,19 @@ elif defined(hotcodereloading) or defined(testNimHcr): proc hcrPerformCodeReload*() {.nimhcr.} - proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} + proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.} - proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} + proc hcrMarkGlobals*() {.raises: [], nimhcr, nimcall, gcsafe.} when declared(nimRegisterGlobalMarker): - nimRegisterGlobalMarker(hcrMarkGlobals) + nimRegisterGlobalMarker(cast[GlobalMarkerProc](hcrMarkGlobals)) else: proc hcrHasModuleChanged*(moduleHash: string): bool = # TODO false - proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) = + proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) = # TODO discard diff --git a/lib/nimrtl.nim b/lib/nimrtl.nim index 7ec5eb981..a2fb6ce60 100644 --- a/lib/nimrtl.nim +++ b/lib/nimrtl.nim @@ -1,3 +1,7 @@ +discard """ +batchable: false +""" + # # # Nim's Runtime Library @@ -8,7 +12,7 @@ # ## Main file to generate a DLL from the standard library. -## The default Nimrtl does not only contain the ``system`` module, but these +## The default Nimrtl does not only contain the `system` module, but these ## too: ## ## * parseutils @@ -22,6 +26,7 @@ ## * unicode ## * pegs ## * ropes +## * cstrutils ## when system.appType != "lib": @@ -31,5 +36,5 @@ when not defined(createNimRtl): {.error: "This file has to be compiled with '-d:createNimRtl'".} import - parseutils, strutils, parseopt, parsecfg, strtabs, unicode, pegs, ropes, - os, osproc, times, cstrutils + std/[parseutils, strutils, parseopt, parsecfg, strtabs, unicode, pegs, ropes, + os, osproc, times, cstrutils] diff --git a/lib/nintendoswitch/switch_memory.nim b/lib/nintendoswitch/switch_memory.nim deleted file mode 100644 index f34bd363a..000000000 --- a/lib/nintendoswitch/switch_memory.nim +++ /dev/null @@ -1,36 +0,0 @@ -## All of these library headers and source can be found in the github repo -## https://github.com/switchbrew/libnx. - -const virtMemHeader = "<switch/kernel/virtmem.h>" -const svcHeader = "<switch/kernel/svc.h>" -const mallocHeader = "<malloc.h>" - -## Aligns a block of memory with request `size` to `bytes` size. For -## example, a request of memalign(0x1000, 0x1001) == 0x2000 bytes allocated -proc memalign*(bytes: csize, size: csize): pointer {.importc: "memalign", - header: mallocHeader.} - -# Should be required, but not needed now because of how -# svcUnmapMemory frees all memory -#proc free*(address: pointer) {.importc: "free", -# header: mallocHeader.} - -## Maps a memaligned block of memory from `src_addr` to `dst_addr`. The -## Nintendo Switch requires this call in order to make use of memory, otherwise -## an invalid memory access occurs. -proc svcMapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {. - importc: "svcMapMemory", header: svcHeader.} - -## Unmaps (frees) all memory from both `dst_addr` and `src_addr`. **Must** be called -## whenever svcMapMemory is used. The Switch will expect all memory to be allocated -## before gfxExit() calls (<switch/gfx/gfx.h>) -proc svcUnmapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {. - importc: "svcUnmapMemory", header: svcHeader.} - -proc virtmemReserveMap*(size: csize): pointer {.importc: "virtmemReserveMap", - header: virtMemHeader.} - -# Should be required, but not needed now because of how -# svcUnmapMemory frees all memory -#proc virtmemFreeMap*(address: pointer; size: csize) {.importc: "virtmemFreeMap", -# header: virtMemHeader.} diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim new file mode 100644 index 000000000..0a41d85b5 --- /dev/null +++ b/lib/packages/docutils/dochelpers.nim @@ -0,0 +1,298 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integration helpers between ``docgen.nim`` and ``rst.nim``. +## +## Function `toLangSymbol(linkText)`_ produces a signature `docLink` of +## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ +## matches it with `generated`, produced from `PNode` by ``docgen.rst``. + +import rstast +import std/strutils + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + + +type + LangSymbol* = object ## symbol signature in Nim + symKind*: string ## "proc", "const", "type", etc + symTypeKind*: string ## ""|enum|object|tuple - + ## valid only when `symKind == "type"` + name*: string ## plain symbol name without any parameters + generics*: string ## generic parameters (without brackets) + isGroup*: bool ## is LangSymbol a group with overloads? + # the following fields are valid iff `isGroup` == false + # (always false when parsed by `toLangSymbol` because link like foo_ + # can point to just a single symbol foo, e.g. proc). + parametersProvided*: bool ## to disambiguate `proc f`_ and `proc f()`_ + parameters*: seq[tuple[name: string, `type`: string]] + ## name-type seq, e.g. for proc + outType*: string ## result type, e.g. for proc + +proc `$`*(s: LangSymbol): string = # for debug + ("(symkind=$1, symTypeKind=$2, name=$3, generics=$4, isGroup=$5, " & + "parametersProvided=$6, parameters=$7, outType=$8)") % [ + s.symKind, s.symTypeKind , s.name, s.generics, $s.isGroup, + $s.parametersProvided, $s.parameters, s.outType] + +func nimIdentBackticksNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## Unlike `nimIdentNormalize` removes spaces and backticks. + ## + ## .. Warning:: No checking (e.g. that identifiers cannot start from + ## digits or '_', or that number of backticks is even) is performed. + runnableExamples: + doAssert nimIdentBackticksNormalize("Foo_bar") == "Foobar" + doAssert nimIdentBackticksNormalize("FoO BAr") == "Foobar" + doAssert nimIdentBackticksNormalize("`Foo BAR`") == "Foobar" + doAssert nimIdentBackticksNormalize("` Foo BAR `") == "Foobar" + # not a valid identifier: + doAssert nimIdentBackticksNormalize("`_x_y`") == "_xy" + result = newString(s.len) + var firstChar = true + var j = 0 + for i in 0..len(s) - 1: + if s[i] in {'A'..'Z'}: + if not firstChar: # to lowercase + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + else: + result[j] = s[i] + firstChar = false + inc j + elif s[i] notin {'_', ' ', '`'}: + result[j] = s[i] + inc j + firstChar = false + elif s[i] == '_' and firstChar: + result[j] = '_' + inc j + firstChar = false + else: discard # just omit '`' or ' ' + if j != s.len: setLen(result, j) + +proc langSymbolGroup*(kind: string, name: string): LangSymbol = + if kind notin ["proc", "func", "macro", "method", "iterator", + "template", "converter"]: + raise newException(ValueError, "unknown symbol kind $1" % [kind]) + result = LangSymbol(symKind: kind, name: name, isGroup: true) + +proc toLangSymbol*(linkText: PRstNode): LangSymbol = + ## Parses `linkText` into a more structured form using a state machine. + ## + ## This proc is designed to allow link syntax with operators even + ## without escaped backticks inside: + ## + ## `proc *`_ + ## `proc []`_ + ## + ## This proc should be kept in sync with the `renderTypes` proc from + ## ``compiler/typesrenderer.nim``. + template fail(msg: string) = + raise newException(ValueError, msg) + if linkText.kind notin {rnRstRef, rnInner}: + fail("toLangSymbol: wrong input kind " & $linkText.kind) + + const NimDefs = ["proc", "func", "macro", "method", "iterator", + "template", "converter", "const", "type", "var", + "enum", "object", "tuple", "module"] + template resolveSymKind(x: string) = + if x in ["enum", "object", "tuple"]: + result.symKind = "type" + result.symTypeKind = x + else: + result.symKind = x + type + State = enum + inBeginning + afterSymKind + beforeSymbolName # auxiliary state to catch situations like `proc []`_ after space + atSymbolName + afterSymbolName + genericsPar + parameterName + parameterType + outType + var state = inBeginning + var curIdent = "" + template flushIdent() = + if curIdent != "": + case state + of inBeginning: fail("incorrect state inBeginning") + of afterSymKind: resolveSymKind curIdent + of beforeSymbolName: fail("incorrect state beforeSymbolName") + of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize + of afterSymbolName: fail("incorrect state afterSymbolName") + of genericsPar: result.generics = curIdent + of parameterName: result.parameters.add (curIdent, "") + of parameterType: + for a in countdown(result.parameters.len - 1, 0): + if result.parameters[a].`type` == "": + result.parameters[a].`type` = curIdent + of outType: result.outType = curIdent + curIdent = "" + var parens = 0 + let L = linkText.sons.len + template s(i: int): string = linkText.sons[i].text + var i = 0 + template nextState = + case s(i) + of " ": + if state == afterSymKind: + flushIdent + state = beforeSymbolName + of "`": + curIdent.add "`" + inc i + while i < L: # add contents between ` ` as a whole + curIdent.add s(i) + if s(i) == "`": + break + inc i + curIdent = curIdent.nimIdentBackticksNormalize + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + flushIdent + state = afterSymbolName + of "[": + if state notin {inBeginning, afterSymKind, beforeSymbolName}: + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + curIdent.add s(i) + elif state in {atSymbolName, afterSymbolName} and parens == 1: + flushIdent + state = genericsPar + curIdent.add s(i) + else: curIdent.add s(i) + of "]": + if state notin {inBeginning, afterSymKind, beforeSymbolName, atSymbolName}: + dec parens + if state == genericsPar and parens == 0: + curIdent.add s(i) + flushIdent + else: curIdent.add s(i) + of "(": + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + result.parametersProvided = true + state = atSymbolName + flushIdent + state = parameterName + elif state in {atSymbolName, afterSymbolName, genericsPar} and parens == 1: + result.parametersProvided = true + flushIdent + state = parameterName + else: curIdent.add s(i) + of ")": + dec parens + if state in {parameterName, parameterType} and parens == 0: + flushIdent + state = outType + else: curIdent.add s(i) + of "{": # remove pragmas + while i < L: + if s(i) == "}": + break + inc i + of ",", ";": + if state in {parameterName, parameterType} and parens == 1: + flushIdent + state = parameterName + else: curIdent.add s(i) + of "*": # skip export symbol + if state == atSymbolName: + flushIdent + state = afterSymbolName + elif state == afterSymbolName: + discard + else: curIdent.add "*" + of ":": + if state == outType: discard + elif state == parameterName: + flushIdent + state = parameterType + else: curIdent.add ":" + else: + let isPostfixSymKind = i > 0 and i == L - 1 and + result.symKind == "" and s(i) in NimDefs + if isPostfixSymKind: # for links like `foo proc`_ + resolveSymKind s(i) + else: + case state + of inBeginning: + if s(i) in NimDefs: + state = afterSymKind + else: + state = atSymbolName + curIdent.add s(i) + of afterSymKind, beforeSymbolName: + state = atSymbolName + curIdent.add s(i) + of parameterType: + case s(i) + of "ref": curIdent.add "ref." + of "ptr": curIdent.add "ptr." + of "var": discard + else: curIdent.add s(i).nimIdentBackticksNormalize + of atSymbolName: + curIdent.add s(i) + else: + curIdent.add s(i).nimIdentBackticksNormalize + while i < L: + nextState + inc i + if state == afterSymKind: # treat `type`_ as link to symbol `type` + state = atSymbolName + flushIdent + result.isGroup = false + +proc match*(generated: LangSymbol, docLink: LangSymbol): bool = + ## Returns true if `generated` can be a target for `docLink`. + ## If `generated` is an overload group then only `symKind` and `name` + ## are compared for success. + result = true + if docLink.symKind != "": + if generated.symKind == "proc": + result = docLink.symKind in ["proc", "func"] + else: + result = generated.symKind == docLink.symKind + if result and docLink.symKind == "type" and docLink.symTypeKind != "": + result = generated.symTypeKind == docLink.symTypeKind + if not result: return + result = generated.name == docLink.name + if not result: return + if generated.isGroup: + # if `()` were added then it's not a reference to the whole group: + return not docLink.parametersProvided + if docLink.generics != "": + result = generated.generics == docLink.generics + if not result: return + if docLink.outType != "": + result = generated.outType == docLink.outType + if not result: return + if docLink.parametersProvided: + result = generated.parameters.len == docLink.parameters.len + if not result: return + var onlyType = false + for i in 0 ..< generated.parameters.len: + let g = generated.parameters[i] + let d = docLink.parameters[i] + if i == 0: + if g.`type` == d.name: + onlyType = true # only types, not names, are provided in `docLink` + if onlyType: + result = g.`type` == d.name + else: + if d.`type` != "": + result = g.`type` == d.`type` + if not result: return + result = g.name == d.name + if not result: return diff --git a/lib/packages/docutils/docutils.nimble b/lib/packages/docutils/docutils.nimble deleted file mode 100644 index f1683c515..000000000 --- a/lib/packages/docutils/docutils.nimble +++ /dev/null @@ -1,4 +0,0 @@ -version = "0.10.0" -author = "Andreas Rumpf" -description = "Nim's reStructuredText processor." -license = "MIT" diff --git a/lib/packages/docutils/docutils.nimble.old b/lib/packages/docutils/docutils.nimble.old new file mode 100644 index 000000000..f97c3bdde --- /dev/null +++ b/lib/packages/docutils/docutils.nimble.old @@ -0,0 +1,7 @@ +# xxx disabled this as this isn't really a nimble package and it affects logic +# used to compute canonical imports, refs https://github.com/nim-lang/Nim/pull/16999#issuecomment-805442914 + +version = "0.10.0" +author = "Andreas Rumpf" +description = "Nim's reStructuredText processor." +license = "MIT" diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index ca6a3e05e..f8376f46c 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -11,11 +11,9 @@ ## Currently only few languages are supported, other languages may be added. ## The interface supports one language nested in another. ## -## **Note:** Import ``packages/docutils/highlite`` to use this module -## ## You can use this to build your own syntax highlighting, check this example: ## -## .. code::nim +## ```Nim ## let code = """for x in $int.high: echo x.ord mod 2 == 0""" ## var toknizr: GeneralTokenizer ## initGeneralTokenizer(toknizr, code) @@ -33,18 +31,43 @@ ## else: ## echo toknizr.kind # All the kinds of tokens can be processed here. ## echo substr(code, toknizr.start, toknizr.length + toknizr.start - 1) +## ``` ## -## The proc ``getSourceLanguage`` can get the language ``enum`` from a string: -## -## .. code::nim +## The proc `getSourceLanguage` can get the language `enum` from a string: +## ```Nim ## for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l) +## ``` +## +## There is also a `Cmd` pseudo-language supported, which is a simple generic +## shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command): +## no escaping, no programming language constructs besides variable definition +## at the beginning of line. It supports these operators: +## ```Cmd +## & && | || ( ) '' "" ; # for comments +## ``` +## +## Instead of escaping always use quotes like here +## `nimgrep --ext:'nim|nims' file.name`:cmd: shows how to input ``|``. +## Any argument that contains ``.`` or ``/`` or ``\`` will be treated +## as a file or directory. ## +## In addition to `Cmd` there is also `Console` language for +## displaying interactive sessions. +## Lines with a command should start with ``$``, other lines are considered +## as program output. import - strutils -from algorithm import binarySearch + std/strutils +from std/algorithm import binarySearch + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + type + SourceLanguage* = enum + langNone, langNim, langCpp, langCsharp, langC, langJava, + langYaml, langPython, langCmd, langConsole TokenClass* = enum gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, @@ -52,28 +75,31 @@ type gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression, gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, - gtReference, gtOther + gtReference, gtPrompt, gtProgramOutput, gtProgram, gtOption, gtOther GeneralTokenizer* = object of RootObj kind*: TokenClass start*, length*: int buf: cstring pos: int state: TokenClass - - SourceLanguage* = enum - langNone, langNim, langCpp, langCsharp, langC, langJava, - langYaml + lang: SourceLanguage const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "C++", "C#", "C", "Java", "Yaml"] + "Nim", "C++", "C#", "C", "Java", "Yaml", "Python", "Cmd", "Console"] + sourceLanguageToAlpha*: array[SourceLanguage, string] = ["none", + "Nim", "cpp", "csharp", "C", "Java", "Yaml", "Python", "Cmd", "Console"] + ## list of languages spelled with alpabetic characters tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment", "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", - "Label", "Reference", "Other"] + "Label", "Reference", "Prompt", "ProgramOutput", + # start from lower-case if there is a corresponding RST role (see rst.nim) + "program", "option", + "Other"] # The following list comes from doc/keywords.txt, make sure it is # synchronized with this array by running the module itself as a test case. @@ -90,9 +116,11 @@ const "xor", "yield"] proc getSourceLanguage*(name: string): SourceLanguage = - for i in countup(succ(low(SourceLanguage)), high(SourceLanguage)): + for i in succ(low(SourceLanguage)) .. high(SourceLanguage): if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: return i + if cmpIgnoreStyle(name, sourceLanguageToAlpha[i]) == 0: + return i result = langNone proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) = @@ -101,9 +129,8 @@ proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) = g.start = 0 g.length = 0 g.state = low(TokenClass) - var pos = 0 # skip initial whitespace: - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) - g.pos = pos + g.lang = low(SourceLanguage) + g.pos = 0 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: string) = initGeneralTokenizer(g, cstring(buf)) @@ -161,7 +188,10 @@ const OpChars = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.', '|', '=', '%', '&', '$', '@', '~', ':'} -proc nimNextToken(g: var GeneralTokenizer) = +proc isKeyword(x: openArray[string], y: string): int = + binarySearch(x, y) + +proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) = const hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'} octChars = {'0'..'7', '_'} @@ -170,36 +200,38 @@ proc nimNextToken(g: var GeneralTokenizer) = var pos = g.pos g.start = g.pos if g.state == gtStringLit: - g.kind = gtStringLit - while true: + if g.buf[pos] == '\\': + g.kind = gtEscapeSequence + inc(pos) case g.buf[pos] - of '\\': - g.kind = gtEscapeSequence + of 'x', 'X': inc(pos) + if g.buf[pos] in hexChars: inc(pos) + if g.buf[pos] in hexChars: inc(pos) + of '0'..'9': + while g.buf[pos] in {'0'..'9'}: inc(pos) + of '\0': + g.state = gtNone + else: inc(pos) + else: + g.kind = gtStringLit + while true: case g.buf[pos] - of 'x', 'X': + of '\\': + break + of '\0', '\r', '\n': + g.state = gtNone + break + of '\"': inc(pos) - if g.buf[pos] in hexChars: inc(pos) - if g.buf[pos] in hexChars: inc(pos) - of '0'..'9': - while g.buf[pos] in {'0'..'9'}: inc(pos) - of '\0': g.state = gtNone + break else: inc(pos) - break - of '\0', '\x0D', '\x0A': - g.state = gtNone - break - of '\"': - inc(pos) - g.state = gtNone - break - else: inc(pos) else: case g.buf[pos] - of ' ', '\x09'..'\x0D': + of ' ', '\t'..'\r': g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) of '#': g.kind = gtComment inc(pos) @@ -207,7 +239,7 @@ proc nimNextToken(g: var GeneralTokenizer) = if g.buf[pos] == '#': inc(pos) isDoc = true - if g.buf[pos] == '[': + if g.buf[pos] == '[' and g.lang == langNim: g.kind = gtLongComment var nesting = 0 while true: @@ -236,7 +268,7 @@ proc nimNextToken(g: var GeneralTokenizer) = else: inc pos else: - while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': var id = "" while g.buf[pos] in SymChars + {'_'}: @@ -260,12 +292,15 @@ proc nimNextToken(g: var GeneralTokenizer) = else: g.kind = gtRawData inc(pos) - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): + while not (g.buf[pos] in {'\0', '\n', '\r'}): if g.buf[pos] == '"' and g.buf[pos+1] != '"': break inc(pos) if g.buf[pos] == '\"': inc(pos) else: - g.kind = nimGetKeyword(id) + if g.lang == langNim: + g.kind = nimGetKeyword(id) + elif isKeyword(keywords, id) >= 0: + g.kind = gtKeyword of '0': inc(pos) case g.buf[pos] @@ -289,17 +324,18 @@ proc nimNextToken(g: var GeneralTokenizer) = pos = nimNumber(g, pos) of '\'': inc(pos) - g.kind = gtCharLit - while true: - case g.buf[pos] - of '\0', '\x0D', '\x0A': - break - of '\'': - inc(pos) - break - of '\\': - inc(pos, 2) - else: inc(pos) + if g.kind != gtPunctuation: + g.kind = gtCharLit + while true: + case g.buf[pos] + of '\0', '\r', '\n': + break + of '\'': + inc(pos) + break + of '\\': + inc(pos, 2) + else: inc(pos) of '\"': inc(pos) if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'): @@ -320,7 +356,7 @@ proc nimNextToken(g: var GeneralTokenizer) = g.kind = gtStringLit while true: case g.buf[pos] - of '\0', '\x0D', '\x0A': + of '\0', '\r', '\n': break of '\"': inc(pos) @@ -342,7 +378,7 @@ proc nimNextToken(g: var GeneralTokenizer) = inc(pos) g.kind = gtNone g.length = pos - g.pos - if g.kind != gtEof and g.length <= 0: + if g.kind != gtEof and g.state != gtNone and g.length <= 0: assert false, "nimNextToken: produced an empty token" g.pos = pos @@ -394,12 +430,6 @@ proc generalStrLit(g: var GeneralTokenizer, position: int): int = inc(pos) result = pos -proc isKeyword(x: openArray[string], y: string): int = - binarySearch(x, y) - -proc isKeywordIgnoreCase(x: openArray[string], y: string): int = - binarySearch(x, y, cmpIgnoreCase) - type TokenizerFlag = enum hasPreprocessor, hasNestedComments @@ -432,7 +462,7 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], g.state = gtNone else: inc(pos) break - of '\0', '\x0D', '\x0A': + of '\0', '\r', '\n': g.state = gtNone break of '\"': @@ -442,14 +472,14 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], else: inc(pos) else: case g.buf[pos] - of ' ', '\x09'..'\x0D': + of ' ', '\t'..'\r': g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) of '/': inc(pos) if g.buf[pos] == '/': g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) + while not (g.buf[pos] in {'\0', '\n', '\r'}): inc(pos) elif g.buf[pos] == '*': g.kind = gtLongComment var nested = 0 @@ -469,6 +499,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], of '\0': break else: inc(pos) + else: + g.kind = gtOperator + while g.buf[pos] in OpChars: inc(pos) of '#': inc(pos) if hasPreprocessor in flags: @@ -589,9 +622,9 @@ proc javaNextToken(g: var GeneralTokenizer) = proc yamlPlainStrLit(g: var GeneralTokenizer, pos: var int) = g.kind = gtStringLit - while g.buf[pos] notin {'\0', '\x09'..'\x0D', ',', ']', '}'}: + while g.buf[pos] notin {'\0', '\t'..'\r', ',', ']', '}'}: if g.buf[pos] == ':' and - g.buf[pos + 1] in {'\0', '\x09'..'\x0D', ' '}: + g.buf[pos + 1] in {'\0', '\t'..'\r', ' '}: break inc(pos) @@ -604,14 +637,14 @@ proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) = while g.buf[pos] in {'0'..'9'}: inc(pos) else: yamlPlainStrLit(g, pos) if g.kind == gtNone: - if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + if g.buf[pos] in {'\0', '\t'..'\r', ' ', ',', ']', '}'}: g.kind = gtDecNumber elif g.buf[pos] == '.': inc(pos) if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) else: while g.buf[pos] in {'0'..'9'}: inc(pos) - if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + if g.buf[pos] in {'\0', '\t'..'\r', ' ', ',', ']', '}'}: g.kind = gtFloatNumber if g.kind == gtNone: if g.buf[pos] in {'e', 'E'}: @@ -620,13 +653,13 @@ proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) = if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos) else: while g.buf[pos] in {'0'..'9'}: inc(pos) - if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}: + if g.buf[pos] in {'\0', '\t'..'\r', ' ', ',', ']', '}'}: g.kind = gtFloatNumber else: yamlPlainStrLit(g, pos) else: yamlPlainStrLit(g, pos) - while g.buf[pos] notin {'\0', ',', ']', '}', '\x0A', '\x0D'}: + while g.buf[pos] notin {'\0', ',', ']', '}', '\n', '\r'}: inc(pos) - if g.buf[pos] notin {'\x09'..'\x0D', ' ', ',', ']', '}'}: + if g.buf[pos] notin {'\t'..'\r', ' ', ',', ']', '}'}: yamlPlainStrLit(g, pos) break # theoretically, we would need to parse indentation (like with block scalars) @@ -651,19 +684,16 @@ proc yamlNextToken(g: var GeneralTokenizer) = of 'x': inc(pos) for i in 1..2: - {.unroll.} if g.buf[pos] in hexChars: inc(pos) break of 'u': inc(pos) for i in 1..4: - {.unroll.} if g.buf[pos] in hexChars: inc(pos) break of 'U': inc(pos) for i in 1..8: - {.unroll.} if g.buf[pos] in hexChars: inc(pos) break else: inc(pos) @@ -698,13 +728,13 @@ proc yamlNextToken(g: var GeneralTokenizer) = while g.buf[pos] in {' ', '\t'}: inc(pos) of '#': g.kind = gtComment - while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) - of '\x0A', '\x0D': discard + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + of '\n', '\r': discard else: # illegal here. just don't parse a block scalar g.kind = gtNone g.state = gtOther - if g.buf[pos] in {'\x0A', '\x0D'} and g.state == gtCommand: + if g.buf[pos] in {'\n', '\r'} and g.state == gtCommand: g.state = gtLongStringLit elif g.state == gtLongStringLit: # beware, this is the only token where we actually have to parse @@ -713,10 +743,10 @@ proc yamlNextToken(g: var GeneralTokenizer) = g.kind = gtLongStringLit # first, we have to find the parent indentation of the block scalar, so that # we know when to stop - assert g.buf[pos] in {'\x0A', '\x0D'} + assert g.buf[pos] in {'\n', '\r'} var lookbehind = pos - 1 var headerStart = -1 - while lookbehind >= 0 and g.buf[lookbehind] notin {'\x0A', '\x0D'}: + while lookbehind >= 0 and g.buf[lookbehind] notin {'\n', '\r'}: if headerStart == -1 and g.buf[lookbehind] in {'|', '>'}: headerStart = lookbehind dec(lookbehind) @@ -727,12 +757,12 @@ proc yamlNextToken(g: var GeneralTokenizer) = # when the header is alone in a line, this line does not show the parent's # indentation, so we must go further. search the first previous line with # non-whitespace content. - while lookbehind >= 0 and g.buf[lookbehind] in {'\x0A', '\x0D'}: + while lookbehind >= 0 and g.buf[lookbehind] in {'\n', '\r'}: dec(lookbehind) while lookbehind >= 0 and g.buf[lookbehind] in {' ', '\t'}: dec(lookbehind) # now, find the beginning of the line... - while lookbehind >= 0 and g.buf[lookbehind] notin {'\x0A', '\x0D'}: + while lookbehind >= 0 and g.buf[lookbehind] notin {'\n', '\r'}: dec(lookbehind) # ... and its indentation indentation = 1 @@ -740,7 +770,7 @@ proc yamlNextToken(g: var GeneralTokenizer) = if lookbehind == -1: indentation = 0 # top level elif g.buf[lookbehind + 1] == '-' and g.buf[lookbehind + 2] == '-' and g.buf[lookbehind + 3] == '-' and - g.buf[lookbehind + 4] in {'\x09'..'\x0D', ' '}: + g.buf[lookbehind + 4] in {'\t'..'\r', ' '}: # this is a document start, therefore, we are at top level indentation = 0 # because lookbehind was at newline char when calculating indentation, we're @@ -748,7 +778,7 @@ proc yamlNextToken(g: var GeneralTokenizer) = let parentIndentation = indentation - 1 # find first content - while g.buf[pos] in {' ', '\x0A', '\x0D'}: + while g.buf[pos] in {' ', '\n', '\r'}: if g.buf[pos] == ' ': inc(indentation) else: indentation = 0 inc(pos) @@ -765,12 +795,12 @@ proc yamlNextToken(g: var GeneralTokenizer) = if (indentation < minIndentation and g.buf[pos] == '#') or (indentation == 0 and g.buf[pos] == '.' and g.buf[pos + 1] == '.' and g.buf[pos + 2] == '.' and - g.buf[pos + 3] in {'\0', '\x09'..'\x0D', ' '}): + g.buf[pos + 3] in {'\0', '\t'..'\r', ' '}): # comment after end of block scalar, or end of document break minIndentation = min(indentation, minIndentation) - while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) - while g.buf[pos] in {' ', '\x0A', '\x0D'}: + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) + while g.buf[pos] in {' ', '\n', '\r'}: if g.buf[pos] == ' ': inc(indentation) else: indentation = 0 inc(pos) @@ -779,30 +809,29 @@ proc yamlNextToken(g: var GeneralTokenizer) = elif g.state == gtOther: # gtOther means 'inside YAML document' case g.buf[pos] - of ' ', '\x09'..'\x0D': + of ' ', '\t'..'\r': g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) of '#': g.kind = gtComment inc(pos) - while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) of '-': inc(pos) - if g.buf[pos] in {'\0', ' ', '\x09'..'\x0D'}: + if g.buf[pos] in {'\0', ' ', '\t'..'\r'}: g.kind = gtPunctuation elif g.buf[pos] == '-' and - (pos == 1 or g.buf[pos - 2] in {'\x0A', '\x0D'}): # start of line + (pos == 1 or g.buf[pos - 2] in {'\n', '\r'}): # start of line inc(pos) - if g.buf[pos] == '-' and g.buf[pos + 1] in {'\0', '\x09'..'\x0D', ' '}: + if g.buf[pos] == '-' and g.buf[pos + 1] in {'\0', '\t'..'\r', ' '}: inc(pos) g.kind = gtKeyword else: yamlPossibleNumber(g, pos) else: yamlPossibleNumber(g, pos) of '.': - if pos == 0 or g.buf[pos - 1] in {'\x0A', '\x0D'}: + if pos == 0 or g.buf[pos - 1] in {'\n', '\r'}: inc(pos) for i in 1..2: - {.unroll.} if g.buf[pos] != '.': break inc(pos) if pos == g.start + 3: @@ -812,12 +841,12 @@ proc yamlNextToken(g: var GeneralTokenizer) = else: yamlPlainStrLit(g, pos) of '?': inc(pos) - if g.buf[pos] in {'\0', ' ', '\x09'..'\x0D'}: + if g.buf[pos] in {'\0', ' ', '\t'..'\r'}: g.kind = gtPunctuation else: yamlPlainStrLit(g, pos) of ':': inc(pos) - if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', '\'', '\"'} or + if g.buf[pos] in {'\0', '\t'..'\r', ' ', '\'', '\"'} or (pos > 0 and g.buf[pos - 2] in {'}', ']', '\"', '\''}): g.kind = gtPunctuation else: yamlPlainStrLit(g, pos) @@ -836,7 +865,7 @@ proc yamlNextToken(g: var GeneralTokenizer) = inc(pos) if g.buf[pos] == '<': # literal tag (e.g. `!<tag:yaml.org,2002:str>`) - while g.buf[pos] notin {'\0', '>', '\x09'..'\x0D', ' '}: inc(pos) + while g.buf[pos] notin {'\0', '>', '\t'..'\r', ' '}: inc(pos) if g.buf[pos] == '>': inc(pos) else: while g.buf[pos] in {'A'..'Z', 'a'..'z', '0'..'9', '-'}: inc(pos) @@ -845,17 +874,17 @@ proc yamlNextToken(g: var GeneralTokenizer) = # prefixed tag (e.g. `!!str`) inc(pos) while g.buf[pos] notin - {'\0', '\x09'..'\x0D', ' ', ',', '[', ']', '{', '}'}: inc(pos) - of '\0', '\x09'..'\x0D', ' ': discard + {'\0', '\t'..'\r', ' ', ',', '[', ']', '{', '}'}: inc(pos) + of '\0', '\t'..'\r', ' ': discard else: # local tag (e.g. `!nim:system:int`) - while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + while g.buf[pos] notin {'\0', '\t'..'\r', ' '}: inc(pos) of '&': g.kind = gtLabel - while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + while g.buf[pos] notin {'\0', '\t'..'\r', ' '}: inc(pos) of '*': g.kind = gtReference - while g.buf[pos] notin {'\0', '\x09'..'\x0D', ' '}: inc(pos) + while g.buf[pos] notin {'\0', '\t'..'\r', ' '}: inc(pos) of '|', '>': # this can lead to incorrect tokenization when | or > appear inside flow # content. checking whether we're inside flow content is not @@ -871,18 +900,18 @@ proc yamlNextToken(g: var GeneralTokenizer) = # outside document case g.buf[pos] of '%': - if pos == 0 or g.buf[pos - 1] in {'\x0A', '\x0D'}: + if pos == 0 or g.buf[pos - 1] in {'\n', '\r'}: g.kind = gtDirective - while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) else: g.state = gtOther yamlPlainStrLit(g, pos) - of ' ', '\x09'..'\x0D': + of ' ', '\t'..'\r': g.kind = gtWhitespace - while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) + while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) of '#': g.kind = gtComment - while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) + while g.buf[pos] notin {'\0', '\n', '\r'}: inc(pos) of '\0': g.kind = gtEof else: g.kind = gtNone @@ -890,7 +919,86 @@ proc yamlNextToken(g: var GeneralTokenizer) = g.length = pos - g.pos g.pos = pos +proc pythonNextToken(g: var GeneralTokenizer) = + const + keywords: array[0..34, string] = [ + "False", "None", "True", "and", "as", "assert", "async", "await", + "break", "class", "continue", "def", "del", "elif", "else", "except", + "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", + "with", "yield"] + nimNextToken(g, keywords) + +proc cmdNextToken(g: var GeneralTokenizer, dollarPrompt = false) = + var pos = g.pos + g.start = g.pos + if g.state == low(TokenClass): + g.state = if dollarPrompt: gtPrompt else: gtProgram + case g.buf[pos] + of ' ', '\t'..'\r': + g.kind = gtWhitespace + while g.buf[pos] in {' ', '\t'..'\r'}: + if g.buf[pos] == '\n': + g.state = if dollarPrompt: gtPrompt else: gtProgram + inc(pos) + of '\'', '"': + g.kind = gtOption + let q = g.buf[pos] + inc(pos) + while g.buf[pos] notin {q, '\0'}: + inc(pos) + if g.buf[pos] == q: inc(pos) + of '#': + g.kind = gtComment + while g.buf[pos] notin {'\n', '\0'}: + inc(pos) + of '&', '|': + g.kind = gtOperator + inc(pos) + if g.buf[pos] == g.buf[pos-1]: inc(pos) + g.state = gtProgram + of '(': + g.kind = gtOperator + g.state = gtProgram + inc(pos) + of ')': + g.kind = gtOperator + inc(pos) + of ';': + g.state = gtProgram + g.kind = gtOperator + inc(pos) + of '\0': g.kind = gtEof + elif dollarPrompt and g.state == gtPrompt: + if g.buf[pos] == '$' and g.buf[pos+1] in {' ', '\t'}: + g.kind = gtPrompt + inc pos, 2 + g.state = gtProgram + else: + g.kind = gtProgramOutput + while g.buf[pos] notin {'\n', '\0'}: + inc(pos) + else: + if g.state == gtProgram: + g.kind = gtProgram + g.state = gtOption + else: + g.kind = gtOption + while g.buf[pos] notin {' ', '\t'..'\r', '&', '|', '(', ')', '\'', '"', '\0'}: + if g.buf[pos] == ';' and g.buf[pos+1] == ' ': + # (check space because ';' can be used inside arguments in Win bat) + break + if g.kind == gtOption and g.buf[pos] in {'/', '\\', '.'}: + g.kind = gtIdentifier # for file/dir name + elif g.kind == gtProgram and g.buf[pos] == '=': + g.kind = gtIdentifier # for env variable setting at beginning of line + g.state = gtProgram + inc(pos) + g.length = pos - g.pos + g.pos = pos + proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = + g.lang = lang case lang of langNone: assert false of langNim: nimNextToken(g) @@ -899,13 +1007,28 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = of langC: cNextToken(g) of langJava: javaNextToken(g) of langYaml: yamlNextToken(g) + of langPython: pythonNextToken(g) + of langCmd: cmdNextToken(g) + of langConsole: cmdNextToken(g, dollarPrompt=true) + +proc tokenize*(text: string, lang: SourceLanguage): seq[(string, TokenClass)] = + var g: GeneralTokenizer + initGeneralTokenizer(g, text) + var prevPos = 0 + while true: + getNextToken(g, lang) + if g.kind == gtEof: + break + var s = text[prevPos ..< g.pos] + result.add (s, g.kind) + prevPos = g.pos when isMainModule: var keywords: seq[string] # Try to work running in both the subdir or at the root. for filename in ["doc/keywords.txt", "../../../doc/keywords.txt"]: try: - let input = string(readFile(filename)) + let input = readFile(filename) keywords = input.splitWhitespace() break except: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 50b855662..706c50689 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -7,23 +7,46 @@ # distribution, for details about the copyright. # -## This module implements a `reStructuredText`:idx: parser. A large -## subset is implemented. Some features of the `markdown`:idx: wiki syntax are -## also supported. +## This module implements a `reStructuredText`:idx: (RST) and +## `Markdown`:idx: parser. +## User's manual on supported markup syntax and command line usage can be +## found in [Nim-flavored Markdown and reStructuredText](markdown_rst.html). ## -## **Note:** Import ``packages/docutils/rst`` to use this module +## * See also [Nim DocGen Tools Guide](docgen.html) for handling of +## ``.nim`` files. +## * See also [packages/docutils/rstgen module](rstgen.html) to know how to +## generate HTML or Latex strings (for embedding them into custom documents). +## +## Choice between Markdown and RST as well as optional additional features are +## turned on by passing ``options:`` [RstParseOptions] to [proc rstParse]. import - os, strutils, rstast + std/[os, strutils, enumutils, algorithm, lists, sequtils, + tables, strscans] +import dochelpers, rstidx, rstast +import std/private/miscdollars +from highlite import SourceLanguage, getSourceLanguage + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + type RstParseOption* = enum ## options for the RST parser - roSkipPounds, ## skip ``#`` at line beginning (documentation - ## embedded in Nim comments) roSupportSmilies, ## make the RST parser support smilies like ``:)`` roSupportRawDirective, ## support the ``raw`` directive (don't support ## it for sandboxing) - roSupportMarkdown ## support additional features of markdown + roSupportMarkdown, ## support additional features of Markdown + roPreferMarkdown, ## parse as Markdown (keeping RST as "extension" + ## to Markdown) -- implies `roSupportMarkdown` + roNimFile ## set for Nim files where default interpreted + ## text role should be :nim: + roSandboxDisabled ## this option enables certain options + ## (e.g. raw, include, importdoc) + ## which are disabled by default as they can + ## enable users to read arbitrary data and + ## perform XSS if the parser is used in a web + ## app. RstParseOptions* = set[RstParseOption] @@ -32,39 +55,41 @@ type mcWarning = "Warning", mcError = "Error" + # keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim: MsgKind* = enum ## the possible messages - meCannotOpenFile, - meExpected, - meGridTableNotImplemented, - meNewSectionExpected, - meGeneralParseError, - meInvalidDirective, - mwRedefinitionOfLabel, - mwUnknownSubstitution, - mwUnsupportedLanguage, - mwUnsupportedField + meCannotOpenFile = "cannot open '$1'", + meExpected = "'$1' expected", + meMissingClosing = "$1", + meGridTableNotImplemented = "grid table is not implemented", + meMarkdownIllformedTable = "illformed delimiter row of a Markdown table", + meIllformedTable = "Illformed table: $1", + meNewSectionExpected = "new section expected $1", + meGeneralParseError = "general parse error", + meInvalidDirective = "invalid directive: '$1'", + meInvalidField = "invalid field: $1", + meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", + mwRedefinitionOfLabel = "redefinition of label '$1'", + mwUnknownSubstitution = "unknown substitution '$1'", + mwAmbiguousLink = "ambiguous doc link $1", + mwBrokenLink = "broken link '$1'", + mwUnsupportedLanguage = "language '$1' not supported", + mwUnsupportedField = "field '$1' not supported", + mwRstStyle = "RST style: $1", + mwUnusedImportdoc = "importdoc for '$1' is not used", + meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.closure, gcsafe.} ## what to do in case of an error FindFileHandler* = proc (filename: string): string {.closure, gcsafe.} - -const - messages: array[MsgKind, string] = [ - meCannotOpenFile: "cannot open '$1'", - meExpected: "'$1' expected", - meGridTableNotImplemented: "grid table is not implemented", - meNewSectionExpected: "new section expected", - meGeneralParseError: "general parse error", - meInvalidDirective: "invalid directive: '$1'", - mwRedefinitionOfLabel: "redefinition of label '$1'", - mwUnknownSubstitution: "unknown substitution '$1'", - mwUnsupportedLanguage: "language '$1' not supported", - mwUnsupportedField: "field '$1' not supported" - ] + FindRefFileHandler* = + proc (targetRelPath: string): + tuple[targetPath: string, linkRelPath: string] {.closure, gcsafe.} + ## returns where .html or .idx file should be found by its relative path; + ## `linkRelPath` is a prefix to be added before a link anchor from such file proc rstnodeToRefname*(n: PRstNode): string proc addNodes*(n: PRstNode): string -proc getFieldValue*(n: PRstNode, fieldname: string): string +proc getFieldValue*(n: PRstNode, fieldname: string): string {.gcsafe.} proc getArgument*(n: PRstNode): string # ----------------------------- scanner part -------------------------------- @@ -111,10 +136,19 @@ const ":geek:": "icon_e_geek", ":ugeek:": "icon_e_ugeek" } + SandboxDirAllowlist = [ + "image", "code", "code-block", "admonition", "attention", "caution", + "container", "contents", "danger", "default-role", "error", "figure", + "hint", "important", "index", "note", "role", "tip", "title", "warning"] type TokType = enum - tkEof, tkIndent, tkWhite, tkWord, tkAdornment, tkPunct, tkOther + tkEof, tkIndent, + tkWhite, tkWord, + tkAdornment, # used for chapter adornment, transitions and + # horizontal table borders + tkPunct, # one or many punctuation characters + tkOther Token = object # a RST token kind*: TokType # the type of the token ival*: int # the indentation or parsed integer value @@ -126,7 +160,8 @@ type buf*: cstring bufpos*: int line*, col*, baseIndent*: int - skipPounds*: bool + adornmentLine*: bool + escapeNext*: bool proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.kind = tkWord @@ -134,57 +169,82 @@ proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.col = L.col var pos = L.bufpos while true: - add(tok.symbol, L.buf[pos]) - inc(pos) + tok.symbol.add(L.buf[pos]) + inc pos if L.buf[pos] notin s: break - inc(L.col, pos - L.bufpos) + inc L.col, pos - L.bufpos L.bufpos = pos -proc getAdornment(L: var Lexer, tok: var Token) = - tok.kind = tkAdornment +proc isCurrentLineAdornment(L: var Lexer): bool = + var pos = L.bufpos + let c = L.buf[pos] + while true: + inc pos + if L.buf[pos] in {'\c', '\l', '\0'}: + break + if c == '+': # grid table + if L.buf[pos] notin {'-', '=', '+'}: + return false + else: # section adornment or table horizontal border + if L.buf[pos] notin {c, ' ', '\t', '\v', '\f'}: + return false + result = true + +proc getPunctAdornment(L: var Lexer, tok: var Token) = + if L.adornmentLine: + tok.kind = tkAdornment + else: + tok.kind = tkPunct tok.line = L.line tok.col = L.col var pos = L.bufpos - var c = L.buf[pos] - while true: - add(tok.symbol, L.buf[pos]) - inc(pos) - if L.buf[pos] != c: break - inc(L.col, pos - L.bufpos) + let c = L.buf[pos] + if not L.escapeNext and (c != '\\' or L.adornmentLine): + while true: + tok.symbol.add(L.buf[pos]) + inc pos + if L.buf[pos] != c: break + elif L.escapeNext: + tok.symbol.add(L.buf[pos]) + inc pos + else: # not L.escapeNext and c == '\\' and not L.adornmentLine + tok.symbol.add '\\' + inc pos + L.escapeNext = true + inc L.col, pos - L.bufpos L.bufpos = pos + if tok.symbol == "\\": tok.kind = tkPunct + # nim extension: standalone \ can not be adornment proc getBracket(L: var Lexer, tok: var Token) = tok.kind = tkPunct tok.line = L.line tok.col = L.col - add(tok.symbol, L.buf[L.bufpos]) + tok.symbol.add(L.buf[L.bufpos]) inc L.col inc L.bufpos proc getIndentAux(L: var Lexer, start: int): int = var pos = start # skip the newline (but include it in the token!) - if L.buf[pos] == '\x0D': - if L.buf[pos + 1] == '\x0A': inc(pos, 2) - else: inc(pos) - elif L.buf[pos] == '\x0A': - inc(pos) - if L.skipPounds: - if L.buf[pos] == '#': inc(pos) - if L.buf[pos] == '#': inc(pos) + if L.buf[pos] == '\r': + if L.buf[pos + 1] == '\n': inc pos, 2 + else: inc pos + elif L.buf[pos] == '\n': + inc pos while true: case L.buf[pos] - of ' ', '\x0B', '\x0C': - inc(pos) - inc(result) - of '\x09': - inc(pos) + of ' ', '\v', '\f': + inc pos + inc result + of '\t': + inc pos result = result - (result mod 8) + 8 else: break # EndOfFile also leaves the loop if L.buf[pos] == '\0': result = 0 - elif (L.buf[pos] == '\x0A') or (L.buf[pos] == '\x0D'): + elif L.buf[pos] == '\n' or L.buf[pos] == '\r': # look at the next line for proper indentation: result = getIndentAux(L, pos) L.bufpos = pos # no need to set back buf @@ -202,22 +262,26 @@ proc getIndent(L: var Lexer, tok: var Token) = proc rawGetTok(L: var Lexer, tok: var Token) = tok.symbol = "" tok.ival = 0 + if L.col == 0: + L.adornmentLine = false var c = L.buf[L.bufpos] case c of 'a'..'z', 'A'..'Z', '\x80'..'\xFF', '0'..'9': getThing(L, tok, SymChars) - of ' ', '\x09', '\x0B', '\x0C': - getThing(L, tok, {' ', '\x09'}) + of ' ', '\t', '\v', '\f': + getThing(L, tok, {' ', '\t'}) tok.kind = tkWhite - if L.buf[L.bufpos] in {'\x0D', '\x0A'}: + if L.buf[L.bufpos] in {'\r', '\n'}: rawGetTok(L, tok) # ignore spaces before \n - of '\x0D', '\x0A': + of '\r', '\n': getIndent(L, tok) + L.adornmentLine = false of '!', '\"', '#', '$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '\\', '^', '_', '`', '|', '~': - getAdornment(L, tok) - if len(tok.symbol) <= 3: tok.kind = tkPunct + if L.col == 0: + L.adornmentLine = L.isCurrentLineAdornment() + getPunctAdornment(L, tok) of '(', ')', '[', ']', '{', '}': getBracket(L, tok) else: @@ -227,179 +291,449 @@ proc rawGetTok(L: var Lexer, tok: var Token) = tok.kind = tkEof else: tok.kind = tkOther - add(tok.symbol, c) - inc(L.bufpos) - inc(L.col) + tok.symbol.add(c) + inc L.bufpos + inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq) = var L: Lexer - var length = len(tokens) + var length = tokens.len L.buf = cstring(buffer) L.line = 0 # skip UTF-8 BOM - if (L.buf[0] == '\xEF') and (L.buf[1] == '\xBB') and (L.buf[2] == '\xBF'): - inc(L.bufpos, 3) - L.skipPounds = skipPounds - if skipPounds: - if L.buf[L.bufpos] == '#': - inc(L.bufpos) - inc(result) - if L.buf[L.bufpos] == '#': - inc(L.bufpos) - inc(result) - L.baseIndent = 0 - while L.buf[L.bufpos] == ' ': - inc(L.bufpos) - inc(L.baseIndent) - inc(result) + if L.buf[0] == '\xEF' and L.buf[1] == '\xBB' and L.buf[2] == '\xBF': + inc L.bufpos, 3 while true: - inc(length) + inc length setLen(tokens, length) + let toEscape = L.escapeNext rawGetTok(L, tokens[length - 1]) + if toEscape: L.escapeNext = false if tokens[length - 1].kind == tkEof: break if tokens[0].kind == tkWhite: # BUGFIX - tokens[0].ival = len(tokens[0].symbol) + tokens[0].ival = tokens[0].symbol.len tokens[0].kind = tkIndent type - LevelMap = array[char, int] + LevelInfo = object + symbol: char # adornment character + hasOverline: bool # has also overline (besides underline)? + line: int # the last line of this style occurrence + # (for error message) + hasPeers: bool # has headings on the same level of hierarchy? + LiteralBlockKind = enum # RST-style literal blocks after `::` + lbNone, + lbIndentedLiteralBlock, + lbQuotedLiteralBlock + LevelMap = seq[LevelInfo] # Saves for each possible title adornment + # style its level in the current document. + SubstitutionKind = enum + rstSubstitution = "substitution", + hyperlinkAlias = "hyperlink alias", + implicitHyperlinkAlias = "implicitly-generated hyperlink alias" Substitution = object + kind*: SubstitutionKind key*: string value*: PRstNode - - SharedState = object - options: RstParseOptions # parsing options - uLevel, oLevel: int # counters for the section levels + info*: TLineInfo # place where the substitution was defined + AnchorRule = enum + arInternalRst, ## For automatically generated RST anchors (from + ## headings, footnotes, inline internal targets): + ## case-insensitive, 1-space-significant (by RST spec) + arExternalRst, ## For external .nim doc comments or .rst/.md + arNim ## For anchors generated by ``docgen.nim``: Nim-style case + ## sensitivity, etc. (see `proc normalizeNimName`_ for details) + arHyperlink, ## For links with manually set anchors in + ## form `text <pagename.html#anchor>`_ + RstAnchorKind = enum + manualDirectiveAnchor = "manual directive anchor", + manualInlineAnchor = "manual inline anchor", + footnoteAnchor = "footnote anchor", + headlineAnchor = "implicitly-generated headline anchor" + AnchorSubst = object + info: TLineInfo # the file where the anchor was defined + priority: int + case kind: range[arInternalRst .. arNim] + of arInternalRst: + anchorType: RstAnchorKind + target: PRstNode + of arExternalRst: + anchorTypeExt: RstAnchorKind + refnameExt: string + of arNim: + module: FileIndex # anchor's module (generally not the same as file) + tooltip: string # displayed tooltip for Nim-generated anchors + langSym: LangSymbol + refname: string # A reference name that will be inserted directly + # into HTML/Latex. + external: bool + AnchorSubstTable = Table[string, seq[AnchorSubst]] + # use `seq` to account for duplicate anchors + FootnoteType = enum + fnManualNumber, # manually numbered footnote like [3] + fnAutoNumber, # auto-numbered footnote [#] + fnAutoNumberLabel, # auto-numbered with label [#label] + fnAutoSymbol, # auto-symbol footnote [*] + fnCitation # simple text label like [citation2021] + FootnoteSubst = tuple + kind: FootnoteType # discriminator + number: int # valid for fnManualNumber (always) and fnAutoNumber, + # fnAutoNumberLabel after resolveSubs is called + autoNumIdx: int # order of occurrence: fnAutoNumber, fnAutoNumberLabel + autoSymIdx: int # order of occurrence: fnAutoSymbol + label: string # valid for fnAutoNumberLabel + RstFileTable* = object + filenameToIdx*: Table[string, FileIndex] + idxToFilename*: seq[string] + ImportdocInfo = object + used: bool # was this import used? + fromInfo: TLineInfo # place of `.. importdoc::` directive + idxPath: string # full path to ``.idx`` file + linkRelPath: string # prefix before target anchor + title: string # document title obtained from ``.idx`` + RstSharedState = object + options*: RstParseOptions # parsing options + hLevels: LevelMap # hierarchy of heading styles + hTitleCnt: int # =0 if no title, =1 if only main title, + # =2 if both title and subtitle are present + hCurLevel: int # current section level + currRole: string # current interpreted text role + currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions - refs: seq[Substitution] # references - underlineToLevel: LevelMap # Saves for each possible title adornment - # character its level in the - # current document. - # This is for single underline adornments. - overlineToLevel: LevelMap # Saves for each possible title adornment - # character its level in the current - # document. - # This is for over-underline adornments. + refs*: seq[Substitution] # references + anchors*: AnchorSubstTable + # internal target substitutions + lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#] + lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_ + currFootnoteNumRef: int # ... their counter for `resolveSubs` + lineFootnoteSym: seq[TLineInfo] # footnote line, auto symbols .. [*] + lineFootnoteSymRef: seq[TLineInfo] # footnote line, their reference [*]_ + currFootnoteSymRef: int # ... their counter for `resolveSubs` + footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, + # number, order of occurrence msgHandler: MsgHandler # How to handle errors. - findFile: FindFileHandler # How to find files. - - PSharedState = ref SharedState + findFile: FindFileHandler # How to find files for include. + findRefFile: FindRefFileHandler + # How to find files imported by importdoc. + filenames*: RstFileTable # map file name <-> FileIndex (for storing + # file names for warnings after 1st stage) + currFileIdx*: FileIndex # current index in `filenames` + tocPart*: seq[PRstNode] # all the headings of a document + hasToc*: bool + idxImports*: Table[string, ImportdocInfo] + # map `importdoc`ed filename -> it's info + nimFileImported*: bool # Was any ``.nim`` module `importdoc`ed ? + + PRstSharedState* = ref RstSharedState + ManualAnchor = object + alias: string # a (short) name that can substitute the `anchor` + anchor: string # anchor = id = refname + info: TLineInfo RstParser = object of RootObj idx*: int tok*: TokenSeq - s*: PSharedState + s*: PRstSharedState indentStack*: seq[int] - filename*: string - line*, col*: int - hasToc*: bool + line*, col*: int ## initial line/column of whole text or + ## documenation fragment that will be added + ## in case of error/warning reporting to + ## (relative) line/column of the token. + curAnchors*: seq[ManualAnchor] + ## seq to accumulate aliases for anchors: + ## because RST can have >1 alias per 1 anchor EParseError* = object of ValueError + SectionParser = proc (p: var RstParser): PRstNode {.nimcall, gcsafe.} + +const + LineRstInit* = 1 ## Initial line number for standalone RST text + ColRstInit* = 0 ## Initial column number for standalone RST text + ## (Nim global reporting adds ColOffset=1) + ColRstOffset* = 1 ## 1: a replica of ColOffset for internal use + +template currentTok(p: RstParser): Token = p.tok[p.idx] +template prevTok(p: RstParser): Token = p.tok[p.idx - 1] +template nextTok(p: RstParser): Token = p.tok[p.idx + 1] proc whichMsgClass*(k: MsgKind): MsgClass = ## returns which message class `k` belongs to. - case ($k)[1] + case k.symbolName[1] of 'e', 'E': result = mcError of 'w', 'W': result = mcWarning of 'h', 'H': result = mcHint else: assert false, "msgkind does not fit naming scheme" proc defaultMsgHandler*(filename: string, line, col: int, msgkind: MsgKind, - arg: string) {.procvar.} = + arg: string) = let mc = msgkind.whichMsgClass - let a = messages[msgkind] % arg - let message = "$1($2, $3) $4: $5" % [filename, $line, $col, $mc, a] + let a = $msgkind % arg + var message: string + toLocation(message, filename, line, col + ColRstOffset) + message.add " $1: $2" % [$mc, a] if mc == mcError: raise newException(EParseError, message) else: writeLine(stdout, message) -proc defaultFindFile*(filename: string): string {.procvar.} = - if existsFile(filename): result = filename +proc defaultFindFile*(filename: string): string = + if fileExists(filename): result = filename else: result = "" -proc newSharedState(options: RstParseOptions, - findFile: FindFileHandler, - msgHandler: MsgHandler): PSharedState = - new(result) - result.subs = @[] - result.refs = @[] - result.options = options - result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler - result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc defaultFindRefFile*(filename: string): (string, string) = + (filename, "") + +proc defaultRole(options: RstParseOptions): string = + if roNimFile in options: "nim" else: "literal" + +proc whichRoleAux(sym: string): RstNodeKind = + let r = sym.toLowerAscii + case r + of "idx": result = rnIdx + of "literal": result = rnInlineLiteral + of "strong": result = rnStrongEmphasis + of "emphasis": result = rnEmphasis + of "sub", "subscript": result = rnSub + of "sup", "superscript": result = rnSup + # literal and code are the same in our implementation + of "code": result = rnInlineLiteral + of "program", "option", "tok": result = rnCodeFragment + # c++ currently can be spelled only as cpp, c# only as csharp + elif getSourceLanguage(r) != langNone: + result = rnInlineCode + else: # unknown role + result = rnUnknownRole + +proc len(filenames: RstFileTable): int = filenames.idxToFilename.len + +proc addFilename*(s: PRstSharedState, file1: string): FileIndex = + ## Returns index of filename, adding it if it has not been used before + let nextIdx = s.filenames.len.FileIndex + result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) + if result == nextIdx: + s.filenames.filenameToIdx[file1] = result + s.filenames.idxToFilename.add file1 + +proc setCurrFilename*(s: PRstSharedState, file1: string) = + s.currFileIdx = addFilename(s, file1) + +proc getFilename(filenames: RstFileTable, fid: FileIndex): string = + doAssert(0 <= fid.int and fid.int < filenames.len, + "incorrect FileIndex $1 (range 0..$2)" % [ + $fid.int, $(filenames.len - 1)]) + result = filenames.idxToFilename[fid.int] + +proc getFilename(s: PRstSharedState, subst: AnchorSubst): string = + getFilename(s.filenames, subst.info.fileIndex) + +proc getModule(s: PRstSharedState, subst: AnchorSubst): string = + result = getFilename(s.filenames, subst.module) + +proc currFilename(s: PRstSharedState): string = + getFilename(s.filenames, s.currFileIdx) + +proc newRstSharedState*(options: RstParseOptions, + filename: string, + findFile: FindFileHandler, + findRefFile: FindRefFileHandler, + msgHandler: MsgHandler, + hasToc: bool): PRstSharedState = + let r = defaultRole(options) + result = PRstSharedState( + currRole: r, + currRoleKind: whichRoleAux(r), + options: options, + msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, + findFile: if not isNil(findFile): findFile else: defaultFindFile, + findRefFile: + if not isNil(findRefFile): findRefFile + else: defaultFindRefFile, + hasToc: hasToc + ) + setCurrFilename(result, filename) + +proc curLine(p: RstParser): int = p.line + currentTok(p).line proc findRelativeFile(p: RstParser; filename: string): string = - result = p.filename.splitFile.dir / filename - if not existsFile(result): + result = p.s.currFilename.splitFile.dir / filename + if not fileExists(result): result = p.s.findFile(filename) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = - p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, - p.col + p.tok[p.idx].col, msgKind, arg) + p.s.msgHandler(p.s.currFilename, curLine(p), + p.col + currentTok(p).col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = + s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) + +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string; + line, col: int) = + s.msgHandler(s.currFilename, line, col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, filename: string, msgKind: MsgKind, + arg: string) = + s.msgHandler(filename, LineRstInit, ColRstInit, msgKind, arg) + +proc rstMessage*(filenames: RstFileTable, f: MsgHandler, + info: TLineInfo, msgKind: MsgKind, arg: string) = + ## Print warnings using `info`, i.e. in 2nd-pass warnings for + ## footnotes/substitutions/references or from ``rstgen.nim``. + let file = getFilename(filenames, info.fileIndex) + f(file, info.line.int, info.col.int, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = - p.s.msgHandler(p.filename, p.line + line, + p.s.msgHandler(p.s.currFilename, p.line + line, p.col + col, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind) = - p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, - p.col + p.tok[p.idx].col, msgKind, - p.tok[p.idx].symbol) + p.s.msgHandler(p.s.currFilename, curLine(p), + p.col + currentTok(p).col, msgKind, + currentTok(p).symbol) + +# Functions `isPureRst` & `stopOrWarn` address differences between +# Markdown and RST: +# * Markdown always tries to continue working. If it is really impossible +# to parse a markup element, its proc just returns `nil` and parsing +# continues for it as for normal text paragraph. +# The downside is that real mistakes/typos are often silently ignored. +# The same applies to legacy `RstMarkdown` mode for nimforum. +# * RST really signals errors. The downside is that it's more intrusive - +# the user must escape special syntax with \ explicitly. +# +# TODO: we need to apply this strategy to all markup elements eventually. + +func isPureRst(p: RstParser): bool = roSupportMarkdown notin p.s.options +func isRst(p: RstParser): bool = roPreferMarkdown notin p.s.options +func isMd(p: RstParser): bool = roPreferMarkdown in p.s.options +func isMd(s: PRstSharedState): bool = roPreferMarkdown in s.options + +proc stopOrWarn(p: RstParser, errorType: MsgKind, arg: string) = + let realMsgKind = if isPureRst(p): errorType else: mwRstStyle + rstMessage(p, realMsgKind, arg) + +proc stopOrWarn(p: RstParser, errorType: MsgKind, arg: string, line, col: int) = + let realMsgKind = if isPureRst(p): errorType else: mwRstStyle + rstMessage(p, realMsgKind, arg, line, col) proc currInd(p: RstParser): int = result = p.indentStack[high(p.indentStack)] proc pushInd(p: var RstParser, ind: int) = - add(p.indentStack, ind) + p.indentStack.add(ind) proc popInd(p: var RstParser) = - if len(p.indentStack) > 1: setLen(p.indentStack, len(p.indentStack) - 1) + if p.indentStack.len > 1: setLen(p.indentStack, p.indentStack.len - 1) + +# Working with indentation in rst.nim +# ----------------------------------- +# +# Every line break has an associated tkIndent. +# The tokenizer writes back the first column of next non-blank line +# in all preceeding tkIndent tokens to the `ival` field of tkIndent. +# +# RST document is separated into body elements (B.E.), every of which +# has a dedicated handler proc (or block of logic when B.E. is a block quote) +# that should follow the next rule: +# Every B.E. handler proc should finish at tkIndent (newline) +# after its B.E. finishes. +# Then its callers (which is `parseSection` or another B.E. handler) +# check for tkIndent ival (without necessity to advance `p.idx`) +# and decide themselves whether they continue processing or also stop. +# +# An example:: +# +# L RST text fragment indentation +# +--------------------+ +# 1 | | <- (empty line at the start of file) no tokens +# 2 |First paragraph. | <- tkIndent has ival=0, and next tkWord has col=0 +# 3 | | <- tkIndent has ival=0 +# 4 |* bullet item and | <- tkIndent has ival=0, and next tkPunct has col=0 +# 5 | its continuation | <- tkIndent has ival=2, and next tkWord has col=2 +# 6 | | <- tkIndent has ival=4 +# 7 | Block quote | <- tkIndent has ival=4, and next tkWord has col=4 +# 8 | | <- tkIndent has ival=0 +# 9 | | <- tkIndent has ival=0 +# 10|Final paragraph | <- tkIndent has ival=0, and tkWord has col=0 +# +--------------------+ +# C:01234 +# +# Here parser starts with initial `indentStack=[0]` and then calls the +# 1st `parseSection`: +# +# - `parseSection` calls `parseParagraph` and "First paragraph" is parsed +# - bullet list handler is started at reaching ``*`` (L4 C0), it +# starts bullet item logic (L4 C2), which calls `pushInd(p, ind=2)`, +# then calls `parseSection` (2nd call, nested) which parses +# paragraph "bullet list and its continuation" and then starts +# a block quote logic (L7 C4). +# The block quote logic calls calls `pushInd(p, ind=4)` and +# calls `parseSection` again, so a (simplified) sequence of calls now is:: +# +# parseSection -> parseBulletList -> +# parseSection (+block quote logic) -> parseSection +# +# 3rd `parseSection` finishes, block quote logic calls `popInd(p)`, +# it returns to bullet item logic, which sees that next tkIndent has +# ival=0 and stops there since the required indentation for a bullet item +# is 2 and 0<2; the bullet item logic calls `popInd(p)`. +# Then bullet list handler checks that next tkWord (L10 C0) has the +# right indentation but does not have ``*`` so stops at tkIndent (L10). +# - 1st `parseSection` invocation calls `parseParagraph` and the +# "Final paragraph" is parsed. +# +# If a B.E. handler has advanced `p.idx` past tkIndent to check +# whether it should continue its processing or not, and decided not to, +# then this B.E. handler should step back (e.g. do `dec p.idx`). -proc initParser(p: var RstParser, sharedState: PSharedState) = +proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.indentStack = @[0] p.tok = @[] p.idx = 0 - p.filename = "" - p.hasToc = false - p.col = 0 - p.line = 1 + p.col = ColRstInit + p.line = LineRstInit p.s = sharedState proc addNodesAux(n: PRstNode, result: var string) = + if n == nil: + return if n.kind == rnLeaf: - add(result, n.text) + result.add(n.text) else: - for i in countup(0, len(n) - 1): addNodesAux(n.sons[i], result) + for i in 0 ..< n.len: addNodesAux(n.sons[i], result) proc addNodes(n: PRstNode): string = - result = "" - addNodesAux(n, result) + n.addNodesAux(result) + +proc linkName(n: PRstNode): string = + ## Returns a normalized reference name, see: + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + n.addNodes.toLowerAscii proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = template special(s) = if b: - add(r, '-') + r.add('-') b = false - add(r, s) + r.add(s) if n == nil: return if n.kind == rnLeaf: - for i in countup(0, len(n.text) - 1): + for i in 0 ..< n.text.len: case n.text[i] of '0'..'9': if b: - add(r, '-') + r.add('-') b = false - if len(r) == 0: add(r, 'Z') - add(r, n.text[i]) + if r.len == 0: r.add('Z') + r.add(n.text[i]) of 'a'..'z', '\128'..'\255': if b: - add(r, '-') + r.add('-') b = false - add(r, n.text[i]) + r.add(n.text[i]) of 'A'..'Z': if b: - add(r, '-') + r.add('-') b = false - add(r, chr(ord(n.text[i]) - ord('A') + ord('a'))) + r.add(chr(ord(n.text[i]) - ord('A') + ord('a'))) of '$': special "dollar" of '%': special "percent" of '&': special "amp" @@ -420,128 +754,446 @@ proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = of '@': special "at" of '|': special "bar" else: - if len(r) > 0: b = true + if r.len > 0: b = true else: - for i in countup(0, len(n) - 1): rstnodeToRefnameAux(n.sons[i], r, b) + for i in 0 ..< n.len: rstnodeToRefnameAux(n.sons[i], r, b) proc rstnodeToRefname(n: PRstNode): string = - result = "" var b = false rstnodeToRefnameAux(n, result, b) -proc findSub(p: var RstParser, n: PRstNode): int = +proc findSub(s: PRstSharedState, n: PRstNode): int = var key = addNodes(n) # the spec says: if no exact match, try one without case distinction: - for i in countup(0, high(p.s.subs)): - if key == p.s.subs[i].key: + for i in countup(0, high(s.subs)): + if key == s.subs[i].key: return i - for i in countup(0, high(p.s.subs)): - if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: + for i in countup(0, high(s.subs)): + if cmpIgnoreStyle(key, s.subs[i].key) == 0: return i result = -1 +proc lineInfo(p: RstParser, iTok: int): TLineInfo = + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx + +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) +# TODO: we need this simplification because we don't preserve exact starting +# token of currently parsed element: +proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1) + proc setSub(p: var RstParser, key: string, value: PRstNode) = - var length = len(p.s.subs) - for i in countup(0, length - 1): + var length = p.s.subs.len + for i in 0 ..< length: if key == p.s.subs[i].key: p.s.subs[i].value = value return - setLen(p.s.subs, length + 1) - p.s.subs[length].key = key - p.s.subs[length].value = value + p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p))) -proc setRef(p: var RstParser, key: string, value: PRstNode) = - var length = len(p.s.refs) - for i in countup(0, length - 1): +proc setRef(p: var RstParser, key: string, value: PRstNode, + refType: SubstitutionKind) = + var length = p.s.refs.len + for i in 0 ..< length: if key == p.s.refs[i].key: if p.s.refs[i].value.addNodes != value.addNodes: rstMessage(p, mwRedefinitionOfLabel, key) - p.s.refs[i].value = value return - setLen(p.s.refs, length + 1) - p.s.refs[length].key = key - p.s.refs[length].value = value + p.s.refs.add(Substitution(kind: refType, key: key, value: value, + info: prevLineInfo(p))) + +proc findRef(s: PRstSharedState, key: string): seq[Substitution] = + for i in countup(0, high(s.refs)): + if key == s.refs[i].key: + result.add s.refs[i] + +# Ambiguity in links: we don't follow procedure of removing implicit targets +# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets +# Instead we just give explicit links a higher priority than to implicit ones +# and report ambiguities as warnings. Hopefully it is easy to remove +# ambiguities manually. Nim auto-generated links from ``docgen.nim`` +# have lowest priority: 1 (for procs) and below for other symbol types. + +proc refPriority(k: SubstitutionKind): int = + case k + of rstSubstitution: result = 8 + of hyperlinkAlias: result = 7 + of implicitHyperlinkAlias: result = 2 + +proc internalRefPriority(k: RstAnchorKind): int = + case k + of manualDirectiveAnchor: result = 6 + of manualInlineAnchor: result = 5 + of footnoteAnchor: result = 4 + of headlineAnchor: result = 3 + +proc `$`(subst: AnchorSubst): string = # for debug + let s = + case subst.kind + of arInternalRst: "type=" & $subst.anchorType + of arExternalRst: "type=" & $subst.anchorTypeExt + of arNim: "langsym=" & $subst.langSym + result = "(kind=$1, priority=$2, $3)" % [$subst.kind, $subst.priority, s] + +proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, + anchorType: RstAnchorKind) = + ## Associates node `target` (which has field `anchor`) with an + ## alias `name` and updates the corresponding aliases in `p.curAnchors`. + let prio = internalRefPriority(anchorType) + for a in p.curAnchors: + p.s.anchors.mgetOrPut(a.alias, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, target: target, priority: prio, + info: a.info, anchorType: manualDirectiveAnchor)) + if name != "": + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, target: target, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) + p.curAnchors.setLen 0 + +proc addAnchorExtRst(s: var PRstSharedState, key: string, refn: string, + anchorType: RstAnchorKind, info: TLineInfo) = + let name = key.toLowerAscii + let prio = internalRefPriority(anchorType) + s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arExternalRst, refnameExt: refn, priority: prio, + info: info, + anchorTypeExt: anchorType)) + +proc addAnchorNim*(s: var PRstSharedState, external: bool, refn: string, tooltip: string, + langSym: LangSymbol, priority: int, + info: TLineInfo, module: FileIndex) = + ## Adds an anchor `refn`, which follows + ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) + s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arNim, external: external, refname: refn, langSym: langSym, + tooltip: tooltip, priority: priority, + info: info)) + +proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, + info: TLineInfo): + seq[AnchorSubst] = + var langSym: LangSymbol + try: + langSym = toLangSymbol(signature) + except ValueError: # parsing failed, not a Nim symbol + return + let substitutions = s.anchors.getOrDefault(langSym.name, + newSeq[AnchorSubst]()) + if substitutions.len == 0: + return + # logic to select only groups instead of concrete symbols + # with overloads, note that the same symbol can be defined + # in multiple modules and `importdoc`ed: + type GroupKey = tuple[symKind: string, origModule: string] + # map (symKind, file) (like "proc", "os.nim") -> found symbols/groups: + var found: Table[GroupKey, seq[AnchorSubst]] + for subst in substitutions: + if subst.kind == arNim: + if match(subst.langSym, langSym): + let key: GroupKey = (subst.langSym.symKind, getModule(s, subst)) + found.mgetOrPut(key, newSeq[AnchorSubst]()).add subst + for key, sList in found: + if sList.len == 1: + result.add sList[0] + else: # > 1, there are overloads, potential ambiguity in this `symKind` + if langSym.parametersProvided: + # there are non-group signatures, select only them + for s in sList: + if not s.langSym.isGroup: + result.add s + else: # when there are many overloads a link like foo_ points to all + # of them, so selecting the group + var foundGroup = false + for s in sList: + if s.langSym.isGroup: + result.add s + foundGroup = true + break + doAssert(foundGroup, + "docgen has not generated the group for $1 (file $2)" % [ + langSym.name, getModule(s, sList[0]) ]) + +proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): + seq[AnchorSubst] = + let name = linkText.toLowerAscii + let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) + for s in substitutions: + if s.kind in {arInternalRst, arExternalRst}: + result.add s + +proc addFootnoteNumManual(p: var RstParser, num: int) = + ## add manually-numbered footnote + for fnote in p.s.footnotes: + if fnote.number == num: + rstMessage(p, mwRedefinitionOfLabel, $num) + return + p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) + +proc addFootnoteNumAuto(p: var RstParser, label: string) = + ## add auto-numbered footnote. + ## Empty label [#] means it'll be resolved by the occurrence. + if label == "": # simple auto-numbered [#] + p.s.lineFootnoteNum.add lineInfo(p) + p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label)) + else: # auto-numbered with label [#label] + for fnote in p.s.footnotes: + if fnote.label == label: + rstMessage(p, mwRedefinitionOfLabel, label) + return + p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label)) + +proc addFootnoteSymAuto(p: var RstParser) = + p.s.lineFootnoteSym.add lineInfo(p) + p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) + +proc orderFootnotes(s: PRstSharedState) = + ## numerate auto-numbered footnotes taking into account that all + ## manually numbered ones always have preference. + ## Save the result back to `s.footnotes`. + + # Report an error if found any mismatch in number of automatic footnotes + proc listFootnotes(locations: seq[TLineInfo]): string = + var lines: seq[string] + for info in locations: + if s.filenames.len > 1: + let file = getFilename(s.filenames, info.fileIndex) + lines.add file & ":" + else: # no need to add file name here if there is only 1 + lines.add "" + lines[^1].add $info.line + result.add $lines.len & " (lines " & join(lines, ", ") & ")" + if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteNum), + listFootnotes(s.lineFootnoteNumRef)] & + " for auto-numbered footnotes") + if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteSym), + listFootnotes(s.lineFootnoteSymRef)] & + " for auto-symbol footnotes") + + var result: seq[FootnoteSubst] + var manuallyN, autoN, autoSymbol: seq[FootnoteSubst] + for fs in s.footnotes: + if fs.kind == fnManualNumber: manuallyN.add fs + elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs + else: autoSymbol.add fs + + if autoN.len == 0: + result = manuallyN + else: + # fill gaps between manually numbered footnotes in ascending order + manuallyN.sort() # sort by number - its first field + var lst = initSinglyLinkedList[FootnoteSubst]() + for elem in manuallyN: lst.append(elem) + var firstAuto = 0 + if lst.head == nil or lst.head.value.number != 1: + # no manual footnote [1], start numeration from 1 for auto-numbered + lst.prepend (autoN[0].kind, 1, autoN[0].autoNumIdx, -1, autoN[0].label) + firstAuto = 1 + var curNode = lst.head + var nextNode: SinglyLinkedNode[FootnoteSubst] + # go simultaneously through `autoN` and `lst` looking for gaps + for (kind, x, autoNumIdx, y, label) in autoN[firstAuto .. ^1]: + while (nextNode = curNode.next; nextNode != nil): + if nextNode.value.number - curNode.value.number > 1: + # gap found, insert new node `n` between curNode and nextNode: + var n = newSinglyLinkedNode((kind, curNode.value.number + 1, + autoNumIdx, -1, label)) + curNode.next = n + n.next = nextNode + curNode = n + break + else: + curNode = nextNode + if nextNode == nil: # no gap found, just append + lst.append (kind, curNode.value.number + 1, autoNumIdx, -1, label) + curNode = lst.tail + result = lst.toSeq + + # we use ASCII symbols instead of those recommended in RST specification: + const footnoteAutoSymbols = ["*", "^", "+", "=", "~", "$", "@", "%", "&"] + for fs in autoSymbol: + # assignment order: *, **, ***, ^, ^^, ^^^, ... &&&, ****, *****, ... + let i = fs.autoSymIdx - 1 + let symbolNum = (i div 3) mod footnoteAutoSymbols.len + let nSymbols = (1 + i mod 3) + 3 * (i div (3 * footnoteAutoSymbols.len)) + let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols) + result.add((fs.kind, -1, -1, fs.autoSymIdx, label)) + + s.footnotes = result + +proc getFootnoteNum(s: PRstSharedState, label: string): int = + ## get number from label. Must be called after `orderFootnotes`. + result = -1 + for fnote in s.footnotes: + if fnote.label == label: + return fnote.number -proc findRef(p: var RstParser, key: string): PRstNode = - for i in countup(0, high(p.s.refs)): - if key == p.s.refs[i].key: - return p.s.refs[i].value +proc getFootnoteNum(s: PRstSharedState, order: int): int = + ## get number from occurrence. Must be called after `orderFootnotes`. + result = -1 + for fnote in s.footnotes: + if fnote.autoNumIdx == order: + return fnote.number + +proc getAutoSymbol(s: PRstSharedState, order: int): string = + ## get symbol from occurrence of auto-symbol footnote. + result = "???" + for fnote in s.footnotes: + if fnote.autoSymIdx == order: + return fnote.label + +proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = + ## create node and consume the current anchor + result = newRstNode(kind) + if p.curAnchors.len > 0: + result.anchor = p.curAnchors[0].anchor + addAnchorRst(p, "", result, manualDirectiveAnchor) + +template newLeaf(s: string): PRstNode = newRstLeaf(s) proc newLeaf(p: var RstParser): PRstNode = - result = newRstNode(rnLeaf, p.tok[p.idx].symbol) + result = newLeaf(currentTok(p).symbol) + +proc validRefnamePunct(x: string): bool = + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + x.len == 1 and x[0] in {'-', '_', '.', ':', '+'} + +func getRefnameIdx(p: RstParser, startIdx: int): int = + ## Gets last token index of a refname ("word" in RST terminology): + ## + ## reference names are single words consisting of alphanumerics plus + ## isolated (no two adjacent) internal hyphens, underscores, periods, + ## colons and plus signs; no whitespace or other characters are allowed. + ## + ## Refnames are used for: + ## - reference names + ## - role names + ## - directive names + ## - footnote labels + ## + # TODO: use this func in all other relevant places + var j = startIdx + if p.tok[j].kind == tkWord: + inc j + while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and + p.tok[j+1].kind == tkWord: + inc j, 2 + result = j - 1 + +func getRefname(p: RstParser, startIdx: int): (string, int) = + let lastIdx = getRefnameIdx(p, startIdx) + result[1] = lastIdx + for j in startIdx..lastIdx: + result[0].add p.tok[j].symbol proc getReferenceName(p: var RstParser, endStr: string): PRstNode = var res = newRstNode(rnInner) while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWord, tkOther, tkWhite: - add(res, newLeaf(p)) + res.add(newLeaf(p)) of tkPunct: - if p.tok[p.idx].symbol == endStr: - inc(p.idx) + if currentTok(p).symbol == endStr: + inc p.idx break else: - add(res, newLeaf(p)) + res.add(newLeaf(p)) else: rstMessage(p, meExpected, endStr) break - inc(p.idx) + inc p.idx result = res proc untilEol(p: var RstParser): PRstNode = result = newRstNode(rnInner) - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - add(result, newLeaf(p)) - inc(p.idx) + while currentTok(p).kind notin {tkIndent, tkEof}: + result.add(newLeaf(p)) + inc p.idx proc expect(p: var RstParser, tok: string) = - if p.tok[p.idx].symbol == tok: inc(p.idx) + if currentTok(p).symbol == tok: inc p.idx else: rstMessage(p, meExpected, tok) -proc isInlineMarkupEnd(p: RstParser, markup: string): bool = - result = p.tok[p.idx].symbol == markup - if not result: - return # Rule 3: - result = not (p.tok[p.idx - 1].kind in {tkIndent, tkWhite}) - if not result: - return # Rule 4: - result = (p.tok[p.idx + 1].kind in {tkIndent, tkWhite, tkEof}) or - (p.tok[p.idx + 1].symbol[0] in - {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', - '?', '_'}) - if not result: - return # Rule 7: +proc inlineMarkdownEnd(p: RstParser): bool = + result = prevTok(p).kind notin {tkIndent, tkWhite} + ## (For a special case of ` we don't allow spaces surrounding it + ## unlike original Markdown because this behavior confusing/useless) + +proc inlineRstEnd(p: RstParser): bool = + # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules + # Rule 2: + result = prevTok(p).kind notin {tkIndent, tkWhite} + if not result: return + # Rule 7: + result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or + nextTok(p).symbol[0] in + {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'} + +proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool = + if exact: + result = currentTok(p).symbol == markup + else: + result = currentTok(p).symbol.endsWith markup + if (not result) and markup == "``": + # check that escaping may have splitted `` to 2 tokens ` and ` + result = currentTok(p).symbol == "`" and prevTok(p).symbol == "`" + if not result: return + # surroundings check + if markup in ["_", "__"]: + result = inlineRstEnd(p) + else: + if roPreferMarkdown in p.s.options: result = inlineMarkdownEnd(p) + else: result = inlineRstEnd(p) + +proc rstRuleSurround(p: RstParser): bool = + result = true + # Rules 4 & 5: if p.idx > 0: - if (markup != "``") and (p.tok[p.idx - 1].symbol == "\\"): - result = false + var d: char + var c = prevTok(p).symbol[0] + case c + of '\'', '\"': d = c + of '(': d = ')' + of '[': d = ']' + of '{': d = '}' + of '<': d = '>' + else: d = '\0' + if d != '\0': result = nextTok(p).symbol[0] != d + +proc inlineMarkdownStart(p: RstParser): bool = + result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof} + if not result: return + # this rst rule is really nice, let us use it in Markdown mode too. + result = rstRuleSurround(p) + +proc inlineRstStart(p: RstParser): bool = + ## rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules + # Rule 6 + result = p.idx == 0 or prevTok(p).kind in {tkIndent, tkWhite} or + prevTok(p).symbol[0] in {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'} + if not result: return + # Rule 1: + result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof} + if not result: return + result = rstRuleSurround(p) proc isInlineMarkupStart(p: RstParser, markup: string): bool = - var d: char - result = p.tok[p.idx].symbol == markup - if not result: - return # Rule 1: - result = (p.idx == 0) or (p.tok[p.idx - 1].kind in {tkIndent, tkWhite}) or - (p.tok[p.idx - 1].symbol[0] in - {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'}) - if not result: - return # Rule 2: - result = not (p.tok[p.idx + 1].kind in {tkIndent, tkWhite, tkEof}) - if not result: - return # Rule 5 & 7: - if p.idx > 0: - if p.tok[p.idx - 1].symbol == "\\": - result = false - else: - var c = p.tok[p.idx - 1].symbol[0] - case c - of '\'', '\"': d = c - of '(': d = ')' - of '[': d = ']' - of '{': d = '}' - of '<': d = '>' - else: d = '\0' - if d != '\0': result = p.tok[p.idx + 1].symbol[0] != d + if markup != "_`": + result = currentTok(p).symbol == markup + else: # _` is a 2 token case + result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`" + if not result: return + # surroundings check + if markup in ["_", "__", "[", "|"]: + # Note: we require space/punctuation even before [markdown link](...) + result = inlineRstStart(p) + else: + if roPreferMarkdown in p.s.options: result = inlineMarkdownStart(p) + else: result = inlineRstStart(p) proc match(p: RstParser, start: int, expr: string): bool = # regular expressions are: @@ -550,97 +1202,159 @@ proc match(p: RstParser, start: int, expr: string): bool = # ' ' tkWhite # 'a' tkAdornment # 'i' tkIndent + # 'I' tkIndent or tkEof # 'p' tkPunct # 'T' always true # 'E' whitespace, indent or eof - # 'e' tkWord or '#' (for enumeration lists) + # 'e' any enumeration sequence or '#' (for enumeration lists) + # 'x' a..z or '#' (for enumeration lists) + # 'n' 0..9 or '#' (for enumeration lists) var i = 0 var j = start - var last = len(expr) - 1 + var last = expr.len - 1 while i <= last: case expr[i] - of 'w': result = p.tok[j].kind == tkWord + of 'w': + let lastIdx = getRefnameIdx(p, j) + result = lastIdx >= j + if result: j = lastIdx of ' ': result = p.tok[j].kind == tkWhite of 'i': result = p.tok[j].kind == tkIndent + of 'I': result = p.tok[j].kind in {tkIndent, tkEof} of 'p': result = p.tok[j].kind == tkPunct of 'a': result = p.tok[j].kind == tkAdornment of 'o': result = p.tok[j].kind == tkOther of 'T': result = true of 'E': result = p.tok[j].kind in {tkEof, tkWhite, tkIndent} - of 'e': - result = (p.tok[j].kind == tkWord) or (p.tok[j].symbol == "#") + of 'e', 'x', 'n': + result = p.tok[j].kind == tkWord or p.tok[j].symbol == "#" if result: case p.tok[j].symbol[0] - of 'a'..'z', 'A'..'Z', '#': result = len(p.tok[j].symbol) == 1 - of '0'..'9': result = allCharsInSet(p.tok[j].symbol, {'0'..'9'}) + of '#': result = true + of 'a'..'z', 'A'..'Z': + result = expr[i] in {'e', 'x'} and p.tok[j].symbol.len == 1 + of '0'..'9': + result = expr[i] in {'e', 'n'} and + allCharsInSet(p.tok[j].symbol, {'0'..'9'}) else: result = false else: var c = expr[i] var length = 0 - while (i <= last) and (expr[i] == c): - inc(i) - inc(length) - dec(i) - result = (p.tok[j].kind in {tkPunct, tkAdornment}) and - (len(p.tok[j].symbol) == length) and (p.tok[j].symbol[0] == c) + while i <= last and expr[i] == c: + inc i + inc length + dec i + result = p.tok[j].kind in {tkPunct, tkAdornment} and + p.tok[j].symbol.len == length and p.tok[j].symbol[0] == c if not result: return - inc(j) - inc(i) + inc j + inc i result = true -proc fixupEmbeddedRef(n, a, b: PRstNode) = +proc safeProtocol*(linkStr: var string): string = + # Returns link's protocol and, if it's not safe, clears `linkStr` + result = "" + if scanf(linkStr, "$w:", result): + # if it has a protocol at all, ensure that it's not 'javascript:' or worse: + if cmpIgnoreCase(result, "http") == 0 or + cmpIgnoreCase(result, "https") == 0 or + cmpIgnoreCase(result, "ftp") == 0: + discard "it's fine" + else: + linkStr = "" + +proc fixupEmbeddedRef(p: var RstParser, n, a, b: PRstNode): bool = + # Returns `true` if the link belongs to an allowed protocol var sep = - 1 - for i in countdown(len(n) - 2, 0): + for i in countdown(n.len - 2, 0): if n.sons[i].text == "<": sep = i break - var incr = if (sep > 0) and (n.sons[sep - 1].text[0] == ' '): 2 else: 1 - for i in countup(0, sep - incr): add(a, n.sons[i]) - for i in countup(sep + 1, len(n) - 2): add(b, n.sons[i]) + var incr = if sep > 0 and n.sons[sep - 1].text[0] == ' ': 2 else: 1 + for i in countup(0, sep - incr): a.add(n.sons[i]) + var linkStr = "" + for i in countup(sep + 1, n.len - 2): linkStr.add(n.sons[i].addNodes) + if linkStr != "": + let protocol = safeProtocol(linkStr) + result = linkStr != "" + if not result: + rstMessage(p, mwBrokenLink, protocol, + p.tok[p.idx-3].line, p.tok[p.idx-3].col) + b.add newLeaf(linkStr) + +proc whichRole(p: RstParser, sym: string): RstNodeKind = + result = whichRoleAux(sym) + if result == rnUnknownRole: + rstMessage(p, mwUnsupportedLanguage, sym) + +proc toInlineCode(n: PRstNode, language: string): PRstNode = + ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). + result = newRstNode(rnInlineCode, info=n.info) + let args = newRstNode(rnDirArg) + var lang = language + if language == "cpp": lang = "c++" + elif language == "csharp": lang = "c#" + args.add newLeaf(lang) + result.add args + result.add PRstNode(nil) + var lb = newRstNode(rnLiteralBlock) + var s: string + for i in n.sons: + assert i.kind == rnLeaf + s.add i.text + lb.add newLeaf(s) + result.add lb + +proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode = + let newN = newRstNode(rnInner, n.sons) + let newSons = @[newN, newLeaf(roleName)] + result = newRstNode(kind, newSons) proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = - result = n - if isInlineMarkupEnd(p, "_") or isInlineMarkupEnd(p, "__"): - inc(p.idx) + ## Finalizes node `n` that was tentatively determined as interpreted text. + var newKind = n.kind + var newSons = n.sons + + proc finalizeInterpreted(node: PRstNode, newKind: RstNodeKind, + newSons: seq[PRstNode], roleName: string): + PRstNode {.nimcall.} = + # fixes interpreted text (`x` or `y`:role:) to proper internal AST format + if newKind in {rnUnknownRole, rnCodeFragment}: + result = node.toOtherRole(newKind, roleName) + elif newKind == rnInlineCode: + result = node.toInlineCode(language=roleName) + else: + result = newRstNode(newKind, newSons) + + if isInlineMarkupEnd(p, "_", exact=true) or + isInlineMarkupEnd(p, "__", exact=true): + inc p.idx if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">": var a = newRstNode(rnInner) var b = newRstNode(rnInner) - fixupEmbeddedRef(n, a, b) - if len(a) == 0: - result = newRstNode(rnStandaloneHyperlink) - add(result, b) - else: - result = newRstNode(rnHyperlink) - add(result, a) - add(result, b) - setRef(p, rstnodeToRefname(a), b) - elif n.kind == rnInterpretedText: - n.kind = rnRef - else: - result = newRstNode(rnRef) - add(result, n) + if fixupEmbeddedRef(p, n, a, b): + if a.len == 0: # e.g. `<a_named_relative_link>`_ + newKind = rnStandaloneHyperlink + newSons = @[b] + else: # e.g. `link title <http://site>`_ + newKind = rnHyperlink + newSons = @[a, b] + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) + else: # include as plain text, not a link + newKind = rnInner + newSons = n.sons + result = newRstNode(newKind, newSons) + else: # some link that will be resolved in `resolveSubs` + newKind = rnRstRef + result = newRstNode(newKind, sons=newSons, info=n.info) elif match(p, p.idx, ":w:"): # a role: - if p.tok[p.idx + 1].symbol == "idx": - n.kind = rnIdx - elif p.tok[p.idx + 1].symbol == "literal": - n.kind = rnInlineLiteral - elif p.tok[p.idx + 1].symbol == "strong": - n.kind = rnStrongEmphasis - elif p.tok[p.idx + 1].symbol == "emphasis": - n.kind = rnEmphasis - elif (p.tok[p.idx + 1].symbol == "sub") or - (p.tok[p.idx + 1].symbol == "subscript"): - n.kind = rnSub - elif (p.tok[p.idx + 1].symbol == "sup") or - (p.tok[p.idx + 1].symbol == "supscript"): - n.kind = rnSup - else: - result = newRstNode(rnGeneralRole) - n.kind = rnInner - add(result, n) - add(result, newRstNode(rnLeaf, p.tok[p.idx + 1].symbol)) - inc(p.idx, 3) + let (roleName, lastIdx) = getRefname(p, p.idx+1) + newKind = whichRole(p, roleName) + result = n.finalizeInterpreted(newKind, newSons, roleName) + p.idx = lastIdx + 2 + else: + result = n.finalizeInterpreted(p.s.currRoleKind, newSons, p.s.currRole) proc matchVerbatim(p: RstParser, start: int, expr: string): int = result = start @@ -652,7 +1366,7 @@ proc matchVerbatim(p: RstParser, start: int, expr: string): int = if j < expr.len: result = 0 proc parseSmiley(p: var RstParser): PRstNode = - if p.tok[p.idx].symbol[0] notin SmileyStartChars: return + if currentTok(p).symbol[0] notin SmileyStartChars: return for key, val in items(Smilies): let m = matchVerbatim(p, p.idx, key) if m > 0: @@ -661,306 +1375,683 @@ proc parseSmiley(p: var RstParser): PRstNode = result.text = val return -when false: - const - urlChars = {'A'..'Z', 'a'..'z', '0'..'9', ':', '#', '@', '%', '/', ';', - '$', '(', ')', '~', '_', '?', '+', '-', '=', '\\', '.', '&', - '\128'..'\255'} - proc isUrl(p: RstParser, i: int): bool = - result = (p.tok[i+1].symbol == ":") and (p.tok[i+2].symbol == "//") and - (p.tok[i+3].kind == tkWord) and - (p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"]) - -proc parseUrl(p: var RstParser, father: PRstNode) = - #if p.tok[p.idx].symbol[strStart] == '<': - if isUrl(p, p.idx): - var n = newRstNode(rnStandaloneHyperlink) - while true: - case p.tok[p.idx].kind - of tkWord, tkAdornment, tkOther: discard - of tkPunct: - if p.tok[p.idx+1].kind notin {tkWord, tkAdornment, tkOther, tkPunct}: + result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and + p.tok[i+3].kind == tkWord and + p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"] + +proc checkParen(token: Token, parensStack: var seq[char]): bool {.inline.} = + ## Returns `true` iff `token` is a closing parenthesis for some + ## previous opening parenthesis saved in `parensStack`. + ## This is according Markdown balanced parentheses rule + ## (https://spec.commonmark.org/0.29/#link-destination) + ## to allow links like + ## https://en.wikipedia.org/wiki/APL_(programming_language), + ## we use it for RST also. + result = false + if token.kind == tkPunct: + let c = token.symbol[0] + if c in {'(', '[', '{'}: # push + parensStack.add c + elif c in {')', ']', '}'}: # try pop + # a case like ([) inside a link is allowed and [ is also `pop`ed: + for i in countdown(parensStack.len - 1, 0): + if (parensStack[i] == '(' and c == ')' or + parensStack[i] == '[' and c == ']' or + parensStack[i] == '{' and c == '}'): + parensStack.setLen i + result = true break - else: break - add(n, newLeaf(p)) - inc(p.idx) - add(father, n) + +proc parseUrl(p: var RstParser): PRstNode = + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#standalone-hyperlinks + result = newRstNode(rnStandaloneHyperlink) + var lastIdx = p.idx + var closedParenIdx = p.idx - 1 # for balanced parens rule + var parensStack: seq[char] + while p.tok[lastIdx].kind in {tkWord, tkPunct, tkOther}: + let isClosing = checkParen(p.tok[lastIdx], parensStack) + if isClosing: + closedParenIdx = lastIdx + inc lastIdx + dec lastIdx + # standalone URL can not end with punctuation in RST + while lastIdx > closedParenIdx and p.tok[lastIdx].kind == tkPunct and + p.tok[lastIdx].symbol != "/": + dec lastIdx + var s = "" + for i in p.idx .. lastIdx: s.add p.tok[i].symbol + result.add s + p.idx = lastIdx + 1 + +proc parseWordOrRef(p: var RstParser, father: PRstNode) = + ## Parses a normal word or may be a reference or URL. + if nextTok(p).kind != tkPunct: # <- main path, a normal word + father.add newLeaf(p) + inc p.idx + elif isUrl(p, p.idx): # URL http://something + father.add parseUrl(p) else: - var n = newLeaf(p) - inc(p.idx) - if p.tok[p.idx].symbol == "_": n = parsePostfix(p, n) - add(father, n) + # check for reference (probably, long one like some.ref.with.dots_ ) + var saveIdx = p.idx + var reference: PRstNode = nil + inc p.idx + while currentTok(p).kind in {tkWord, tkPunct}: + if currentTok(p).kind == tkPunct: + if isInlineMarkupEnd(p, "_", exact=true): + reference = newRstNode(rnRstRef, info=lineInfo(p, saveIdx)) + break + if not validRefnamePunct(currentTok(p).symbol): + break + inc p.idx + if reference != nil: + for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol) + father.add reference + inc p.idx # skip final _ + else: # 1 normal word + father.add newLeaf(p.tok[saveIdx].symbol) + p.idx = saveIdx + 1 proc parseBackslash(p: var RstParser, father: PRstNode) = - assert(p.tok[p.idx].kind == tkPunct) - if p.tok[p.idx].symbol == "\\\\": - add(father, newRstNode(rnLeaf, "\\")) - inc(p.idx) - elif p.tok[p.idx].symbol == "\\": + assert(currentTok(p).kind == tkPunct) + if currentTok(p).symbol == "\\": # XXX: Unicode? - inc(p.idx) - if p.tok[p.idx].kind != tkWhite: add(father, newLeaf(p)) - if p.tok[p.idx].kind != tkEof: inc(p.idx) + inc p.idx + if currentTok(p).kind != tkWhite: father.add(newLeaf(p)) + if currentTok(p).kind != tkEof: inc p.idx else: - add(father, newLeaf(p)) - inc(p.idx) - -when false: - proc parseAdhoc(p: var RstParser, father: PRstNode, verbatim: bool) = - if not verbatim and isURL(p, p.idx): - var n = newRstNode(rnStandaloneHyperlink) - while true: - case p.tok[p.idx].kind - of tkWord, tkAdornment, tkOther: nil - of tkPunct: - if p.tok[p.idx+1].kind notin {tkWord, tkAdornment, tkOther, tkPunct}: - break - else: break - add(n, newLeaf(p)) - inc(p.idx) - add(father, n) - elif not verbatim and roSupportSmilies in p.sharedState.options: - let n = parseSmiley(p) - if s != nil: - add(father, n) - else: - var n = newLeaf(p) - inc(p.idx) - if p.tok[p.idx].symbol == "_": n = parsePostfix(p, n) - add(father, n) + father.add(newLeaf(p)) + inc p.idx proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, interpretBackslash: bool) = let - line = p.tok[p.idx].line - col = p.tok[p.idx].col + line = currentTok(p).line + col = currentTok(p).col inc p.idx while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkPunct: - if isInlineMarkupEnd(p, postfix): - inc(p.idx) + if isInlineMarkupEnd(p, postfix, exact=false): + let l = currentTok(p).symbol.len + if l > postfix.len: + # handle cases like *emphasis with stars****. (It's valid RST!) + father.add newLeaf(currentTok(p).symbol[0 ..< l - postfix.len]) + elif postfix == "``" and currentTok(p).symbol == "`" and + prevTok(p).symbol == "`": + # handle cases like ``literal\`` - delete ` already added after \ + father.sons.setLen(father.sons.len - 1) + inc p.idx break - elif interpretBackslash: - parseBackslash(p, father) else: - add(father, newLeaf(p)) - inc(p.idx) + if postfix == "`": + if currentTok(p).symbol == "\\": + if nextTok(p).symbol == "\\": + father.add newLeaf("\\") + father.add newLeaf("\\") + inc p.idx, 2 + elif nextTok(p).symbol == "`": # escape ` + father.add newLeaf("`") + inc p.idx, 2 + else: + father.add newLeaf("\\") + inc p.idx + else: + father.add(newLeaf(p)) + inc p.idx + else: + if interpretBackslash: + parseBackslash(p, father) + else: + father.add(newLeaf(p)) + inc p.idx of tkAdornment, tkWord, tkOther: - add(father, newLeaf(p)) - inc(p.idx) + father.add(newLeaf(p)) + inc p.idx of tkIndent: - add(father, newRstNode(rnLeaf, " ")) - inc(p.idx) - if p.tok[p.idx].kind == tkIndent: + father.add newLeaf(" ") + inc p.idx + if currentTok(p).kind == tkIndent: rstMessage(p, meExpected, postfix, line, col) break of tkWhite: - add(father, newRstNode(rnLeaf, " ")) - inc(p.idx) + father.add newLeaf(" ") + inc p.idx else: rstMessage(p, meExpected, postfix, line, col) +proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode = + ## Parses additional (after language string) code block parameters + ## in a format *suggested* in the `CommonMark Spec`_ with handling of `"`. + if currentTok(p).kind == tkIndent: + result = nil + else: + result = newRstNode(rnFieldList) + while currentTok(p).kind notin {tkIndent, tkEof}: + if currentTok(p).kind == tkWhite: + inc p.idx + else: + let field = newRstNode(rnField) + var fieldName = "" + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof} and + currentTok(p).symbol != "=": + fieldName.add currentTok(p).symbol + inc p.idx + field.add(newRstNode(rnFieldName, @[newLeaf(fieldName)])) + if currentTok(p).kind == tkWhite: inc p.idx + let fieldBody = newRstNode(rnFieldBody) + if currentTok(p).symbol == "=": + inc p.idx + if currentTok(p).kind == tkWhite: inc p.idx + var fieldValue = "" + if currentTok(p).symbol == "\"": + while true: + fieldValue.add currentTok(p).symbol + inc p.idx + if currentTok(p).kind == tkEof: + rstMessage(p, meExpected, "\"") + elif currentTok(p).symbol == "\"": + fieldValue.add "\"" + inc p.idx + break + else: + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof}: + fieldValue.add currentTok(p).symbol + inc p.idx + fieldBody.add newLeaf(fieldValue) + field.add(fieldBody) + result.add(field) + +proc mayLoadFile(p: RstParser, result: var PRstNode) = + var filename = strip(getFieldValue(result, "file"), + chars = Whitespace + {'"'}) + if filename != "": + if roSandboxDisabled notin p.s.options: + let tok = p.tok[p.idx-2] + rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col) + var path = p.findRelativeFile(filename) + if path == "": rstMessage(p, meCannotOpenFile, filename) + var n = newRstNode(rnLiteralBlock) + n.add newLeaf(readFile(path)) + result.sons[2] = n + +proc defaultCodeLangNim(p: RstParser, result: var PRstNode) = + # Create a field block if the input block didn't have any. + if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) + assert result.sons[1].kind == rnFieldList + # Hook the extra field and specify the Nim language as value. + var extraNode = newRstNode(rnField, info=lineInfo(p)) + extraNode.add(newRstNode(rnFieldName)) + extraNode.add(newRstNode(rnFieldBody)) + extraNode.sons[0].add newLeaf("default-language") + extraNode.sons[1].add newLeaf("Nim") + result.sons[1].add(extraNode) + proc parseMarkdownCodeblock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnCodeBlock) + result.sons.setLen(3) + let line = curLine(p) + let baseCol = currentTok(p).col + let baseSym = currentTok(p).symbol # usually just ``` + inc p.idx + result.info = lineInfo(p) var args = newRstNode(rnDirArg) - if p.tok[p.idx].kind == tkWord: - add(args, newLeaf(p)) - inc(p.idx) + if currentTok(p).kind == tkWord: + args.add(newLeaf(p)) + inc p.idx + result.sons[1] = parseMarkdownCodeblockFields(p) + mayLoadFile(p, result) else: args = nil - var n = newRstNode(rnLeaf, "") + var n = newLeaf("") + var isFirstLine = true while true: - case p.tok[p.idx].kind - of tkEof: - rstMessage(p, meExpected, "```") + if currentTok(p).kind == tkEof: + rstMessage(p, meMissingClosing, + "$1 (started at line $2)" % [baseSym, $line]) break - of tkPunct: - if p.tok[p.idx].symbol == "```": - inc(p.idx) - break - else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) + elif nextTok(p).kind in {tkPunct, tkAdornment} and + nextTok(p).symbol[0] == baseSym[0] and + nextTok(p).symbol.len >= baseSym.len: + inc p.idx, 2 + break + elif currentTok(p).kind == tkIndent: + if not isFirstLine: + n.text.add "\n" + if currentTok(p).ival > baseCol: + n.text.add " ".repeat(currentTok(p).ival - baseCol) + elif currentTok(p).ival < baseCol: + rstMessage(p, mwRstStyle, + "unexpected de-indentation in Markdown code block") + inc p.idx else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) - var lb = newRstNode(rnLiteralBlock) - add(lb, n) - result = newRstNode(rnCodeBlock) - add(result, args) - add(result, PRstNode(nil)) - add(result, lb) + n.text.add(currentTok(p).symbol) + inc p.idx + isFirstLine = false + result.sons[0] = args + if result.sons[2] == nil: + var lb = newRstNode(rnLiteralBlock) + lb.add(n) + result.sons[2] = lb + if result.sons[0].isNil and roNimFile in p.s.options: + defaultCodeLangNim(p, result) proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = - result = true - var desc, link = "" + # Parses Markdown link. If it's Pandoc auto-link then its second + # son (target) will be in tokenized format (rnInner with leafs). + var desc = newRstNode(rnInner) var i = p.idx + var parensStack: seq[char] template parse(endToken, dest) = + parensStack.setLen 0 inc i # skip begin token while true: - if p.tok[i].kind in {tkEof, tkIndent}: return false - if p.tok[i].symbol == endToken: break - dest.add p.tok[i].symbol + if p.tok[i].kind == tkEof: return false + if p.tok[i].kind == tkIndent and p.tok[i+1].kind == tkIndent: + return false + let isClosing = checkParen(p.tok[i], parensStack) + if p.tok[i].symbol == endToken and not isClosing: + break + let symbol = if p.tok[i].kind == tkIndent: " " else: p.tok[i].symbol + when dest is string: dest.add symbol + else: dest.add newLeaf(symbol) inc i inc i # skip end token parse("]", desc) - if p.tok[i].symbol != "(": return false - parse(")", link) - let child = newRstNode(rnHyperlink) - child.add desc - child.add link - # only commit if we detected no syntax error: - father.add child + if p.tok[i].symbol == "(": + var link = "" + let linkIdx = i + 1 + parse(")", link) + # only commit if we detected no syntax error: + let protocol = safeProtocol(link) + if link == "": + result = false + rstMessage(p, mwBrokenLink, protocol, + p.tok[linkIdx].line, p.tok[linkIdx].col) + else: + let child = newRstNode(rnHyperlink) + child.add newLeaf(desc.addNodes) + child.add link + father.add child + p.idx = i + result = true + elif roPreferMarkdown in p.s.options: + # Use Pandoc's implicit_header_references extension + var n = newRstNode(rnPandocRef) + if p.tok[i].symbol == "[": + var link = newRstNode(rnInner) + let targetIdx = i + 1 + parse("]", link) + n.add desc + if link.len != 0: # [description][target] + n.add link + n.info = lineInfo(p, targetIdx) + else: # [description=target][] + n.add desc + n.info = lineInfo(p, p.idx + 1) + else: # [description=target] + n.add desc + n.add desc # target is the same as description + n.info = lineInfo(p, p.idx + 1) + father.add n + p.idx = i + result = true + else: + result = false + +proc getRstFootnoteType(label: PRstNode): (FootnoteType, int) = + if label.sons.len >= 1 and label.sons[0].kind == rnLeaf and + label.sons[0].text == "#": + if label.sons.len == 1: + result = (fnAutoNumber, -1) + else: + result = (fnAutoNumberLabel, -1) + elif label.len == 1 and label.sons[0].kind == rnLeaf and + label.sons[0].text == "*": + result = (fnAutoSymbol, -1) + elif label.len == 1 and label.sons[0].kind == rnLeaf: + try: + result = (fnManualNumber, parseInt(label.sons[0].text)) + except ValueError: + result = (fnCitation, -1) + else: + result = (fnCitation, -1) + +proc getMdFootnoteType(label: PRstNode): (FootnoteType, int) = + try: + result = (fnManualNumber, parseInt(label.sons[0].text)) + except ValueError: + result = (fnAutoNumberLabel, -1) + +proc getFootnoteType(s: PRstSharedState, label: PRstNode): (FootnoteType, int) = + ## Returns footnote/citation type and manual number (if present). + if isMd(s): getMdFootnoteType(label) + else: getRstFootnoteType(label) + +proc parseRstFootnoteName(p: var RstParser, reference: bool): PRstNode = + ## parse footnote/citation label. Precondition: start at `[`. + ## Label text should be valid ref. name symbol, otherwise nil is returned. + var i = p.idx + 1 + result = newRstNode(rnInner) + while true: + if p.tok[i].kind in {tkEof, tkIndent, tkWhite}: + return nil + if p.tok[i].kind == tkPunct: + case p.tok[i].symbol: + of "]": + if i > p.idx + 1 and (not reference or (p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol == "_")): + inc i # skip ] + if reference: inc i # skip _ + break # to succeed, it's a footnote/citation indeed + else: + return nil + of "#": + if i != p.idx + 1: + return nil + of "*": + if i != p.idx + 1 and p.tok[i].kind != tkPunct and p.tok[i+1].symbol != "]": + return nil + else: + if not validRefnamePunct(p.tok[i].symbol): + return nil + result.add newLeaf(p.tok[i].symbol) + inc i p.idx = i - result = true + +proc isMdFootnoteName(p: RstParser, reference: bool): bool = + ## Pandoc Markdown footnote extension. + let j = p.idx + result = p.tok[j].symbol == "[" and p.tok[j+1].symbol == "^" and + p.tok[j+2].kind == tkWord + +proc parseMdFootnoteName(p: var RstParser, reference: bool): PRstNode = + if isMdFootnoteName(p, reference): + result = newRstNode(rnInner) + var j = p.idx + 2 + while p.tok[j].kind in {tkWord, tkOther} or + validRefnamePunct(p.tok[j].symbol): + result.add newLeaf(p.tok[j].symbol) + inc j + if j == p.idx + 2: + return nil + if p.tok[j].symbol == "]": + if reference: + p.idx = j + 1 # skip ] + else: + if p.tok[j+1].symbol == ":": + p.idx = j + 2 # skip ]: + else: + result = nil + else: + result = nil + else: + result = nil + +proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode = + if isMd(p): parseMdFootnoteName(p, reference) + else: + if isInlineMarkupStart(p, "["): parseRstFootnoteName(p, reference) + else: nil + +proc isMarkdownCodeBlock(p: RstParser, idx: int): bool = + let tok = p.tok[idx] + template allowedSymbol: bool = + (tok.symbol[0] == '`' or + roPreferMarkdown in p.s.options and tok.symbol[0] == '~') + result = (roSupportMarkdown in p.s.options and + tok.kind in {tkPunct, tkAdornment} and + allowedSymbol and + tok.symbol.len >= 3) + +proc isMarkdownCodeBlock(p: RstParser): bool = + isMarkdownCodeBlock(p, p.idx) proc parseInline(p: var RstParser, father: PRstNode) = - case p.tok[p.idx].kind + var n: PRstNode # to be used in `if` condition + let saveIdx = p.idx + case currentTok(p).kind of tkPunct: if isInlineMarkupStart(p, "***"): var n = newRstNode(rnTripleEmphasis) parseUntil(p, n, "***", true) - add(father, n) + father.add(n) elif isInlineMarkupStart(p, "**"): var n = newRstNode(rnStrongEmphasis) parseUntil(p, n, "**", true) - add(father, n) + father.add(n) elif isInlineMarkupStart(p, "*"): var n = newRstNode(rnEmphasis) parseUntil(p, n, "*", true) - add(father, n) - elif roSupportMarkdown in p.s.options and p.tok[p.idx].symbol == "```": - inc(p.idx) - add(father, parseMarkdownCodeblock(p)) + father.add(n) + elif isInlineMarkupStart(p, "_`"): + var n = newRstNode(rnInlineTarget) + inc p.idx + parseUntil(p, n, "`", false) + n.anchor = rstnodeToRefname(n) + addAnchorRst(p, name = linkName(n), target = n, + anchorType=manualInlineAnchor) + father.add(n) + elif isMarkdownCodeBlock(p): + father.add(parseMarkdownCodeblock(p)) elif isInlineMarkupStart(p, "``"): var n = newRstNode(rnInlineLiteral) parseUntil(p, n, "``", false) - add(father, n) + father.add(n) + elif match(p, p.idx, ":w:") and + (var lastIdx = getRefnameIdx(p, p.idx + 1); + p.tok[lastIdx+2].symbol == "`"): + let (roleName, _) = getRefname(p, p.idx+1) + let k = whichRole(p, roleName) + var n = newRstNode(k) + p.idx = lastIdx + 2 + if k == rnInlineCode: + n = n.toInlineCode(language=roleName) + parseUntil(p, n, "`", false) # bug #17260 + if k in {rnUnknownRole, rnCodeFragment}: + n = n.toOtherRole(k, roleName) + father.add(n) elif isInlineMarkupStart(p, "`"): - var n = newRstNode(rnInterpretedText) - parseUntil(p, n, "`", true) + var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1)) + parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) - add(father, n) + father.add(n) elif isInlineMarkupStart(p, "|"): - var n = newRstNode(rnSubstitutionReferences) + var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1)) parseUntil(p, n, "|", false) - add(father, n) + father.add(n) + elif currentTok(p).symbol == "[" and nextTok(p).symbol != "[" and + (n = parseFootnoteName(p, reference=true); n != nil): + var nn = newRstNode(rnFootnoteRef) + nn.info = lineInfo(p, saveIdx+1) + nn.add n + let (fnType, _) = getFootnoteType(p.s, n) + case fnType + of fnAutoSymbol: + p.s.lineFootnoteSymRef.add lineInfo(p) + of fnAutoNumber: + p.s.lineFootnoteNumRef.add lineInfo(p) + else: discard + father.add(nn) elif roSupportMarkdown in p.s.options and - p.tok[p.idx].symbol == "[" and p.tok[p.idx+1].symbol != "[" and + currentTok(p).symbol == "[" and nextTok(p).symbol != "[" and parseMarkdownLink(p, father): discard "parseMarkdownLink already processed it" else: if roSupportSmilies in p.s.options: let n = parseSmiley(p) if n != nil: - add(father, n) + father.add(n) return parseBackslash(p, father) of tkWord: if roSupportSmilies in p.s.options: let n = parseSmiley(p) if n != nil: - add(father, n) + father.add(n) return - parseUrl(p, father) + parseWordOrRef(p, father) of tkAdornment, tkOther, tkWhite: + if isMarkdownCodeBlock(p): + father.add(parseMarkdownCodeblock(p)) + return if roSupportSmilies in p.s.options: let n = parseSmiley(p) if n != nil: - add(father, n) + father.add(n) return - add(father, newLeaf(p)) - inc(p.idx) + father.add(newLeaf(p)) + inc p.idx else: discard proc getDirective(p: var RstParser): string = - if p.tok[p.idx].kind == tkWhite and p.tok[p.idx+1].kind == tkWord: - var j = p.idx - inc(p.idx) - result = p.tok[p.idx].symbol - inc(p.idx) - while p.tok[p.idx].kind in {tkWord, tkPunct, tkAdornment, tkOther}: - if p.tok[p.idx].symbol == "::": break - add(result, p.tok[p.idx].symbol) - inc(p.idx) - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - if p.tok[p.idx].symbol == "::": - inc(p.idx) - if (p.tok[p.idx].kind == tkWhite): inc(p.idx) - else: - p.idx = j # set back - result = "" # error - else: - result = "" - -proc parseComment(p: var RstParser): PRstNode = - case p.tok[p.idx].kind - of tkIndent, tkEof: - if p.tok[p.idx].kind != tkEof and p.tok[p.idx + 1].kind == tkIndent: - inc(p.idx) # empty comment - else: - var indent = p.tok[p.idx].ival - while true: - case p.tok[p.idx].kind - of tkEof: - break - of tkIndent: - if (p.tok[p.idx].ival < indent): break - else: - discard - inc(p.idx) + result = "" + if currentTok(p).kind == tkWhite: + let (name, lastIdx) = getRefname(p, p.idx + 1) + let afterIdx = lastIdx + 1 + if name.len > 0: + if p.tok[afterIdx].symbol == "::": + result = name + p.idx = afterIdx + 1 + if currentTok(p).kind == tkWhite: + inc p.idx + elif currentTok(p).kind != tkIndent: + rstMessage(p, mwRstStyle, + "whitespace or newline expected after directive " & name) + result = result.toLowerAscii() + elif p.tok[afterIdx].symbol == ":": + rstMessage(p, mwRstStyle, + "double colon :: may be missing at end of '" & name & "'", + p.tok[afterIdx].line, p.tok[afterIdx].col) + elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':': + rstMessage(p, mwRstStyle, + "too many colons for a directive (should be ::)", + p.tok[afterIdx].line, p.tok[afterIdx].col) + +proc parseComment(p: var RstParser, col: int): PRstNode = + if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: + inc p.idx # empty comment else: - while p.tok[p.idx].kind notin {tkIndent, tkEof}: inc(p.idx) + while currentTok(p).kind != tkEof: + if currentTok(p).kind == tkIndent and currentTok(p).ival > col or + currentTok(p).kind != tkIndent and currentTok(p).col > col: + inc p.idx + else: + break result = nil -type - DirKind = enum # must be ordered alphabetically! - dkNone, dkAuthor, dkAuthors, dkCode, dkCodeBlock, dkContainer, dkContents, - dkFigure, dkImage, dkInclude, dkIndex, dkRaw, dkTitle - -const - DirIds: array[0..12, string] = ["", "author", "authors", "code", - "code-block", "container", "contents", "figure", "image", "include", - "index", "raw", "title"] - -proc getDirKind(s: string): DirKind = - let i = find(DirIds, s) - if i >= 0: result = DirKind(i) - else: result = dkNone - proc parseLine(p: var RstParser, father: PRstNode) = while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWhite, tkWord, tkOther, tkPunct: parseInline(p, father) else: break proc parseUntilNewline(p: var RstParser, father: PRstNode) = while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWhite, tkWord, tkAdornment, tkOther, tkPunct: parseInline(p, father) of tkEof, tkIndent: break proc parseSection(p: var RstParser, result: PRstNode) {.gcsafe.} + +proc tokenAfterNewline(p: RstParser, start: int): int = + result = start + while true: + case p.tok[result].kind + of tkEof: + break + of tkIndent: + inc result + break + else: inc result + +proc tokenAfterNewline(p: RstParser): int {.inline.} = + result = tokenAfterNewline(p, p.idx) + +proc getWrappableIndent(p: RstParser): int = + ## Gets baseline indentation for bodies of field lists and directives. + ## Handles situations like this (with possible de-indent in [case.3]):: + ## + ## :field: definition [case.1] + ## + ## currInd currentTok(p).col + ## | | + ## v v + ## + ## .. Note:: defItem: [case.2] + ## definition + ## + ## ^ + ## | + ## nextIndent + ## + ## .. Note:: - point1 [case.3] + ## - point 2 + ## + ## ^ + ## | + ## nextIndent + if currentTok(p).kind == tkIndent: + result = currentTok(p).ival + else: + var nextIndent = p.tok[tokenAfterNewline(p)-1].ival + if nextIndent <= currInd(p): # parse only this line [case.1] + result = currentTok(p).col + elif nextIndent >= currentTok(p).col: # may be a definition list [case.2] + result = currentTok(p).col + else: + result = nextIndent # allow parsing next lines [case.3] + +proc getMdBlockIndent(p: RstParser): int = + ## Markdown version of `getWrappableIndent`. + if currentTok(p).kind == tkIndent: + result = currentTok(p).ival + else: + var nextIndent = p.tok[tokenAfterNewline(p)-1].ival + # TODO: Markdown-compliant definition should allow nextIndent == currInd(p): + if nextIndent <= currInd(p): # parse only this line + result = currentTok(p).col + else: + result = nextIndent # allow parsing next lines [case.3] + +proc indFollows(p: RstParser): bool = + result = currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) + +proc parseBlockContent(p: var RstParser, father: var PRstNode, + contentParser: SectionParser): bool {.gcsafe.} = + ## parse the final content part of explicit markup blocks (directives, + ## footnotes, etc). Returns true if succeeded. + if currentTok(p).kind != tkIndent or indFollows(p): + let blockIndent = getWrappableIndent(p) + pushInd(p, blockIndent) + let content = contentParser(p) + popInd(p) + father.add content + result = true + +proc parseSectionWrapper(p: var RstParser): PRstNode = + result = newRstNode(rnInner) + parseSection(p, result) + while result.kind == rnInner and result.len == 1: + result = result.sons[0] + proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. - result = newRstNode(rnField) - var col = p.tok[p.idx].col + result = newRstNode(rnField, info=lineInfo(p)) + var col = currentTok(p).col var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) var fieldbody = newRstNode(rnFieldBody) - if p.tok[p.idx].kind != tkIndent: parseLine(p, fieldbody) - if p.tok[p.idx].kind == tkIndent: - var indent = p.tok[p.idx].ival - if indent > col: - pushInd(p, indent) - parseSection(p, fieldbody) - popInd(p) - add(result, fieldname) - add(result, fieldbody) + if currentTok(p).kind == tkWhite: inc p.idx + let indent = getWrappableIndent(p) + if indent > col: + pushInd(p, indent) + parseSection(p, fieldbody) + popInd(p) + result.add(fieldname) + result.add(fieldbody) proc parseFields(p: var RstParser): PRstNode = ## Parses fields for a section or directive block. @@ -969,16 +2060,16 @@ proc parseFields(p: var RstParser): PRstNode = ## otherwise it will return a node of rnFieldList type with children. result = nil var atStart = p.idx == 0 and p.tok[0].symbol == ":" - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx + 1].symbol == ":") or + if currentTok(p).kind == tkIndent and nextTok(p).symbol == ":" or atStart: - var col = if atStart: p.tok[p.idx].col else: p.tok[p.idx].ival - result = newRstNode(rnFieldList) - if not atStart: inc(p.idx) + var col = if atStart: currentTok(p).col else: currentTok(p).ival + result = newRstNodeA(p, rnFieldList) + if not atStart: inc p.idx while true: - add(result, parseField(p)) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == ":"): - inc(p.idx) + result.add(parseField(p)) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).symbol == ":": + inc p.idx else: break @@ -995,13 +2086,12 @@ proc getFieldValue*(n: PRstNode): string = result = addNodes(n.sons[1]).strip proc getFieldValue(n: PRstNode, fieldname: string): string = - result = "" if n.sons[1] == nil: return - if (n.sons[1].kind != rnFieldList): + if n.sons[1].kind != rnFieldList: #InternalError("getFieldValue (2): " & $n.sons[1].kind) # We don't like internal errors here anymore as that would break the forum! return - for i in countup(0, len(n.sons[1]) - 1): + for i in 0 ..< n.sons[1].len: var f = n.sons[1].sons[i] if cmpIgnoreStyle(addNodes(f.sons[0]), fieldname) == 0: result = addNodes(f.sons[1]) @@ -1014,64 +2104,215 @@ proc getArgument(n: PRstNode): string = proc parseDotDot(p: var RstParser): PRstNode {.gcsafe.} proc parseLiteralBlock(p: var RstParser): PRstNode = - result = newRstNode(rnLiteralBlock) - var n = newRstNode(rnLeaf, "") - if p.tok[p.idx].kind == tkIndent: - var indent = p.tok[p.idx].ival - inc(p.idx) + result = newRstNodeA(p, rnLiteralBlock) + var n = newLeaf("") + if currentTok(p).kind == tkIndent: + var indent = currentTok(p).ival + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkEof: break of tkIndent: - if (p.tok[p.idx].ival < indent): + if currentTok(p).ival < indent: break else: - add(n.text, "\n") - add(n.text, spaces(p.tok[p.idx].ival - indent)) - inc(p.idx) + n.text.add("\n") + n.text.add(spaces(currentTok(p).ival - indent)) + inc p.idx else: - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) + n.text.add(currentTok(p).symbol) + inc p.idx else: - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - add(n.text, p.tok[p.idx].symbol) - inc(p.idx) - add(result, n) - -proc getLevel(map: var LevelMap, lvl: var int, c: char): int = - if map[c] == 0: - inc(lvl) - map[c] = lvl - result = map[c] - -proc tokenAfterNewline(p: RstParser): int = - result = p.idx - while true: - case p.tok[result].kind - of tkEof: - break - of tkIndent: - inc(result) - break - else: inc(result) + while currentTok(p).kind notin {tkIndent, tkEof}: + n.text.add(currentTok(p).symbol) + inc p.idx + result.add(n) + +proc parseQuotedLiteralBlock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnLiteralBlock) + var n = newLeaf("") + if currentTok(p).kind == tkIndent: + var indent = currInd(p) + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines + var quoteSym = currentTok(p).symbol[0] + while true: + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if currentTok(p).ival < indent: + break + elif currentTok(p).ival == indent: + if nextTok(p).kind == tkPunct and nextTok(p).symbol[0] == quoteSym: + n.text.add("\n") + inc p.idx + elif nextTok(p).kind == tkIndent: + break + else: + rstMessage(p, mwRstStyle, "no newline after quoted literal block") + break + else: + rstMessage(p, mwRstStyle, + "unexpected indentation in quoted literal block") + break + else: + n.text.add(currentTok(p).symbol) + inc p.idx + result.add(n) + +proc parseRstLiteralBlock(p: var RstParser, kind: LiteralBlockKind): PRstNode = + if kind == lbIndentedLiteralBlock: + result = parseLiteralBlock(p) + else: + result = parseQuotedLiteralBlock(p) + +proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = + ## Returns (preliminary) heading level corresponding to `c` and + ## `hasOverline`. If level does not exist, add it first. + for i, hType in p.s.hLevels: + if hType.symbol == c and hType.hasOverline == hasOverline: + p.s.hLevels[i].line = curLine(p) + p.s.hLevels[i].hasPeers = true + return i + p.s.hLevels.add LevelInfo(symbol: c, hasOverline: hasOverline, + line: curLine(p), hasPeers: false) + result = p.s.hLevels.len - 1 + +proc countTitles(s: PRstSharedState, n: PRstNode) = + ## Fill `s.hTitleCnt` + if n == nil: return + for node in n.sons: + if node != nil: + if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}: + break + if node.kind == rnOverline: + if s.hLevels[s.hTitleCnt].hasPeers: + break + inc s.hTitleCnt + if s.hTitleCnt >= 2: + break + +proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = + ## check that underline/overline length is enough for the heading. + ## No support for Unicode. + if p.tok[adornmentIdx].symbol in ["::", "..", "|"]: + return false + if isMarkdownCodeBlock(p, adornmentIdx): + return false + var headlineLen = 0 + var failure = "" + if p.idx < adornmentIdx: # check for underline + if p.idx > 0: + headlineLen = currentTok(p).col - p.tok[adornmentIdx].col + if headlineLen > 0: + rstMessage(p, mwRstStyle, "indentation of heading text allowed" & + " only for overline titles") + for i in p.idx ..< adornmentIdx-1: # adornmentIdx-1 is a linebreak + headlineLen += p.tok[i].symbol.len + result = p.tok[adornmentIdx].symbol.len >= headlineLen and headlineLen != 0 + if not result: + failure = "(underline '" & p.tok[adornmentIdx].symbol & "' is too short)" + else: # p.idx == adornmentIdx, at overline. Check overline and underline + var i = p.idx + 2 + headlineLen = p.tok[i].col - p.tok[adornmentIdx].col + while p.tok[i].kind notin {tkEof, tkIndent}: + headlineLen += p.tok[i].symbol.len + inc i + if p.tok[i].kind == tkIndent and + p.tok[i+1].kind == tkAdornment and + p.tok[i+1].symbol[0] == p.tok[adornmentIdx].symbol[0]: + result = p.tok[adornmentIdx].symbol.len >= headlineLen and + headlineLen != 0 + if result: + result = p.tok[i+1].symbol == p.tok[adornmentIdx].symbol + if not result: + failure = "(underline '" & p.tok[i+1].symbol & "' does not match " & + "overline '" & p.tok[adornmentIdx].symbol & "')" + else: + failure = "(overline '" & p.tok[adornmentIdx].symbol & "' is too short)" + else: # it's not overline/underline section, not reporting error + return false + if not result: + rstMessage(p, meNewSectionExpected, failure) proc isLineBlock(p: RstParser): bool = var j = tokenAfterNewline(p) - result = (p.tok[p.idx].col == p.tok[j].col) and (p.tok[j].symbol == "|") or - (p.tok[j].col > p.tok[p.idx].col) + result = currentTok(p).col == p.tok[j].col and p.tok[j].symbol == "|" or + p.tok[j].col > currentTok(p).col or + p.tok[j].symbol == "\n" + +proc isMarkdownBlockQuote(p: RstParser): bool = + result = currentTok(p).symbol[0] == '>' + +proc whichRstLiteralBlock(p: RstParser): LiteralBlockKind = + ## Checks that the following tokens are either Indented Literal Block or + ## Quoted Literal Block (which is not quite the same as Markdown quote block). + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks + if currentTok(p).symbol == "::" and nextTok(p).kind == tkIndent: + if currInd(p) > nextTok(p).ival: + result = lbNone + if currInd(p) < nextTok(p).ival: + result = lbIndentedLiteralBlock + elif currInd(p) == nextTok(p).ival: + var i = p.idx + 1 + while p.tok[i].kind == tkIndent: inc i + const validQuotingCharacters = { + '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', + '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', + '_', '`', '{', '|', '}', '~'} + if p.tok[i].kind in {tkPunct, tkAdornment} and + p.tok[i].symbol[0] in validQuotingCharacters: + result = lbQuotedLiteralBlock + else: + result = lbNone + else: + result = lbNone proc predNL(p: RstParser): bool = result = true if p.idx > 0: - result = p.tok[p.idx-1].kind == tkIndent and - p.tok[p.idx-1].ival == currInd(p) + result = prevTok(p).kind == tkIndent and + prevTok(p).ival == currInd(p) proc isDefList(p: RstParser): bool = var j = tokenAfterNewline(p) - result = (p.tok[p.idx].col < p.tok[j].col) and - (p.tok[j].kind in {tkWord, tkOther, tkPunct}) and - (p.tok[j - 2].symbol != "::") + result = currentTok(p).col < p.tok[j].col and + p.tok[j].kind in {tkWord, tkOther, tkPunct} and + p.tok[j - 2].symbol != "::" + +proc `$`(t: Token): string = # for debugging only + result = "(" & $t.kind & " line=" & $t.line & " col=" & $t.col + if t.kind == tkIndent: result = result & " ival=" & $t.ival & ")" + else: result = result & " symbol=" & t.symbol & ")" + +proc skipNewlines(p: RstParser, j: int): int = + result = j + while p.tok[result].kind != tkEof and p.tok[result].kind == tkIndent: + inc result # skip blank lines + +proc skipNewlines(p: var RstParser) = + p.idx = skipNewlines(p, p.idx) + +const maxMdRelInd = 3 ## In Markdown: maximum indentation that does not yet + ## make the indented block a code + +proc isMdRelInd(outerInd, nestedInd: int): bool = + result = outerInd <= nestedInd and nestedInd <= outerInd + maxMdRelInd + +proc isMdDefBody(p: RstParser, j: int, termCol: int): bool = + let defCol = p.tok[j].col + result = p.tok[j].symbol == ":" and + isMdRelInd(termCol, defCol) and + p.tok[j+1].kind == tkWhite and + p.tok[j+2].kind in {tkWord, tkOther, tkPunct} + +proc isMdDefListItem(p: RstParser, idx: int): bool = + var j = tokenAfterNewline(p, idx) + j = skipNewlines(p, j) + let termCol = p.tok[j].col + result = isMdRelInd(currInd(p), termCol) and + isMdDefBody(p, j, termCol) proc isOptionList(p: RstParser): bool = result = match(p, p.idx, "-w") or match(p, p.idx, "--w") or @@ -1085,99 +2326,240 @@ proc isMarkdownHeadlinePattern(s: string): bool = proc isMarkdownHeadline(p: RstParser): bool = if roSupportMarkdown in p.s.options: - if isMarkdownHeadlinePattern(p.tok[p.idx].symbol) and p.tok[p.idx+1].kind == tkWhite: + if isMarkdownHeadlinePattern(currentTok(p).symbol) and nextTok(p).kind == tkWhite: if p.tok[p.idx+2].kind in {tkWord, tkOther, tkPunct}: result = true +proc findPipe(p: RstParser, start: int): bool = + var i = start + while true: + if p.tok[i].symbol == "|": return true + if p.tok[i].kind in {tkIndent, tkEof}: return false + inc i + proc whichSection(p: RstParser): RstNodeKind = - case p.tok[p.idx].kind + if currentTok(p).kind in {tkAdornment, tkPunct}: + # for punctuation sequences that can be both tkAdornment and tkPunct + if isMarkdownCodeBlock(p): + return rnCodeBlock + elif isRst(p) and currentTok(p).symbol == "::": + return rnLiteralBlock + elif currentTok(p).symbol == ".." and + nextTok(p).kind in {tkWhite, tkIndent}: + return rnDirective + case currentTok(p).kind of tkAdornment: - if match(p, p.idx + 1, "ii"): result = rnTransition + if match(p, p.idx + 1, "iI") and currentTok(p).symbol.len >= 4: + result = rnTransition + elif match(p, p.idx, "+a+"): + result = rnGridTable + rstMessage(p, meGridTableNotImplemented) elif match(p, p.idx + 1, " a"): result = rnTable - elif match(p, p.idx + 1, "i"): result = rnOverline - elif isMarkdownHeadline(p): - result = rnHeadline + elif currentTok(p).symbol == "|" and isLineBlock(p): + result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote + elif (match(p, p.idx + 1, "i") and not match(p, p.idx + 2, "I")) and + isAdornmentHeadline(p, p.idx): + result = rnOverline else: - result = rnLeaf + result = rnParagraph of tkPunct: if isMarkdownHeadline(p): + result = rnMarkdownHeadline + elif roSupportMarkdown in p.s.options and predNL(p) and + match(p, p.idx, "| w") and findPipe(p, p.idx+3): + result = rnMarkdownTable + elif isMd(p) and isMdFootnoteName(p, reference=false): + result = rnFootnote + elif currentTok(p).symbol == "|" and isLineBlock(p): + result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote + elif match(p, tokenAfterNewline(p), "aI") and + isAdornmentHeadline(p, tokenAfterNewline(p)): result = rnHeadline - elif match(p, tokenAfterNewline(p), "ai"): - result = rnHeadline - elif p.tok[p.idx].symbol == "::": - result = rnLiteralBlock - elif predNL(p) and - ((p.tok[p.idx].symbol == "+") or (p.tok[p.idx].symbol == "*") or - (p.tok[p.idx].symbol == "-")) and (p.tok[p.idx + 1].kind == tkWhite): + elif currentTok(p).symbol in ["+", "*", "-"] and nextTok(p).kind == tkWhite: result = rnBulletList - elif (p.tok[p.idx].symbol == "|") and isLineBlock(p): - result = rnLineBlock - elif (p.tok[p.idx].symbol == "..") and predNL(p): - result = rnDirective - elif match(p, p.idx, ":w:") and predNL(p): - # (p.tok[p.idx].symbol == ":") + elif match(p, p.idx, ":w:E"): + # (currentTok(p).symbol == ":") result = rnFieldList - elif match(p, p.idx, "(e) ") or match(p, p.idx, "e. "): + elif match(p, p.idx, "(e) ") or match(p, p.idx, "e) ") or + match(p, p.idx, "e. "): result = rnEnumList - elif match(p, p.idx, "+a+"): - result = rnGridTable - rstMessage(p, meGridTableNotImplemented) - elif isDefList(p): - result = rnDefList elif isOptionList(p): result = rnOptionList + elif isRst(p) and isDefList(p): + result = rnDefList + elif isMd(p) and isMdDefListItem(p, p.idx): + result = rnMdDefList else: result = rnParagraph of tkWord, tkOther, tkWhite: - if match(p, tokenAfterNewline(p), "ai"): result = rnHeadline + let tokIdx = tokenAfterNewline(p) + if match(p, tokIdx, "aI"): + if isAdornmentHeadline(p, tokIdx): result = rnHeadline + else: result = rnParagraph elif match(p, p.idx, "e) ") or match(p, p.idx, "e. "): result = rnEnumList - elif isDefList(p): result = rnDefList + elif isRst(p) and isDefList(p): result = rnDefList + elif isMd(p) and isMdDefListItem(p, p.idx): + result = rnMdDefList else: result = rnParagraph else: result = rnLeaf proc parseLineBlock(p: var RstParser): PRstNode = + ## Returns rnLineBlock with all sons of type rnLineBlockItem result = nil - if p.tok[p.idx + 1].kind == tkWhite: - var col = p.tok[p.idx].col - result = newRstNode(rnLineBlock) - pushInd(p, p.tok[p.idx + 2].col) - inc(p.idx, 2) + if nextTok(p).kind in {tkWhite, tkIndent}: + var col = currentTok(p).col + result = newRstNodeA(p, rnLineBlock) while true: var item = newRstNode(rnLineBlockItem) - parseSection(p, item) - add(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == "|") and - (p.tok[p.idx + 2].kind == tkWhite): - inc(p.idx, 3) + if nextTok(p).kind == tkWhite: + if nextTok(p).symbol.len > 1: # pass additional indentation after '| ' + item.lineIndent = nextTok(p).symbol + inc p.idx, 2 + pushInd(p, p.tok[p.idx].col) + parseSection(p, item) + popInd(p) + else: # tkIndent => add an empty line + item.lineIndent = "\n" + inc p.idx, 1 + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).symbol == "|" and + p.tok[p.idx + 2].kind in {tkWhite, tkIndent}: + inc p.idx, 1 else: break - popInd(p) + +proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} + +proc getQuoteSymbol(p: RstParser, idx: int): tuple[sym: string, depth: int, tokens: int] = + result = ("", 0, 0) + var i = idx + result.sym &= p.tok[i].symbol + result.depth += p.tok[i].symbol.len + inc result.tokens + inc i + while p.tok[i].kind == tkWhite and i+1 < p.tok.len and + p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol[0] == '>': + result.sym &= p.tok[i].symbol + result.sym &= p.tok[i+1].symbol + result.depth += p.tok[i+1].symbol.len + inc result.tokens, 2 + inc i, 2 + +proc parseMarkdownQuoteSegment(p: var RstParser, curSym: string, col: int): + PRstNode = + ## We define *segment* as a group of lines that starts with exactly the + ## same quote symbol. If the following lines don't contain any `>` (*lazy* + ## continuation) they considered as continuation of the current segment. + var q: RstParser # to delete `>` at a start of line and then parse normally + initParser(q, p.s) + q.col = p.col + q.line = p.line + var minCol = int.high # minimum colum num in the segment + while true: # move tokens of segment from `p` to `q` skipping `curSym` + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if nextTok(p).kind in {tkIndent, tkEof}: + break + else: + if nextTok(p).symbol[0] == '>': + var (quoteSym, _, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + if quoteSym == curSym: # the segment continues + var iTok = tokenAfterNewline(p, p.idx+1) + if p.tok[iTok].kind notin {tkEof, tkIndent} and + p.tok[iTok].symbol[0] != '>': + rstMessage(p, mwRstStyle, + "two or more quoted lines are followed by unquoted line " & + $(curLine(p) + 1)) + break + q.tok.add currentTok(p) + var ival = currentTok(p).ival + quoteSym.len + inc p.idx, (1 + quoteTokens) # skip newline and > > > + if currentTok(p).kind == tkWhite: + ival += currentTok(p).symbol.len + inc p.idx + # fix up previous `tkIndent`s to ival (as if >>> were not there) + var j = q.tok.len - 1 + while j >= 0 and q.tok[j].kind == tkIndent: + q.tok[j].ival = ival + dec j + else: # next segment started + break + elif currentTok(p).ival < col: + break + else: # the segment continues, a case like: + # > beginning + # continuation + q.tok.add currentTok(p) + inc p.idx + else: + if currentTok(p).col < minCol: minCol = currentTok(p).col + q.tok.add currentTok(p) + inc p.idx + q.indentStack = @[minCol] + # if initial indentation `minCol` is > 0 then final newlines + # should be omitted so that parseDoc could advance to the end of tokens: + var j = q.tok.len - 1 + while q.tok[j].kind == tkIndent: dec j + q.tok.setLen (j+1) + q.tok.add Token(kind: tkEof, line: currentTok(p).line) + result = parseDoc(q) + +proc parseMarkdownBlockQuote(p: var RstParser): PRstNode = + var (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx) + let col = currentTok(p).col + result = newRstNodeA(p, rnMarkdownBlockQuote) + inc p.idx, quoteTokens # skip first > + while true: + var item = newRstNode(rnMarkdownBlockQuoteItem) + item.quotationDepth = quotationDepth + if currentTok(p).kind == tkWhite: inc p.idx + item.add parseMarkdownQuoteSegment(p, curSym, col) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).kind != tkEof and nextTok(p).symbol[0] == '>': + (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + inc p.idx, (1 + quoteTokens) # skip newline and > > > + else: + break proc parseParagraph(p: var RstParser, result: PRstNode) = while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkIndent: - if p.tok[p.idx + 1].kind == tkIndent: - inc(p.idx) - break - elif (p.tok[p.idx].ival == currInd(p)): - inc(p.idx) + if nextTok(p).kind == tkIndent: + inc p.idx + break # blank line breaks paragraph for both Md & Rst + elif currentTok(p).ival == currInd(p) or ( + isMd(p) and currentTok(p).ival > currInd(p)): + # (Md allows adding additional indentation inside paragraphs) + inc p.idx case whichSection(p) - of rnParagraph, rnLeaf, rnHeadline, rnOverline, rnDirective: - add(result, newRstNode(rnLeaf, " ")) + of rnParagraph, rnLeaf, rnHeadline, rnMarkdownHeadline, + rnOverline, rnDirective: + result.add newLeaf(" ") of rnLineBlock: - addIfNotNil(result, parseLineBlock(p)) - else: break + result.addIfNotNil(parseLineBlock(p)) + of rnMarkdownBlockQuote: + result.addIfNotNil(parseMarkdownBlockQuote(p)) + else: + dec p.idx # allow subsequent block to be parsed as another section + break else: break of tkPunct: - if (p.tok[p.idx].symbol == "::") and - (p.tok[p.idx + 1].kind == tkIndent) and - (currInd(p) < p.tok[p.idx + 1].ival): - add(result, newRstNode(rnLeaf, ":")) - inc(p.idx) # skip '::' - add(result, parseLiteralBlock(p)) + if isRst(p) and ( + let literalBlockKind = whichRstLiteralBlock(p); + literalBlockKind != lbNone): + result.add newLeaf(":") + inc p.idx # skip '::' + result.add(parseRstLiteralBlock(p, literalBlockKind)) break else: parseInline(p, result) @@ -1185,375 +2567,725 @@ proc parseParagraph(p: var RstParser, result: PRstNode) = parseInline(p, result) else: break +proc checkHeadingHierarchy(p: RstParser, lvl: int) = + if lvl - p.s.hCurLevel > 1: # broken hierarchy! + proc descr(l: int): string = + (if p.s.hLevels[l].hasOverline: "overline " else: "underline ") & + repeat(p.s.hLevels[l].symbol, 5) + var msg = "(section level inconsistent: " + msg.add descr(lvl) & " unexpectedly found, " & + "while the following intermediate section level(s) are missing on lines " + msg.add $p.s.hLevels[p.s.hCurLevel].line & ".." & $curLine(p) & ":" + for l in p.s.hCurLevel+1 .. lvl-1: + msg.add " " & descr(l) + if l != lvl-1: msg.add "," + rstMessage(p, meNewSectionExpected, msg & ")") + proc parseHeadline(p: var RstParser): PRstNode = - result = newRstNode(rnHeadline) if isMarkdownHeadline(p): - result.level = p.tok[p.idx].symbol.len - assert(p.tok[p.idx+1].kind == tkWhite) + result = newRstNode(rnMarkdownHeadline) + # Note that level hierarchy is not checked for markdown headings + result.level = currentTok(p).symbol.len + assert(nextTok(p).kind == tkWhite) inc p.idx, 2 parseUntilNewline(p, result) else: + result = newRstNode(rnHeadline) + parseUntilNewline(p, result) + assert(currentTok(p).kind == tkIndent) + assert(nextTok(p).kind == tkAdornment) + var c = nextTok(p).symbol[0] + inc p.idx, 2 + result.level = getLevel(p, c, hasOverline=false) + checkHeadingHierarchy(p, result.level) + p.s.hCurLevel = result.level + addAnchorRst(p, linkName(result), result, anchorType=headlineAnchor) + p.s.tocPart.add result + +proc parseOverline(p: var RstParser): PRstNode = + var c = currentTok(p).symbol[0] + inc p.idx, 2 + result = newRstNode(rnOverline) + while true: parseUntilNewline(p, result) - assert(p.tok[p.idx].kind == tkIndent) - assert(p.tok[p.idx + 1].kind == tkAdornment) - var c = p.tok[p.idx + 1].symbol[0] - inc(p.idx, 2) - result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c) + if currentTok(p).kind == tkIndent: + inc p.idx + if prevTok(p).ival > currInd(p): + result.add newLeaf(" ") + else: + break + else: + break + result.level = getLevel(p, c, hasOverline=true) + checkHeadingHierarchy(p, result.level) + p.s.hCurLevel = result.level + if currentTok(p).kind == tkAdornment: + inc p.idx + if currentTok(p).kind == tkIndent: inc p.idx + addAnchorRst(p, linkName(result), result, anchorType=headlineAnchor) + p.s.tocPart.add result + +proc fixHeadlines(s: PRstSharedState) = + # Fix up section levels depending on presence of a title and subtitle: + for n in s.tocPart: + if n.kind in {rnHeadline, rnOverline}: + if s.hTitleCnt == 2: + if n.level == 1: # it's the subtitle + n.level = 0 + elif n.level >= 2: # normal sections, start numbering from 1 + n.level -= 1 + elif s.hTitleCnt == 0: + n.level += 1 + # Set headline anchors: + for iHeading in 0 .. s.tocPart.high: + let n: PRstNode = s.tocPart[iHeading] + if n.level >= 1: + n.anchor = rstnodeToRefname(n) + # Fix anchors for uniqueness if `.. contents::` is present + if s.hasToc: + # Find the last higher level section for unique reference name + var sectionPrefix = "" + for i in countdown(iHeading - 1, 0): + if s.tocPart[i].level >= 1 and s.tocPart[i].level < n.level: + sectionPrefix = rstnodeToRefname(s.tocPart[i]) & "-" + break + if sectionPrefix != "": + n.anchor = sectionPrefix & n.anchor + s.tocPart.setLen 0 type - IntSeq = seq[int] + ColSpec = object + start, stop: int + RstCols = seq[ColSpec] + ColumnLimits = tuple # for Markdown + first, last: int + ColSeq = seq[ColumnLimits] + +proc tokStart(p: RstParser, idx: int): int = + result = p.tok[idx].col + +proc tokStart(p: RstParser): int = + result = tokStart(p, p.idx) + +proc tokEnd(p: RstParser, idx: int): int = + result = p.tok[idx].col + p.tok[idx].symbol.len - 1 proc tokEnd(p: RstParser): int = - result = p.tok[p.idx].col + len(p.tok[p.idx].symbol) - 1 + result = tokEnd(p, p.idx) -proc getColumns(p: var RstParser, cols: var IntSeq) = +proc getColumns(p: RstParser, cols: var RstCols, startIdx: int): int = + # Fills table column specification (or separator) `cols` and returns + # the next parser index after it. var L = 0 + result = startIdx while true: - inc(L) + inc L setLen(cols, L) - cols[L - 1] = tokEnd(p) - assert(p.tok[p.idx].kind == tkAdornment) - inc(p.idx) - if p.tok[p.idx].kind != tkWhite: break - inc(p.idx) - if p.tok[p.idx].kind != tkAdornment: break - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - # last column has no limit: - cols[L - 1] = 32000 + cols[L - 1].start = tokStart(p, result) + cols[L - 1].stop = tokEnd(p, result) + assert(p.tok[result].kind == tkAdornment) + inc result + if p.tok[result].kind != tkWhite: break + inc result + if p.tok[result].kind != tkAdornment: break + if p.tok[result].kind == tkIndent: inc result -proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} +proc checkColumns(p: RstParser, cols: RstCols) = + var i = p.idx + if p.tok[i].symbol[0] != '=': + stopOrWarn(p, meIllformedTable, + "only tables with `=` columns specification are allowed") + for col in 0 ..< cols.len: + if tokEnd(p, i) != cols[col].stop: + stopOrWarn(p, meIllformedTable, + "end of table column #$1 should end at position $2" % [ + $(col+1), $(cols[col].stop+ColRstOffset)], + p.tok[i].line, tokEnd(p, i)) + inc i + if col == cols.len - 1: + if p.tok[i].kind == tkWhite: + inc i + if p.tok[i].kind notin {tkIndent, tkEof}: + stopOrWarn(p, meIllformedTable, "extraneous column specification") + elif p.tok[i].kind == tkWhite: + inc i + else: + stopOrWarn(p, meIllformedTable, + "no enough table columns", p.tok[i].line, p.tok[i].col) + +proc getSpans(p: RstParser, nextLine: int, + cols: RstCols, unitedCols: RstCols): seq[int] = + ## Calculates how many columns a joined cell occupies. + if unitedCols.len > 0: + result = newSeq[int](unitedCols.len) + var + iCell = 0 + jCell = 0 + uCell = 0 + while jCell < cols.len: + if cols[jCell].stop < unitedCols[uCell].stop: + inc jCell + elif cols[jCell].stop == unitedCols[uCell].stop: + result[uCell] = jCell - iCell + 1 + iCell = jCell + 1 + jCell = jCell + 1 + inc uCell + else: + rstMessage(p, meIllformedTable, + "spanning underline does not match main table columns", + p.tok[nextLine].line, p.tok[nextLine].col) + +proc parseSimpleTableRow(p: var RstParser, cols: RstCols, colChar: char): PRstNode = + ## Parses 1 row in RST simple table. + # Consider that columns may be spanning (united by using underline like ----): + let nextLine = tokenAfterNewline(p) + var unitedCols: RstCols + var afterSpan: int + if p.tok[nextLine].kind == tkAdornment and p.tok[nextLine].symbol[0] == '-': + afterSpan = getColumns(p, unitedCols, nextLine) + if unitedCols == cols and p.tok[nextLine].symbol[0] == colChar: + # legacy rst.nim compat.: allow punctuation like `----` in main boundaries + afterSpan = nextLine + unitedCols.setLen 0 + else: + afterSpan = nextLine + template colEnd(i): int = + if i == cols.len - 1: high(int) # last column has no limit + elif unitedCols.len > 0: unitedCols[i].stop else: cols[i].stop + template colStart(i): int = + if unitedCols.len > 0: unitedCols[i].start else: cols[i].start + var row = newSeq[string](if unitedCols.len > 0: unitedCols.len else: cols.len) + var spans: seq[int] = getSpans(p, nextLine, cols, unitedCols) + + let line = currentTok(p).line + # Iterate over the lines a single cell may span: + while true: + var nCell = 0 + # distribute tokens between cells in the current line: + while currentTok(p).kind notin {tkIndent, tkEof}: + if tokEnd(p) <= colEnd(nCell): + if tokStart(p) < colStart(nCell): + if currentTok(p).kind != tkWhite: + stopOrWarn(p, meIllformedTable, + "this word crosses table column from the left") + row[nCell].add(currentTok(p).symbol) + else: + row[nCell].add(currentTok(p).symbol) + inc p.idx + else: + if tokStart(p) < colEnd(nCell) and currentTok(p).kind != tkWhite: + stopOrWarn(p, meIllformedTable, + "this word crosses table column from the right") + row[nCell].add(currentTok(p).symbol) + inc p.idx + inc nCell + if currentTok(p).kind == tkIndent: inc p.idx + if tokEnd(p) <= colEnd(0): break + # Continued current cells because the 1st column is empty. + if currentTok(p).kind in {tkEof, tkAdornment}: + break + for nCell in countup(1, high(row)): row[nCell].add('\n') + result = newRstNode(rnTableRow) + var q: RstParser + for uCell in 0 ..< row.len: + initParser(q, p.s) + q.col = colStart(uCell) + q.line = line - 1 + getTokens(row[uCell], q.tok) + let cell = newRstNode(rnTableDataCell) + cell.span = if spans.len == 0: 0 else: spans[uCell] + cell.add(parseDoc(q)) + result.add(cell) + if afterSpan > p.idx: + p.idx = afterSpan proc parseSimpleTable(p: var RstParser): PRstNode = - var - cols: IntSeq - row: seq[string] - i, last, line: int - c: char - q: RstParser - a, b: PRstNode - result = newRstNode(rnTable) - cols = @[] - row = @[] - a = nil - c = p.tok[p.idx].symbol[0] + var cols: RstCols + result = newRstNodeA(p, rnTable) + let startIdx = getColumns(p, cols, p.idx) + let colChar = currentTok(p).symbol[0] + checkColumns(p, cols) + p.idx = startIdx + result.colCount = cols.len while true: - if p.tok[p.idx].kind == tkAdornment: - last = tokenAfterNewline(p) - if p.tok[last].kind in {tkEof, tkIndent}: + if currentTok(p).kind == tkAdornment: + checkColumns(p, cols) + p.idx = tokenAfterNewline(p) + if currentTok(p).kind in {tkEof, tkIndent}: # skip last adornment line: - p.idx = last break - getColumns(p, cols) - setLen(row, len(cols)) - if a != nil: - for j in 0..len(a)-1: a.sons[j].kind = rnTableHeaderCell - if p.tok[p.idx].kind == tkEof: break - for j in countup(0, high(row)): row[j] = "" - # the following while loop iterates over the lines a single cell may span: - line = p.tok[p.idx].line - while true: - i = 0 - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - if (tokEnd(p) <= cols[i]): - add(row[i], p.tok[p.idx].symbol) - inc(p.idx) - else: - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - inc(i) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - if tokEnd(p) <= cols[0]: break - if p.tok[p.idx].kind in {tkEof, tkAdornment}: break - for j in countup(1, high(row)): add(row[j], '\x0A') + if result.sons.len > 0: result.sons[^1].endsHeader = true + # fix rnTableDataCell -> rnTableHeaderCell for previous table rows: + for nRow in 0 ..< result.sons.len: + for nCell in 0 ..< result.sons[nRow].len: + template cell: PRstNode = result.sons[nRow].sons[nCell] + cell = PRstNode(kind: rnTableHeaderCell, sons: cell.sons, + span: cell.span, anchor: cell.anchor) + if currentTok(p).kind == tkEof: break + let tabRow = parseSimpleTableRow(p, cols, colChar) + result.add tabRow + +proc readTableRow(p: var RstParser): ColSeq = + if currentTok(p).symbol == "|": inc p.idx + while currentTok(p).kind notin {tkIndent, tkEof}: + var limits: ColumnLimits + limits.first = p.idx + while currentTok(p).kind notin {tkIndent, tkEof}: + if currentTok(p).symbol == "|" and prevTok(p).symbol != "\\": break + inc p.idx + limits.last = p.idx + result.add(limits) + if currentTok(p).kind in {tkIndent, tkEof}: break + inc p.idx + p.idx = tokenAfterNewline(p) + +proc getColContents(p: var RstParser, colLim: ColumnLimits): string = + for i in colLim.first ..< colLim.last: + result.add(p.tok[i].symbol) + result.strip + +proc isValidDelimiterRow(p: var RstParser, colNum: int): bool = + let row = readTableRow(p) + if row.len != colNum: return false + for limits in row: + let content = getColContents(p, limits) + if content.len < 3 or not (content.startsWith("--") or content.startsWith(":-")): + return false + return true + +proc parseMarkdownTable(p: var RstParser): PRstNode = + var + row: ColSeq + a, b: PRstNode + q: RstParser + result = newRstNodeA(p, rnMarkdownTable) + + proc parseRow(p: var RstParser, cellKind: RstNodeKind, result: PRstNode) = + row = readTableRow(p) + if result.colCount == 0: result.colCount = row.len # table header + elif row.len < result.colCount: row.setLen(result.colCount) a = newRstNode(rnTableRow) - for j in countup(0, high(row)): + for j in 0 ..< result.colCount: + b = newRstNode(cellKind) initParser(q, p.s) - q.col = cols[j] - q.line = line - 1 - q.filename = p.filename - q.col += getTokens(row[j], false, q.tok) - b = newRstNode(rnTableDataCell) - add(b, parseDoc(q)) - add(a, b) - add(result, a) + q.col = p.col + q.line = currentTok(p).line - 1 + getTokens(getColContents(p, row[j]), q.tok) + b.add(parseDoc(q)) + a.add(b) + result.add(a) + + parseRow(p, rnTableHeaderCell, result) + if not isValidDelimiterRow(p, result.colCount): + rstMessage(p, meMarkdownIllformedTable) + while predNL(p) and currentTok(p).symbol == "|": + parseRow(p, rnTableDataCell, result) proc parseTransition(p: var RstParser): PRstNode = - result = newRstNode(rnTransition) - inc(p.idx) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - if p.tok[p.idx].kind == tkIndent: inc(p.idx) - -proc parseOverline(p: var RstParser): PRstNode = - var c = p.tok[p.idx].symbol[0] - inc(p.idx, 2) - result = newRstNode(rnOverline) - while true: - parseUntilNewline(p, result) - if p.tok[p.idx].kind == tkIndent: - inc(p.idx) - if p.tok[p.idx - 1].ival > currInd(p): - add(result, newRstNode(rnLeaf, " ")) - else: - break - else: - break - result.level = getLevel(p.s.overlineToLevel, p.s.oLevel, c) - if p.tok[p.idx].kind == tkAdornment: - inc(p.idx) # XXX: check? - if p.tok[p.idx].kind == tkIndent: inc(p.idx) + result = newRstNodeA(p, rnTransition) + inc p.idx + if currentTok(p).kind == tkIndent: inc p.idx + if currentTok(p).kind == tkIndent: inc p.idx proc parseBulletList(p: var RstParser): PRstNode = result = nil - if p.tok[p.idx + 1].kind == tkWhite: - var bullet = p.tok[p.idx].symbol - var col = p.tok[p.idx].col - result = newRstNode(rnBulletList) + if nextTok(p).kind == tkWhite: + var bullet = currentTok(p).symbol + var col = currentTok(p).col + result = newRstNodeA(p, rnBulletList) pushInd(p, p.tok[p.idx + 2].col) - inc(p.idx, 2) + inc p.idx, 2 while true: var item = newRstNode(rnBulletItem) parseSection(p, item) - add(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - (p.tok[p.idx + 1].symbol == bullet) and - (p.tok[p.idx + 2].kind == tkWhite): - inc(p.idx, 3) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).symbol == bullet and + p.tok[p.idx + 2].kind == tkWhite: + inc p.idx, 3 else: break popInd(p) proc parseOptionList(p: var RstParser): PRstNode = - result = newRstNode(rnOptionList) + result = newRstNodeA(p, rnOptionList) + let col = currentTok(p).col + var order = 1 while true: - if isOptionList(p): + if currentTok(p).col == col and isOptionList(p): var a = newRstNode(rnOptionGroup) var b = newRstNode(rnDescription) var c = newRstNode(rnOptionListItem) - if match(p, p.idx, "//w"): inc(p.idx) - while not (p.tok[p.idx].kind in {tkIndent, tkEof}): - if (p.tok[p.idx].kind == tkWhite) and (len(p.tok[p.idx].symbol) > 1): - inc(p.idx) + if match(p, p.idx, "//w"): inc p.idx + while currentTok(p).kind notin {tkIndent, tkEof}: + if currentTok(p).kind == tkWhite and currentTok(p).symbol.len > 1: + inc p.idx break - add(a, newLeaf(p)) - inc(p.idx) + a.add(newLeaf(p)) + inc p.idx var j = tokenAfterNewline(p) - if (j > 0) and (p.tok[j - 1].kind == tkIndent) and - (p.tok[j - 1].ival > currInd(p)): + if j > 0 and p.tok[j - 1].kind == tkIndent and p.tok[j - 1].ival > currInd(p): pushInd(p, p.tok[j - 1].ival) parseSection(p, b) popInd(p) else: parseLine(p, b) - if (p.tok[p.idx].kind == tkIndent): inc(p.idx) - add(c, a) - add(c, b) - add(result, c) + while currentTok(p).kind == tkIndent: inc p.idx + c.add(a) + c.add(b) + c.order = order; inc order + result.add(c) + else: + if currentTok(p).kind != tkEof: dec p.idx # back to tkIndent + break + +proc parseMdDefinitionList(p: var RstParser): PRstNode = + ## Parses (Pandoc/kramdown/PHPextra) Markdown definition lists. + result = newRstNodeA(p, rnMdDefList) + let termCol = currentTok(p).col + while true: + var item = newRstNode(rnDefItem) + var term = newRstNode(rnDefName) + parseLine(p, term) + skipNewlines(p) + inc p.idx, 2 # skip ":" and space + item.add(term) + while true: + var def = newRstNode(rnDefBody) + let indent = getMdBlockIndent(p) + pushInd(p, indent) + parseSection(p, def) + popInd(p) + item.add(def) + let j = skipNewlines(p, p.idx) + if isMdDefBody(p, j, termCol): # parse next definition body + p.idx = j + 2 # skip ":" and space + else: + break + result.add(item) + let j = skipNewlines(p, p.idx) + if p.tok[j].col == termCol and isMdDefListItem(p, j): + p.idx = j # parse next item else: break proc parseDefinitionList(p: var RstParser): PRstNode = result = nil var j = tokenAfterNewline(p) - 1 - if (j >= 1) and (p.tok[j].kind == tkIndent) and - (p.tok[j].ival > currInd(p)) and (p.tok[j - 1].symbol != "::"): - var col = p.tok[p.idx].col - result = newRstNode(rnDefList) + if j >= 1 and p.tok[j].kind == tkIndent and + p.tok[j].ival > currInd(p) and p.tok[j - 1].symbol != "::": + var col = currentTok(p).col + result = newRstNodeA(p, rnDefList) while true: + if isOptionList(p): + break # option list has priority over def.list j = p.idx var a = newRstNode(rnDefName) parseLine(p, a) - if (p.tok[p.idx].kind == tkIndent) and - (p.tok[p.idx].ival > currInd(p)) and - (p.tok[p.idx + 1].symbol != "::") and - not (p.tok[p.idx + 1].kind in {tkIndent, tkEof}): - pushInd(p, p.tok[p.idx].ival) + if currentTok(p).kind == tkIndent and + currentTok(p).ival > currInd(p) and + nextTok(p).symbol != "::" and + nextTok(p).kind notin {tkIndent, tkEof}: + pushInd(p, currentTok(p).ival) var b = newRstNode(rnDefBody) parseSection(p, b) var c = newRstNode(rnDefItem) - add(c, a) - add(c, b) - add(result, c) + c.add(a) + c.add(b) + result.add(c) popInd(p) else: p.idx = j break - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col): - inc(p.idx) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col: + inc p.idx j = tokenAfterNewline(p) - 1 if j >= 1 and p.tok[j].kind == tkIndent and p.tok[j].ival > col and p.tok[j-1].symbol != "::" and p.tok[j+1].kind != tkIndent: discard else: break - if len(result) == 0: result = nil + if result.len == 0: result = nil proc parseEnumList(p: var RstParser): PRstNode = const - wildcards: array[0..2, string] = ["(e) ", "e) ", "e. "] - wildpos: array[0..2, int] = [1, 0, 0] - result = nil + wildcards: array[0..5, string] = ["(n) ", "n) ", "n. ", + "(x) ", "x) ", "x. "] + # enumerator patterns, where 'x' means letter and 'n' means number + wildToken: array[0..5, int] = [4, 3, 3, 4, 3, 3] # number of tokens + wildIndex: array[0..5, int] = [1, 0, 0, 1, 0, 0] + # position of enumeration sequence (number/letter) in enumerator + let col = currentTok(p).col var w = 0 - while w <= 2: + while w < wildcards.len: if match(p, p.idx, wildcards[w]): break - inc(w) - if w <= 2: - var col = p.tok[p.idx].col - result = newRstNode(rnEnumList) - inc(p.idx, wildpos[w] + 3) - var j = tokenAfterNewline(p) - if (p.tok[j].col == p.tok[p.idx].col) or match(p, j, wildcards[w]): - pushInd(p, p.tok[p.idx].col) - while true: - var item = newRstNode(rnEnumItem) - parseSection(p, item) - add(result, item) - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival == col) and - match(p, p.idx + 1, wildcards[w]): - inc(p.idx, wildpos[w] + 4) - else: + inc w + assert w < wildcards.len + + proc checkAfterNewline(p: RstParser, report: bool): bool = + ## If no indentation on the next line then parse as a normal paragraph + ## according to the RST spec. And report a warning with suggestions + let j = tokenAfterNewline(p, start=p.idx+1) + let requiredIndent = p.tok[p.idx+wildToken[w]].col + if p.tok[j].kind notin {tkIndent, tkEof} and + p.tok[j].col < requiredIndent and + (p.tok[j].col > col or + (p.tok[j].col == col and not match(p, j, wildcards[w]))): + if report: + let n = p.line + p.tok[j].line + let msg = "\n" & """ + not enough indentation on line $2 + (should be at column $3 if it's a continuation of enum. list), + or no blank line after line $1 (if it should be the next paragraph), + or no escaping \ at the beginning of line $1 + (if lines $1..$2 are a normal paragraph, not enum. list)""".dedent + let c = p.col + requiredIndent + ColRstOffset + rstMessage(p, mwRstStyle, msg % [$(n-1), $n, $c], + p.tok[j].line, p.tok[j].col) + result = false + else: + result = true + + if not checkAfterNewline(p, report = true): + return nil + result = newRstNodeA(p, rnEnumList) + let autoEnums = if roSupportMarkdown in p.s.options: @["#", "1"] else: @["#"] + var prevAE = "" # so as not allow mixing auto-enumerators `1` and `#` + var curEnum = 1 + for i in 0 ..< wildToken[w]-1: # add first enumerator with (, ), and . + if p.tok[p.idx + i].symbol == "#": + prevAE = "#" + result.labelFmt.add "1" + else: + result.labelFmt.add p.tok[p.idx + i].symbol + var prevEnum = p.tok[p.idx + wildIndex[w]].symbol + inc p.idx, wildToken[w] + while true: + var item = newRstNode(rnEnumItem) + pushInd(p, currentTok(p).col) + parseSection(p, item) + popInd(p) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + match(p, p.idx+1, wildcards[w]): + # don't report to avoid duplication of warning since for + # subsequent enum. items parseEnumList will be called second time: + if not checkAfterNewline(p, report = false): + break + let enumerator = p.tok[p.idx + 1 + wildIndex[w]].symbol + # check that it's in sequence: enumerator == next(prevEnum) + if "n" in wildcards[w]: # arabic numeral + let prevEnumI = try: parseInt(prevEnum) except ValueError: 1 + if enumerator in autoEnums: + if prevAE != "" and enumerator != prevAE: + break + prevAE = enumerator + curEnum = prevEnumI + 1 + else: curEnum = (try: parseInt(enumerator) except ValueError: 1) + if curEnum - prevEnumI != 1: break - popInd(p) + prevEnum = enumerator + else: # a..z + let prevEnumI = ord(prevEnum[0]) + if enumerator == "#": curEnum = prevEnumI + 1 + else: curEnum = ord(enumerator[0]) + if curEnum - prevEnumI != 1: + break + prevEnum = $chr(curEnum) + inc p.idx, 1 + wildToken[w] else: - dec(p.idx, wildpos[w] + 3) - result = nil + break + +proc prefix(ftnType: FootnoteType): string = + case ftnType + of fnManualNumber: result = "footnote-" + of fnAutoNumber: result = "footnoteauto-" + of fnAutoNumberLabel: result = "footnote-" + of fnAutoSymbol: result = "footnotesym-" + of fnCitation: result = "citation-" + +proc parseFootnote(p: var RstParser): PRstNode {.gcsafe.} = + ## Parses footnotes and citations, always returns 2 sons: + ## + ## 1) footnote label, always containing rnInner with 1 or more sons + ## 2) footnote body, which may be nil + var label: PRstNode + if isRst(p): + inc p.idx # skip space after `..` + label = parseFootnoteName(p, reference=false) + if label == nil: + if isRst(p): + dec p.idx + return nil + result = newRstNode(rnFootnote) + result.add label + let (fnType, i) = getFootnoteType(p.s, label) + var name = "" + var anchor = fnType.prefix + case fnType + of fnManualNumber: + addFootnoteNumManual(p, i) + anchor.add $i + of fnAutoNumber, fnAutoNumberLabel: + name = rstnodeToRefname(label) + addFootnoteNumAuto(p, name) + if fnType == fnAutoNumberLabel: + anchor.add name + else: # fnAutoNumber + result.order = p.s.lineFootnoteNum.len + anchor.add $result.order + of fnAutoSymbol: + addFootnoteSymAuto(p) + result.order = p.s.lineFootnoteSym.len + anchor.add $p.s.lineFootnoteSym.len + of fnCitation: + anchor.add rstnodeToRefname(label) + addAnchorRst(p, anchor, target = result, anchorType = footnoteAnchor) + result.anchor = anchor + if currentTok(p).kind == tkWhite: inc p.idx + discard parseBlockContent(p, result, parseSectionWrapper) + if result.len < 2: + result.add nil proc sonKind(father: PRstNode, i: int): RstNodeKind = result = rnLeaf - if i < len(father): result = father.sons[i].kind + if i < father.len: result = father.sons[i].kind proc parseSection(p: var RstParser, result: PRstNode) = + ## parse top-level RST elements: sections, transitions and body elements. while true: var leave = false assert(p.idx >= 0) - while p.tok[p.idx].kind == tkIndent: - if currInd(p) == p.tok[p.idx].ival: - inc(p.idx) - elif p.tok[p.idx].ival > currInd(p): - pushInd(p, p.tok[p.idx].ival) - var a = newRstNode(rnBlockQuote) - parseSection(p, a) - add(result, a) - popInd(p) + while currentTok(p).kind == tkIndent: + if currInd(p) == currentTok(p).ival: + inc p.idx + elif currentTok(p).ival > currInd(p): + if roPreferMarkdown in p.s.options: # Markdown => normal paragraphs + if currentTok(p).ival - currInd(p) >= 4: + result.add parseLiteralBlock(p) + else: + pushInd(p, currentTok(p).ival) + parseSection(p, result) + popInd(p) + else: # RST mode => block quotes + pushInd(p, currentTok(p).ival) + var a = newRstNodeA(p, rnBlockQuote) + parseSection(p, a) + result.add(a) + popInd(p) else: + while currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: + inc p.idx # skip blank lines leave = true break - if leave or p.tok[p.idx].kind == tkEof: break + if leave or currentTok(p).kind == tkEof: break var a: PRstNode = nil var k = whichSection(p) case k of rnLiteralBlock: - inc(p.idx) # skip '::' + inc p.idx # skip '::' a = parseLiteralBlock(p) of rnBulletList: a = parseBulletList(p) of rnLineBlock: a = parseLineBlock(p) + of rnMarkdownBlockQuote: a = parseMarkdownBlockQuote(p) of rnDirective: a = parseDotDot(p) + of rnFootnote: a = parseFootnote(p) of rnEnumList: a = parseEnumList(p) - of rnLeaf: rstMessage(p, meNewSectionExpected) + of rnLeaf: rstMessage(p, meNewSectionExpected, "(syntax error)") of rnParagraph: discard of rnDefList: a = parseDefinitionList(p) + of rnMdDefList: a = parseMdDefinitionList(p) of rnFieldList: - if p.idx > 0: dec(p.idx) + if p.idx > 0: dec p.idx a = parseFields(p) of rnTransition: a = parseTransition(p) - of rnHeadline: a = parseHeadline(p) + of rnHeadline, rnMarkdownHeadline: a = parseHeadline(p) of rnOverline: a = parseOverline(p) of rnTable: a = parseSimpleTable(p) + of rnMarkdownTable: a = parseMarkdownTable(p) of rnOptionList: a = parseOptionList(p) else: #InternalError("rst.parseSection()") discard if a == nil and k != rnDirective: - a = newRstNode(rnParagraph) + a = newRstNodeA(p, rnParagraph) parseParagraph(p, a) - addIfNotNil(result, a) + result.addIfNotNil(a) if sonKind(result, 0) == rnParagraph and sonKind(result, 1) != rnParagraph: - result.sons[0].kind = rnInner - -proc parseSectionWrapper(p: var RstParser): PRstNode = - result = newRstNode(rnInner) - parseSection(p, result) - while (result.kind == rnInner) and (len(result) == 1): - result = result.sons[0] - -proc `$`(t: Token): string = - result = $t.kind & ' ' & t.symbol + result.sons[0] = newRstNode(rnInner, result.sons[0].sons, + anchor=result.sons[0].anchor) proc parseDoc(p: var RstParser): PRstNode = result = parseSectionWrapper(p) - if p.tok[p.idx].kind != tkEof: - when false: - assert isAllocatedPtr(cast[pointer](p.tok)) - for i in 0 .. high(p.tok): - assert isNil(p.tok[i].symbol) or - isAllocatedPtr(cast[pointer](p.tok[i].symbol)) - echo "index: ", p.idx, " length: ", high(p.tok), "##", - p.tok[p.idx-1], p.tok[p.idx], p.tok[p.idx+1] - #assert isAllocatedPtr(cast[pointer](p.indentStack)) + if currentTok(p).kind != tkEof: rstMessage(p, meGeneralParseError) type DirFlag = enum hasArg, hasOptions, argIsFile, argIsWord DirFlags = set[DirFlag] - SectionParser = proc (p: var RstParser): PRstNode {.nimcall.} -proc parseDirective(p: var RstParser, flags: DirFlags): PRstNode = +proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode = ## Parses arguments and options for a directive block. ## ## A directive block will always have three sons: the arguments for the - ## directive (rnDirArg), the options (rnFieldList) and the block - ## (rnLineBlock). This proc parses the two first nodes, the block is left to + ## directive (rnDirArg), the options (rnFieldList) and the directive + ## content block. This proc parses the two first nodes, the 3rd is left to ## the outer `parseDirective` call. ## ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## check them before accessing. - result = newRstNode(rnDirective) + result = newRstNodeA(p, k) + if k == rnCodeBlock: result.info = lineInfo(p) var args: PRstNode = nil var options: PRstNode = nil if hasArg in flags: args = newRstNode(rnDirArg) if argIsFile in flags: while true: - case p.tok[p.idx].kind + case currentTok(p).kind of tkWord, tkOther, tkPunct, tkAdornment: - add(args, newLeaf(p)) - inc(p.idx) + args.add(newLeaf(p)) + inc p.idx else: break elif argIsWord in flags: - while p.tok[p.idx].kind == tkWhite: inc(p.idx) - if p.tok[p.idx].kind == tkWord: - add(args, newLeaf(p)) - inc(p.idx) + while currentTok(p).kind == tkWhite: inc p.idx + if currentTok(p).kind == tkWord: + args.add(newLeaf(p)) + inc p.idx else: args = nil else: parseLine(p, args) - add(result, args) + result.add(args) if hasOptions in flags: - if (p.tok[p.idx].kind == tkIndent) and (p.tok[p.idx].ival >= 3) and - (p.tok[p.idx + 1].symbol == ":"): + if currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) and + nextTok(p).symbol == ":": + pushInd(p, currentTok(p).ival) options = parseFields(p) - add(result, options) - -proc indFollows(p: RstParser): bool = - result = p.tok[p.idx].kind == tkIndent and p.tok[p.idx].ival > currInd(p) + popInd(p) + result.add(options) -proc parseDirective(p: var RstParser, flags: DirFlags, +proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags, contentParser: SectionParser): PRstNode = - ## Returns a generic rnDirective tree. + ## A helper proc that does main work for specific directive procs. + ## Always returns a generic rnDirective tree with these 3 children: ## - ## The children are rnDirArg, rnFieldList and rnLineBlock. Any might be nil. - result = parseDirective(p, flags) - if not isNil(contentParser) and indFollows(p): - pushInd(p, p.tok[p.idx].ival) - var content = contentParser(p) - popInd(p) - add(result, content) + ## 1) rnDirArg + ## 2) rnFieldList + ## 3) a node returned by `contentParser`. + ## + ## .. warning:: Any of the 3 children may be nil. + result = parseDirective(p, k, flags) + if not isNil(contentParser) and + parseBlockContent(p, result, contentParser): + discard "result is updated by parseBlockContent" else: - add(result, PRstNode(nil)) + result.add(PRstNode(nil)) proc parseDirBody(p: var RstParser, contentParser: SectionParser): PRstNode = if indFollows(p): - pushInd(p, p.tok[p.idx].ival) + pushInd(p, currentTok(p).ival) result = contentParser(p) popInd(p) @@ -1580,7 +3312,7 @@ proc dirInclude(p: var RstParser): PRstNode = # encoding (if specified). # result = nil - var n = parseDirective(p, {hasArg, argIsFile, hasOptions}, nil) + var n = parseDirective(p, rnDirective, {hasArg, argIsFile, hasOptions}, nil) var filename = strip(addNodes(n.sons[0])) var path = p.findRelativeFile(filename) if path == "": @@ -1589,15 +3321,15 @@ proc dirInclude(p: var RstParser): PRstNode = # XXX: error handling; recursive file inclusion! if getFieldValue(n, "literal") != "": result = newRstNode(rnLiteralBlock) - add(result, newRstNode(rnLeaf, readFile(path))) + result.add newLeaf(readFile(path)) else: - let inputString = readFile(path).string() + let inputString = readFile(path) let startPosition = block: let searchFor = n.getFieldValue("start-after").strip() if searchFor != "": let pos = inputString.find(searchFor) - if pos != -1: pos + searchFor.len() + if pos != -1: pos + searchFor.len else: 0 else: 0 @@ -1614,15 +3346,16 @@ proc dirInclude(p: var RstParser): PRstNode = var q: RstParser initParser(q, p.s) - q.filename = path - q.col += getTokens( - inputString[startPosition..endPosition].strip(), - false, + let saveFileIdx = p.s.currFileIdx + setCurrFilename(p.s, path) + getTokens( + inputString[startPosition..endPosition], q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: # InternalError("Too many binary zeros in include file") result = parseDoc(q) + p.s.currFileIdx = saveFileIdx proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = ## Parses a code block. @@ -1640,57 +3373,54 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = ## ## As an extension this proc will process the ``file`` extension field and if ## present will replace the code block with the contents of the referenced - ## file. - result = parseDirective(p, {hasArg, hasOptions}, parseLiteralBlock) - var filename = strip(getFieldValue(result, "file")) - if filename != "": - var path = p.findRelativeFile(filename) - if path == "": rstMessage(p, meCannotOpenFile, filename) - var n = newRstNode(rnLiteralBlock) - add(n, newRstNode(rnLeaf, readFile(path))) - result.sons[2] = n + ## file. This behaviour is disabled in sandboxed mode and can be re-enabled + ## with the `roSandboxDisabled` flag. + result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock) + mayLoadFile(p, result) - # Extend the field block if we are using our custom extension. + # Extend the field block if we are using our custom Nim extension. if nimExtension: - # Create a field block if the input block didn't have any. - if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) - assert result.sons[1].kind == rnFieldList - # Hook the extra field and specify the Nim language as value. - var extraNode = newRstNode(rnField) - extraNode.add(newRstNode(rnFieldName)) - extraNode.add(newRstNode(rnFieldBody)) - extraNode.sons[0].add(newRstNode(rnLeaf, "default-language")) - extraNode.sons[1].add(newRstNode(rnLeaf, "Nim")) - result.sons[1].add(extraNode) - - result.kind = rnCodeBlock + defaultCodeLangNim(p, result) proc dirContainer(p: var RstParser): PRstNode = - result = parseDirective(p, {hasArg}, parseSectionWrapper) - assert(result.kind == rnDirective) - assert(len(result) == 3) - result.kind = rnContainer + result = parseDirective(p, rnContainer, {hasArg}, parseSectionWrapper) + assert(result.len == 3) proc dirImage(p: var RstParser): PRstNode = - result = parseDirective(p, {hasOptions, hasArg, argIsFile}, nil) - result.kind = rnImage + result = parseDirective(p, rnImage, {hasOptions, hasArg, argIsFile}, nil) proc dirFigure(p: var RstParser): PRstNode = - result = parseDirective(p, {hasOptions, hasArg, argIsFile}, + result = parseDirective(p, rnFigure, {hasOptions, hasArg, argIsFile}, parseSectionWrapper) - result.kind = rnFigure proc dirTitle(p: var RstParser): PRstNode = - result = parseDirective(p, {hasArg}, nil) - result.kind = rnTitle + result = parseDirective(p, rnTitle, {hasArg}, nil) proc dirContents(p: var RstParser): PRstNode = - result = parseDirective(p, {hasArg}, nil) - result.kind = rnContents + result = parseDirective(p, rnContents, {hasArg}, nil) + p.s.hasToc = true proc dirIndex(p: var RstParser): PRstNode = - result = parseDirective(p, {}, parseSectionWrapper) - result.kind = rnIndex + result = parseDirective(p, rnIndex, {}, parseSectionWrapper) + +proc dirAdmonition(p: var RstParser, d: string): PRstNode = + result = parseDirective(p, rnAdmonition, {}, parseSectionWrapper) + result.adType = d + +proc dirDefaultRole(p: var RstParser): PRstNode = + result = parseDirective(p, rnDefaultRole, {hasArg}, nil) + if result.sons[0].len == 0: p.s.currRole = defaultRole(p.s.options) + else: + assert result.sons[0].sons[0].kind == rnLeaf + p.s.currRole = result.sons[0].sons[0].text + p.s.currRoleKind = whichRole(p, p.s.currRole) + +proc dirRole(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {hasArg, hasOptions}, nil) + # just check that language is supported, TODO: real role association + let lang = getFieldValue(result, "language").strip + if lang != "" and getSourceLanguage(lang) == langNone: + rstMessage(p, mwUnsupportedLanguage, lang) proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind, contentParser: SectionParser) = @@ -1702,10 +3432,10 @@ proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind, else: var f = readFile(path) result = newRstNode(kind) - add(result, newRstNode(rnLeaf, f)) + result.add newLeaf(f) else: - result.kind = kind - add(result, parseDirBody(p, contentParser)) + result = newRstNode(kind, result.sons) + result.add(parseDirBody(p, contentParser)) proc dirRaw(p: var RstParser): PRstNode = # @@ -1716,7 +3446,7 @@ proc dirRaw(p: var RstParser): PRstNode = # # html # latex - result = parseDirective(p, {hasOptions, hasArg, argIsWord}) + result = parseDirective(p, rnDirective, {hasOptions, hasArg, argIsWord}) if result.sons[0] != nil: if cmpIgnoreCase(result.sons[0].sons[0].text, "html") == 0: dirRawAux(p, result, rnRawHtml, parseLiteralBlock) @@ -1727,99 +3457,430 @@ proc dirRaw(p: var RstParser): PRstNode = else: dirRawAux(p, result, rnRaw, parseSectionWrapper) +proc dirImportdoc(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {}, parseLiteralBlock) + assert result.sons[2].kind == rnLiteralBlock + assert result.sons[2].sons[0].kind == rnLeaf + let filenames: seq[string] = split(result.sons[2].sons[0].text, seps = {','}) + proc rmSpaces(s: string): string = s.split.join("") + for origFilename in filenames: + p.s.idxImports[origFilename.rmSpaces] = ImportdocInfo(fromInfo: lineInfo(p)) + +proc selectDir(p: var RstParser, d: string): PRstNode = + result = nil + let tok = p.tok[p.idx-2] # report on directive in ".. directive::" + if roSandboxDisabled notin p.s.options: + if d notin SandboxDirAllowlist: + rstMessage(p, meSandboxedDirective, d, tok.line, tok.col) + + case d + of "admonition", "attention", "caution": result = dirAdmonition(p, d) + of "code": result = dirCodeBlock(p) + of "code-block": result = dirCodeBlock(p, nimExtension = true) + of "container": result = dirContainer(p) + of "contents": result = dirContents(p) + of "danger": result = dirAdmonition(p, d) + of "default-role": result = dirDefaultRole(p) + of "error": result = dirAdmonition(p, d) + of "figure": result = dirFigure(p) + of "hint": result = dirAdmonition(p, d) + of "image": result = dirImage(p) + of "important": result = dirAdmonition(p, d) + of "importdoc": result = dirImportdoc(p) + of "include": result = dirInclude(p) + of "index": result = dirIndex(p) + of "note": result = dirAdmonition(p, d) + of "raw": + if roSupportRawDirective in p.s.options: + result = dirRaw(p) + else: + rstMessage(p, meInvalidDirective, d) + of "role": result = dirRole(p) + of "tip": result = dirAdmonition(p, d) + of "title": result = dirTitle(p) + of "warning": result = dirAdmonition(p, d) + else: + rstMessage(p, meInvalidDirective, d, tok.line, tok.col) + proc parseDotDot(p: var RstParser): PRstNode = + # parse "explicit markup blocks" result = nil - var col = p.tok[p.idx].col - inc(p.idx) + var n: PRstNode # to store result, workaround for bug 16855 + var col = currentTok(p).col + inc p.idx var d = getDirective(p) if d != "": pushInd(p, col) - case getDirKind(d) - of dkInclude: result = dirInclude(p) - of dkImage: result = dirImage(p) - of dkFigure: result = dirFigure(p) - of dkTitle: result = dirTitle(p) - of dkContainer: result = dirContainer(p) - of dkContents: result = dirContents(p) - of dkRaw: - if roSupportRawDirective in p.s.options: - result = dirRaw(p) - else: - rstMessage(p, meInvalidDirective, d) - of dkCode: result = dirCodeBlock(p) - of dkCodeBlock: result = dirCodeBlock(p, nimExtension = true) - of dkIndex: result = dirIndex(p) - else: rstMessage(p, meInvalidDirective, d) + result = selectDir(p, d) popInd(p) elif match(p, p.idx, " _"): # hyperlink target: - inc(p.idx, 2) - var a = getReferenceName(p, ":") - if p.tok[p.idx].kind == tkWhite: inc(p.idx) + inc p.idx, 2 + var ending = ":" + if currentTok(p).symbol == "`": + inc p.idx + ending = "`" + var a = getReferenceName(p, ending) + if ending == "`": + if currentTok(p).symbol == ":": + inc p.idx + else: + rstMessage(p, meExpected, ":") + if currentTok(p).kind == tkWhite: inc p.idx var b = untilEol(p) - setRef(p, rstnodeToRefname(a), b) + if len(b) == 0: # set internal anchor + p.curAnchors.add ManualAnchor( + alias: linkName(a), anchor: rstnodeToRefname(a), info: prevLineInfo(p) + ) + else: # external hyperlink + setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias) elif match(p, p.idx, " |"): # substitution definitions: - inc(p.idx, 2) + inc p.idx, 2 var a = getReferenceName(p, "|") var b: PRstNode - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - if cmpIgnoreStyle(p.tok[p.idx].symbol, "replace") == 0: - inc(p.idx) + if currentTok(p).kind == tkWhite: inc p.idx + if cmpIgnoreStyle(currentTok(p).symbol, "replace") == 0: + inc p.idx expect(p, "::") b = untilEol(p) - elif cmpIgnoreStyle(p.tok[p.idx].symbol, "image") == 0: - inc(p.idx) + elif cmpIgnoreStyle(currentTok(p).symbol, "image") == 0: + inc p.idx b = dirImage(p) else: - rstMessage(p, meInvalidDirective, p.tok[p.idx].symbol) + rstMessage(p, meInvalidDirective, currentTok(p).symbol) setSub(p, addNodes(a), b) - elif match(p, p.idx, " ["): - # footnotes, citations - inc(p.idx, 2) - var a = getReferenceName(p, "]") - if p.tok[p.idx].kind == tkWhite: inc(p.idx) - var b = untilEol(p) - setRef(p, rstnodeToRefname(a), b) + elif match(p, p.idx, " [") and + (n = parseFootnote(p); n != nil): + result = n else: - result = parseComment(p) - -proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = + result = parseComment(p, col) + +proc rstParsePass1*(fragment: string, + line, column: int, + sharedState: PRstSharedState): PRstNode = + ## Parses an RST `fragment`. + ## The result should be further processed by + ## preparePass2_ and resolveSubs_ (which is pass 2). + var p: RstParser + initParser(p, sharedState) + p.line = line + p.col = column + getTokens(fragment, p.tok) + result = parseDoc(p) + +proc extractLinkEnd(x: string): string = + ## From links like `path/to/file.html#/%` extract `file.html#/%`. + let i = find(x, '#') + let last = + if i >= 0: i + else: x.len - 1 + let j = rfind(x, '/', start=0, last=last) + if j >= 0: + result = x[j+1 .. ^1] + else: + result = x + +proc loadIdxFile(s: var PRstSharedState, origFilename: string) = + doAssert roSandboxDisabled in s.options + var info: TLineInfo + info.fileIndex = addFilename(s, origFilename) + var (dir, basename, ext) = origFilename.splitFile + if ext notin [".md", ".rst", ".nim", ""]: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, origFilename & ": unknown extension") + let idxFilename = dir / basename & ".idx" + let (idxPath, linkRelPath) = s.findRefFile(idxFilename) + s.idxImports[origFilename].linkRelPath = linkRelPath + var + fileEntries: seq[IndexEntry] + title: IndexEntry + try: + (fileEntries, title) = parseIdxFile(idxPath) + except IOError: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, idxPath) + except ValueError as e: + s.msgHandler(idxPath, LineRstInit, ColRstInit, meInvalidField, e.msg) + + var isMarkup = false # for sanity check to avoid mixing .md <-> .nim + for entry in fileEntries: + # Though target .idx already has inside it the path to HTML relative + # project's root, we won't rely on it and use `linkRelPath` instead. + let refn = extractLinkEnd(entry.link) + # select either markup (rst/md) or Nim cases: + if entry.kind in {ieMarkupTitle, ieNimTitle}: + s.idxImports[origFilename].title = entry.keyword + case entry.kind + of ieIdxRole, ieHeading, ieMarkupTitle: + if ext == ".nim" and entry.kind == ieMarkupTitle: + rstMessage(s, idxPath, meInvalidField, + $ieMarkupTitle & " in supposedly .nim-derived file") + if entry.kind == ieMarkupTitle: + isMarkup = true + info.line = entry.line.uint16 + addAnchorExtRst(s, key = entry.keyword, refn = refn, + anchorType = headlineAnchor, info=info) + of ieNim, ieNimGroup, ieNimTitle: + if ext in [".md", ".rst"] or isMarkup: + rstMessage(s, idxPath, meInvalidField, + $entry.kind & " in supposedly markup-derived file") + s.nimFileImported = true + var langSym: LangSymbol + if entry.kind in {ieNim, ieNimTitle}: + var q: RstParser + initParser(q, s) + info.line = entry.line.uint16 + setLen(q.tok, 0) + q.idx = 0 + getTokens(entry.linkTitle, q.tok) + var sons = newSeq[PRstNode](q.tok.len) + for i in 0 ..< q.tok.len: sons[i] = newLeaf(q.tok[i].symbol) + let linkTitle = newRstNode(rnInner, sons) + langSym = linkTitle.toLangSymbol + else: # entry.kind == ieNimGroup + langSym = langSymbolGroup(kind=entry.linkTitle, name=entry.keyword) + addAnchorNim(s, external = true, refn = refn, tooltip = entry.linkDesc, + langSym = langSym, priority = -4, # lowest + info = info, module = info.fileIndex) + doAssert s.idxImports[origFilename].title != "" + +proc preparePass2*(s: var PRstSharedState, mainNode: PRstNode, importdoc = true) = + ## Records titles in node `mainNode` and orders footnotes. + countTitles(s, mainNode) + fixHeadlines(s) + orderFootnotes(s) + if importdoc: + for origFilename in s.idxImports.keys: + loadIdxFile(s, origFilename) + +proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = + # Associate this link alias with its target and change node kind to + # rnHyperlink or rnInternalRef appropriately. + var desc, alias: PRstNode + if n.kind == rnPandocRef: # link like [desc][alias] + desc = n.sons[0] + alias = n.sons[1] + else: # n.kind == rnRstRef, link like `desc=alias`_ + desc = n + alias = n + type LinkDef = object + ar: AnchorRule + priority: int + tooltip: string + target: PRstNode + info: TLineInfo + externFilename: string + # when external anchor: origin filename where anchor was defined + isTitle: bool + proc cmp(x, y: LinkDef): int = + result = cmp(x.priority, y.priority) + if result == 0: + result = cmp(x.target, y.target) + var foundLinks: seq[LinkDef] + let refn = rstnodeToRefname(alias) + var hyperlinks = findRef(s, refn) + for y in hyperlinks: + foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind), + target: y.value, info: y.info, + tooltip: "(" & $y.kind & ")") + let substRst = findMainAnchorRst(s, alias.addNodes, n.info) + template getExternFilename(subst: AnchorSubst): string = + if subst.kind == arExternalRst or + (subst.kind == arNim and subst.external): + getFilename(s, subst) + else: "" + for subst in substRst: + var refname, fullRefname: string + if subst.kind == arInternalRst: + refname = subst.target.anchor + fullRefname = refname + else: # arExternalRst + refname = subst.refnameExt + fullRefname = s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & refname + let anchorType = + if subst.kind == arInternalRst: subst.anchorType + else: subst.anchorTypeExt # arExternalRst + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + info: subst.info, + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(refname), + tooltip: "(" & $anchorType & ")") + # find anchors automatically generated from Nim symbols + if roNimFile in s.options or s.nimFileImported: + let substNim = findMainAnchorNim(s, signature=alias, n.info) + for subst in substNim: + let fullRefname = + if subst.external: + s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & subst.refname + else: subst.refname + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(subst.refname), + info: subst.info, tooltip: subst.tooltip) + foundLinks.sort(cmp = cmp, order = Descending) + let aliasStr = addNodes(alias) + if foundLinks.len >= 1: + if foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].used = true + let kind = if foundLinks[0].ar in {arHyperlink, arExternalRst}: rnHyperlink + elif foundLinks[0].ar == arNim: + if foundLinks[0].externFilename == "": rnNimdocRef + else: rnHyperlink + else: rnInternalRef + result = newRstNode(kind) + let documentName = # filename without ext for `.nim`, title for `.md` + if foundLinks[0].ar == arNim: + changeFileExt(foundLinks[0].externFilename.extractFilename, "") + elif foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].title + else: foundLinks[0].externFilename.extractFilename + let linkText = + if foundLinks[0].externFilename != "": + if foundLinks[0].isTitle: newLeaf(addNodes(desc)) + else: newLeaf(documentName & ": " & addNodes(desc)) + else: + newRstNode(rnInner, desc.sons) + result.sons = @[linkText, foundLinks[0].target] + if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip + if foundLinks.len > 1: # report ambiguous link + var targets = newSeq[string]() + for l in foundLinks: + var t = " " + if s.filenames.len > 1: + t.add getFilename(s.filenames, l.info.fileIndex) + let n = l.info.line + let c = l.info.col + ColRstOffset + t.add "($1, $2): $3" % [$n, $c, l.tooltip] + targets.add t + rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink, + "`$1`\n clash:\n$2" % [ + aliasStr, targets.join("\n")]) + else: # nothing found + result = n + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, aliasStr) + +proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = + ## Makes pass 2 of RST parsing. + ## Resolves substitutions and anchor aliases, groups footnotes. + ## Takes input node `n` and returns the same node with recursive + ## substitutions in `n.sons` to `result`. result = n if n == nil: return case n.kind of rnSubstitutionReferences: - var x = findSub(p, n) + var x = findSub(s, n) if x >= 0: - result = p.s.subs[x].value + result = s.subs[x].value else: var key = addNodes(n) var e = getEnv(key) - if e != "": result = newRstNode(rnLeaf, e) - else: rstMessage(p, mwUnknownSubstitution, key) - of rnRef: - var y = findRef(p, rstnodeToRefname(n)) - if y != nil: - result = newRstNode(rnHyperlink) - n.kind = rnInner - add(result, n) - add(result, y) + if e != "": result = newLeaf(e) + else: rstMessage(s.filenames, s.msgHandler, n.info, + mwUnknownSubstitution, key) + of rnRstRef, rnPandocRef: + result = resolveLink(s, n) + of rnFootnote: + var (fnType, num) = getFootnoteType(s, n.sons[0]) + case fnType + of fnManualNumber, fnCitation: + discard "no need to alter fixed text" + of fnAutoNumberLabel, fnAutoNumber: + if fnType == fnAutoNumberLabel: + let labelR = rstnodeToRefname(n.sons[0]) + num = getFootnoteNum(s, labelR) + else: + num = getFootnoteNum(s, n.order) + var nn = newRstNode(rnInner) + nn.add newLeaf($num) + result.sons[0] = nn + of fnAutoSymbol: + let sym = getAutoSymbol(s, n.order) + n.sons[0].sons[0].text = sym + n.sons[1] = resolveSubs(s, n.sons[1]) + of rnFootnoteRef: + var (fnType, num) = getFootnoteType(s, n.sons[0]) + template addLabel(number: int | string) = + var nn = newRstNode(rnInner) + nn.add newLeaf($number) + result.add(nn) + var refn = fnType.prefix + # create new rnFootnoteRef, add final label, and finalize target refn: + result = newRstNode(rnFootnoteRef, info = n.info) + case fnType + of fnManualNumber: + addLabel num + refn.add $num + of fnAutoNumber: + inc s.currFootnoteNumRef + addLabel getFootnoteNum(s, s.currFootnoteNumRef) + refn.add $s.currFootnoteNumRef + of fnAutoNumberLabel: + addLabel getFootnoteNum(s, rstnodeToRefname(n)) + refn.add rstnodeToRefname(n) + of fnAutoSymbol: + inc s.currFootnoteSymRef + addLabel getAutoSymbol(s, s.currFootnoteSymRef) + refn.add $s.currFootnoteSymRef + of fnCitation: + result.add n.sons[0] + refn.add rstnodeToRefname(n) + # TODO: correctly report ambiguities + let anchorInfo = findMainAnchorRst(s, refn, n.info) + if anchorInfo.len != 0: + result.add newLeaf(anchorInfo[0].target.anchor) # add link + else: + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) + result.add newLeaf(refn) # add link of rnLeaf: discard - of rnContents: - p.hasToc = true else: - for i in countup(0, len(n) - 1): n.sons[i] = resolveSubs(p, n.sons[i]) + var regroup = false + for i in 0 ..< n.len: + n.sons[i] = resolveSubs(s, n.sons[i]) + if n.sons[i] != nil and n.sons[i].kind == rnFootnote: + regroup = true + if regroup: # group footnotes together into rnFootnoteGroup + var newSons: seq[PRstNode] + var i = 0 + while i < n.len: + if n.sons[i] != nil and n.sons[i].kind == rnFootnote: + var grp = newRstNode(rnFootnoteGroup) + while i < n.len and n.sons[i].kind == rnFootnote: + grp.sons.add n.sons[i] + inc i + newSons.add grp + else: + newSons.add n.sons[i] + inc i + result.sons = newSons + +proc completePass2*(s: PRstSharedState) = + for (filename, importdocInfo) in s.idxImports.pairs: + if not importdocInfo.used: + rstMessage(s.filenames, s.msgHandler, importdocInfo.fromInfo, + mwUnusedImportdoc, filename) proc rstParse*(text, filename: string, - line, column: int, hasToc: var bool, + line, column: int, options: RstParseOptions, findFile: FindFileHandler = nil, - msgHandler: MsgHandler = nil): PRstNode = - var p: RstParser - initParser(p, newSharedState(options, findFile, msgHandler)) - p.filename = filename - p.line = line - p.col = column + getTokens(text, roSkipPounds in options, p.tok) - result = resolveSubs(p, parseDoc(p)) - hasToc = p.hasToc + findRefFile: FindRefFileHandler = nil, + msgHandler: MsgHandler = nil): + tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] = + ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, + ## note that 2nd tuple element should be fed to `initRstGenerator` + ## argument `filenames` (it is being filled here at least with `filename` + ## and possibly with other files from RST ``.. include::`` statement). + var sharedState = newRstSharedState(options, filename, findFile, findRefFile, + msgHandler, hasToc=false) + let unresolved = rstParsePass1(text, line, column, sharedState) + preparePass2(sharedState, unresolved) + result.node = resolveSubs(sharedState, unresolved) + completePass2(sharedState) + result.filenames = sharedState.filenames + result.hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 044ea2c14..2bbb0d0b8 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -8,23 +8,26 @@ # ## This module implements an AST for the `reStructuredText`:idx: parser. -## -## **Note:** Import ``packages/docutils/rstast`` to use this module -import strutils, json +import std/[strutils, json] + +when defined(nimPreviewSlimSystem): + import std/assertions + type RstNodeKind* = enum ## the possible node kinds of an PRstNode rnInner, # an inner node or a root rnHeadline, # a headline rnOverline, # an over- and underlined headline + rnMarkdownHeadline, # a Markdown headline rnTransition, # a transition (the ------------- <hr> thingie) rnParagraph, # a paragraph rnBulletList, # a bullet list rnBulletItem, # a bullet item rnEnumList, # an enumerated list rnEnumItem, # an enumerated item - rnDefList, # a definition list + rnDefList, rnMdDefList, # a definition list (RST/Markdown) rnDefItem, # an item of a definition list consisting of ... rnDefName, # ... a name part ... rnDefBody, # ... and a body part ... @@ -33,16 +36,28 @@ type rnFieldName, # consisting of a field name ... rnFieldBody, # ... and a field body rnOptionList, rnOptionListItem, rnOptionGroup, rnOption, rnOptionString, - rnOptionArgument, rnDescription, rnLiteralBlock, rnQuotedLiteralBlock, + rnOptionArgument, rnDescription, rnLiteralBlock, + rnMarkdownBlockQuote, # a quote starting from punctuation like >>> + rnMarkdownBlockQuoteItem, # a quotation block, quote lines starting with + # the same number of chars rnLineBlock, # the | thingie - rnLineBlockItem, # sons of the | thing + rnLineBlockItem, # a son of rnLineBlock - one line inside it. + # When `RstNode` lineIndent="\n" the line's empty rnBlockQuote, # text just indented - rnTable, rnGridTable, rnTableRow, rnTableHeaderCell, rnTableDataCell, - rnLabel, # used for footnotes and other things + rnTable, rnGridTable, rnMarkdownTable, rnTableRow, rnTableHeaderCell, rnTableDataCell, rnFootnote, # a footnote - rnCitation, # similar to footnote - rnStandaloneHyperlink, rnHyperlink, rnRef, rnDirective, # a directive - rnDirArg, rnRaw, rnTitle, rnContents, rnImage, rnFigure, rnCodeBlock, + rnCitation, # similar to footnote, so use rnFootnote instead + rnFootnoteGroup, # footnote group - exists for a purely stylistic + # reason: to display a few footnotes as 1 block + rnStandaloneHyperlink, rnHyperlink, + rnRstRef, # RST reference like `section name`_ + rnPandocRef, # Pandoc Markdown reference like [section name] + rnInternalRef, rnFootnoteRef, + rnNimdocRef, # reference to automatically generated Nim symbol + rnDirective, # a general directive + rnDirArg, # a directive argument (for some directives). + # here are directives that are not rnDirective: + rnRaw, rnTitle, rnContents, rnImage, rnFigure, rnCodeBlock, rnAdmonition, rnRawHtml, rnRawLatex, rnContainer, # ``container`` directive rnIndex, # index directve: @@ -51,40 +66,94 @@ type # * `file#id <file#id>`_ # * `file#id <file#id>'_ rnSubstitutionDef, # a definition of a substitution - rnGeneralRole, # Inline markup: + # Inline markup: + rnInlineCode, # interpreted text with code in a known language + rnCodeFragment, # inline code for highlighting with the specified + # class (which cannot be inferred from context) + rnUnknownRole, # interpreted text with an unknown role rnSub, rnSup, rnIdx, rnEmphasis, # "*" rnStrongEmphasis, # "**" rnTripleEmphasis, # "***" - rnInterpretedText, # "`" + rnInterpretedText, # "`" an auxiliary role for parsing that will + # be converted into other kinds like rnInlineCode rnInlineLiteral, # "``" + rnInlineTarget, # "_`target`" rnSubstitutionReferences, # "|" rnSmiley, # some smiley + rnDefaultRole, # .. default-role:: code rnLeaf # a leaf; the node's text field contains the # leaf val + FileIndex* = distinct int32 + TLineInfo* = object + line*: uint16 + col*: int16 + fileIndex*: FileIndex PRstNode* = ref RstNode ## an RST node RstNodeSeq* = seq[PRstNode] - RstNode* {.acyclic, final.} = object ## an RST node's description - kind*: RstNodeKind ## the node's kind - text*: string ## valid for leafs in the AST; and the title of - ## the document or the section - level*: int ## valid for some node kinds + RstNode* {.acyclic, final.} = object ## AST node (result of RST parsing) + case kind*: RstNodeKind ## the node's kind + of rnLeaf, rnSmiley: + text*: string ## string that is expected to be displayed + of rnEnumList: + labelFmt*: string ## label format like "(1)" + of rnLineBlockItem: + lineIndent*: string ## a few spaces or newline at the line beginning + of rnAdmonition: + adType*: string ## admonition type: "note", "caution", etc. This + ## text will set the style and also be displayed + of rnOverline, rnHeadline, rnMarkdownHeadline: + level*: int ## level of headings starting from 1 (main + ## chapter) to larger ones (minor sub-sections) + ## level=0 means it's document title or subtitle + of rnFootnote, rnCitation, rnOptionListItem: + order*: int ## footnote order (for auto-symbol footnotes and + ## auto-numbered ones without a label) + of rnMarkdownBlockQuoteItem: + quotationDepth*: int ## number of characters in line prefix + of rnRstRef, rnPandocRef, rnSubstitutionReferences, + rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: + info*: TLineInfo ## To have line/column info for warnings at + ## nodes that are post-processed after parsing + of rnNimdocRef: + tooltip*: string + of rnTable, rnGridTable, rnMarkdownTable: + colCount*: int ## Number of (not-united) cells in the table + of rnTableRow: + endsHeader*: bool ## Is last row in the header of table? + of rnTableHeaderCell, rnTableDataCell: + span*: int ## Number of table columns that the cell occupies + else: + discard + anchor*: string ## anchor, internal link target + ## (aka HTML id tag, aka Latex label/hypertarget) sons*: RstNodeSeq ## the node's sons +proc `==`*(a, b: FileIndex): bool {.borrow.} + proc len*(n: PRstNode): int = result = len(n.sons) -proc newRstNode*(kind: RstNodeKind): PRstNode = - new(result) - result.sons = @[] - result.kind = kind +proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], + anchor = ""): PRstNode = + result = PRstNode(kind: kind, sons: sons, anchor: anchor) + +proc newRstNode*(kind: RstNodeKind, info: TLineInfo, + sons: seq[PRstNode] = @[]): PRstNode = + result = PRstNode(kind: kind, sons: sons) + result.info = info -proc newRstNode*(kind: RstNodeKind, s: string): PRstNode = +proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = + assert kind in {rnLeaf, rnSmiley} result = newRstNode(kind) result.text = s +proc newRstLeaf*(s: string): PRstNode = + result = newRstNode(rnLeaf) + result.text = s + proc lastSon*(n: PRstNode): PRstNode = result = n.sons[len(n.sons)-1] @@ -92,7 +161,7 @@ proc add*(father, son: PRstNode) = add(father.sons, son) proc add*(father: PRstNode; s: string) = - add(father.sons, newRstNode(rnLeaf, s)) + add(father.sons, newRstLeaf(s)) proc addIfNotNil*(father, son: PRstNode) = if son != nil: add(father, son) @@ -215,7 +284,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) = inc(d.indent, 2) renderRstSons(d, n, result) dec(d.indent, 2) - of rnRef: + of rnRstRef: result.add("`") renderRstSons(d, n, result) result.add("`_") @@ -225,7 +294,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) = result.add(" <") renderRstToRst(d, n.sons[1], result) result.add(">`_") - of rnGeneralRole: + of rnUnknownRole: result.add('`') renderRstToRst(d, n.sons[0],result) result.add("`:") @@ -298,7 +367,7 @@ proc renderRstToJsonNode(node: PRstNode): JsonNode = (key: "kind", val: %($node.kind)), (key: "level", val: %BiggestInt(node.level)) ] - if node.text.len > 0: + if node.kind in {rnLeaf, rnSmiley} and node.text.len > 0: result.add("text", %node.text) if len(node.sons) > 0: var accm = newSeq[JsonNode](len(node.sons)) @@ -308,11 +377,68 @@ proc renderRstToJsonNode(node: PRstNode): JsonNode = proc renderRstToJson*(node: PRstNode): string = ## Writes the given RST node as JSON that is in the form - ## :: - ## { - ## "kind":string node.kind, - ## "text":optional string node.text, - ## "level":optional int node.level, - ## "sons":optional node array - ## } + ## + ## { + ## "kind":string node.kind, + ## "text":optional string node.text, + ## "level":optional int node.level, + ## "sons":optional node array + ## } renderRstToJsonNode(node).pretty + +proc renderRstToText*(node: PRstNode): string = + ## minimal text representation of markup node + const code = {rnCodeFragment, rnInterpretedText, rnInlineLiteral, rnInlineCode} + if node == nil: + return "" + case node.kind + of rnLeaf, rnSmiley: + result.add node.text + else: + if node.kind in code: result.add "`" + for i in 0 ..< node.sons.len: + if node.kind in {rnInlineCode, rnCodeBlock} and i == 0: + continue # omit language specifier + result.add renderRstToText(node.sons[i]) + if node.kind in code: result.add "`" + +proc treeRepr*(node: PRstNode, indent=0): string = + ## Writes the parsed RST `node` into an AST tree with compact string + ## representation in the format (one line per every sub-node): + ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)`` + ## (suitable for debugging of RST parsing). + if node == nil: + result.add " ".repeat(indent) & "[nil]\n" + return + result.add " ".repeat(indent) & $node.kind + case node.kind + of rnLeaf, rnSmiley: + result.add (if node.text == "": "" else: " '" & node.text & "'") + of rnEnumList: + result.add " labelFmt=" & node.labelFmt + of rnLineBlockItem: + var txt: string + if node.lineIndent == "\n": txt = " (blank line)" + else: txt = " lineIndent=" & $node.lineIndent.len + result.add txt + of rnAdmonition: + result.add " adType=" & node.adType + of rnHeadline, rnOverline, rnMarkdownHeadline: + result.add " level=" & $node.level + of rnFootnote, rnCitation, rnOptionListItem: + result.add (if node.order == 0: "" else: " order=" & $node.order) + of rnMarkdownBlockQuoteItem: + result.add " quotationDepth=" & $node.quotationDepth + of rnTable, rnGridTable, rnMarkdownTable: + result.add " colCount=" & $node.colCount + of rnTableHeaderCell, rnTableDataCell: + if node.span > 0: + result.add " span=" & $node.span + of rnTableRow: + if node.endsHeader: result.add " endsHeader" + else: + discard + result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") + result.add "\n" + for son in node.sons: + result.add treeRepr(son, indent=indent+2) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index abbf99fed..7fc0ac03a 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -23,10 +23,32 @@ ## many options and tweaking, but you are not limited to snippets and can ## generate `LaTeX documents <https://en.wikipedia.org/wiki/LaTeX>`_ too. ## -## **Note:** Import ``packages/docutils/rstgen`` to use this module +## `Docutils configuration files`_ are not supported. Instead HTML generation +## can be tweaked by editing file ``config/nimdoc.cfg``. +## +## .. _Docutils configuration files: https://docutils.sourceforge.io/docs/user/config.htm +## +## There are stylistic difference between how this module renders some elements +## and how original Python Docutils does: +## +## * Backreferences to TOC in section headings are not generated. +## In HTML each section is also a link that points to the section itself: +## this is done for user to be able to copy the link into clipboard. +## +## * The same goes for footnotes/citations links: they point to themselves. +## No backreferences are generated since finding all references of a footnote +## can be done by simply searching for ``[footnoteName]``. + +import std/[strutils, os, hashes, strtabs, tables, sequtils, + algorithm, parseutils, strbasics] + +import rstast, rst, rstidx, highlite -import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio, formatfloat] + + +import ../../std/private/since const HtmlExt = "html" @@ -37,25 +59,27 @@ type outHtml, # output is HTML outLatex # output is Latex - TocEntry = object - n*: PRstNode - refname*, header*: string - MetaEnum* = enum - metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion + metaNone, metaTitleRaw, metaTitle, metaSubtitle, metaAuthor, metaVersion + + EscapeMode* = enum # in Latex text inside options [] and URLs is + # escaped slightly differently than in normal text + emText, emOption, emUrl # emText is currently used for code also RstGenerator* = object of RootObj target*: OutputTarget config*: StringTableRef splitAfter*: int # split too long entries in the TOC listingCounter*: int - tocPart*: seq[TocEntry] + tocPart*: seq[PRstNode] # headings for Table of Contents hasToc*: bool theIndex: string # Contents of the index file to be dumped at the end. - options*: RstParseOptions findFile*: FindFileHandler msgHandler*: MsgHandler - filename*: string + outDir*: string ## output directory, initialized by docgen.nim + destFile*: string ## output (HTML) file, initialized by docgen.nim + filenames*: RstFileTable + filename*: string ## source Nim or Rst file meta*: array[MetaEnum, string] currentSection: string ## \ ## Stores the empty string or the last headline/overline found in the rst @@ -65,7 +89,9 @@ type ## for hyperlinks. See renderIndexTerm proc for details. id*: int ## A counter useful for generating IDs. onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int; - content: string) + content: string) {.gcsafe.} + escMode*: EscapeMode + curQuotationDepth: int PDoc = var RstGenerator ## Alias to type less. @@ -78,6 +104,9 @@ type testCmd: string status: int +proc prettyLink*(file: string): string = + changeFileExt(file, "").replace("_._", "..") + proc init(p: var CodeBlockParams) = ## Default initialisation of CodeBlockParams to sane values. p.startLine = 1 @@ -86,9 +115,10 @@ proc init(p: var CodeBlockParams) = proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, config: StringTableRef, filename: string, - options: RstParseOptions, findFile: FindFileHandler = nil, - msgHandler: MsgHandler = nil) = + msgHandler: MsgHandler = nil, + filenames = default(RstFileTable), + hasToc = false) = ## Initializes a ``RstGenerator``. ## ## You need to call this before using a ``RstGenerator`` with any other @@ -105,7 +135,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, ## it helps to prettify the generated index if no title is found. ## ## The ``RstParseOptions``, ``FindFileHandler`` and ``MsgHandler`` types - ## are defined in the the `packages/docutils/rst module <rst.html>`_. + ## are defined in the `packages/docutils/rst module <rst.html>`_. ## ``options`` selects the behaviour of the rst parser. ## ## ``findFile`` is a proc used by the rst ``include`` directive among others. @@ -124,22 +154,25 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, ## ## Example: ## - ## .. code-block:: nim - ## + ## ```nim ## import packages/docutils/rstgen ## ## var gen: RstGenerator ## gen.initRstGenerator(outHtml, defaultConfig(), "filename", {}) + ## ``` g.config = config g.target = target g.tocPart = @[] + g.hasToc = hasToc g.filename = filename + g.filenames = filenames g.splitAfter = 20 g.theIndex = "" - g.options = options g.findFile = findFile g.currentSection = "" g.id = 0 + g.escMode = emText + g.curQuotationDepth = 0 let fileParts = filename.splitFile if fileParts.ext == ".nim": g.currentSection = "Module " & fileParts.name @@ -158,7 +191,9 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) = ## If the index is empty the file won't be created. if g.theIndex.len > 0: writeFile(outfile, g.theIndex) -proc addXmlChar(dest: var string, c: char) = +proc addHtmlChar(dest: var string, c: char) = + # Escapes HTML characters. Note that single quote ' is not escaped as + # ' -- unlike XML (for standards pre HTML5 it was even forbidden). case c of '&': add(dest, "&") of '<': add(dest, "<") @@ -166,35 +201,36 @@ proc addXmlChar(dest: var string, c: char) = of '\"': add(dest, """) else: add(dest, c) -proc addRtfChar(dest: var string, c: char) = +proc addTexChar(dest: var string, c: char, escMode: EscapeMode) = + ## Escapes 10 special Latex characters and sometimes ` and [, ]. + ## TODO: @ is always a normal symbol (besides the header), am I wrong? + ## All escapes that need to work in text and code blocks (`emText` mode) + ## should start from \ (to be compatible with fancyvrb/fvextra). case c - of '{': add(dest, "\\{") - of '}': add(dest, "\\}") - of '\\': add(dest, "\\\\") + of '_', '&', '#', '%': add(dest, "\\" & c) + # commands \label and \pageref don't accept \$ by some reason but OK with $: + of '$': (if escMode == emUrl: add(dest, c) else: add(dest, "\\" & c)) + # \~ and \^ have a special meaning unless they are followed by {} + of '~', '^': add(dest, "\\" & c & "{}") + # Latex loves to substitute ` to opening quote, even in texttt mode! + of '`': add(dest, "\\textasciigrave{}") + # add {} to avoid gobbling up space by \textbackslash + of '\\': add(dest, "\\textbackslash{}") + # Using { and } in URL in Latex: https://tex.stackexchange.com/a/469175 + of '{': + add(dest, if escMode == emUrl: "\\%7B" else: "\\{") + of '}': + add(dest, if escMode == emUrl: "\\%7D" else: "\\}") + of ']': + # escape ] inside an optional argument in e.g. \section[static[T]]{.. + add(dest, if escMode == emOption: "\\text{]}" else: "]") else: add(dest, c) -proc addTexChar(dest: var string, c: char) = - case c - of '_': add(dest, "\\_") - of '{': add(dest, "\\symbol{123}") - of '}': add(dest, "\\symbol{125}") - of '[': add(dest, "\\symbol{91}") - of ']': add(dest, "\\symbol{93}") - of '\\': add(dest, "\\symbol{92}") - of '$': add(dest, "\\$") - of '&': add(dest, "\\&") - of '#': add(dest, "\\#") - of '%': add(dest, "\\%") - of '~': add(dest, "\\symbol{126}") - of '@': add(dest, "\\symbol{64}") - of '^': add(dest, "\\symbol{94}") - of '`': add(dest, "\\symbol{96}") - else: add(dest, c) - -proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} = +proc escChar*(target: OutputTarget, dest: var string, + c: char, escMode: EscapeMode) {.inline.} = case target - of outHtml: addXmlChar(dest, c) - of outLatex: addTexChar(dest, c) + of outHtml: addHtmlChar(dest, c) + of outLatex: addTexChar(dest, c, escMode) proc addSplitter(target: OutputTarget; dest: var string) {.inline.} = case target @@ -213,7 +249,7 @@ proc nextSplitPoint*(s: string, start: int): int = inc(result) dec(result) # last valid index -proc esc*(target: OutputTarget, s: string, splitAfter = -1): string = +proc esc*(target: OutputTarget, s: string, splitAfter = -1, escMode = emText): string = ## Escapes the HTML. result = "" if splitAfter >= 0: @@ -224,11 +260,11 @@ proc esc*(target: OutputTarget, s: string, splitAfter = -1): string = #if (splitter != " ") or (partLen + k - j + 1 > splitAfter): partLen = 0 addSplitter(target, result) - for i in countup(j, k): escChar(target, result, s[i]) + for i in countup(j, k): escChar(target, result, s[i], escMode) inc(partLen, k - j + 1) j = k + 1 else: - for i in countup(0, len(s) - 1): escChar(target, result, s[i]) + for i in countup(0, len(s) - 1): escChar(target, result, s[i], escMode) proc disp(target: OutputTarget, xml, tex: string): string = @@ -248,58 +284,46 @@ proc dispA(target: OutputTarget, dest: var string, proc `or`(x, y: string): string {.inline.} = result = if x.len == 0: y else: x -proc renderRstToOut*(d: var RstGenerator, n: PRstNode, result: var string) +proc renderRstToOut*(d: var RstGenerator, n: PRstNode, result: var string) {.gcsafe.} ## Writes into ``result`` the rst ast ``n`` using the ``d`` configuration. ## ## Before using this proc you need to initialise a ``RstGenerator`` with ## ``initRstGenerator`` and parse a rst file with ``rstParse`` from the ## `packages/docutils/rst module <rst.html>`_. Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## # ...configure gen and rst vars... ## var generatedHtml = "" ## renderRstToOut(gen, rst, generatedHtml) ## echo generatedHtml + ## ``` proc renderAux(d: PDoc, n: PRstNode, result: var string) = for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], result) -proc renderAux(d: PDoc, n: PRstNode, frmtA, frmtB: string, result: var string) = +template idS(txt: string): string = + if txt == "": "" + else: + case d.target + of outHtml: + " id=\"" & txt & "\"" + of outLatex: + "\\label{" & txt & "}\\hypertarget{" & txt & "}{}" + # we add \label for page number references via \pageref, while + # \hypertarget is for clickable links via \hyperlink. + +proc renderAux(d: PDoc, n: PRstNode, html, tex: string, result: var string) = + # formats sons of `n` as substitution variable $1 inside strings `html` and + # `tex`, internal target (anchor) is provided as substitute $2. var tmp = "" for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], tmp) - if d.target != outLatex: - result.addf(frmtA, [tmp]) - else: - result.addf(frmtB, [tmp]) + case d.target + of outHtml: result.addf(html, [tmp, n.anchor.idS]) + of outLatex: result.addf(tex, [tmp, n.anchor.idS]) # ---------------- index handling -------------------------------------------- -proc quoteIndexColumn(text: string): string = - ## Returns a safe version of `text` for serialization to the ``.idx`` file. - ## - ## The returned version can be put without worries in a line based tab - ## separated column text file. The following character sequence replacements - ## will be performed for that goal: - ## - ## * ``"\\"`` => ``"\\\\"`` - ## * ``"\n"`` => ``"\\n"`` - ## * ``"\t"`` => ``"\\t"`` - result = newStringOfCap(text.len + 3) - for c in text: - case c - of '\\': result.add "\\" - of '\L': result.add "\\n" - of '\C': discard - of '\t': result.add "\\t" - else: result.add c - -proc unquoteIndexColumn(text: string): string = - ## Returns the unquoted version generated by ``quoteIndexColumn``. - result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) - -proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string, - linkTitle, linkDesc = "") = +proc setIndexTerm*(d: var RstGenerator; k: IndexEntryKind, htmlFile, id, term: string, + linkTitle, linkDesc = "", line = 0) = ## Adds a `term` to the index using the specified hyperlink identifier. ## ## A new entry will be added to the index using the format @@ -322,21 +346,8 @@ proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string, ## <#writeIndexFile,RstGenerator,string>`_. The purpose of the index is ## documented in the `docgen tools guide ## <docgen.html#related-options-index-switch>`_. - var - entry = term - isTitle = false - entry.add('\t') - entry.add(htmlFile) - if id.len > 0: - entry.add('#') - entry.add(id) - else: - isTitle = true - if linkTitle.len > 0 or linkDesc.len > 0: - entry.add('\t' & linkTitle.quoteIndexColumn) - entry.add('\t' & linkDesc.quoteIndexColumn) - entry.add("\n") - + let (entry, isTitle) = formatIndexEntry(k, htmlFile, id, term, + linkTitle, linkDesc, line) if isTitle: d.theIndex.insert(entry) else: d.theIndex.add(entry) @@ -349,6 +360,15 @@ proc hash(n: PRstNode): int = result = result !& hash(n.sons[i]) result = !$result +proc htmlFileRelPath(d: PDoc): string = + if d.outDir.len == 0: + # /foo/bar/zoo.nim -> zoo.html + changeFileExt(extractFilename(d.filename), HtmlExt) + else: # d is initialized in docgen.nim + # outDir = /foo -\ + # destFile = /foo/bar/zoo.html -|-> bar/zoo.html + d.destFile.relativePath(d.outDir, '/') + proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = ## Renders the string decorated within \`foobar\`\:idx\: markers. ## @@ -365,18 +385,13 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) - setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) - dispA(d.target, result, "<span id=\"$1\">$2</span>", "$2\\label{$1}", + setIndexTerm(d, ieIdxRole, + htmlFileRelPath(d), id, term, d.currentSection) + dispA(d.target, result, "<span id=\"$1\">$2</span>", "\\nimindexterm{$1}{$2}", [id, term]) type - IndexEntry = object - keyword: string - link: string - linkTitle: string ## contains a prettier text for the href - linkDesc: string ## the title attribute of the final href - - IndexedDocs = Table[IndexEntry, seq[IndexEntry]] ## \ + IndexedDocs* = Table[IndexEntry, seq[IndexEntry]] ## \ ## Contains the index sequences for doc types. ## ## The key is a *fake* IndexEntry which will contain the title of the @@ -386,21 +401,6 @@ type ## The value indexed by this IndexEntry is a sequence with the real index ## entries found in the ``.idx`` file. -proc cmp(a, b: IndexEntry): int = - ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. - result = cmpIgnoreStyle(a.keyword, b.keyword) - if result == 0: - result = cmpIgnoreStyle(a.link, b.link) - -proc hash(x: IndexEntry): Hash = - ## Returns the hash for the combined fields of the type. - ## - ## The hash is computed as the chained hash of the individual string hashes. - result = x.keyword.hash !& x.link.hash - result = result !& x.linkTitle.hash - result = result !& x.linkDesc.hash - result = !$result - when defined(gcDestructors): template `<-`(a, b: var IndexEntry) = a = move(b) else: @@ -409,6 +409,7 @@ else: shallowCopy a.link, b.link shallowCopy a.linkTitle, b.linkTitle shallowCopy a.linkDesc, b.linkDesc + shallowCopy a.module, b.module proc sortIndex(a: var openArray[IndexEntry]) = # we use shellsort here; fast and simple @@ -448,16 +449,20 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = result = "<dl>" var i = 0 while i < symbols.len: - let keyword = symbols[i].keyword + let keyword = esc(outHtml, symbols[i].keyword) let cleanedKeyword = keyword.escapeLink result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><dd><ul class=\"simple\">\n", [keyword, cleanedKeyword]) var j = i - while j < symbols.len and keyword == symbols[j].keyword: + while j < symbols.len and symbols[i].keyword == symbols[j].keyword: let url = symbols[j].link.escapeLink - text = if symbols[j].linkTitle.len > 0: symbols[j].linkTitle else: url - desc = if symbols[j].linkDesc.len > 0: symbols[j].linkDesc else: "" + module = symbols[j].module + text = + if symbols[j].linkTitle.len > 0: + esc(outHtml, module & ": " & symbols[j].linkTitle) + else: url + desc = symbols[j].linkDesc if desc.len > 0: result.addf("""<li><a class="reference external" title="$3" data-doc-search-tag="$2" href="$1">$2</a></li> @@ -471,13 +476,6 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = i = j result.add("</dl>") -proc isDocumentationTitle(hyperlink: string): bool = - ## Returns true if the hyperlink is actually a documentation title. - ## - ## Documentation titles lack the hash. See `mergeIndexes() - ## <#mergeIndexes,string>`_ for a more detailed explanation. - result = hyperlink.find('#') < 0 - proc stripTocLevel(s: string): tuple[level: int, text: string] = ## Returns the *level* of the toc along with the text without it. for c in 0 ..< s.len: @@ -511,17 +509,15 @@ proc generateDocumentationToc(entries: seq[IndexEntry]): string = level = 1 levels.newSeq(entries.len) for entry in entries: - let (rawLevel, rawText) = stripTocLevel(entry.linkTitle or entry.keyword) + let (rawLevel, rawText) = stripTocLevel(entry.linkTitle) if rawLevel < 1: # This is a normal symbol, push it *inside* one level from the last one. levels[L].level = level + 1 - # Also, ignore the linkTitle and use directly the keyword. - levels[L].text = entry.keyword else: # The level did change, update the level indicator. level = rawLevel levels[L].level = rawLevel - levels[L].text = rawText + levels[L].text = rawText inc L # Now generate hierarchical lists based on the precalculated levels. @@ -552,7 +548,7 @@ proc generateDocumentationIndex(docs: IndexedDocs): string = for title in titles: let tocList = generateDocumentationToc(docs.getOrDefault(title)) result.add("<ul><li><a href=\"" & - title.link & "\">" & title.keyword & "</a>\n" & tocList & "</li></ul>\n") + title.link & "\">" & title.linkTitle & "</a>\n" & tocList & "</li></ul>\n") proc generateDocumentationJumps(docs: IndexedDocs): string = ## Returns a plain list of hyperlinks to documentation TOCs in HTML. @@ -564,7 +560,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string = var chunks: seq[string] = @[] for title in titles: - chunks.add("<a href=\"" & title.link & "\">" & title.keyword & "</a>") + chunks.add("<a href=\"" & title.link & "\">" & title.linkTitle & "</a>") result.add(chunks.join(", ") & ".<br/>") @@ -574,11 +570,11 @@ proc generateModuleJumps(modules: seq[string]): string = var chunks: seq[string] = @[] for name in modules: - chunks.add("<a href=\"" & name & ".html\">" & name & "</a>") + chunks.add("<a href=\"$1.html\">$2</a>" % [name, name.prettyLink]) result.add(chunks.join(", ") & ".<br/>") -proc readIndexDir(dir: string): +proc readIndexDir*(dir: string): tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] = ## Walks `dir` reading ``.idx`` files converting them in IndexEntry items. ## @@ -593,39 +589,12 @@ proc readIndexDir(dir: string): # Scan index files and build the list of symbols. for path in walkDirRec(dir): if path.endsWith(IndexExt): - var - fileEntries: seq[IndexEntry] - title: IndexEntry - f = 0 - newSeq(fileEntries, 500) - setLen(fileEntries, 0) - for line in lines(path): - let s = line.find('\t') - if s < 0: continue - setLen(fileEntries, f+1) - fileEntries[f].keyword = line.substr(0, s-1) - fileEntries[f].link = line.substr(s+1) - # See if we detect a title, a link without a `#foobar` trailing part. - if title.keyword.len == 0 and fileEntries[f].link.isDocumentationTitle: - title.keyword = fileEntries[f].keyword - title.link = fileEntries[f].link - - if fileEntries[f].link.find('\t') > 0: - let extraCols = fileEntries[f].link.split('\t') - fileEntries[f].link = extraCols[0] - assert extraCols.len == 3 - fileEntries[f].linkTitle = extraCols[1].unquoteIndexColumn - fileEntries[f].linkDesc = extraCols[2].unquoteIndexColumn - else: - fileEntries[f].linkTitle = "" - fileEntries[f].linkDesc = "" - inc f + var (fileEntries, title) = parseIdxFile(path) # Depending on type add this to the list of symbols or table of APIs. - if title.keyword.len == 0: - for i in 0 ..< f: - # Don't add to symbols TOC entries (they start with a whitespace). - let toc = fileEntries[i].linkTitle - if toc.len > 0 and toc[0] == ' ': + + if title.kind == ieNimTitle: + for i in 0 ..< fileEntries.len: + if fileEntries[i].kind != ieNim: continue # Ok, non TOC entry, add it. setLen(result.symbols, L + 1) @@ -635,16 +604,22 @@ proc readIndexDir(dir: string): var x = fileEntries[0].link let i = find(x, '#') if i > 0: - x = x.substr(0, i-1) + x.setLen(i) if i != 0: # don't add entries starting with '#' result.modules.add(x.changeFileExt("")) else: # Generate the symbolic anchor for index quickjumps. - title.linkTitle = "doc_toc_" & $result.docs.len + title.aux = "doc_toc_" & $result.docs.len result.docs[title] = fileEntries - sort(result.modules, system.cmp) + for i in 0 ..< fileEntries.len: + if fileEntries[i].kind != ieIdxRole: + continue + + setLen(result.symbols, L + 1) + result.symbols[L] = fileEntries[i] + inc L proc mergeIndexes*(dir: string): string = ## Merges all index files in `dir` and returns the generated index as HTML. @@ -675,6 +650,7 @@ proc mergeIndexes*(dir: string): string = ## Returns the merged and sorted indices into a single HTML block which can ## be further embedded into nimdoc templates. var (modules, symbols, docs) = readIndexDir(dir) + sort(modules, system.cmp) result = "" # Generate a quick jump list of documents. @@ -702,63 +678,36 @@ proc mergeIndexes*(dir: string): string = # ---------------------------------------------------------------------------- -proc stripTocHtml(s: string): string = - ## Ugly quick hack to remove HTML tags from TOC titles. - ## - ## A TocEntry.header field already contains rendered HTML tags. Instead of - ## implementing a proper version of renderRstToOut() which recursively - ## renders an rst tree to plain text, we simply remove text found between - ## angled brackets. Given the limited possibilities of rst inside TOC titles - ## this should be enough. - result = s - var first = result.find('<') - while first >= 0: - let last = result.find('>', first) - if last < 0: - # Abort, since we didn't found a closing angled bracket. - return - result.delete(first, last) - first = result.find('<', first) - proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) d.currentSection = tmp - # Find the last higher level section for unique reference name - var sectionPrefix = "" - for i in countdown(d.tocPart.high, 0): - let n2 = d.tocPart[i].n - if n2.level < n.level: - sectionPrefix = rstnodeToRefname(n2) & "-" - break - var refname = sectionPrefix & rstnodeToRefname(n) + var tocName = esc(d.target, renderRstToText(n), escMode = emOption) + # for Latex: simple text without commands that may break TOC/hyperref if d.hasToc: - var length = len(d.tocPart) - setLen(d.tocPart, length + 1) - d.tocPart[length].refname = refname - d.tocPart[length].n = n - d.tocPart[length].header = tmp - - dispA(d.target, result, "\n<h$1><a class=\"toc-backref\" " & - "id=\"$2\" href=\"#$2\">$3</a></h$1>", "\\rsth$4{$3}\\label{$2}\n", - [$n.level, d.tocPart[length].refname, tmp, $chr(n.level - 1 + ord('A'))]) + d.tocPart.add n + dispA(d.target, result, "\n<h$1><a class=\"toc-backref\"" & + "$2 href=\"#$5\">$3</a></h$1>", "\\rsth$4[$6]{$3}$2\n", + [$n.level, n.anchor.idS, tmp, + $chr(n.level - 1 + ord('A')), n.anchor, tocName]) else: - dispA(d.target, result, "\n<h$1 id=\"$2\">$3</h$1>", - "\\rsth$4{$3}\\label{$2}\n", [ - $n.level, refname, tmp, - $chr(n.level - 1 + ord('A'))]) + dispA(d.target, result, "\n<h$1$2>$3</h$1>", + "\\rsth$4[$5]{$3}$2\n", [ + $n.level, n.anchor.idS, tmp, + $chr(n.level - 1 + ord('A')), tocName]) # Generate index entry using spaces to indicate TOC level for the output HTML. assert n.level >= 0 - setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), refname, tmp.stripTocHtml, - spaces(max(0, n.level)) & tmp) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) proc renderOverline(d: PDoc, n: PRstNode, result: var string) = - if d.meta[metaTitle].len == 0: + if n.level == 0 and d.meta[metaTitle].len == 0: + d.meta[metaTitleRaw] = n.addNodes for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], d.meta[metaTitle]) d.currentSection = d.meta[metaTitle] - elif d.meta[metaSubtitle].len == 0: + elif n.level == 0 and d.meta[metaSubtitle].len == 0: for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], d.meta[metaSubtitle]) d.currentSection = d.meta[metaSubtitle] @@ -766,21 +715,25 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) d.currentSection = tmp - dispA(d.target, result, "<h$1 id=\"$2\"><center>$3</center></h$1>", - "\\rstov$4{$3}\\label{$2}\n", [$n.level, - rstnodeToRefname(n), tmp, $chr(n.level - 1 + ord('A'))]) - - -proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) = + var tocName = esc(d.target, renderRstToText(n), escMode=emOption) + dispA(d.target, result, "<h$1$2><center>$3</center></h$1>", + "\\rstov$4[$5]{$3}$2\n", [$n.level, + n.anchor.idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) + +proc renderTocEntry(d: PDoc, n: PRstNode, result: var string) = + var header = "" + for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], header) dispA(d.target, result, "<li><a class=\"reference\" id=\"$1_toc\" href=\"#$1\">$2</a></li>\n", - "\\item\\label{$1_toc} $2\\ref{$1}\n", [e.refname, e.header]) + "\\item\\label{$1_toc} $2\\ref{$1}\n", [n.anchor, header]) proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int, result: var string) = var tmp = "" while j <= high(d.tocPart): - var a = abs(d.tocPart[j].n.level) + var a = abs(d.tocPart[j].level) if a == lvl: renderTocEntry(d, d.tocPart[j], tmp) inc(j) @@ -826,14 +779,28 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) = if arg.endsWith(".mp4") or arg.endsWith(".ogg") or arg.endsWith(".webm"): htmlOut = """ - <video src="$1"$2 autoPlay='true' loop='true' muted='true'> + <video$3 src="$1"$2 autoPlay='true' loop='true' muted='true'> Sorry, your browser doesn't support embedded videos </video> """ else: - htmlOut = "<img src=\"$1\"$2/>" - dispA(d.target, result, htmlOut, "\\includegraphics$2{$1}", - [esc(d.target, arg), options]) + htmlOut = "<img$3 src=\"$1\"$2/>" + + # support for `:target:` links for images: + var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl) + discard safeProtocol(target) + + if target.len > 0: + # `htmlOut` needs to be of the following format for link to work for images: + # <a class="reference external" href="target"><img src=\"$1\"$2/></a> + var htmlOutWithLink = "" + dispA(d.target, htmlOutWithLink, + "<a class=\"reference external\" href=\"$2\">$1</a>", + "\\href{$2}{$1}", [htmlOut, target]) + htmlOut = htmlOutWithLink + + dispA(d.target, result, htmlOut, "$3\\includegraphics$2{$1}", + [esc(d.target, arg), options, n.anchor.idS]) if len(n) >= 3: renderRstToOut(d, n.sons[2], result) proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = @@ -843,6 +810,25 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = "\\includegraphics{$1}", [d.config.getOrDefault"doc.smiley_format" % n.text]) +proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = + template err(msg: string) = + rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg) + let value = n.getFieldValue + var number: int + let nChars = parseInt(value, number) + if nChars == 0: + if value.len == 0: + # use a good default value: + result = 1 + else: + err("field $1 requires an integer, but '$2' was given" % + [fieldName, value]) + elif nChars < value.len: + err("extra arguments were given to $1: '$2'" % + [fieldName, value[nChars..^1]]) + else: + result = number + proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = ## Parses useful fields which can appear before a code block. ## @@ -852,9 +838,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = of "number-lines": params.numberLines = true # See if the field has a parameter specifying a different line than 1. - var number: int - if parseInt(n.getFieldValue, number) > 0: - params.startLine = number + params.startLine = getField1Int(d, n, "number-lines") of "file", "filename": # The ``file`` option is a Nim extension to the official spec, it acts # like it would for other directives like ``raw`` or ``cvs-table``. This @@ -865,18 +849,20 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = of "test": params.testCmd = n.getFieldValue.strip if params.testCmd.len == 0: - params.testCmd = "nim c -r $1" + # factor with D20210224T221756. Note that `$docCmd` should appear before `$file` + # but after all other options, but currently `$options` merges both options and `$file` so it's tricky. + params.testCmd = "$nim r --backend:$backend --lib:$libpath $docCmd $options" else: + # consider whether `$docCmd` should be appended here too params.testCmd = unescape(params.testCmd) of "status", "exitcode": - var status: int - if parseInt(n.getFieldValue, status) > 0: - params.status = status + params.status = getField1Int(d, n, n.getArgument) of "default-language": params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage else: - d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument) + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField, + n.getArgument) proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = ## Iterates over all code block fields and returns processed params. @@ -886,8 +872,7 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = result.init if n.isNil: return - assert n.kind == rnCodeBlock - assert(not n.sons[2].isNil) + assert n.kind in {rnCodeBlock, rnInlineCode} # Parse the field list for rendering parameters if there are any. if not n.sons[1].isNil: @@ -898,7 +883,8 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = if result.langStr != "": result.lang = getSourceLanguage(result.langStr) -proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string): +proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string, + idStr: string): tuple[beginTable, endTable: string] = ## Returns the necessary tags to start/end a code block in HTML. ## @@ -910,13 +896,14 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string): let id = $d.listingCounter if not params.numberLines: result = (d.config.getOrDefault"doc.listing_start" % - [id, sourceLanguageToStr[params.lang]], + [id, sourceLanguageToStr[params.lang], idStr], d.config.getOrDefault"doc.listing_end" % id) return var codeLines = code.strip.countLines assert codeLines > 0 - result.beginTable = """<table class="line-nums-table"><tbody><tr><td class="blob-line-nums"><pre class="line-nums">""" + result.beginTable = """<table$1 class="line-nums-table">""" % [idStr] & + """<tbody><tr><td class="blob-line-nums"><pre class="line-nums">""" var line = params.startLine while codeLines > 0: result.beginTable.add($line & "\n") @@ -924,13 +911,32 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string): codeLines.dec result.beginTable.add("</pre></td><td>" & ( d.config.getOrDefault"doc.listing_start" % - [id, sourceLanguageToStr[params.lang]])) + [id, sourceLanguageToStr[params.lang], idStr])) result.endTable = (d.config.getOrDefault"doc.listing_end" % id) & "</td></tr></tbody></table>" & ( d.config.getOrDefault"doc.listing_button" % id) -proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = - ## Renders a code block, appending it to `result`. +proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string, + target: OutputTarget) = + var g: GeneralTokenizer + initGeneralTokenizer(g, code) + while true: + getNextToken(g, lang) + case g.kind + of gtEof: break + of gtNone, gtWhitespace: + add(result, substr(code, g.start, g.length + g.start - 1)) + else: + dispA(target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [ + esc(target, substr(code, g.start, g.length+g.start-1)), + tokenClassToStr[g.kind]]) + deinitGeneralTokenizer(g) + +proc renderNimCode*(result: var string, code: string, target: OutputTarget) = + renderCodeLang(result, langNim, code, target) + +proc renderCode(d: PDoc, n: PRstNode, result: var string) {.gcsafe.} = + ## Renders a code (code block or inline code), appending it to `result`. ## ## If the code block uses the ``number-lines`` option, a table will be ## generated with two columns, the first being a list of numbers and the @@ -939,37 +945,40 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = ## may also come from the parser through the internal ``default-language`` ## option to differentiate between a plain code block and Nim's code block ## extension. - assert n.kind == rnCodeBlock - if n.sons[2] == nil: return + assert n.kind in {rnCodeBlock, rnInlineCode} var params = d.parseCodeBlockParams(n) + if n.sons[2] == nil: return var m = n.sons[2].sons[0] assert m.kind == rnLeaf if params.testCmd.len > 0 and d.onTestSnippet != nil: d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text) - let (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text) - - dispA(d.target, result, blockStart, "\\begin{rstpre}\n", []) + var blockStart, blockEnd: string + case d.target + of outHtml: + if n.kind == rnCodeBlock: + (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text, + n.anchor.idS) + else: # rnInlineCode + blockStart = "<tt class=\"docutils literal\"><span class=\"pre\">" + blockEnd = "</span></tt>" + of outLatex: + if n.kind == rnCodeBlock: + blockStart = "\n\n" & n.anchor.idS & "\\begin{rstpre}\n" + blockEnd = "\n\\end{rstpre}\n\n" + else: # rnInlineCode + blockStart = "\\rstcode{" + blockEnd = "}" + dispA(d.target, result, blockStart, blockStart, []) if params.lang == langNone: - if len(params.langStr) > 0: - d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr) - for letter in m.text: escChar(d.target, result, letter) + if len(params.langStr) > 0 and params.langStr.toLowerAscii != "none": + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage, + params.langStr) + for letter in m.text: escChar(d.target, result, letter, emText) else: - var g: GeneralTokenizer - initGeneralTokenizer(g, m.text) - while true: - getNextToken(g, params.lang) - case g.kind - of gtEof: break - of gtNone, gtWhitespace: - add(result, substr(m.text, g.start, g.length + g.start - 1)) - else: - dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [ - esc(d.target, substr(m.text, g.start, g.length+g.start-1)), - tokenClassToStr[g.kind]]) - deinitGeneralTokenizer(g) - dispA(d.target, result, blockEnd, "\n\\end{rstpre}\n") + renderCodeLang(result, params.lang, m.text, d.target) + dispA(d.target, result, blockEnd, blockEnd) proc renderContainer(d: PDoc, n: PRstNode, result: var string) = var tmp = "" @@ -980,10 +989,6 @@ proc renderContainer(d: PDoc, n: PRstNode, result: var string) = else: dispA(d.target, result, "<div class=\"$1\">$2</div>", "$2", [arg, tmp]) -proc texColumns(n: PRstNode): string = - result = "" - for i in countup(1, len(n)): add(result, "|X") - proc renderField(d: PDoc, n: PRstNode, result: var string) = var b = false if d.target == outLatex: @@ -1001,41 +1006,150 @@ proc renderField(d: PDoc, n: PRstNode, result: var string) = if not b: renderAux(d, n, "<tr>$1</tr>\n", "$1", result) +proc renderEnumList(d: PDoc, n: PRstNode, result: var string) = + var + specifier = "" + specStart = "" + i1 = 0 + pre = "" + i2 = n.labelFmt.len - 1 + post = "" + if n.labelFmt[0] == '(': + i1 = 1 + pre = "(" + if n.labelFmt[^1] == ')' or n.labelFmt[^1] == '.': + i2 = n.labelFmt.len - 2 + post = $n.labelFmt[^1] + let enumR = i1 .. i2 # enumerator range without surrounding (, ), . + if d.target == outLatex: + result.add ("\n%" & n.labelFmt & "\n") + # use enumerate parameters from package enumitem + if n.labelFmt[i1].isDigit: + var labelDef = "" + if pre != "" or post != "": + labelDef = "label=" & pre & "\\arabic*" & post & "," + if n.labelFmt[enumR] != "1": + specStart = "start=$1" % [n.labelFmt[enumR]] + if labelDef != "" or specStart != "": + specifier = "[$1$2]" % [labelDef, specStart] + else: + let (first, labelDef) = + if n.labelFmt[i1].isUpperAscii: ('A', "label=" & pre & "\\Alph*" & post) + else: ('a', "label=" & pre & "\\alph*" & post) + if n.labelFmt[i1] != first: + specStart = ",start=" & $(ord(n.labelFmt[i1]) - ord(first) + 1) + specifier = "[$1$2]" % [labelDef, specStart] + else: # HTML + # TODO: implement enumerator formatting using pre and post ( and ) for HTML + if n.labelFmt[i1].isDigit: + if n.labelFmt[enumR] != "1": + specStart = " start=\"$1\"" % [n.labelFmt[enumR]] + specifier = "class=\"simple\"" & specStart + else: + let (first, labelDef) = + if n.labelFmt[i1].isUpperAscii: ('A', "class=\"upperalpha simple\"") + else: ('a', "class=\"loweralpha simple\"") + if n.labelFmt[i1] != first: + specStart = " start=\"$1\"" % [ $(ord(n.labelFmt[i1]) - ord(first) + 1) ] + specifier = labelDef & specStart + renderAux(d, n, "<ol$2 " & specifier & ">$1</ol>\n", + "\\begin{enumerate}" & specifier & "$2$1\\end{enumerate}\n", + result) + +proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = + var + htmlCls = "admonition_warning" + texSz = "\\large" + texColor = "orange" + case n.adType + of "hint", "note", "tip": + htmlCls = "admonition-info"; texSz = "\\normalsize"; texColor = "green" + of "attention", "admonition", "important", "warning", "caution": + htmlCls = "admonition-warning"; texSz = "\\large"; texColor = "orange" + of "danger", "error": + htmlCls = "admonition-error"; texSz = "\\Large"; texColor = "red" + else: discard + let txt = n.adType.capitalizeAscii() + let htmlHead = "<div class=\"admonition " & htmlCls & "\">" + renderAux(d, n, + htmlHead & "<span$2 class=\"" & htmlCls & "-text\"><b>" & txt & + ":</b></span>\n" & "$1</div>\n", + "\n\n\\begin{rstadmonition}[borderline west={0.2em}{0pt}{" & + texColor & "}]$2\n" & + "{" & texSz & "\\color{" & texColor & "}{\\textbf{" & txt & ":}}} " & + "$1\n\\end{rstadmonition}\n", + result) + +proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, + external: bool, nimdoc = false, tooltip="") = + var linkStr = "" + block: + let mode = d.escMode + d.escMode = emUrl + renderRstToOut(d, link, linkStr) + d.escMode = mode + discard safeProtocol(linkStr) + var textStr = "" + renderRstToOut(d, text, textStr) + let nimDocStr = if nimdoc: " nimdoc" else: "" + var tooltipStr = "" + if tooltip != "": + tooltipStr = """ title="$1"""" % [ esc(d.target, tooltip) ] + if external: + dispA(d.target, result, + "<a class=\"reference external$3\"$4 href=\"$2\">$1</a>", + "\\href{$2}{$1}", [textStr, linkStr, nimDocStr, tooltipStr]) + else: + dispA(d.target, result, + "<a class=\"reference internal$3\"$4 href=\"#$2\">$1</a>", + "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", + [textStr, linkStr, nimDocStr, tooltipStr]) + +proc traverseForIndex*(d: PDoc, n: PRstNode) = + ## A version of [renderRstToOut] that only fills entries for ``.idx`` files. + var discarded: string + if n == nil: return + case n.kind + of rnIdx: renderIndexTerm(d, n, discarded) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, discarded) + of rnOverline: renderOverline(d, n, discarded) + else: + for i in 0 ..< len(n): + traverseForIndex(d, n.sons[i]) + proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return case n.kind of rnInner: renderAux(d, n, result) - of rnHeadline: renderHeadline(d, n, result) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result) of rnOverline: renderOverline(d, n, result) - of rnTransition: renderAux(d, n, "<hr />\n", "\\hrule\n", result) - of rnParagraph: renderAux(d, n, "<p>$1</p>\n", "$1\n\n", result) + of rnTransition: renderAux(d, n, "<hr$2 />\n", "\n\n\\vspace{0.6em}\\hrule$2\n", result) + of rnParagraph: renderAux(d, n, "<p$2>$1</p>\n", "\n\n$2\n$1\n\n", result) of rnBulletList: - renderAux(d, n, "<ul class=\"simple\">$1</ul>\n", - "\\begin{itemize}$1\\end{itemize}\n", result) + renderAux(d, n, "<ul$2 class=\"simple\">$1</ul>\n", + "\\begin{itemize}\n$2\n$1\\end{itemize}\n", result) of rnBulletItem, rnEnumItem: - renderAux(d, n, "<li>$1</li>\n", "\\item $1\n", result) - of rnEnumList: - renderAux(d, n, "<ol class=\"simple\">$1</ol>\n", - "\\begin{enumerate}$1\\end{enumerate}\n", result) - of rnDefList: - renderAux(d, n, "<dl class=\"docutils\">$1</dl>\n", - "\\begin{description}$1\\end{description}\n", result) + renderAux(d, n, "<li$2>$1</li>\n", "\\item $2$1\n", result) + of rnEnumList: renderEnumList(d, n, result) + of rnDefList, rnMdDefList: + renderAux(d, n, "<dl$2 class=\"docutils\">$1</dl>\n", + "\\begin{description}\n$2\n$1\\end{description}\n", result) of rnDefItem: renderAux(d, n, result) - of rnDefName: renderAux(d, n, "<dt>$1</dt>\n", "\\item[$1] ", result) - of rnDefBody: renderAux(d, n, "<dd>$1</dd>\n", "$1\n", result) + of rnDefName: renderAux(d, n, "<dt$2>$1</dt>\n", "$2\\item[$1]\\ ", result) + of rnDefBody: renderAux(d, n, "<dd$2>$1</dd>\n", "$2\n$1\n", result) of rnFieldList: var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) if tmp.len != 0: dispA(d.target, result, - "<table class=\"docinfo\" frame=\"void\" rules=\"none\">" & + "<table$2 class=\"docinfo\" frame=\"void\" rules=\"none\">" & "<col class=\"docinfo-name\" />" & "<col class=\"docinfo-content\" />" & "<tbody valign=\"top\">$1" & "</tbody></table>", - "\\begin{description}$1\\end{description}\n", - [tmp]) + "\\begin{description}\n$2\n$1\\end{description}\n", + [tmp, n.anchor.idS]) of rnField: renderField(d, n, result) of rnFieldName: renderAux(d, n, "<th class=\"docinfo-name\">$1:</th>", @@ -1045,96 +1159,190 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnIndex: renderRstToOut(d, n.sons[2], result) of rnOptionList: - renderAux(d, n, "<table frame=\"void\">$1</table>", - "\\begin{description}\n$1\\end{description}\n", result) + renderAux(d, n, "<div$2 class=\"option-list\">$1</div>", + "\\begin{rstoptlist}$2\n$1\\end{rstoptlist}", result) of rnOptionListItem: - renderAux(d, n, "<tr>$1</tr>\n", "$1", result) + var addclass = if n.order mod 2 == 1: " odd" else: "" + renderAux(d, n, + "<div class=\"option-list-item" & addclass & "\">$1</div>\n", + "$1", result) of rnOptionGroup: - renderAux(d, n, "<th align=\"left\">$1</th>", "\\item[$1]", result) + renderAux(d, n, + "<div class=\"option-list-label\"><tt><span class=\"option\">" & + "$1</span></tt></div>", + "\\item[\\rstcodeitem{\\spanoption{$1}}]", result) of rnDescription: - renderAux(d, n, "<td align=\"left\">$1</td>\n", " $1\n", result) + renderAux(d, n, "<div class=\"option-list-description\">$1</div>", + " $1\n", result) of rnOption, rnOptionString, rnOptionArgument: - doAssert false, "renderRstToOut" + raiseAssert "renderRstToOut" of rnLiteralBlock: - renderAux(d, n, "<pre>$1</pre>\n", - "\\begin{rstpre}\n$1\n\\end{rstpre}\n", result) - of rnQuotedLiteralBlock: - doAssert false, "renderRstToOut" + renderAux(d, n, "<pre$2>$1</pre>\n", + "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) + of rnMarkdownBlockQuote: + d.curQuotationDepth = 1 + var tmp = "" + renderAux(d, n, "$1", "$1", tmp) + let itemEnding = + if d.target == outHtml: "</blockquote>" else: "\\end{rstquote}" + tmp.add itemEnding.repeat(d.curQuotationDepth - 1) + dispA(d.target, result, + "<blockquote$2 class=\"markdown-quote\">$1</blockquote>\n", + "\n\\begin{rstquote}\n$2\n$1\\end{rstquote}\n", [tmp, n.anchor.idS]) + of rnMarkdownBlockQuoteItem: + let addQuotationDepth = n.quotationDepth - d.curQuotationDepth + var itemPrefix: string # start or ending (quotation grey bar on the left) + if addQuotationDepth >= 0: + let s = + if d.target == outHtml: "<blockquote class=\"markdown-quote\">" + else: "\\begin{rstquote}" + itemPrefix = s.repeat(addQuotationDepth) + else: + let s = + if d.target == outHtml: "</blockquote>" + else: "\\end{rstquote}" + itemPrefix = s.repeat(-addQuotationDepth) + renderAux(d, n, itemPrefix & "<p>$1</p>", itemPrefix & "\n$1", result) + d.curQuotationDepth = n.quotationDepth of rnLineBlock: - renderAux(d, n, "<p>$1</p>", "$1\n\n", result) + if n.sons.len == 1 and n.sons[0].lineIndent == "\n": + # whole line block is one empty line, no need to add extra spacing + renderAux(d, n, "<p$2>$1</p> ", "\n\n$2\n$1", result) + else: # add extra spacing around the line block for Latex + renderAux(d, n, "<p$2>$1</p>", + "\n\\vspace{0.5em}$2\n$1\\vspace{0.5em}\n", result) of rnLineBlockItem: - renderAux(d, n, "$1<br />", "$1\\\\\n", result) + if n.lineIndent.len == 0: # normal case - no additional indentation + renderAux(d, n, "$1<br/>", "\\noindent $1\n\n", result) + elif n.lineIndent == "\n": # add one empty line + renderAux(d, n, "<br/>", "\\vspace{1em}\n", result) + else: # additional indentation w.r.t. '| ' + let indent = $(0.5 * (n.lineIndent.len - 1).toFloat) & "em" + renderAux(d, n, + "<span style=\"margin-left: " & indent & "\">$1</span><br/>", + "\\noindent\\hspace{" & indent & "}$1\n\n", result) of rnBlockQuote: - renderAux(d, n, "<blockquote><p>$1</p></blockquote>\n", - "\\begin{quote}$1\\end{quote}\n", result) - of rnTable, rnGridTable: + renderAux(d, n, "<blockquote$2><p>$1</p></blockquote>\n", + "\\begin{quote}\n$2\n$1\\end{quote}\n", result) + of rnAdmonition: renderAdmonition(d, n, result) + of rnTable, rnGridTable, rnMarkdownTable: renderAux(d, n, - "<table border=\"1\" class=\"docutils\">$1</table>", - "\\begin{table}\\begin{rsttab}{" & - texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result) + "<table$2 border=\"1\" class=\"docutils\">$1</table>", + "\n$2\n\\begin{rsttab}{" & + "L".repeat(n.colCount) & "}\n\\toprule\n$1" & + "\\addlinespace[0.1em]\\bottomrule\n\\end{rsttab}", result) of rnTableRow: if len(n) >= 1: - if d.target == outLatex: - #var tmp = "" - renderRstToOut(d, n.sons[0], result) - for i in countup(1, len(n) - 1): - result.add(" & ") - renderRstToOut(d, n.sons[i], result) - result.add("\\\\\n\\hline\n") - else: + case d.target + of outHtml: result.add("<tr>") renderAux(d, n, result) result.add("</tr>\n") - of rnTableDataCell: - renderAux(d, n, "<td>$1</td>", "$1", result) - of rnTableHeaderCell: - renderAux(d, n, "<th>$1</th>", "\\textbf{$1}", result) - of rnLabel: - doAssert false, "renderRstToOut" # used for footnotes and other - of rnFootnote: - doAssert false, "renderRstToOut" # a footnote - of rnCitation: - doAssert false, "renderRstToOut" # similar to footnote - of rnRef: - var tmp = "" - renderAux(d, n, tmp) + of outLatex: + if n.sons[0].kind == rnTableHeaderCell: + result.add "\\rowcolor{gray!15} " + var spanLines: seq[(int, int)] + var nCell = 0 + for uCell in 0 .. n.len - 1: + renderRstToOut(d, n.sons[uCell], result) + if n.sons[uCell].span > 0: + spanLines.add (nCell + 1, nCell + n.sons[uCell].span) + nCell += n.sons[uCell].span + else: + nCell += 1 + if uCell != n.len - 1: + result.add(" & ") + result.add("\\\\") + if n.endsHeader: result.add("\\midrule\n") + for (start, stop) in spanLines: + result.add("\\cmidrule(lr){$1-$2}" % [$start, $stop]) + result.add("\n") + of rnTableHeaderCell, rnTableDataCell: + case d.target + of outHtml: + let tag = if n.kind == rnTableHeaderCell: "th" else: "td" + var spanSpec: string + if n.span <= 1: spanSpec = "" + else: + spanSpec = " colspan=\"" & $n.span & "\" style=\"text-align: center\"" + renderAux(d, n, "<$1$2>$$1</$1>" % [tag, spanSpec], "", result) + of outLatex: + let text = if n.kind == rnTableHeaderCell: "\\textbf{$1}" else: "$1" + var latexStr: string + if n.span <= 1: latexStr = text + else: latexStr = "\\multicolumn{" & $n.span & "}{c}{" & text & "}" + renderAux(d, n, "", latexStr, result) + of rnFootnoteGroup: + renderAux(d, n, + "<hr class=\"footnote\">" & + "<div class=\"footnote-group\">\n$1</div>\n", + "\n\n\\noindent\\rule{0.25\\linewidth}{.4pt}\n" & + "\\begin{rstfootnote}\n$1\\end{rstfootnote}\n\n", + result) + of rnFootnote, rnCitation: + var mark = "" + renderAux(d, n.sons[0], mark) + var body = "" + renderRstToOut(d, n.sons[1], body) dispA(d.target, result, - "<a class=\"reference external\" href=\"#$2\">$1</a>", - "$1\\ref{$2}", [tmp, rstnodeToRefname(n)]) + "<div$2><div class=\"footnote-label\">" & + "<sup><strong><a href=\"#$4\">[$3]</a></strong></sup>" & + "</div>   $1\n</div>\n", + "\\item[\\textsuperscript{[$3]}]$2 $1\n", + [body, n.anchor.idS, mark, n.anchor]) + of rnPandocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnRstRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=false) of rnStandaloneHyperlink: - renderAux(d, n, - "<a class=\"reference external\" href=\"$1\">$1</a>", - "\\href{$1}{$1}", result) + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) + of rnInternalRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnNimdocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false, + nimdoc=true, tooltip=n.tooltip) of rnHyperlink: - var tmp0 = "" - var tmp1 = "" - renderRstToOut(d, n.sons[0], tmp0) - renderRstToOut(d, n.sons[1], tmp1) + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) + of rnFootnoteRef: + var tmp = "[" + renderAux(d, n.sons[0], tmp) + tmp.add "]" dispA(d.target, result, - "<a class=\"reference external\" href=\"$2\">$1</a>", - "\\href{$2}{$1}", [tmp0, tmp1]) + "<sup><strong><a class=\"reference internal\" href=\"#$2\">" & + "$1</a></strong></sup>", + "\\textsuperscript{\\hyperlink{$2}{\\textbf{$1}}}", + [tmp, n.sons[1].text]) of rnDirArg, rnRaw: renderAux(d, n, result) of rnRawHtml: - if d.target != outLatex: + if d.target != outLatex and not lastSon(n).isNil: result.add addNodes(lastSon(n)) of rnRawLatex: - if d.target == outLatex: + if d.target == outLatex and not lastSon(n).isNil: result.add addNodes(lastSon(n)) of rnImage, rnFigure: renderImage(d, n, result) - of rnCodeBlock: renderCodeBlock(d, n, result) + of rnCodeBlock, rnInlineCode: renderCode(d, n, result) of rnContainer: renderContainer(d, n, result) of rnSubstitutionReferences, rnSubstitutionDef: renderAux(d, n, "|$1|", "|$1|", result) of rnDirective: renderAux(d, n, "", "", result) - of rnGeneralRole: + of rnUnknownRole, rnCodeFragment: var tmp0 = "" var tmp1 = "" renderRstToOut(d, n.sons[0], tmp0) renderRstToOut(d, n.sons[1], tmp1) - dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", - [tmp0, tmp1]) + var class = tmp1 + # don't allow missing role break latex compilation: + if d.target == outLatex and n.kind == rnUnknownRole: class = "Other" + if n.kind == rnCodeFragment: + dispA(d.target, result, + "<tt class=\"docutils literal\"><span class=\"pre $2\">" & + "$1</span></tt>", + "\\rstcode{\\span$2{$1}}", [tmp0, class]) + else: # rnUnknownRole, not necessarily code/monospace font + dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", + [tmp0, class]) of rnSub: renderAux(d, n, "<sub>$1</sub>", "\\rstsub{$1}", result) of rnSup: renderAux(d, n, "<sup>$1</sup>", "\\rstsup{$1}", result) of rnEmphasis: renderAux(d, n, "<em>$1</em>", "\\emph{$1}", result) @@ -1148,13 +1356,22 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnInlineLiteral, rnInterpretedText: renderAux(d, n, "<tt class=\"docutils literal\"><span class=\"pre\">$1</span></tt>", - "\\texttt{$1}", result) + "\\rstcode{$1}", result) + of rnInlineTarget: + var tmp = "" + renderAux(d, n, tmp) + dispA(d.target, result, + "<span class=\"target\" id=\"$2\">$1</span>", + "\\label{$2}\\hypertarget{$2}{$1}", + [tmp, rstnodeToRefname(n)]) of rnSmiley: renderSmiley(d, n, result) - of rnLeaf: result.add(esc(d.target, n.text)) + of rnLeaf: result.add(esc(d.target, n.text, escMode=d.escMode)) of rnContents: d.hasToc = true + of rnDefaultRole: discard of rnTitle: d.meta[metaTitle] = "" renderRstToOut(d, n.sons[0], d.meta[metaTitle]) + d.meta[metaTitleRaw] = n.sons[0].addNodes # ----------------------------------------------------------------------------- @@ -1284,7 +1501,7 @@ $moduledesc $content </div> """) - setConfigVar("doc.listing_start", "<pre class = \"listing\">") + setConfigVar("doc.listing_start", "<pre$3 class = \"listing\">") setConfigVar("doc.listing_end", "</pre>") setConfigVar("doc.listing_button", "</pre>") setConfigVar("doc.body_no_toc", "$moduledesc $content") @@ -1294,7 +1511,8 @@ $content # ---------- forum --------------------------------------------------------- proc rstToHtml*(s: string, options: RstParseOptions, - config: StringTableRef): string = + config: StringTableRef, + msgHandler: MsgHandler = rst.defaultMsgHandler): string {.gcsafe.} = ## Converts an input rst string into embeddable HTML. ## ## This convenience proc parses any input string using rst markup (it doesn't @@ -1304,12 +1522,13 @@ proc rstToHtml*(s: string, options: RstParseOptions, ## work. For an explanation of the ``config`` parameter see the ## ``initRstGenerator`` proc. Example: ## - ## .. code-block:: nim + ## ```nim ## import packages/docutils/rstgen, strtabs ## ## echo rstToHtml("*Hello* **world**!", {}, ## newStringTable(modeStyleInsensitive)) ## # --> <em>Hello</em> <strong>world</strong>! + ## ``` ## ## If you need to allow the rst ``include`` directive or tweak the generated ## output you have to create your own ``RstGenerator`` with @@ -1318,18 +1537,30 @@ proc rstToHtml*(s: string, options: RstParseOptions, proc myFindFile(filename: string): string = # we don't find any files in online mode: result = "" + proc myFindRefFile(filename: string): (string, string) = + result = ("", "") const filen = "input" + let (rst, filenames, t) = rstParse(s, filen, + line=LineRstInit, column=ColRstInit, + options, myFindFile, myFindRefFile, msgHandler) var d: RstGenerator - initRstGenerator(d, outHtml, config, filen, options, myFindFile, - rst.defaultMsgHandler) - var dummyHasToc = false - var rst = rstParse(s, filen, 0, 1, dummyHasToc, options) + initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, + filenames, hasToc = t) result = "" renderRstToOut(d, rst, result) - - -when isMainModule: - assert rstToHtml("*Hello* **world**!", {}, - newStringTable(modeStyleInsensitive)) == - "<em>Hello</em> <strong>world</strong>!" + strbasics.strip(result) + + +proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} = + ## Convenience proc for `renderRstToOut` and `initRstGenerator`. + runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" + if rstSource.len == 0: return + let (rst, filenames, t) = rstParse(rstSource, "", + line=LineRstInit, column=ColRstInit, + options) + var rstGenera: RstGenerator + rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", + filenames=filenames, hasToc = t) + rstGenera.renderRstToOut(rst, result) + strbasics.strip(result) diff --git a/lib/packages/docutils/rstidx.nim b/lib/packages/docutils/rstidx.nim new file mode 100644 index 000000000..1472d28fd --- /dev/null +++ b/lib/packages/docutils/rstidx.nim @@ -0,0 +1,141 @@ +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## Nim `idx`:idx: file format related definitions. + +import std/[strutils, syncio, hashes] +from std/os import splitFile + +type + IndexEntryKind* = enum ## discriminator tag + ieMarkupTitle = "markupTitle" + ## RST/Markdown title, text in `keyword` + + ## HTML text in `linkTitle` + ieNimTitle = "nimTitle" + ## Nim title + ieHeading = "heading" ## RST/Markdown markup heading, escaped + ieIdxRole = "idx" ## RST :idx: definition, escaped + ieNim = "nim" ## Nim symbol, unescaped + ieNimGroup = "nimgrp" ## Nim overload group, unescaped + IndexEntry* = object + kind*: IndexEntryKind ## 0. + keyword*: string ## 1. + link*: string ## 2. + linkTitle*: string ## 3. contains a prettier text for the href + linkDesc*: string ## 4. the title attribute of the final href + line*: int ## 5. + module*: string ## origin file, NOT a field in ``.idx`` file + aux*: string ## auxuliary field, NOT a field in ``.idx`` file + +proc isDocumentationTitle*(hyperlink: string): bool = + ## Returns true if the hyperlink is actually a documentation title. + ## + ## Documentation titles lack the hash. See `mergeIndexes() + ## <#mergeIndexes,string>`_ for a more detailed explanation. + result = hyperlink.find('#') < 0 + +proc `$`*(e: IndexEntry): string = + """("$1", "$2", "$3", "$4", $5)""" % [ + e.keyword, e.link, e.linkTitle, e.linkDesc, $e.line] + +proc quoteIndexColumn(text: string): string = + ## Returns a safe version of `text` for serialization to the ``.idx`` file. + ## + ## The returned version can be put without worries in a line based tab + ## separated column text file. The following character sequence replacements + ## will be performed for that goal: + ## + ## * ``"\\"`` => ``"\\\\"`` + ## * ``"\n"`` => ``"\\n"`` + ## * ``"\t"`` => ``"\\t"`` + result = newStringOfCap(text.len + 3) + for c in text: + case c + of '\\': result.add "\\" + of '\L': result.add "\\n" + of '\C': discard + of '\t': result.add "\\t" + else: result.add c + +proc unquoteIndexColumn*(text: string): string = + ## Returns the unquoted version generated by ``quoteIndexColumn``. + result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) + +proc formatIndexEntry*(kind: IndexEntryKind; htmlFile, id, term, linkTitle, + linkDesc: string, line: int): + tuple[entry: string, isTitle: bool] = + result.entry = $kind + result.entry.add('\t') + result.entry.add term + result.entry.add('\t') + result.entry.add(htmlFile) + if id.len > 0: + result.entry.add('#') + result.entry.add(id) + result.isTitle = false + else: + result.isTitle = true + result.entry.add('\t' & linkTitle.quoteIndexColumn) + result.entry.add('\t' & linkDesc.quoteIndexColumn) + result.entry.add('\t' & $line) + result.entry.add("\n") + +proc parseIndexEntryKind(s: string): IndexEntryKind = + result = case s: + of "nim": ieNim + of "nimgrp": ieNimGroup + of "heading": ieHeading + of "idx": ieIdxRole + of "nimTitle": ieNimTitle + of "markupTitle": ieMarkupTitle + else: raise newException(ValueError, "unknown index entry value $1" % [s]) + +proc parseIdxFile*(path: string): + tuple[fileEntries: seq[IndexEntry], title: IndexEntry] = + var + f = 0 + newSeq(result.fileEntries, 500) + setLen(result.fileEntries, 0) + let (_, base, _) = path.splitFile + for line in lines(path): + let s = line.find('\t') + if s < 0: continue + setLen(result.fileEntries, f+1) + let cols = line.split('\t') + result.fileEntries[f].kind = parseIndexEntryKind(cols[0]) + result.fileEntries[f].keyword = cols[1] + result.fileEntries[f].link = cols[2] + if result.fileEntries[f].kind == ieIdxRole: + result.fileEntries[f].module = base + else: + if result.title.keyword.len == 0: + result.fileEntries[f].module = base + else: + result.fileEntries[f].module = result.title.keyword + + result.fileEntries[f].linkTitle = cols[3].unquoteIndexColumn + result.fileEntries[f].linkDesc = cols[4].unquoteIndexColumn + result.fileEntries[f].line = parseInt(cols[5]) + + if result.fileEntries[f].kind in {ieNimTitle, ieMarkupTitle}: + result.title = result.fileEntries[f] + inc f + +proc cmp*(a, b: IndexEntry): int = + ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. + result = cmpIgnoreStyle(a.keyword, b.keyword) + if result == 0: + result = cmpIgnoreStyle(a.link, b.link) + +proc hash*(x: IndexEntry): Hash = + ## Returns the hash for the combined fields of the type. + ## + ## The hash is computed as the chained hash of the individual string hashes. + result = x.keyword.hash !& x.link.hash + result = result !& x.linkTitle.hash + result = result !& x.linkDesc.hash + result = !$result diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index 2d38137bb..007488354 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -7,9 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated - -from posix import SocketHandle +from std/posix import SocketHandle const EPOLLIN* = 0x00000001 @@ -23,6 +21,7 @@ const EPOLLWRBAND* = 0x00000200 EPOLLMSG* = 0x00000400 EPOLLRDHUP* = 0x00002000 + EPOLLEXCLUSIVE* = 1 shl 28 EPOLLWAKEUP* = 1 shl 29 EPOLLONESHOT* = 1 shl 30 EPOLLET* = 1 shl 31 @@ -34,56 +33,67 @@ const EPOLL_CTL_DEL* = 2 # Remove a file descriptor from the interface. EPOLL_CTL_MOD* = 3 # Change file descriptor epoll_event structure. +# https://github.com/torvalds/linux/blob/ff6992735ade75aae3e35d16b17da1008d753d28/include/uapi/linux/eventpoll.h#L77 +when defined(linux) and defined(amd64): + {.pragma: epollPacked, packed.} +else: + {.pragma: epollPacked.} + type - EpollData* {.importc: "union epoll_data", - header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union. + EpollData* {.importc: "epoll_data_t", + header: "<sys/epoll.h>", pure, final, union.} = object + `ptr`* {.importc: "ptr".}: pointer + fd* {.importc: "fd".}: cint + u32* {.importc: "u32".}: uint32 u64* {.importc: "u64".}: uint64 - EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final.} = object + EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final, epollPacked.} = object events*: uint32 # Epoll events data*: EpollData # User data variable proc epoll_create*(size: cint): cint {.importc: "epoll_create", header: "<sys/epoll.h>".} ## Creates an epoll instance. Returns an fd for the new instance. - ## The "size" parameter is a hint specifying the number of file - ## descriptors to be associated with the new instance. The fd - ## returned by epoll_create() should be closed with close(). + ## + ## The "size" parameter is a hint specifying the number of file + ## descriptors to be associated with the new instance. The fd + ## returned by epoll_create() should be closed with close(). proc epoll_create1*(flags: cint): cint {.importc: "epoll_create1", header: "<sys/epoll.h>".} ## Same as epoll_create but with an FLAGS parameter. The unused SIZE - ## parameter has been dropped. + ## parameter has been dropped. proc epoll_ctl*(epfd: cint; op: cint; fd: cint | SocketHandle; event: ptr EpollEvent): cint {. importc: "epoll_ctl", header: "<sys/epoll.h>".} - ## Manipulate an epoll instance "epfd". Returns 0 in case of success, - ## -1 in case of error ( the "errno" variable will contain the - ## specific error code ) The "op" parameter is one of the EPOLL_CTL_* - ## constants defined above. The "fd" parameter is the target of the - ## operation. The "event" parameter describes which events the caller - ## is interested in and any associated user data. + ## Manipulate an epoll instance "epfd". Returns `0` in case of success, + ## `-1` in case of error (the "errno" variable will contain the specific error code). + ## + ## The "op" parameter is one of the `EPOLL_CTL_*` + ## constants defined above. The "fd" parameter is the target of the + ## operation. The "event" parameter describes which events the caller + ## is interested in and any associated user data. proc epoll_wait*(epfd: cint; events: ptr EpollEvent; maxevents: cint; timeout: cint): cint {.importc: "epoll_wait", header: "<sys/epoll.h>".} ## Wait for events on an epoll instance "epfd". Returns the number of - ## triggered events returned in "events" buffer. Or -1 in case of - ## error with the "errno" variable set to the specific error code. The - ## "events" parameter is a buffer that will contain triggered - ## events. The "maxevents" is the maximum number of events to be - ## returned ( usually size of "events" ). The "timeout" parameter - ## specifies the maximum wait time in milliseconds (-1 == infinite). + ## triggered events returned in "events" buffer. Or -1 in case of + ## error with the "errno" variable set to the specific error code. The + ## "events" parameter is a buffer that will contain triggered + ## events. The "maxevents" is the maximum number of events to be + ## returned ( usually size of "events" ). The "timeout" parameter + ## specifies the maximum wait time in milliseconds (-1 == infinite). ## - ## This function is a cancellation point and therefore not marked with - ## __THROW. + ## This function is a cancellation point and therefore not marked with + ## __THROW. #proc epoll_pwait*(epfd: cint; events: ptr EpollEvent; maxevents: cint; # timeout: cint; ss: ptr sigset_t): cint {. # importc: "epoll_pwait", header: "<sys/epoll.h>".} # Same as epoll_wait, but the thread's signal mask is temporarily -# and atomically replaced with the one provided as parameter. +# and atomically replaced with the one provided as parameter. # -# This function is a cancellation point and therefore not marked with -# __THROW. +# This function is a cancellation point and therefore not marked with +# __THROW. diff --git a/lib/posix/inotify.nim b/lib/posix/inotify.nim index f88e48a2d..575accc18 100644 --- a/lib/posix/inotify.nim +++ b/lib/posix/inotify.nim @@ -7,66 +7,105 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated +when defined(nimPreviewSlimSystem): + import std/syncio # Get the platform-dependent flags. # Structure describing an inotify event. type - InotifyEvent*{.pure, final, importc: "struct inotify_event", - header: "<sys/inotify.h>".} = object - wd*{.importc: "wd".}: cint # Watch descriptor. - mask*{.importc: "mask".}: uint32 # Watch mask. - cookie*{.importc: "cookie".}: uint32 # Cookie to synchronize two events. - len*{.importc: "len".}: uint32 # Length (including NULs) of name. - name*{.importc: "name".}: char # Name. + InotifyEvent* {.pure, final, importc: "struct inotify_event", + header: "<sys/inotify.h>", + completeStruct.} = object ## An Inotify event. + wd* {.importc: "wd".}: FileHandle ## Watch descriptor. + mask* {.importc: "mask".}: uint32 ## Watch mask. + cookie* {.importc: "cookie".}: uint32 ## Cookie to synchronize two events. + len* {.importc: "len".}: uint32 ## Length (including NULs) of name. + name* {.importc: "name".}: UncheckedArray[char] ## Name. # Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH. const - IN_ACCESS* = 0x00000001 # File was accessed. - IN_MODIFY* = 0x00000002 # File was modified. - IN_ATTRIB* = 0x00000004 # Metadata changed. - IN_CLOSE_WRITE* = 0x00000008 # Writtable file was closed. - IN_CLOSE_NOWRITE* = 0x00000010 # Unwrittable file closed. - IN_CLOSE* = (IN_CLOSE_WRITE or IN_CLOSE_NOWRITE) # Close. - IN_OPEN* = 0x00000020 # File was opened. - IN_MOVED_FROM* = 0x00000040 # File was moved from X. - IN_MOVED_TO* = 0x00000080 # File was moved to Y. - IN_MOVE* = (IN_MOVED_FROM or IN_MOVED_TO) # Moves. - IN_CREATE* = 0x00000100 # Subfile was created. - IN_DELETE* = 0x00000200 # Subfile was deleted. - IN_DELETE_SELF* = 0x00000400 # Self was deleted. - IN_MOVE_SELF* = 0x00000800 # Self was moved. + IN_ACCESS* = 0x00000001 ## File was accessed. + IN_MODIFY* = 0x00000002 ## File was modified. + IN_ATTRIB* = 0x00000004 ## Metadata changed. + IN_CLOSE_WRITE* = 0x00000008 ## Writtable file was closed. + IN_CLOSE_NOWRITE* = 0x00000010 ## Unwrittable file closed. + IN_CLOSE* = (IN_CLOSE_WRITE or IN_CLOSE_NOWRITE) ## Close. + IN_OPEN* = 0x00000020 ## File was opened. + IN_MOVED_FROM* = 0x00000040 ## File was moved from X. + IN_MOVED_TO* = 0x00000080 ## File was moved to Y. + IN_MOVE* = (IN_MOVED_FROM or IN_MOVED_TO) ## Moves. + IN_CREATE* = 0x00000100 ## Subfile was created. + IN_DELETE* = 0x00000200 ## Subfile was deleted. + IN_DELETE_SELF* = 0x00000400 ## Self was deleted. + IN_MOVE_SELF* = 0x00000800 ## Self was moved. + # Events sent by the kernel. const - IN_UNMOUNT* = 0x00002000 # Backing fs was unmounted. - IN_Q_OVERFLOW* = 0x00004000 # Event queued overflowed. - IN_IGNORED* = 0x00008000 # File was ignored. + IN_UNMOUNT* = 0x00002000 ## Backing fs was unmounted. + IN_Q_OVERFLOW* = 0x00004000 ## Event queued overflowed. + IN_IGNORED* = 0x00008000 ## File was ignored. + # Special flags. const - IN_ONLYDIR* = 0x01000000 # Only watch the path if it is a - # directory. - IN_DONT_FOLLOW* = 0x02000000 # Do not follow a sym link. - IN_EXCL_UNLINK* = 0x04000000 # Exclude events on unlinked - # objects. - IN_MASK_ADD* = 0x20000000 # Add to the mask of an already - # existing watch. - IN_ISDIR* = 0x40000000 # Event occurred against dir. - IN_ONESHOT* = 0x80000000 # Only send event once. + IN_ONLYDIR* = 0x01000000 ## Only watch the path if it is a directory. + IN_DONT_FOLLOW* = 0x02000000 ## Do not follow a sym link. + IN_EXCL_UNLINK* = 0x04000000 ## Exclude events on unlinked objects. + IN_MASK_ADD* = 0x20000000 ## Add to the mask of an already existing watch. + IN_ISDIR* = 0x40000000 ## Event occurred against dir. + IN_ONESHOT* = 0x80000000 ## Only send event once. + # All events which a program can wait on. const IN_ALL_EVENTS* = (IN_ACCESS or IN_MODIFY or IN_ATTRIB or IN_CLOSE_WRITE or IN_CLOSE_NOWRITE or IN_OPEN or IN_MOVED_FROM or IN_MOVED_TO or IN_CREATE or IN_DELETE or IN_DELETE_SELF or IN_MOVE_SELF) -# Create and initialize inotify instance. -proc inotify_init*(): cint{.cdecl, importc: "inotify_init", - header: "<sys/inotify.h>".} -# Create and initialize inotify instance. -proc inotify_init1*(flags: cint): cint{.cdecl, importc: "inotify_init1", + + +proc inotify_init*(): FileHandle {.cdecl, importc: "inotify_init", + header: "<sys/inotify.h>".} + ## Create and initialize inotify instance. + +proc inotify_init1*(flags: cint): FileHandle {.cdecl, importc: "inotify_init1", header: "<sys/inotify.h>".} -# Add watch of object NAME to inotify instance FD. Notify about -# events specified by MASK. -proc inotify_add_watch*(fd: cint; name: cstring; mask: uint32): cint{. - cdecl, importc: "inotify_add_watch", header: "<sys/inotify.h>".} -# Remove the watch specified by WD from the inotify instance FD. -proc inotify_rm_watch*(fd: cint; wd: cint): cint{.cdecl, + ## Like `inotify_init<#inotify_init>`_ , + ## but has a flags argument that provides access to some extra functionality. + +proc inotify_add_watch*(fd: cint; name: cstring; mask: uint32): cint {.cdecl, + importc: "inotify_add_watch", header: "<sys/inotify.h>".} + ## Add watch of object NAME to inotify instance FD. Notify about events specified by MASK. + +proc inotify_rm_watch*(fd: cint; wd: cint): cint {.cdecl, importc: "inotify_rm_watch", header: "<sys/inotify.h>".} + ## Remove the watch specified by WD from the inotify instance FD. + +iterator inotify_events*(evs: pointer, n: int): ptr InotifyEvent = + ## Abstract the packed buffer interface to yield event object pointers. + runnableExamples("-r:off"): + when defined(linux): + import std/posix # needed for FileHandle read procedure + const MaxWatches = 8192 + + let inotifyFd = inotify_init() # create new inotify instance and get it's FileHandle + let wd = inotifyFd.inotify_add_watch("/tmp", IN_CREATE or IN_DELETE) # Add new watch + + var events: array[MaxWatches, byte] # event buffer + while (let n = read(inotifyFd, addr events, MaxWatches); n) > 0: # blocks until any events have been read + for e in inotify_events(addr events, n): + echo (e[].wd, e[].mask, cast[cstring](addr e[].name)) # echo watch id, mask, and name value of each event + var ev: ptr InotifyEvent = cast[ptr InotifyEvent](evs) + var n = n + while n > 0: + yield ev + let sz = InotifyEvent.sizeof + int(ev[].len) + n -= sz + ev = cast[ptr InotifyEvent](cast[uint](ev) + uint(sz)) + +runnableExamples: + when defined(linux): + let inotifyFd = inotify_init() # create and get new inotify FileHandle + doAssert inotifyFd >= 0 # check for errors + + let wd = inotifyFd.inotify_add_watch("/tmp", IN_CREATE or IN_DELETE) # Add new watch + doAssert wd >= 0 # check for errors + + discard inotifyFd.inotify_rm_watch(wd) # remove watch diff --git a/lib/posix/kqueue.nim b/lib/posix/kqueue.nim index 18b47f5d5..2450cdb42 100644 --- a/lib/posix/kqueue.nim +++ b/lib/posix/kqueue.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -from posix import Timespec +from std/posix import Timespec when defined(macosx) or defined(freebsd) or defined(openbsd) or defined(dragonfly): @@ -153,7 +153,7 @@ proc kevent*(kqFD: cint, changelist: ptr KEvent, nchanges: cint, eventlist: ptr KEvent, nevents: cint, timeout: ptr Timespec): cint {.importc: "kevent", header: "<sys/event.h>".} - ## Manipulates queue for given ``kqFD`` descriptor. + ## Manipulates queue for given `kqFD` descriptor. proc EV_SET*(event: ptr KEvent, ident: uint, filter: cshort, flags: cushort, fflags: cuint, data: int, udata: pointer) diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim index 25c7c7979..29fd4288d 100644 --- a/lib/posix/linux.nim +++ b/lib/posix/linux.nim @@ -1,28 +1,37 @@ -{.deadCodeElim: on.} # dce option deprecated - -import posix +import std/posix +## Flags of `clone` syscall. +## See `clone syscall manual +## <https://man7.org/linux/man-pages/man2/clone.2.html>`_ for more information. const - CSIGNAL* = 0x000000FF - CLONE_VM* = 0x00000100 - CLONE_FS* = 0x00000200 - CLONE_FILES* = 0x00000400 - CLONE_SIGHAND* = 0x00000800 - CLONE_PTRACE* = 0x00002000 - CLONE_VFORK* = 0x00004000 - CLONE_PARENT* = 0x00008000 - CLONE_THREAD* = 0x00010000 - CLONE_NEWNS* = 0x00020000 - CLONE_SYSVSEM* = 0x00040000 - CLONE_SETTLS* = 0x00080000 - CLONE_PARENT_SETTID* = 0x00100000 - CLONE_CHILD_CLEARTID* = 0x00200000 - CLONE_DETACHED* = 0x00400000 - CLONE_UNTRACED* = 0x00800000 - CLONE_CHILD_SETTID* = 0x01000000 - CLONE_STOPPED* = 0x02000000 + CSIGNAL* = 0x000000FF'i32 + CLONE_VM* = 0x00000100'i32 + CLONE_FS* = 0x00000200'i32 + CLONE_FILES* = 0x00000400'i32 + CLONE_SIGHAND* = 0x00000800'i32 + CLONE_PIDFD* = 0x00001000'i32 + CLONE_PTRACE* = 0x00002000'i32 + CLONE_VFORK* = 0x00004000'i32 + CLONE_PARENT* = 0x00008000'i32 + CLONE_THREAD* = 0x00010000'i32 + CLONE_NEWNS* = 0x00020000'i32 + CLONE_SYSVSEM* = 0x00040000'i32 + CLONE_SETTLS* = 0x00080000'i32 + CLONE_PARENT_SETTID* = 0x00100000'i32 + CLONE_CHILD_CLEARTID* = 0x00200000'i32 + CLONE_DETACHED* = 0x00400000'i32 + CLONE_UNTRACED* = 0x00800000'i32 + CLONE_CHILD_SETTID* = 0x01000000'i32 + CLONE_NEWCGROUP* = 0x02000000'i32 + CLONE_NEWUTS* = 0x04000000'i32 + CLONE_NEWIPC* = 0x08000000'i32 + CLONE_NEWUSER* = 0x10000000'i32 + CLONE_NEWPID* = 0x20000000'i32 + CLONE_NEWNET* = 0x40000000'i32 + CLONE_IO* = 0x80000000'i32 + -# fn should be of type proc (a2: pointer): void {.cdecl.} +# fn should be of type proc (a2: pointer) {.cdecl.} proc clone*(fn: pointer; child_stack: pointer; flags: cint; arg: pointer; ptid: ptr Pid; tls: pointer; ctid: ptr Pid): cint {.importc, header: "<sched.h>".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 70ba53a2e..fbe945df3 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -27,17 +27,19 @@ ## the \`identifier\` notation is used. ## ## This library relies on the header files of your C compiler. The -## resulting C code will just ``#include <XYZ.h>`` and *not* define the +## resulting C code will just `#include <XYZ.h>` and *not* define the ## symbols declared here. # Dead code elimination ensures that we don't accidentally generate #includes # for files that might not exist on a specific platform! The user will get an # error only if they actually try to use the missing declaration -{.deadCodeElim: on.} # dce option deprecated when defined(nimHasStyleChecks): {.push styleChecks: off.} +when defined(nimPreviewSlimSystem): + import std/syncio + # TODO these constants don't seem to be fetched from a header file for unknown # platforms - where do they come from and why are they here? when false: @@ -89,10 +91,10 @@ const type Sighandler = proc (a: cint) {.noconv.} const StatHasNanoseconds* = defined(linux) or defined(freebsd) or - defined(osx) or defined(openbsd) or defined(dragonfly) ## \ + defined(osx) or defined(openbsd) or defined(dragonfly) or defined(haiku) ## \ ## Boolean flag that indicates if the system supports nanosecond time - ## resolution in the fields of ``Stat``. Note that the nanosecond based fields - ## (``Stat.st_atim``, ``Stat.st_mtim`` and ``Stat.st_ctim``) can be accessed + ## resolution in the fields of `Stat`. Note that the nanosecond based fields + ## (`Stat.st_atim`, `Stat.st_mtim` and `Stat.st_ctim`) can be accessed ## without checking this flag, because this module defines fallback procs ## when they are not available. @@ -106,6 +108,8 @@ elif (defined(macos) or defined(macosx) or defined(bsd)) and defined(cpu64): include posix_macos_amd64 elif defined(nintendoswitch): include posix_nintendoswitch +elif defined(haiku): + include posix_haiku else: include posix_other @@ -150,11 +154,13 @@ proc htons*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".} proc ntohl*(a1: uint32): uint32 {.importc, header: "<arpa/inet.h>".} proc ntohs*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".} -proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".} -proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".} -proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. +when not defined(zephyr): + proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".} + proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".} + +proc inet_ntop*(a1: cint, a2: pointer | ptr InAddr | ptr In6Addr, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "<arpa/inet.h>".} -proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. +proc inet_pton*(a1: cint, a2: cstring, a3: pointer | ptr InAddr | ptr In6Addr): cint {. importc, header: "<arpa/inet.h>".} var @@ -166,29 +172,54 @@ proc IN6ADDR_LOOPBACK_INIT* (): In6Addr {.importc, header: "<netinet/in.h>".} # dirent.h proc closedir*(a1: ptr DIR): cint {.importc, header: "<dirent.h>".} -proc opendir*(a1: cstring): ptr DIR {.importc, header: "<dirent.h>".} -proc readdir*(a1: ptr DIR): ptr Dirent {.importc, header: "<dirent.h>".} +proc opendir*(a1: cstring): ptr DIR {.importc, header: "<dirent.h>", sideEffect.} +proc readdir*(a1: ptr DIR): ptr Dirent {.importc, header: "<dirent.h>", sideEffect.} proc readdir_r*(a1: ptr DIR, a2: ptr Dirent, a3: ptr ptr Dirent): cint {. - importc, header: "<dirent.h>".} + importc, header: "<dirent.h>", sideEffect.} proc rewinddir*(a1: ptr DIR) {.importc, header: "<dirent.h>".} proc seekdir*(a1: ptr DIR, a2: int) {.importc, header: "<dirent.h>".} proc telldir*(a1: ptr DIR): int {.importc, header: "<dirent.h>".} # dlfcn.h -proc dlclose*(a1: pointer): cint {.importc, header: "<dlfcn.h>".} -proc dlerror*(): cstring {.importc, header: "<dlfcn.h>".} -proc dlopen*(a1: cstring, a2: cint): pointer {.importc, header: "<dlfcn.h>".} -proc dlsym*(a1: pointer, a2: cstring): pointer {.importc, header: "<dlfcn.h>".} - -proc creat*(a1: cstring, a2: Mode): cint {.importc, header: "<fcntl.h>".} -proc fcntl*(a1: cint | SocketHandle, a2: cint): cint {.varargs, importc, header: "<fcntl.h>".} -proc open*(a1: cstring, a2: cint): cint {.varargs, importc, header: "<fcntl.h>".} +proc dlclose*(a1: pointer): cint {.importc, header: "<dlfcn.h>", sideEffect.} +proc dlerror*(): cstring {.importc, header: "<dlfcn.h>", sideEffect.} +proc dlopen*(a1: cstring, a2: cint): pointer {.importc, header: "<dlfcn.h>", sideEffect.} +proc dlsym*(a1: pointer, a2: cstring): pointer {.importc, header: "<dlfcn.h>", sideEffect.} + +proc creat*(a1: cstring, a2: Mode): cint {.importc, header: "<fcntl.h>", sideEffect.} +proc fcntl*(a1: cint | SocketHandle, a2: cint): cint {.varargs, importc, header: "<fcntl.h>", sideEffect.} +proc openImpl(a1: cstring, a2: cint): cint {.varargs, importc: "open", header: "<fcntl.h>", sideEffect.} +proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = + # prevents bug #17888 + openImpl(a1, a2, mode) + proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "<fcntl.h>".} -proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. - importc, header: "<fcntl.h>".} -when not defined(haiku) and not defined(OpenBSD): +proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "<unistd.h>".} +when defined(osx): # 2001 POSIX evidently does not concern Apple + type FStore {.importc: "fstore_t", header: "<fcntl.h>", bycopy.} = object + fst_flags: uint32 ## IN: flags word + fst_posmode: cint ## IN: indicates offset field + fst_offset, ## IN: start of the region + fst_length, ## IN: size of the region + fst_bytesalloc: Off ## OUT: number of bytes allocated + var F_PEOFPOSMODE {.importc, header: "<fcntl.h>".}: cint + var F_ALLOCATEALL {.importc, header: "<fcntl.h>".}: uint32 + var F_PREALLOCATE {.importc, header: "<fcntl.h>".}: cint + proc posix_fallocate*(a1: cint, a2, a3: Off): cint = + var fst = FStore(fst_flags: F_ALLOCATEALL, fst_posmode: F_PEOFPOSMODE, + fst_offset: a2, fst_length: a3) + # Must also call ftruncate to match what POSIX does. Unlike posix_fallocate, + # this can shrink files. Could guard w/getFileSize, but caller likely knows + # present size & has no good reason to call this unless it is growing. + if fcntl(a1, F_PREALLOCATE, fst.addr) != cint(-1): ftruncate(a1, a2 + a3) + else: cint(-1) +else: + proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. + importc, header: "<fcntl.h>".} + +when not defined(haiku) and not defined(openbsd): proc fmtmsg*(a1: int, a2: cstring, a3: cint, a4, a5, a6: cstring): cint {.importc, header: "<fmtmsg.h>".} @@ -205,7 +236,7 @@ when not (defined(linux) and defined(amd64)) and not defined(nintendoswitch): proc glob*(a1: cstring, a2: cint, a3: proc (x1: cstring, x2: cint): cint {.noconv.}, - a4: ptr Glob): cint {.importc, header: "<glob.h>".} + a4: ptr Glob): cint {.importc, header: "<glob.h>", sideEffect.} ## Filename globbing. Use `os.walkPattern() <os.html#glob_1>`_ and similar. proc globfree*(a1: ptr Glob) {.importc, header: "<glob.h>".} @@ -234,31 +265,57 @@ proc dirname*(a1: cstring): cstring {.importc, header: "<libgen.h>".} proc localeconv*(): ptr Lconv {.importc, header: "<locale.h>".} proc setlocale*(a1: cint, a2: cstring): cstring {. - importc, header: "<locale.h>".} + importc, header: "<locale.h>", sideEffect.} proc strfmon*(a1: cstring, a2: int, a3: cstring): int {.varargs, importc, header: "<monetary.h>".} -when not defined(nintendoswitch): - proc mq_close*(a1: Mqd): cint {.importc, header: "<mqueue.h>".} - proc mq_getattr*(a1: Mqd, a2: ptr MqAttr): cint {. - importc, header: "<mqueue.h>".} - proc mq_notify*(a1: Mqd, a2: ptr SigEvent): cint {. +when not (defined(nintendoswitch) or defined(macos) or defined(macosx)): + proc mq_notify*(mqdes: Mqd, event: ptr SigEvent): cint {. importc, header: "<mqueue.h>".} - proc mq_open*(a1: cstring, a2: cint): Mqd {. + + proc mq_open*(name: cstring, flags: cint): Mqd {. varargs, importc, header: "<mqueue.h>".} - proc mq_receive*(a1: Mqd, a2: cstring, a3: int, a4: var int): int {. - importc, header: "<mqueue.h>".} - proc mq_send*(a1: Mqd, a2: cstring, a3: int, a4: int): cint {. + + proc mq_close*(mqdes: Mqd): cint {.importc, header: "<mqueue.h>".} + + proc mq_receive*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: var cuint + ): int {.importc, header: "<mqueue.h>".} + + proc mq_timedreceive*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint, + timeout: ptr Timespec + ): int {.importc, header: "<mqueue.h>".} + + proc mq_send*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint + ): cint {.importc, header: "<mqueue.h>".} + + proc mq_timedsend*( + mqdes: Mqd, + buffer: cstring, + length: csize_t, + priority: cuint, + timeout: ptr Timespec + ): cint {.importc, header: "<mqueue.h>".} + + proc mq_getattr*(mqdes: Mqd, attribute: ptr MqAttr): cint {. importc, header: "<mqueue.h>".} - proc mq_setattr*(a1: Mqd, a2, a3: ptr MqAttr): cint {. + + proc mq_setattr*(mqdes: Mqd, newAttribute, oldAttribute: ptr MqAttr): cint {. importc, header: "<mqueue.h>".} - proc mq_timedreceive*(a1: Mqd, a2: cstring, a3: int, a4: int, - a5: ptr Timespec): int {.importc, header: "<mqueue.h>".} - proc mq_timedsend*(a1: Mqd, a2: cstring, a3: int, a4: int, - a5: ptr Timespec): cint {.importc, header: "<mqueue.h>".} - proc mq_unlink*(a1: cstring): cint {.importc, header: "<mqueue.h>".} + proc mq_unlink*(mqdes: cstring): cint {.importc, header: "<mqueue.h>".} proc getpwnam*(a1: cstring): ptr Passwd {.importc, header: "<pwd.h>".} @@ -446,7 +503,7 @@ proc pthread_spin_unlock*(a1: ptr Pthread_spinlock): cint {. proc pthread_testcancel*() {.importc, header: "<pthread.h>".} -proc exitnow*(code: int): void {.importc: "_exit", header: "<unistd.h>".} +proc exitnow*(code: int) {.importc: "_exit", header: "<unistd.h>".} proc access*(a1: cstring, a2: cint): cint {.importc, header: "<unistd.h>".} proc alarm*(a1: cint): cint {.importc, header: "<unistd.h>".} proc chdir*(a1: cstring): cint {.importc, header: "<unistd.h>".} @@ -459,54 +516,53 @@ proc dup*(a1: cint): cint {.importc, header: "<unistd.h>".} proc dup2*(a1, a2: cint): cint {.importc, header: "<unistd.h>".} proc encrypt*(a1: array[0..63, char], a2: cint) {.importc, header: "<unistd.h>".} -proc execl*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>".} -proc execle*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>".} -proc execlp*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>".} -proc execv*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>".} +proc execl*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>", sideEffect.} +proc execle*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>", sideEffect.} +proc execlp*(a1, a2: cstring): cint {.varargs, importc, header: "<unistd.h>", sideEffect.} +proc execv*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>", sideEffect.} proc execve*(a1: cstring, a2, a3: cstringArray): cint {. - importc, header: "<unistd.h>".} -proc execvp*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>".} -proc execvpe*(a1: cstring, a2: cstringArray, a3: cstringArray): cint {.importc, header: "<unistd.h>".} -proc fchown*(a1: cint, a2: Uid, a3: Gid): cint {.importc, header: "<unistd.h>".} -proc fchdir*(a1: cint): cint {.importc, header: "<unistd.h>".} + importc, header: "<unistd.h>", sideEffect.} +proc execvp*(a1: cstring, a2: cstringArray): cint {.importc, header: "<unistd.h>", sideEffect.} +proc execvpe*(a1: cstring, a2: cstringArray, a3: cstringArray): cint {.importc, header: "<unistd.h>", sideEffect.} +proc fchown*(a1: cint, a2: Uid, a3: Gid): cint {.importc, header: "<unistd.h>", sideEffect.} +proc fchdir*(a1: cint): cint {.importc, header: "<unistd.h>", sideEffect.} proc fdatasync*(a1: cint): cint {.importc, header: "<unistd.h>".} -proc fork*(): Pid {.importc, header: "<unistd.h>".} +proc fork*(): Pid {.importc, header: "<unistd.h>", sideEffect.} proc fpathconf*(a1, a2: cint): int {.importc, header: "<unistd.h>".} proc fsync*(a1: cint): cint {.importc, header: "<unistd.h>".} ## synchronize a file's buffer cache to the storage device -proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "<unistd.h>".} -proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "<unistd.h>".} -proc getuid*(): Uid {.importc, header: "<unistd.h>".} +proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "<unistd.h>", sideEffect.} +proc getuid*(): Uid {.importc, header: "<unistd.h>", sideEffect.} ## returns the real user ID of the calling process -proc geteuid*(): Uid {.importc, header: "<unistd.h>".} +proc geteuid*(): Uid {.importc, header: "<unistd.h>", sideEffect.} ## returns the effective user ID of the calling process -proc getgid*(): Gid {.importc, header: "<unistd.h>".} +proc getgid*(): Gid {.importc, header: "<unistd.h>", sideEffect.} ## returns the real group ID of the calling process -proc getegid*(): Gid {.importc, header: "<unistd.h>".} +proc getegid*(): Gid {.importc, header: "<unistd.h>", sideEffect.} ## returns the effective group ID of the calling process proc getgroups*(a1: cint, a2: ptr array[0..255, Gid]): cint {. importc, header: "<unistd.h>".} -proc gethostid*(): int {.importc, header: "<unistd.h>".} -proc gethostname*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>".} -proc getlogin*(): cstring {.importc, header: "<unistd.h>".} -proc getlogin_r*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>".} +proc gethostid*(): int {.importc, header: "<unistd.h>", sideEffect.} +proc gethostname*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>", sideEffect.} +proc getlogin*(): cstring {.importc, header: "<unistd.h>", sideEffect.} +proc getlogin_r*(a1: cstring, a2: int): cint {.importc, header: "<unistd.h>", sideEffect.} proc getopt*(a1: cint, a2: cstringArray, a3: cstring): cint {. importc, header: "<unistd.h>".} proc getpgid*(a1: Pid): Pid {.importc, header: "<unistd.h>".} proc getpgrp*(): Pid {.importc, header: "<unistd.h>".} -proc getpid*(): Pid {.importc, header: "<unistd.h>".} +proc getpid*(): Pid {.importc, header: "<unistd.h>", sideEffect.} ## returns the process ID (PID) of the calling process -proc getppid*(): Pid {.importc, header: "<unistd.h>".} +proc getppid*(): Pid {.importc, header: "<unistd.h>", sideEffect.} ## returns the process ID of the parent of the calling process -proc getsid*(a1: Pid): Pid {.importc, header: "<unistd.h>".} +proc getsid*(a1: Pid): Pid {.importc, header: "<unistd.h>", sideEffect.} ## returns the session ID of the calling process proc getwd*(a1: cstring): cstring {.importc, header: "<unistd.h>".} @@ -528,7 +584,8 @@ proc pread*(a1: cint, a2: pointer, a3: int, a4: Off): int {. proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "<unistd.h>".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "<unistd.h>".} -proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".} +when not defined(nintendoswitch): + proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".} proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", header: "<sys/ioctl.h>", varargs, tags: [WriteIOEffect].} ## A system call for device-specific input/output operations and other @@ -547,7 +604,10 @@ proc setsid*(): Pid {.importc, header: "<unistd.h>".} proc setuid*(a1: Uid): cint {.importc, header: "<unistd.h>".} proc sleep*(a1: cint): cint {.importc, header: "<unistd.h>".} proc swab*(a1, a2: pointer, a3: int) {.importc, header: "<unistd.h>".} -proc symlink*(a1, a2: cstring): cint {.importc, header: "<unistd.h>".} +when not defined(nintendoswitch): + proc symlink*(a1, a2: cstring): cint {.importc, header: "<unistd.h>".} +else: + proc symlink*(a1, a2: cstring): cint = -1 proc sync*() {.importc, header: "<unistd.h>".} proc sysconf*(a1: cint): int {.importc, header: "<unistd.h>".} proc tcgetpgrp*(a1: cint): Pid {.importc, header: "<unistd.h>".} @@ -584,11 +644,13 @@ proc statvfs*(a1: cstring, a2: var Statvfs): cint {. proc fstatvfs*(a1: cint, a2: var Statvfs): cint {. importc, header: "<sys/statvfs.h>".} -proc chmod*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>".} -proc fchmod*(a1: cint, a2: Mode): cint {.importc, header: "<sys/stat.h>".} -proc fstat*(a1: cint, a2: var Stat): cint {.importc, header: "<sys/stat.h>".} -proc lstat*(a1: cstring, a2: var Stat): cint {.importc, header: "<sys/stat.h>".} -proc mkdir*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>".} +proc chmod*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} +when defined(osx) or defined(freebsd): + proc lchmod*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc fchmod*(a1: cint, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc fstat*(a1: cint, a2: var Stat): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc lstat*(a1: cstring, a2: var Stat): cint {.importc, header: "<sys/stat.h>", sideEffect.} +proc mkdir*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>", sideEffect.} ## Use `os.createDir() <os.html#createDir,string>`_ and similar. proc mkfifo*(a1: cstring, a2: Mode): cint {.importc, header: "<sys/stat.h>".} @@ -637,7 +699,8 @@ proc posix_madvise*(a1: pointer, a2: int, a3: cint): cint {. importc, header: "<sys/mman.h>".} proc posix_mem_offset*(a1: pointer, a2: int, a3: var Off, a4: var int, a5: var cint): cint {.importc, header: "<sys/mman.h>".} -when not (defined(linux) and defined(amd64)) and not defined(nintendoswitch): +when not (defined(linux) and defined(amd64)) and not defined(nintendoswitch) and + not defined(haiku): proc posix_typed_mem_get_info*(a1: cint, a2: var Posix_typed_mem_info): cint {.importc, header: "<sys/mman.h>".} proc posix_typed_mem_open*(a1: cstring, a2, a3: cint): cint {. @@ -649,17 +712,17 @@ proc shm_unlink*(a1: cstring): cint {.importc, header: "<sys/mman.h>".} proc asctime*(a1: var Tm): cstring{.importc, header: "<time.h>".} proc asctime_r*(a1: var Tm, a2: cstring): cstring {.importc, header: "<time.h>".} -proc clock*(): Clock {.importc, header: "<time.h>".} +proc clock*(): Clock {.importc, header: "<time.h>", sideEffect.} proc clock_getcpuclockid*(a1: Pid, a2: var ClockId): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc clock_getres*(a1: ClockId, a2: var Timespec): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc clock_gettime*(a1: ClockId, a2: var Timespec): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc clock_nanosleep*(a1: ClockId, a2: cint, a3: var Timespec, - a4: var Timespec): cint {.importc, header: "<time.h>".} + a4: var Timespec): cint {.importc, header: "<time.h>", sideEffect.} proc clock_settime*(a1: ClockId, a2: var Timespec): cint {. - importc, header: "<time.h>".} + importc, header: "<time.h>", sideEffect.} proc `==`*(a, b: Time): bool {.borrow.} proc `-`*(a, b: Time): Time {.borrow.} @@ -673,11 +736,11 @@ proc localtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".} proc localtime_r*(a1: var Time, a2: var Tm): ptr Tm {.importc, header: "<time.h>".} proc mktime*(a1: var Tm): Time {.importc, header: "<time.h>".} proc timegm*(a1: var Tm): Time {.importc, header: "<time.h>".} -proc nanosleep*(a1, a2: var Timespec): cint {.importc, header: "<time.h>".} +proc nanosleep*(a1, a2: var Timespec): cint {.importc, header: "<time.h>", sideEffect.} proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "<time.h>".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "<time.h>".} -proc time*(a1: var Time): Time {.importc, header: "<time.h>".} +proc time*(a1: var Time): Time {.importc, header: "<time.h>", sideEffect.} proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "<time.h>".} proc timer_delete*(a1: Timer): cint {.importc, header: "<time.h>".} @@ -689,11 +752,11 @@ proc timer_settime*(a1: Timer, a2: cint, a3: var Itimerspec, proc tzset*() {.importc, header: "<time.h>".} -proc wait*(a1: ptr cint): Pid {.importc, discardable, header: "<sys/wait.h>".} +proc wait*(a1: ptr cint): Pid {.importc, discardable, header: "<sys/wait.h>", sideEffect.} proc waitid*(a1: cint, a2: Id, a3: var SigInfo, a4: cint): cint {. - importc, header: "<sys/wait.h>".} + importc, header: "<sys/wait.h>", sideEffect.} proc waitpid*(a1: Pid, a2: var cint, a3: cint): Pid {. - importc, header: "<sys/wait.h>".} + importc, header: "<sys/wait.h>", sideEffect.} type Rusage* {.importc: "struct rusage", header: "<sys/resource.h>", bycopy.} = object @@ -704,7 +767,7 @@ type Rusage* {.importc: "struct rusage", header: "<sys/resource.h>", ru_nsignals*, ru_nvcsw*, ru_nivcsw*: clong # switching activity proc wait4*(pid: Pid, status: ptr cint, options: cint, rusage: ptr Rusage): Pid - {.importc, header: "<sys/wait.h>".} + {.importc, header: "<sys/wait.h>", sideEffect.} const RUSAGE_SELF* = cint(0) @@ -717,8 +780,8 @@ proc getrusage*(who: cint, rusage: ptr Rusage): cint proc bsd_signal*(a1: cint, a2: proc (x: pointer) {.noconv.}) {. importc, header: "<signal.h>".} -proc kill*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>".} -proc killpg*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>".} +proc kill*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>", sideEffect.} +proc killpg*(a1: Pid, a2: cint): cint {.importc, header: "<signal.h>", sideEffect.} proc pthread_kill*(a1: Pthread, a2: cint): cint {.importc, header: "<signal.h>".} proc pthread_sigmask*(a1: cint, a2, a3: var Sigset): cint {. importc, header: "<signal.h>".} @@ -762,6 +825,13 @@ else: proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint {.importc, header: "<signal.h>".} +when defined(sunos) or defined(solaris): + # The following compile time flag is needed on Illumos/Solaris to use the POSIX + # `sigwait` implementation. See the documentation here: + # https://docs.oracle.com/cd/E19455-01/806-5257/6je9h033k/index.html + # https://www.illumos.org/man/2/sigwait + {.passc: "-D_POSIX_PTHREAD_SEMANTICS".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "<signal.h>".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. @@ -874,15 +944,9 @@ proc CMSG_NXTHDR*(mhdr: ptr Tmsghdr, cmsg: ptr Tcmsghdr): ptr Tcmsghdr {. proc CMSG_FIRSTHDR*(mhdr: ptr Tmsghdr): ptr Tcmsghdr {. importc, header: "<sys/socket.h>".} -proc CMSG_SPACE*(len: csize): csize {. - importc, header: "<sys/socket.h>", deprecated: "argument `len` should be of type `csize_t`".} - proc CMSG_SPACE*(len: csize_t): csize_t {. importc, header: "<sys/socket.h>".} -proc CMSG_LEN*(len: csize): csize {. - importc, header: "<sys/socket.h>", deprecated: "argument `len` should be of type `csize_t`".} - proc CMSG_LEN*(len: csize_t): csize_t {. importc, header: "<sys/socket.h>".} @@ -892,11 +956,15 @@ const proc `==`*(x, y: SocketHandle): bool {.borrow.} proc accept*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen): SocketHandle {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} + +when defined(linux) or defined(bsd) or defined(nuttx): + proc accept4*(a1: SocketHandle, a2: ptr SockAddr, a3: ptr SockLen, + flags: cint): SocketHandle {.importc, header: "<sys/socket.h>".} proc bindSocket*(a1: SocketHandle, a2: ptr SockAddr, a3: SockLen): cint {. importc: "bind", header: "<sys/socket.h>".} - ## is Posix's ``bind``, because ``bind`` is a reserved word + ## is Posix's `bind`, because `bind` is a reserved word proc connect*(a1: SocketHandle, a2: ptr SockAddr, a3: SockLen): cint {. importc, header: "<sys/socket.h>".} @@ -909,21 +977,21 @@ proc getsockopt*(a1: SocketHandle, a2, a3: cint, a4: pointer, a5: ptr SockLen): importc, header: "<sys/socket.h>".} proc listen*(a1: SocketHandle, a2: cint): cint {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc recv*(a1: SocketHandle, a2: pointer, a3: int, a4: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc recvfrom*(a1: SocketHandle, a2: pointer, a3: int, a4: cint, a5: ptr SockAddr, a6: ptr SockLen): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc recvmsg*(a1: SocketHandle, a2: ptr Tmsghdr, a3: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc send*(a1: SocketHandle, a2: pointer, a3: int, a4: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc sendmsg*(a1: SocketHandle, a2: ptr Tmsghdr, a3: cint): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc sendto*(a1: SocketHandle, a2: pointer, a3: int, a4: cint, a5: ptr SockAddr, a6: SockLen): int {. - importc, header: "<sys/socket.h>".} + importc, header: "<sys/socket.h>", sideEffect.} proc setsockopt*(a1: SocketHandle, a2, a3: cint, a4: pointer, a5: SockLen): cint {. importc, header: "<sys/socket.h>".} proc shutdown*(a1: SocketHandle, a2: cint): cint {. @@ -956,9 +1024,15 @@ proc IN6_IS_ADDR_LINKLOCAL* (a1: ptr In6Addr): cint {. proc IN6_IS_ADDR_SITELOCAL* (a1: ptr In6Addr): cint {. importc, header: "<netinet/in.h>".} ## Unicast site-local address. -proc IN6_IS_ADDR_V4MAPPED* (a1: ptr In6Addr): cint {. - importc, header: "<netinet/in.h>".} - ## IPv4 mapped address. +when defined(lwip): + proc IN6_IS_ADDR_V4MAPPED*(ipv6_address: ptr In6Addr): cint = + var bits32: ptr array[4, uint32] = cast[ptr array[4, uint32]](ipv6_address) + return (bits32[1] == 0'u32 and bits32[2] == htonl(0x0000FFFF)).cint +else: + proc IN6_IS_ADDR_V4MAPPED* (a1: ptr In6Addr): cint {. + importc, header: "<netinet/in.h>".} + ## IPv4 mapped address. + proc IN6_IS_ADDR_V4COMPAT* (a1: ptr In6Addr): cint {. importc, header: "<netinet/in.h>".} ## IPv4-compatible address. @@ -982,7 +1056,7 @@ proc endhostent*() {.importc, header: "<netdb.h>".} proc endnetent*() {.importc, header: "<netdb.h>".} proc endprotoent*() {.importc, header: "<netdb.h>".} proc endservent*() {.importc, header: "<netdb.h>".} -proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "<netdb.h>".} +proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "<netdb.h>".} proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h>".} @@ -1020,22 +1094,37 @@ proc setnetent*(a1: cint) {.importc, header: "<netdb.h>".} proc setprotoent*(a1: cint) {.importc, header: "<netdb.h>".} proc setservent*(a1: cint) {.importc, header: "<netdb.h>".} -proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. - importc, header: "<poll.h>".} +when not defined(lwip): + proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. + importc, header: "<poll.h>", sideEffect.} proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "<stdlib.h>".} -proc mkstemp*(tmpl: cstring): cint {.importc, header: "<stdlib.h>".} +proc mkstemp*(tmpl: cstring): cint {.importc, header: "<stdlib.h>", sideEffect.} ## Creates a unique temporary file. ## - ## **Warning**: The `tmpl` argument is written to by `mkstemp` and thus - ## can't be a string literal. If in doubt copy the string before passing it. + ## .. warning:: The `tmpl` argument is written to by `mkstemp` and thus + ## can't be a string literal. If in doubt make a copy of the cstring before + ## passing it in. -proc mkdtemp*(tmpl: cstring): pointer {.importc, header: "<stdlib.h>".} +proc mkstemps*(tmpl: cstring, suffixlen: int): cint {.importc, header: "<stdlib.h>", sideEffect.} + ## Creates a unique temporary file. + ## + ## .. warning:: The `tmpl` argument is written to by `mkstemps` and thus + ## can't be a string literal. If in doubt make a copy of the cstring before + ## passing it in. + +proc mkdtemp*(tmpl: cstring): pointer {.importc, header: "<stdlib.h>", sideEffect.} + +when defined(linux) or defined(bsd) or defined(osx): + proc mkostemp*(tmpl: cstring, oflags: cint): cint {.importc, header: "<stdlib.h>", sideEffect.} + proc mkostemps*(tmpl: cstring, suffixlen: cint, oflags: cint): cint {.importc, header: "<stdlib.h>", sideEffect.} + + proc posix_memalign*(memptr: pointer, alignment: csize_t, size: csize_t): cint {.importc, header: "<stdlib.h>".} proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. - importc: "utimes", header: "<sys/time.h>".} + importc: "utimes", header: "<sys/time.h>", sideEffect.} ## Sets file access and modification times. ## ## Pass the filename and an array of times to set the access and modification @@ -1050,15 +1139,15 @@ proc handle_signal(sig: cint, handler: proc (a: cint) {.noconv.}) {.importc: "si template onSignal*(signals: varargs[cint], body: untyped) = ## Setup code to be executed when Unix signals are received. The - ## currently handled signal is injected as ``sig`` into the calling + ## currently handled signal is injected as `sig` into the calling ## scope. ## ## Example: - ## - ## .. code-block:: - ## from posix import SIGINT, SIGTERM, onSignal + ## ```Nim + ## from std/posix import SIGINT, SIGTERM, onSignal ## onSignal(SIGINT, SIGTERM): ## echo "bye from signal ", sig + ## ``` for s in signals: handle_signal(s, @@ -1075,12 +1164,12 @@ type ## The getrlimit() and setrlimit() system calls get and set resource limits respectively. ## Each resource has an associated soft and hard limit, as defined by the RLimit structure -proc setrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "setrlimit",header: "<sys/resource.h>".} +proc setrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "setrlimit", header: "<sys/resource.h>".} ## The setrlimit() system calls sets resource limits. -proc getrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "getrlimit",header: "<sys/resource.h>".} +proc getrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "getrlimit", header: "<sys/resource.h>".} ## The getrlimit() system call gets resource limits. when defined(nimHasStyleChecks): diff --git a/lib/posix/posix_freertos_consts.nim b/lib/posix/posix_freertos_consts.nim new file mode 100644 index 000000000..0f0fc0aae --- /dev/null +++ b/lib/posix/posix_freertos_consts.nim @@ -0,0 +1,506 @@ +# Generated by detect.nim + + + +# <errno.h> +var E2BIG* {.importc: "E2BIG", header: "<errno.h>".}: cint +var EACCES* {.importc: "EACCES", header: "<errno.h>".}: cint +var EADDRINUSE* {.importc: "EADDRINUSE", header: "<errno.h>".}: cint +var EADDRNOTAVAIL* {.importc: "EADDRNOTAVAIL", header: "<errno.h>".}: cint +var EAFNOSUPPORT* {.importc: "EAFNOSUPPORT", header: "<errno.h>".}: cint +var EAGAIN* {.importc: "EAGAIN", header: "<errno.h>".}: cint +var EALREADY* {.importc: "EALREADY", header: "<errno.h>".}: cint +var EBADF* {.importc: "EBADF", header: "<errno.h>".}: cint +var EBADMSG* {.importc: "EBADMSG", header: "<errno.h>".}: cint +var EBUSY* {.importc: "EBUSY", header: "<errno.h>".}: cint +var ECANCELED* {.importc: "ECANCELED", header: "<errno.h>".}: cint +var ECHILD* {.importc: "ECHILD", header: "<errno.h>".}: cint +var ECONNABORTED* {.importc: "ECONNABORTED", header: "<errno.h>".}: cint +var ECONNREFUSED* {.importc: "ECONNREFUSED", header: "<errno.h>".}: cint +var ECONNRESET* {.importc: "ECONNRESET", header: "<errno.h>".}: cint +var EDEADLK* {.importc: "EDEADLK", header: "<errno.h>".}: cint +var EDESTADDRREQ* {.importc: "EDESTADDRREQ", header: "<errno.h>".}: cint +var EDOM* {.importc: "EDOM", header: "<errno.h>".}: cint +var EDQUOT* {.importc: "EDQUOT", header: "<errno.h>".}: cint +var EEXIST* {.importc: "EEXIST", header: "<errno.h>".}: cint +var EFAULT* {.importc: "EFAULT", header: "<errno.h>".}: cint +var EFBIG* {.importc: "EFBIG", header: "<errno.h>".}: cint +var EHOSTUNREACH* {.importc: "EHOSTUNREACH", header: "<errno.h>".}: cint +var EIDRM* {.importc: "EIDRM", header: "<errno.h>".}: cint +var EILSEQ* {.importc: "EILSEQ", header: "<errno.h>".}: cint +var EINPROGRESS* {.importc: "EINPROGRESS", header: "<errno.h>".}: cint +var EINTR* {.importc: "EINTR", header: "<errno.h>".}: cint +var EINVAL* {.importc: "EINVAL", header: "<errno.h>".}: cint +var EIO* {.importc: "EIO", header: "<errno.h>".}: cint +var EISCONN* {.importc: "EISCONN", header: "<errno.h>".}: cint +var EISDIR* {.importc: "EISDIR", header: "<errno.h>".}: cint +var ELOOP* {.importc: "ELOOP", header: "<errno.h>".}: cint +var EMFILE* {.importc: "EMFILE", header: "<errno.h>".}: cint +var EMLINK* {.importc: "EMLINK", header: "<errno.h>".}: cint +var EMSGSIZE* {.importc: "EMSGSIZE", header: "<errno.h>".}: cint +var EMULTIHOP* {.importc: "EMULTIHOP", header: "<errno.h>".}: cint +var ENAMETOOLONG* {.importc: "ENAMETOOLONG", header: "<errno.h>".}: cint +var ENETDOWN* {.importc: "ENETDOWN", header: "<errno.h>".}: cint +var ENETRESET* {.importc: "ENETRESET", header: "<errno.h>".}: cint +var ENETUNREACH* {.importc: "ENETUNREACH", header: "<errno.h>".}: cint +var ENFILE* {.importc: "ENFILE", header: "<errno.h>".}: cint +var ENOBUFS* {.importc: "ENOBUFS", header: "<errno.h>".}: cint +var ENODATA* {.importc: "ENODATA", header: "<errno.h>".}: cint +var ENODEV* {.importc: "ENODEV", header: "<errno.h>".}: cint +var ENOENT* {.importc: "ENOENT", header: "<errno.h>".}: cint +var ENOEXEC* {.importc: "ENOEXEC", header: "<errno.h>".}: cint +var ENOLCK* {.importc: "ENOLCK", header: "<errno.h>".}: cint +var ENOLINK* {.importc: "ENOLINK", header: "<errno.h>".}: cint +var ENOMEM* {.importc: "ENOMEM", header: "<errno.h>".}: cint +var ENOMSG* {.importc: "ENOMSG", header: "<errno.h>".}: cint +var ENOPROTOOPT* {.importc: "ENOPROTOOPT", header: "<errno.h>".}: cint +var ENOSPC* {.importc: "ENOSPC", header: "<errno.h>".}: cint +var ENOSR* {.importc: "ENOSR", header: "<errno.h>".}: cint +var ENOSTR* {.importc: "ENOSTR", header: "<errno.h>".}: cint +var ENOSYS* {.importc: "ENOSYS", header: "<errno.h>".}: cint +var ENOTCONN* {.importc: "ENOTCONN", header: "<errno.h>".}: cint +var ENOTDIR* {.importc: "ENOTDIR", header: "<errno.h>".}: cint +var ENOTEMPTY* {.importc: "ENOTEMPTY", header: "<errno.h>".}: cint +var ENOTSOCK* {.importc: "ENOTSOCK", header: "<errno.h>".}: cint +var ENOTSUP* {.importc: "ENOTSUP", header: "<errno.h>".}: cint +var ENOTTY* {.importc: "ENOTTY", header: "<errno.h>".}: cint +var ENXIO* {.importc: "ENXIO", header: "<errno.h>".}: cint +var EOPNOTSUPP* {.importc: "EOPNOTSUPP", header: "<errno.h>".}: cint +var EOVERFLOW* {.importc: "EOVERFLOW", header: "<errno.h>".}: cint +var EPERM* {.importc: "EPERM", header: "<errno.h>".}: cint +var EPIPE* {.importc: "EPIPE", header: "<errno.h>".}: cint +var EPROTO* {.importc: "EPROTO", header: "<errno.h>".}: cint +var EPROTONOSUPPORT* {.importc: "EPROTONOSUPPORT", header: "<errno.h>".}: cint +var EPROTOTYPE* {.importc: "EPROTOTYPE", header: "<errno.h>".}: cint +var ERANGE* {.importc: "ERANGE", header: "<errno.h>".}: cint +var EROFS* {.importc: "EROFS", header: "<errno.h>".}: cint +var ESPIPE* {.importc: "ESPIPE", header: "<errno.h>".}: cint +var ESRCH* {.importc: "ESRCH", header: "<errno.h>".}: cint +var ESTALE* {.importc: "ESTALE", header: "<errno.h>".}: cint +var ETIME* {.importc: "ETIME", header: "<errno.h>".}: cint +var ETIMEDOUT* {.importc: "ETIMEDOUT", header: "<errno.h>".}: cint +var ETXTBSY* {.importc: "ETXTBSY", header: "<errno.h>".}: cint +var EWOULDBLOCK* {.importc: "EWOULDBLOCK", header: "<errno.h>".}: cint +var EXDEV* {.importc: "EXDEV", header: "<errno.h>".}: cint + +# <sys/fcntl.h> +var F_GETFD* {.importc: "F_GETFD", header: "<sys/fcntl.h>".}: cint +# var F_SETFD* {.importc: "F_SETFD", header: "<sys/fcntl.h>".}: cint +var F_GETFL* {.importc: "F_GETFL", header: "<sys/fcntl.h>".}: cint +var F_SETFL* {.importc: "F_SETFL", header: "<sys/fcntl.h>".}: cint +# var F_GETLK* {.importc: "F_GETLK", header: "<sys/fcntl.h>".}: cint +# var F_SETLK* {.importc: "F_SETLK", header: "<sys/fcntl.h>".}: cint +# var F_SETLKW* {.importc: "F_SETLKW", header: "<sys/fcntl.h>".}: cint +# var F_GETOWN* {.importc: "F_GETOWN", header: "<sys/fcntl.h>".}: cint +# var F_SETOWN* {.importc: "F_SETOWN", header: "<sys/fcntl.h>".}: cint +# var FD_CLOEXEC* {.importc: "FD_CLOEXEC", header: "<sys/fcntl.h>".}: cint +# var F_RDLCK* {.importc: "F_RDLCK", header: "<sys/fcntl.h>".}: cint +# var F_UNLCK* {.importc: "F_UNLCK", header: "<sys/fcntl.h>".}: cint +# var F_WRLCK* {.importc: "F_WRLCK", header: "<sys/fcntl.h>".}: cint +var O_CREAT* {.importc: "O_CREAT", header: "<sys/fcntl.h>".}: cint +var O_EXCL* {.importc: "O_EXCL", header: "<sys/fcntl.h>".}: cint +# var O_NOCTTY* {.importc: "O_NOCTTY", header: "<sys/fcntl.h>".}: cint +var O_TRUNC* {.importc: "O_TRUNC", header: "<sys/fcntl.h>".}: cint +var O_APPEND* {.importc: "O_APPEND", header: "<sys/fcntl.h>".}: cint +# var O_DSYNC* {.importc: "O_DSYNC", header: "<sys/fcntl.h>".}: cint +var O_NONBLOCK* {.importc: "O_NONBLOCK", header: "<sys/fcntl.h>".}: cint +# var O_RSYNC* {.importc: "O_RSYNC", header: "<sys/fcntl.h>".}: cint +# var O_SYNC* {.importc: "O_SYNC", header: "<sys/fcntl.h>".}: cint +# var O_ACCMODE* {.importc: "O_ACCMODE", header: "<sys/fcntl.h>".}: cint +var O_RDONLY* {.importc: "O_RDONLY", header: "<sys/fcntl.h>".}: cint +var O_RDWR* {.importc: "O_RDWR", header: "<sys/fcntl.h>".}: cint +var O_WRONLY* {.importc: "O_WRONLY", header: "<sys/fcntl.h>".}: cint +# var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "<sys/fcntl.h>".}: cint + +# # <locale.h> +# var LC_ALL* {.importc: "LC_ALL", header: "<locale.h>".}: cint +# var LC_COLLATE* {.importc: "LC_COLLATE", header: "<locale.h>".}: cint +# var LC_CTYPE* {.importc: "LC_CTYPE", header: "<locale.h>".}: cint +# var LC_MESSAGES* {.importc: "LC_MESSAGES", header: "<locale.h>".}: cint +# var LC_MONETARY* {.importc: "LC_MONETARY", header: "<locale.h>".}: cint +# var LC_NUMERIC* {.importc: "LC_NUMERIC", header: "<locale.h>".}: cint +# var LC_TIME* {.importc: "LC_TIME", header: "<locale.h>".}: cint + +# <netdb.h> +var HOST_NOT_FOUND* {.importc: "HOST_NOT_FOUND", header: "<netdb.h>".}: cint +var NO_DATA* {.importc: "NO_DATA", header: "<netdb.h>".}: cint +var NO_RECOVERY* {.importc: "NO_RECOVERY", header: "<netdb.h>".}: cint +var TRY_AGAIN* {.importc: "TRY_AGAIN", header: "<netdb.h>".}: cint +var AI_PASSIVE* {.importc: "AI_PASSIVE", header: "<netdb.h>".}: cint +var AI_CANONNAME* {.importc: "AI_CANONNAME", header: "<netdb.h>".}: cint +var AI_NUMERICHOST* {.importc: "AI_NUMERICHOST", header: "<netdb.h>".}: cint +var AI_NUMERICSERV* {.importc: "AI_NUMERICSERV", header: "<netdb.h>".}: cint +var AI_V4MAPPED* {.importc: "AI_V4MAPPED", header: "<netdb.h>".}: cint +var AI_ALL* {.importc: "AI_ALL", header: "<netdb.h>".}: cint +var AI_ADDRCONFIG* {.importc: "AI_ADDRCONFIG", header: "<netdb.h>".}: cint +var NI_NOFQDN* {.importc: "NI_NOFQDN", header: "<netdb.h>".}: cint +var NI_NUMERICHOST* {.importc: "NI_NUMERICHOST", header: "<netdb.h>".}: cint +var NI_NAMEREQD* {.importc: "NI_NAMEREQD", header: "<netdb.h>".}: cint +var NI_NUMERICSERV* {.importc: "NI_NUMERICSERV", header: "<netdb.h>".}: cint +var NI_DGRAM* {.importc: "NI_DGRAM", header: "<netdb.h>".}: cint +var EAI_AGAIN* {.importc: "EAI_AGAIN", header: "<netdb.h>".}: cint +var EAI_BADFLAGS* {.importc: "EAI_BADFLAGS", header: "<netdb.h>".}: cint +var EAI_FAIL* {.importc: "EAI_FAIL", header: "<netdb.h>".}: cint +var EAI_FAMILY* {.importc: "EAI_FAMILY", header: "<netdb.h>".}: cint +var EAI_MEMORY* {.importc: "EAI_MEMORY", header: "<netdb.h>".}: cint +var EAI_NONAME* {.importc: "EAI_NONAME", header: "<netdb.h>".}: cint +var EAI_SERVICE* {.importc: "EAI_SERVICE", header: "<netdb.h>".}: cint +var EAI_SOCKTYPE* {.importc: "EAI_SOCKTYPE", header: "<netdb.h>".}: cint + +var LWIP_DNS_API_DECLARE_H_ERRNO* {.importc: "LWIP_DNS_API_DECLARE_H_ERRNO", header: "<netdb.h>".}: cint +var LWIP_DNS_API_DEFINE_ERRORS* {.importc: "LWIP_DNS_API_DEFINE_ERRORS", header: "<netdb.h>".}: cint +var LWIP_DNS_API_DEFINE_FLAGS* {.importc: "LWIP_DNS_API_DEFINE_FLAGS", header: "<netdb.h>".}: cint +var LWIP_DNS_API_DECLARE_STRUCTS* {.importc: "LWIP_DNS_API_DECLARE_STRUCTS", header: "<netdb.h>".}: cint +var NETDB_ELEM_SIZE* {.importc: "NETDB_ELEM_SIZE", header: "<netdb.h>".}: cint + + +# <netif.h> +var ENABLE_LOOPBACK* {.importc: "ENABLE_LOOPBACK", header: "<netif.h>".}: cint +var NETIF_MAX_HWADDR_LEN* {.importc: "NETIF_MAX_HWADDR_LEN", header: "<netif.h>".}: cint +var NETIF_NAMESIZE* {.importc: "NETIF_NAMESIZE", header: "<netif.h>".}: cint +var NETIF_FLAG_UP* {.importc: "NETIF_FLAG_UP", header: "<netif.h>".}: cint +var NETIF_FLAG_BROADCAST* {.importc: "NETIF_FLAG_BROADCAST", header: "<netif.h>".}: cint +var NETIF_FLAG_LINK_UP* {.importc: "NETIF_FLAG_LINK_UP", header: "<netif.h>".}: cint +var NETIF_FLAG_ETHARP* {.importc: "NETIF_FLAG_ETHARP", header: "<netif.h>".}: cint +var NETIF_FLAG_ETHERNET* {.importc: "NETIF_FLAG_ETHERNET", header: "<netif.h>".}: cint +var NETIF_FLAG_IGMP* {.importc: "NETIF_FLAG_IGMP", header: "<netif.h>".}: cint +var NETIF_FLAG_MLD6* {.importc: "NETIF_FLAG_MLD6", header: "<netif.h>".}: cint +var NETIF_FLAG_GARP* {.importc: "NETIF_FLAG_GARP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_IP* {.importc: "NETIF_CHECKSUM_GEN_IP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_UDP* {.importc: "NETIF_CHECKSUM_GEN_UDP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_TCP* {.importc: "NETIF_CHECKSUM_GEN_TCP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_ICMP* {.importc: "NETIF_CHECKSUM_GEN_ICMP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_GEN_ICMP6* {.importc: "NETIF_CHECKSUM_GEN_ICMP6", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_IP* {.importc: "NETIF_CHECKSUM_CHECK_IP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_UDP* {.importc: "NETIF_CHECKSUM_CHECK_UDP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_TCP* {.importc: "NETIF_CHECKSUM_CHECK_TCP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_ICMP* {.importc: "NETIF_CHECKSUM_CHECK_ICMP", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_CHECK_ICMP6* {.importc: "NETIF_CHECKSUM_CHECK_ICMP6", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_ENABLE_ALL* {.importc: "NETIF_CHECKSUM_ENABLE_ALL", header: "<netif.h>".}: cint +var NETIF_CHECKSUM_DISABLE_ALL* {.importc: "NETIF_CHECKSUM_DISABLE_ALL", header: "<netif.h>".}: cint +var NETIF_ADDR_IDX_MAX* {.importc: "NETIF_ADDR_IDX_MAX", header: "<netif.h>".}: cint +var LWIP_NETIF_USE_HINTS* {.importc: "LWIP_NETIF_USE_HINTS", header: "<netif.h>".}: cint +var NETIF_NO_INDEX* {.importc: "NETIF_NO_INDEX", header: "<netif.h>".}: cint +var LWIP_NSC_NONE* {.importc: "LWIP_NSC_NONE", header: "<netif.h>".}: cint +var LWIP_NSC_NETIF_ADDED* {.importc: "LWIP_NSC_NETIF_ADDED", header: "<netif.h>".}: cint +var LWIP_NSC_NETIF_REMOVED* {.importc: "LWIP_NSC_NETIF_REMOVED", header: "<netif.h>".}: cint +var LWIP_NSC_LINK_CHANGED* {.importc: "LWIP_NSC_LINK_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_STATUS_CHANGED* {.importc: "LWIP_NSC_STATUS_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_ADDRESS_CHANGED* {.importc: "LWIP_NSC_IPV4_ADDRESS_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_GATEWAY_CHANGED* {.importc: "LWIP_NSC_IPV4_GATEWAY_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_NETMASK_CHANGED* {.importc: "LWIP_NSC_IPV4_NETMASK_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV4_SETTINGS_CHANGED* {.importc: "LWIP_NSC_IPV4_SETTINGS_CHANGED", header: "<netif.h>".}: cint +var LWIP_NSC_IPV6_SET* {.importc: "LWIP_NSC_IPV6_SET", header: "<netif.h>".}: cint +var LWIP_NSC_IPV6_ADDR_STATE_CHANGED* {.importc: "LWIP_NSC_IPV6_ADDR_STATE_CHANGED", header: "<netif.h>".}: cint +#define NETIF_DECLARE_EXT_CALLBACK(name) + + +# <net/if.h> +var IF_NAMESIZE* {.importc: "IF_NAMESIZE", header: "<net/if.h>".}: cint + +# <sys/socket.h> +var IPPROTO_IP* {.importc: "IPPROTO_IP", header: "<sys/socket.h>".}: cint +var IPPROTO_IPV6* {.importc: "IPPROTO_IPV6", header: "<sys/socket.h>".}: cint +var IPPROTO_ICMP* {.importc: "IPPROTO_ICMP", header: "<sys/socket.h>".}: cint +var IPPROTO_ICMPV6* {.importc: "IPPROTO_ICMPV6", header: "<sys/socket.h>".}: cint +var IPPROTO_RAW* {.importc: "IPPROTO_RAW", header: "<sys/socket.h>".}: cint +var IPPROTO_TCP* {.importc: "IPPROTO_TCP", header: "<sys/socket.h>".}: cint +var IPPROTO_UDP* {.importc: "IPPROTO_UDP", header: "<sys/socket.h>".}: cint +var INADDR_ANY* {.importc: "INADDR_ANY", header: "<sys/socket.h>".}: InAddrScalar +var INADDR_LOOPBACK* {.importc: "INADDR_LOOPBACK", header: "<sys/socket.h>".}: InAddrScalar +var INADDR_BROADCAST* {.importc: "INADDR_BROADCAST", header: "<sys/socket.h>".}: InAddrScalar +var INET_ADDRSTRLEN* {.importc: "INET_ADDRSTRLEN", header: "<sys/socket.h>".}: cint +var INET6_ADDRSTRLEN* {.importc: "INET6_ADDRSTRLEN", header: "<sys/socket.h>".}: cint +var IPV6_JOIN_GROUP* {.importc: "IPV6_JOIN_GROUP", header: "<sys/socket.h>".}: cint +var IPV6_LEAVE_GROUP* {.importc: "IPV6_LEAVE_GROUP", header: "<sys/socket.h>".}: cint +var IPV6_MULTICAST_HOPS* {.importc: "IPV6_MULTICAST_HOPS", header: "<sys/socket.h>".}: cint +var IPV6_MULTICAST_IF* {.importc: "IPV6_MULTICAST_IF", header: "<sys/socket.h>".}: cint +var IPV6_MULTICAST_LOOP* {.importc: "IPV6_MULTICAST_LOOP", header: "<sys/socket.h>".}: cint +var IPV6_UNICAST_HOPS* {.importc: "IPV6_UNICAST_HOPS", header: "<sys/socket.h>".}: cint +var IPV6_V6ONLY* {.importc: "IPV6_V6ONLY", header: "<sys/socket.h>".}: cint + +# <netinet/tcp.h> +const TCP_NODELAY* = 0x01 # don't delay send to coalesce packets +const TCP_KEEPALIVE* = 0x02 # send KEEPALIVE probes when idle for pcb->keep_idle milliseconds +const TCP_KEEPIDLE* = 0x03 # set pcb->keep_idle - Same as TCP_KEEPALIVE, but use seconds for get/setsockopt +const TCP_KEEPINTVL* = 0x04 # set pcb->keep_intvl - Use seconds for get/setsockopt +const TCP_KEEPCNT* = 0x05 # set pcb->keep_cnt - Use number of probes sent for get/setsockopt + +# <nl_types.h> +# var NL_SETD* {.importc: "NL_SETD", header: "<nl_types.h>".}: cint +# var NL_CAT_LOCALE* {.importc: "NL_CAT_LOCALE", header: "<nl_types.h>".}: cint + +# <sys/poll.h> +# var POLLIN* {.importc: "POLLIN", header: "<sys/poll.h>".}: cshort +# var POLLRDNORM* {.importc: "POLLRDNORM", header: "<sys/poll.h>".}: cshort +# var POLLRDBAND* {.importc: "POLLRDBAND", header: "<sys/poll.h>".}: cshort +# var POLLPRI* {.importc: "POLLPRI", header: "<sys/poll.h>".}: cshort +# var POLLOUT* {.importc: "POLLOUT", header: "<sys/poll.h>".}: cshort +# var POLLWRNORM* {.importc: "POLLWRNORM", header: "<sys/poll.h>".}: cshort +# var POLLWRBAND* {.importc: "POLLWRBAND", header: "<sys/poll.h>".}: cshort +# var POLLERR* {.importc: "POLLERR", header: "<sys/poll.h>".}: cshort +# var POLLHUP* {.importc: "POLLHUP", header: "<sys/poll.h>".}: cshort +# var POLLNVAL* {.importc: "POLLNVAL", header: "<sys/poll.h>".}: cshort + +# <pthread.h> +var PTHREAD_STACK_MIN* {.importc: "PTHREAD_STACK_MIN", header: "<pthread.h>".}: cint +# var PTHREAD_BARRIER_SERIAL_THREAD* {.importc: "PTHREAD_BARRIER_SERIAL_THREAD", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_ASYNCHRONOUS* {.importc: "PTHREAD_CANCEL_ASYNCHRONOUS", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_ENABLE* {.importc: "PTHREAD_CANCEL_ENABLE", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_DEFERRED* {.importc: "PTHREAD_CANCEL_DEFERRED", header: "<pthread.h>".}: cint +# var PTHREAD_CANCEL_DISABLE* {.importc: "PTHREAD_CANCEL_DISABLE", header: "<pthread.h>".}: cint +# var PTHREAD_CREATE_DETACHED* {.importc: "PTHREAD_CREATE_DETACHED", header: "<pthread.h>".}: cint +# var PTHREAD_CREATE_JOINABLE* {.importc: "PTHREAD_CREATE_JOINABLE", header: "<pthread.h>".}: cint +# var PTHREAD_EXPLICIT_SCHED* {.importc: "PTHREAD_EXPLICIT_SCHED", header: "<pthread.h>".}: cint +# var PTHREAD_INHERIT_SCHED* {.importc: "PTHREAD_INHERIT_SCHED", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_DEFAULT* {.importc: "PTHREAD_MUTEX_DEFAULT", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_ERRORCHECK* {.importc: "PTHREAD_MUTEX_ERRORCHECK", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_NORMAL* {.importc: "PTHREAD_MUTEX_NORMAL", header: "<pthread.h>".}: cint +# var PTHREAD_MUTEX_RECURSIVE* {.importc: "PTHREAD_MUTEX_RECURSIVE", header: "<pthread.h>".}: cint +# var PTHREAD_PRIO_INHERIT* {.importc: "PTHREAD_PRIO_INHERIT", header: "<pthread.h>".}: cint +# var PTHREAD_PRIO_NONE* {.importc: "PTHREAD_PRIO_NONE", header: "<pthread.h>".}: cint +# var PTHREAD_PRIO_PROTECT* {.importc: "PTHREAD_PRIO_PROTECT", header: "<pthread.h>".}: cint +# var PTHREAD_PROCESS_SHARED* {.importc: "PTHREAD_PROCESS_SHARED", header: "<pthread.h>".}: cint +# var PTHREAD_PROCESS_PRIVATE* {.importc: "PTHREAD_PROCESS_PRIVATE", header: "<pthread.h>".}: cint +# var PTHREAD_SCOPE_PROCESS* {.importc: "PTHREAD_SCOPE_PROCESS", header: "<pthread.h>".}: cint +# var PTHREAD_SCOPE_SYSTEM* {.importc: "PTHREAD_SCOPE_SYSTEM", header: "<pthread.h>".}: cint + +# # <sched.h> +# var SCHED_FIFO* {.importc: "SCHED_FIFO", header: "<sched.h>".}: cint +# var SCHED_RR* {.importc: "SCHED_RR", header: "<sched.h>".}: cint +# var SCHED_SPORADIC* {.importc: "SCHED_SPORADIC", header: "<sched.h>".}: cint +# var SCHED_OTHER* {.importc: "SCHED_OTHER", header: "<sched.h>".}: cint + +# <semaphore.h> +var SEM_FAILED* {.importc: "SEM_FAILED", header: "<semaphore.h>".}: pointer + +# # <signal.h> +# var SIGEV_NONE* {.importc: "SIGEV_NONE", header: "<signal.h>".}: cint +# var SIGEV_SIGNAL* {.importc: "SIGEV_SIGNAL", header: "<signal.h>".}: cint +# var SIGEV_THREAD* {.importc: "SIGEV_THREAD", header: "<signal.h>".}: cint +# var SIGABRT* {.importc: "SIGABRT", header: "<signal.h>".}: cint +# var SIGALRM* {.importc: "SIGALRM", header: "<signal.h>".}: cint +# var SIGBUS* {.importc: "SIGBUS", header: "<signal.h>".}: cint +# var SIGCHLD* {.importc: "SIGCHLD", header: "<signal.h>".}: cint +# var SIGCONT* {.importc: "SIGCONT", header: "<signal.h>".}: cint +# var SIGFPE* {.importc: "SIGFPE", header: "<signal.h>".}: cint +# var SIGHUP* {.importc: "SIGHUP", header: "<signal.h>".}: cint +# var SIGILL* {.importc: "SIGILL", header: "<signal.h>".}: cint +# var SIGINT* {.importc: "SIGINT", header: "<signal.h>".}: cint +# var SIGKILL* {.importc: "SIGKILL", header: "<signal.h>".}: cint +# var SIGPIPE* {.importc: "SIGPIPE", header: "<signal.h>".}: cint +# var SIGQUIT* {.importc: "SIGQUIT", header: "<signal.h>".}: cint +# var SIGSEGV* {.importc: "SIGSEGV", header: "<signal.h>".}: cint +# var SIGSTOP* {.importc: "SIGSTOP", header: "<signal.h>".}: cint +# var SIGTERM* {.importc: "SIGTERM", header: "<signal.h>".}: cint +# var SIGTSTP* {.importc: "SIGTSTP", header: "<signal.h>".}: cint +# var SIGTTIN* {.importc: "SIGTTIN", header: "<signal.h>".}: cint +# var SIGTTOU* {.importc: "SIGTTOU", header: "<signal.h>".}: cint +# var SIGUSR1* {.importc: "SIGUSR1", header: "<signal.h>".}: cint +# var SIGUSR2* {.importc: "SIGUSR2", header: "<signal.h>".}: cint +# var SIGPOLL* {.importc: "SIGPOLL", header: "<signal.h>".}: cint +# var SIGPROF* {.importc: "SIGPROF", header: "<signal.h>".}: cint +# var SIGSYS* {.importc: "SIGSYS", header: "<signal.h>".}: cint +# var SIGTRAP* {.importc: "SIGTRAP", header: "<signal.h>".}: cint +# var SIGURG* {.importc: "SIGURG", header: "<signal.h>".}: cint +# var SIGVTALRM* {.importc: "SIGVTALRM", header: "<signal.h>".}: cint +# var SIGXCPU* {.importc: "SIGXCPU", header: "<signal.h>".}: cint +# var SIGXFSZ* {.importc: "SIGXFSZ", header: "<signal.h>".}: cint +# var SA_NOCLDSTOP* {.importc: "SA_NOCLDSTOP", header: "<signal.h>".}: cint +# var SIG_BLOCK* {.importc: "SIG_BLOCK", header: "<signal.h>".}: cint +# var SIG_UNBLOCK* {.importc: "SIG_UNBLOCK", header: "<signal.h>".}: cint +# var SIG_SETMASK* {.importc: "SIG_SETMASK", header: "<signal.h>".}: cint +# var SA_ONSTACK* {.importc: "SA_ONSTACK", header: "<signal.h>".}: cint +# var SA_RESETHAND* {.importc: "SA_RESETHAND", header: "<signal.h>".}: cint +# var SA_RESTART* {.importc: "SA_RESTART", header: "<signal.h>".}: cint +# var SA_SIGINFO* {.importc: "SA_SIGINFO", header: "<signal.h>".}: cint +# var SA_NOCLDWAIT* {.importc: "SA_NOCLDWAIT", header: "<signal.h>".}: cint +# var SA_NODEFER* {.importc: "SA_NODEFER", header: "<signal.h>".}: cint +# var SS_ONSTACK* {.importc: "SS_ONSTACK", header: "<signal.h>".}: cint +# var SS_DISABLE* {.importc: "SS_DISABLE", header: "<signal.h>".}: cint +# var MINSIGSTKSZ* {.importc: "MINSIGSTKSZ", header: "<signal.h>".}: cint +# var SIGSTKSZ* {.importc: "SIGSTKSZ", header: "<signal.h>".}: cint +# var SIG_HOLD* {.importc: "SIG_HOLD", header: "<signal.h>".}: Sighandler +# var SIG_DFL* {.importc: "SIG_DFL", header: "<signal.h>".}: Sighandler +# var SIG_ERR* {.importc: "SIG_ERR", header: "<signal.h>".}: Sighandler +# var SIG_IGN* {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler + +# # <sys/ipc.h> +# var IPC_CREAT* {.importc: "IPC_CREAT", header: "<sys/ipc.h>".}: cint +# var IPC_EXCL* {.importc: "IPC_EXCL", header: "<sys/ipc.h>".}: cint +# var IPC_NOWAIT* {.importc: "IPC_NOWAIT", header: "<sys/ipc.h>".}: cint +# var IPC_PRIVATE* {.importc: "IPC_PRIVATE", header: "<sys/ipc.h>".}: cint +# var IPC_RMID* {.importc: "IPC_RMID", header: "<sys/ipc.h>".}: cint +# var IPC_SET* {.importc: "IPC_SET", header: "<sys/ipc.h>".}: cint +# var IPC_STAT* {.importc: "IPC_STAT", header: "<sys/ipc.h>".}: cint + +# # <sys/mman.h> +# var PROT_READ* {.importc: "PROT_READ", header: "<sys/mman.h>".}: cint +# var PROT_WRITE* {.importc: "PROT_WRITE", header: "<sys/mman.h>".}: cint +# var PROT_EXEC* {.importc: "PROT_EXEC", header: "<sys/mman.h>".}: cint +# var PROT_NONE* {.importc: "PROT_NONE", header: "<sys/mman.h>".}: cint +# var MAP_ANONYMOUS* {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint +# var MAP_FIXED_NOREPLACE* {.importc: "MAP_FIXED_NOREPLACE", header: "<sys/mman.h>".}: cint +# var MAP_NORESERVE* {.importc: "MAP_NORESERVE", header: "<sys/mman.h>".}: cint +# var MAP_SHARED* {.importc: "MAP_SHARED", header: "<sys/mman.h>".}: cint +# var MAP_PRIVATE* {.importc: "MAP_PRIVATE", header: "<sys/mman.h>".}: cint +# var MAP_FIXED* {.importc: "MAP_FIXED", header: "<sys/mman.h>".}: cint +# var MS_ASYNC* {.importc: "MS_ASYNC", header: "<sys/mman.h>".}: cint +# var MS_SYNC* {.importc: "MS_SYNC", header: "<sys/mman.h>".}: cint +# var MS_INVALIDATE* {.importc: "MS_INVALIDATE", header: "<sys/mman.h>".}: cint +# var MCL_CURRENT* {.importc: "MCL_CURRENT", header: "<sys/mman.h>".}: cint +# var MCL_FUTURE* {.importc: "MCL_FUTURE", header: "<sys/mman.h>".}: cint +# var MAP_FAILED* {.importc: "MAP_FAILED", header: "<sys/mman.h>".}: pointer +# var POSIX_MADV_NORMAL* {.importc: "POSIX_MADV_NORMAL", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_SEQUENTIAL* {.importc: "POSIX_MADV_SEQUENTIAL", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_RANDOM* {.importc: "POSIX_MADV_RANDOM", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_WILLNEED* {.importc: "POSIX_MADV_WILLNEED", header: "<sys/mman.h>".}: cint +# var POSIX_MADV_DONTNEED* {.importc: "POSIX_MADV_DONTNEED", header: "<sys/mman.h>".}: cint +# var POSIX_TYPED_MEM_ALLOCATE* {.importc: "POSIX_TYPED_MEM_ALLOCATE", header: "<sys/mman.h>".}: cint +# var POSIX_TYPED_MEM_ALLOCATE_CONTIG* {.importc: "POSIX_TYPED_MEM_ALLOCATE_CONTIG", header: "<sys/mman.h>".}: cint +# var POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc: "POSIX_TYPED_MEM_MAP_ALLOCATABLE", header: "<sys/mman.h>".}: cint + +# <sys/resource.h> +# var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint + +var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "<lwipopts.h>".}: cint + +# <sys/select.h> +var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint + +# <sys/socket.h> +# struct msghdr->msg_flags bit field values +const MSG_TRUNC* = 0x04 +const MSG_CTRUNC* = 0x08 + +# Flags we can use with send and recv. +const MSG_PEEK* = 0x01 # Peeks at an incoming message +const MSG_WAITALL* = 0x02 # Unimplemented: Requests that the function block until the full amount of data requested can be returned +const MSG_OOB* = 0x04 # Unimplemented: Requests out-of-band data. The significance and semantics of out-of-band data are protocol-specific +const MSG_DONTWAIT* = 0x08 # Nonblocking i/o for this operation only +const MSG_MORE* = 0x10 # Sender will send more +# const MSG_NOSIGNAL* = 0x20 # Uninmplemented: Requests not to send the SIGPIPE signal if an attempt to send is made on a stream-oriented socket that is no longer connected. + +# Alternately, they can defined like this, but the above seems better when they're known/stable values: +# var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "<sys/socket.h>".}: cint +# var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "<sys/socket.h>".}: cint +# var MSG_DONTROUTE* {.importc: "MSG_DONTROUTE", header: "<sys/socket.h>".}: cint # not defined in lwip +# var MSG_EOR* {.importc: "MSG_EOR", header: "<sys/socket.h>".}: cint # not defined in lwip +# var MSG_OOB* {.importc: "MSG_OOB", header: "<sys/socket.h>".}: cint + + +# var SCM_RIGHTS* {.importc: "SCM_RIGHTS", header: "<sys/socket.h>".}: cint +var SO_ACCEPTCONN* {.importc: "SO_ACCEPTCONN", header: "<sys/socket.h>".}: cint +var SO_BROADCAST* {.importc: "SO_BROADCAST", header: "<sys/socket.h>".}: cint +var SO_DEBUG* {.importc: "SO_DEBUG", header: "<sys/socket.h>".}: cint +var SO_DONTROUTE* {.importc: "SO_DONTROUTE", header: "<sys/socket.h>".}: cint +var SO_ERROR* {.importc: "SO_ERROR", header: "<sys/socket.h>".}: cint +var SO_KEEPALIVE* {.importc: "SO_KEEPALIVE", header: "<sys/socket.h>".}: cint +var SO_LINGER* {.importc: "SO_LINGER", header: "<sys/socket.h>".}: cint +var SO_OOBINLINE* {.importc: "SO_OOBINLINE", header: "<sys/socket.h>".}: cint +var SO_RCVBUF* {.importc: "SO_RCVBUF", header: "<sys/socket.h>".}: cint +var SO_RCVLOWAT* {.importc: "SO_RCVLOWAT", header: "<sys/socket.h>".}: cint +var SO_RCVTIMEO* {.importc: "SO_RCVTIMEO", header: "<sys/socket.h>".}: cint +var SO_REUSEADDR* {.importc: "SO_REUSEADDR", header: "<sys/socket.h>".}: cint +var SO_SNDBUF* {.importc: "SO_SNDBUF", header: "<sys/socket.h>".}: cint +var SO_SNDLOWAT* {.importc: "SO_SNDLOWAT", header: "<sys/socket.h>".}: cint +var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "<sys/socket.h>".}: cint +var SO_TYPE* {.importc: "SO_TYPE", header: "<sys/socket.h>".}: cint +var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "<sys/socket.h>".}: cint +var SOCK_RAW* {.importc: "SOCK_RAW", header: "<sys/socket.h>".}: cint + +# var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint +const SOCK_SEQPACKET* = cint(5) + +var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "<sys/socket.h>".}: cint +var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "<sys/socket.h>".}: cint + +const SocketMaxConnections {.intdefine.}: int = 32 +var SOMAXCONN*: cint = SocketMaxConnections.cint + +var AF_INET* {.importc: "AF_INET", header: "<sys/socket.h>".}: cint +var AF_INET6* {.importc: "AF_INET6", header: "<sys/socket.h>".}: cint +# var AF_UNIX* {.importc: "AF_UNIX", header: "<sys/socket.h>".}: cint +const AF_UNIX*: cint = 1 # for compat with Nim libraries, doesn't exist on freertos +var AF_UNSPEC* {.importc: "AF_UNSPEC", header: "<sys/socket.h>".}: cint +var SHUT_RD* {.importc: "SHUT_RD", header: "<sys/socket.h>".}: cint +var SHUT_RDWR* {.importc: "SHUT_RDWR", header: "<sys/socket.h>".}: cint +var SHUT_WR* {.importc: "SHUT_WR", header: "<sys/socket.h>".}: cint + +# # <sys/stat.h> +# <sys/stat.h> + +# var S_IFBLK* {.importc: "S_IFBLK", header: "<sys/stat.h>".}: cint +# var S_IFCHR* {.importc: "S_IFCHR", header: "<sys/stat.h>".}: cint +# var S_IFDIR* {.importc: "S_IFDIR", header: "<sys/stat.h>".}: cint +# var S_IFIFO* {.importc: "S_IFIFO", header: "<sys/stat.h>".}: cint +# var S_IFLNK* {.importc: "S_IFLNK", header: "<sys/stat.h>".}: cint +# var S_IFMT* {.importc: "S_IFMT", header: "<sys/stat.h>".}: cint +# var S_IFREG* {.importc: "S_IFREG", header: "<sys/stat.h>".}: cint +# var S_IFSOCK* {.importc: "S_IFSOCK", header: "<sys/stat.h>".}: cint +var S_IRGRP* {.importc: "S_IRGRP", header: "<sys/stat.h>".}: cint +var S_IROTH* {.importc: "S_IROTH", header: "<sys/stat.h>".}: cint +var S_IRUSR* {.importc: "S_IRUSR", header: "<sys/stat.h>".}: cint +# var S_IRWXG* {.importc: "S_IRWXG", header: "<sys/stat.h>".}: cint +# var S_IRWXO* {.importc: "S_IRWXO", header: "<sys/stat.h>".}: cint +# var S_IRWXU* {.importc: "S_IRWXU", header: "<sys/stat.h>".}: cint +# var S_ISGID* {.importc: "S_ISGID", header: "<sys/stat.h>".}: cint +# var S_ISUID* {.importc: "S_ISUID", header: "<sys/stat.h>".}: cint +# var S_ISVTX* {.importc: "S_ISVTX", header: "<sys/stat.h>".}: cint +var S_IWGRP* {.importc: "S_IWGRP", header: "<sys/stat.h>".}: cint +var S_IWOTH* {.importc: "S_IWOTH", header: "<sys/stat.h>".}: cint +var S_IWUSR* {.importc: "S_IWUSR", header: "<sys/stat.h>".}: cint +var S_IXGRP* {.importc: "S_IXGRP", header: "<sys/stat.h>".}: cint +var S_IXOTH* {.importc: "S_IXOTH", header: "<sys/stat.h>".}: cint +var S_IXUSR* {.importc: "S_IXUSR", header: "<sys/stat.h>".}: cint + +# # <sys/statvfs.h> +# var ST_RDONLY* {.importc: "ST_RDONLY", header: "<sys/statvfs.h>".}: cint +# var ST_NOSUID* {.importc: "ST_NOSUID", header: "<sys/statvfs.h>".}: cint + +# # <sys/wait.h> +# var WNOHANG* {.importc: "WNOHANG", header: "<sys/wait.h>".}: cint +# var WUNTRACED* {.importc: "WUNTRACED", header: "<sys/wait.h>".}: cint +# var WEXITED* {.importc: "WEXITED", header: "<sys/wait.h>".}: cint +# var WSTOPPED* {.importc: "WSTOPPED", header: "<sys/wait.h>".}: cint +# var WCONTINUED* {.importc: "WCONTINUED", header: "<sys/wait.h>".}: cint +# var WNOWAIT* {.importc: "WNOWAIT", header: "<sys/wait.h>".}: cint +# var P_ALL* {.importc: "P_ALL", header: "<sys/wait.h>".}: cint +# var P_PID* {.importc: "P_PID", header: "<sys/wait.h>".}: cint +# var P_PGID* {.importc: "P_PGID", header: "<sys/wait.h>".}: cint + +# # <spawn.h> +# var POSIX_SPAWN_RESETIDS* {.importc: "POSIX_SPAWN_RESETIDS", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETPGROUP* {.importc: "POSIX_SPAWN_SETPGROUP", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSCHEDPARAM* {.importc: "POSIX_SPAWN_SETSCHEDPARAM", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSCHEDULER* {.importc: "POSIX_SPAWN_SETSCHEDULER", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSIGDEF* {.importc: "POSIX_SPAWN_SETSIGDEF", header: "<spawn.h>".}: cint +# var POSIX_SPAWN_SETSIGMASK* {.importc: "POSIX_SPAWN_SETSIGMASK", header: "<spawn.h>".}: cint + +## <stdio.h> +# var IOFBF* {.importc: "_IOFBF", header: "<stdio.h>".}: cint +# var IONBF* {.importc: "_IONBF", header: "<stdio.h>".}: cint + +# <time.h> +var CLOCKS_PER_SEC* {.importc: "CLOCKS_PER_SEC", header: "<time.h>".}: clong +var CLOCK_PROCESS_CPUTIME_ID* {.importc: "CLOCK_PROCESS_CPUTIME_ID", header: "<time.h>".}: cint +var CLOCK_THREAD_CPUTIME_ID* {.importc: "CLOCK_THREAD_CPUTIME_ID", header: "<time.h>".}: cint +var CLOCK_REALTIME* {.importc: "CLOCK_REALTIME", header: "<time.h>".}: cint +var TIMER_ABSTIME* {.importc: "TIMER_ABSTIME", header: "<time.h>".}: cint +var CLOCK_MONOTONIC* {.importc: "CLOCK_MONOTONIC", header: "<time.h>".}: cint + +# <unistd.h> + +const F_OK* = cint(0) +const R_OK* = cint(4) +const W_OK* = cint(2) +const X_OK* = cint(1) +const F_LOCK* = cint(1) +const F_TEST* = cint(3) +const F_TLOCK* = cint(2) +const F_ULOCK* = cint(0) + +# <stdio.h> +const SEEK_SET* = cint(0) +const SEEK_CUR* = cint(1) +const SEEK_END* = cint(2) diff --git a/lib/posix/posix_haiku.nim b/lib/posix/posix_haiku.nim new file mode 100644 index 000000000..32a6d24e2 --- /dev/null +++ b/lib/posix/posix_haiku.nim @@ -0,0 +1,603 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +const + hasSpawnH = true # should exist for every Posix system nowadays + hasAioH = defined(linux) + +when defined(linux) and not defined(android): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passl: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passl: "-lresolv".} + +type + DIR* {.importc: "DIR", header: "<dirent.h>", + incompleteStruct.} = object + ## A type representing a directory stream. + +type + SocketHandle* = distinct cint # The type used to represent socket descriptors + +type + Time* {.importc: "time_t", header: "<time.h>".} = distinct ( + when defined(i386): + int32 + else: + int64 + ) + Timespec* {.importc: "struct timespec", + header: "<time.h>", final, pure.} = object ## struct timespec + tv_sec*: Time ## Seconds. + tv_nsec*: int ## Nanoseconds. + + Dirent* {.importc: "struct dirent", + header: "<dirent.h>", final, pure.} = object ## dirent_t struct + when defined(haiku): + d_dev*: Dev ## Device (not POSIX) + d_pdev*: Dev ## Parent device (only for queries) (not POSIX) + d_ino*: Ino ## File serial number. + when defined(dragonfly): + # DragonflyBSD doesn't have `d_reclen` field. + d_type*: uint8 + elif defined(linux) or defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or defined(genode): + d_reclen*: cshort ## Length of this record. (not POSIX) + d_type*: int8 ## Type of file; not supported by all filesystem types. + ## (not POSIX) + when defined(linux) or defined(openbsd): + d_off*: Off ## Not an offset. Value that `telldir()` would return. + elif defined(haiku): + d_pino*: Ino ## Parent inode (only for queries) (not POSIX) + d_reclen*: cushort ## Length of this record. (not POSIX) + + when not defined(haiku): + d_name*: array[0..255, char] ## Name of entry. + else: + d_name*: UncheckedArray[char] ## Name of entry (NUL terminated). + + Tflock* {.importc: "struct flock", final, pure, + header: "<fcntl.h>".} = object ## flock type + l_type*: cshort ## Type of lock; F_RDLCK, F_WRLCK, F_UNLCK. + l_whence*: cshort ## Flag for starting offset. + l_start*: Off ## Relative offset in bytes. + l_len*: Off ## Size; if 0 then until EOF. + l_pid*: Pid ## Process ID of the process holding the lock; + ## returned with F_GETLK. + + FTW* {.importc: "struct FTW", header: "<ftw.h>", final, pure.} = object + base*: cint + level*: cint + + Glob* {.importc: "glob_t", header: "<glob.h>", + final, pure.} = object ## glob_t + gl_pathc*: int ## Count of paths matched by pattern. + gl_pathv*: cstringArray ## Pointer to a list of matched pathnames. + gl_offs*: int ## Slots to reserve at the beginning of gl_pathv. + + Group* {.importc: "struct group", header: "<grp.h>", + final, pure.} = object ## struct group + gr_name*: cstring ## The name of the group. + gr_gid*: Gid ## Numerical group ID. + gr_mem*: cstringArray ## Pointer to a null-terminated array of character + ## pointers to member names. + + Iconv* {.importc: "iconv_t", header: "<iconv.h>", final, pure.} = + object ## Identifies the conversion from one codeset to another. + + Lconv* {.importc: "struct lconv", header: "<locale.h>", final, + pure.} = object + currency_symbol*: cstring + decimal_point*: cstring + frac_digits*: char + grouping*: cstring + int_curr_symbol*: cstring + int_frac_digits*: char + int_n_cs_precedes*: char + int_n_sep_by_space*: char + int_n_sign_posn*: char + int_p_cs_precedes*: char + int_p_sep_by_space*: char + int_p_sign_posn*: char + mon_decimal_point*: cstring + mon_grouping*: cstring + mon_thousands_sep*: cstring + negative_sign*: cstring + n_cs_precedes*: char + n_sep_by_space*: char + n_sign_posn*: char + positive_sign*: cstring + p_cs_precedes*: char + p_sep_by_space*: char + p_sign_posn*: char + thousands_sep*: cstring + + Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + + Passwd* {.importc: "struct passwd", header: "<pwd.h>", + final, pure.} = object ## struct passwd + pw_name*: cstring ## User's login name. + pw_uid*: Uid ## Numerical user ID. + pw_gid*: Gid ## Numerical group ID. + pw_dir*: cstring ## Initial working directory. + pw_shell*: cstring ## Program to use as shell. + + Blkcnt* {.importc: "blkcnt_t", header: "<sys/types.h>".} = int64 + ## used for file block counts + Blksize* {.importc: "blksize_t", header: "<sys/types.h>".} = int32 + ## used for block sizes + Clock* {.importc: "clock_t", header: "<time.h>".} = int32 + ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int32 + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = int32 + Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int64 + Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int64 + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = uint32 + Id* {.importc: "id_t", header: "<sys/types.h>".} = int32 + Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int64 + Key* {.importc: "key_t", header: "<sys/types.h>".} = int32 + Mode* {.importc: "mode_t", header: "<sys/types.h>".} = ( + when defined(android) or defined(macos) or defined(macosx) or + (defined(bsd) and not defined(openbsd) and not defined(netbsd)): + uint16 + else: + uint32 + ) + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int32 + Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 + Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 + Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = object + Pthread_barrier* {.importc: "pthread_barrier_t", + header: "<sys/types.h>".} = object + Pthread_barrierattr* {.importc: "pthread_barrierattr_t", + header: "<sys/types.h>".} = object + Pthread_cond* {.importc: "pthread_cond_t", header: "<sys/types.h>".} = object + Pthread_condattr* {.importc: "pthread_condattr_t", + header: "<sys/types.h>".} = object + Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = object + Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>".} = object + Pthread_mutexattr* {.importc: "pthread_mutexattr_t", + header: "<sys/types.h>".} = object + Pthread_once* {.importc: "pthread_once_t", header: "<sys/types.h>".} = object + Pthread_rwlock* {.importc: "pthread_rwlock_t", + header: "<sys/types.h>".} = object + Pthread_rwlockattr* {.importc: "pthread_rwlockattr_t", + header: "<sys/types.h>".} = object + Pthread_spinlock* {.importc: "pthread_spinlock_t", + header: "<sys/types.h>".} = object + Pthread* {.importc: "pthread_t", header: "<sys/types.h>".} = object + Suseconds* {.importc: "suseconds_t", header: "<sys/types.h>".} = int32 + #Ttime* {.importc: "time_t", header: "<sys/types.h>".} = int + Timer* {.importc: "timer_t", header: "<sys/types.h>".} = object + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = uint32 + Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = uint32 + + Utsname* {.importc: "struct utsname", + header: "<sys/utsname.h>", + final, pure.} = object ## struct utsname + sysname*, ## Name of this implementation of the operating system. + nodename*, ## Name of this node within the communications + ## network to which this node is attached, if any. + release*, ## Current release level of this implementation. + version*, ## Current version level of this release. + machine*: array[32, char] ## Name of the hardware type on which the + ## system is running. + + Sem* {.importc: "sem_t", header: "<semaphore.h>", final, pure.} = object + Ipc_perm* {.importc: "struct ipc_perm", + header: "<sys/ipc.h>", final, pure.} = object ## struct ipc_perm + uid*: Uid ## Owner's user ID. + gid*: Gid ## Owner's group ID. + cuid*: Uid ## Creator's user ID. + cgid*: Gid ## Creator's group ID. + mode*: Mode ## Read/write permission. + + Stat* {.importc: "struct stat", + header: "<sys/stat.h>", final, pure.} = object ## struct stat + st_dev*: Dev ## Device ID of device containing file. + st_ino*: Ino ## File serial number. + st_mode*: Mode ## Mode of file (see below). + st_nlink*: Nlink ## Number of hard links to the file. + st_uid*: Uid ## User ID of file. + st_gid*: Gid ## Group ID of file. + st_rdev*: Dev ## Device ID (if file is character or block special). + st_size*: Off ## For regular files, the file size in bytes. + ## For symbolic links, the length in bytes of the + ## pathname contained in the symbolic link. + ## For a shared memory object, the length in bytes. + ## For a typed memory object, the length in bytes. + ## For other file types, the use of this field is + ## unspecified. + when StatHasNanoseconds: + st_atim*: Timespec ## Time of last access. + st_mtim*: Timespec ## Time of last data modification. + st_ctim*: Timespec ## Time of last status change. + else: + st_atime*: Time ## Time of last access. + st_mtime*: Time ## Time of last data modification. + st_ctime*: Time ## Time of last status change. + st_blksize*: Blksize ## A file system-specific preferred I/O block size + ## for this object. In some file system types, this + ## may vary from file to file. + st_blocks*: Blkcnt ## Number of blocks allocated for this object. + + + Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", + final, pure.} = object ## struct statvfs + f_bsize*: culong ## File system block size. + f_frsize*: culong ## Fundamental file system block size. + f_blocks*: Fsblkcnt ## Total number of blocks on file system + ## in units of f_frsize. + f_bfree*: Fsblkcnt ## Total number of free blocks. + f_bavail*: Fsblkcnt ## Number of free blocks available to + ## non-privileged process. + f_files*: Fsfilcnt ## Total number of file serial numbers. + f_ffree*: Fsfilcnt ## Total number of free file serial numbers. + f_favail*: Fsfilcnt ## Number of file serial numbers available to + ## non-privileged process. + f_fsid*: culong ## File system ID. + f_flag*: culong ## Bit mask of f_flag values. + f_namemax*: culong ## Maximum filename length. + + Tm* {.importc: "struct tm", header: "<time.h>", + final, pure.} = object ## struct tm + tm_sec*: cint ## Seconds [0,60]. + tm_min*: cint ## Minutes [0,59]. + tm_hour*: cint ## Hour [0,23]. + tm_mday*: cint ## Day of month [1,31]. + tm_mon*: cint ## Month of year [0,11]. + tm_year*: cint ## Years since 1900. + tm_wday*: cint ## Day of week [0,6] (Sunday =0). + tm_yday*: cint ## Day of year [0,365]. + tm_isdst*: cint ## Daylight Savings flag. + Itimerspec* {.importc: "struct itimerspec", header: "<time.h>", + final, pure.} = object ## struct itimerspec + it_interval*: Timespec ## Timer period. + it_value*: Timespec ## Timer expiration. + + Sig_atomic* {.importc: "sig_atomic_t", header: "<signal.h>".} = cint + ## Possibly volatile-qualified integer type of an object that can be + ## accessed as an atomic entity, even in the presence of asynchronous + ## interrupts. + Sigset* {.importc: "sigset_t", header: "<signal.h>".} = uint64 + + SigEvent* {.importc: "struct sigevent", + header: "<signal.h>", final, pure.} = object ## struct sigevent + sigev_notify*: cint ## Notification type. + sigev_signo*: cint ## Signal number. + sigev_value*: SigVal ## Signal value. + sigev_notify_function*: proc (x: SigVal) {.noconv.} ## Notification func. + sigev_notify_attributes*: ptr Pthread_attr ## Notification attributes. + + SigVal* {.importc: "union sigval", + header: "<signal.h>", final, pure.} = object ## struct sigval + sival_ptr*: pointer ## pointer signal value; + ## integer signal value not defined! + Sigaction* {.importc: "struct sigaction", + header: "<signal.h>", final, pure.} = object ## struct sigaction + sa_handler*: proc (x: cint) {.noconv.} ## Pointer to a signal-catching + ## function or one of the macros + ## SIG_IGN or SIG_DFL. + sa_mask*: Sigset ## Set of signals to be blocked during execution of + ## the signal handling function. + sa_flags*: cint ## Special flags. + sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + + Stack* {.importc: "stack_t", + header: "<signal.h>", final, pure.} = object ## stack_t + ss_sp*: pointer ## Stack base or pointer. + ss_size*: csize_t ## Stack size. + ss_flags*: cint ## Flags. + + SigInfo* {.importc: "siginfo_t", + header: "<signal.h>", final, pure.} = object ## siginfo_t + si_signo*: cint ## Signal number. + si_code*: cint ## Signal code. + si_errno*: cint ## If non-zero, an errno value associated with + ## this signal, as defined in <errno.h>. + si_pid*: Pid ## Sending process ID. + si_uid*: Uid ## Real user ID of sending process. + si_addr*: pointer ## Address of faulting instruction. + si_status*: cint ## Exit value or signal. + si_band*: int ## Band event for SIGPOLL. + si_value*: SigVal ## Signal value. + + Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint + Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = cint + + Sched_param* {.importc: "struct sched_param", + header: "<sched.h>", + final, pure.} = object ## struct sched_param + sched_priority*: cint + + Timeval* {.importc: "struct timeval", header: "<sys/select.h>", + final, pure.} = object ## struct timeval + tv_sec*: Time ## Seconds. + tv_usec*: Suseconds ## Microseconds. + TFdSet* {.importc: "fd_set", header: "<sys/select.h>", + final, pure.} = object + Mcontext* {.importc: "mcontext_t", header: "<ucontext.h>", + final, pure.} = object + Ucontext* {.importc: "ucontext_t", header: "<ucontext.h>", + final, pure.} = object ## ucontext_t + uc_link*: ptr Ucontext ## Pointer to the context that is resumed + ## when this context returns. + uc_sigmask*: Sigset ## The set of signals that are blocked when this + ## context is active. + uc_stack*: Stack ## The stack used by this context. + uc_mcontext*: Mcontext ## A machine-specific representation of the saved + ## context. + +when hasAioH: + type + Taiocb* {.importc: "struct aiocb", header: "<aio.h>", + final, pure.} = object ## struct aiocb + aio_fildes*: cint ## File descriptor. + aio_offset*: Off ## File offset. + aio_buf*: pointer ## Location of buffer. + aio_nbytes*: int ## Length of transfer. + aio_reqprio*: cint ## Request priority offset. + aio_sigevent*: SigEvent ## Signal number and value. + aio_lio_opcode: cint ## Operation to be performed. + +when hasSpawnH: + type + Tposix_spawnattr* {.importc: "posix_spawnattr_t", + header: "<spawn.h>", final, pure.} = object + Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", + header: "<spawn.h>", final, pure.} = object + +when defined(linux): + # from sys/un.h + const Sockaddr_un_path_length* = 108 +elif defined(haiku): + # from sys/un.h + const Sockaddr_un_path_length* = 126 +else: + # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html + # this is >=92 + const Sockaddr_un_path_length* = 92 + +type + SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = uint32 + TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = uint8 + + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..255, char] ## Socket address (variable-length data). + + Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", + pure, final.} = object ## struct sockaddr_un + sun_family*: TSa_Family ## Address family. + sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + + Tif_nameindex* {.importc: "struct if_nameindex", final, + pure, header: "<net/if.h>".} = object ## struct if_nameindex + if_index*: cuint ## Numeric index of the interface. + if_name*: cstring ## Null-terminated name of the interface. + + + IOVec* {.importc: "struct iovec", pure, final, + header: "<sys/uio.h>".} = object ## struct iovec + iov_base*: pointer ## Base address of a memory region for input or output. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. + + Tmsghdr* {.importc: "struct msghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct msghdr + msg_name*: pointer ## Optional address. + msg_namelen*: SockLen ## Size of address. + msg_iov*: ptr IOVec ## Scatter/gather array. + msg_iovlen*: cint ## Members in msg_iov. + msg_control*: pointer ## Ancillary data; see below. + msg_controllen*: SockLen ## Ancillary data buffer len. + msg_flags*: cint ## Flags on received message. + + + Tcmsghdr* {.importc: "struct cmsghdr", pure, final, + header: "<sys/socket.h>".} = object ## struct cmsghdr + cmsg_len*: SockLen ## Data byte count, including the cmsghdr. + cmsg_level*: cint ## Originating protocol. + cmsg_type*: cint ## Protocol-specific type. + + TLinger* {.importc: "struct linger", pure, final, + header: "<sys/socket.h>".} = object ## struct linger + l_onoff*: cint ## Indicates whether linger option is enabled. + l_linger*: cint ## Linger time, in seconds. + + InPort* = uint16 + InAddrScalar* = uint32 + + InAddrT* {.importc: "in_addr_t", pure, final, + header: "<netinet/in.h>".} = uint32 + + InAddr* {.importc: "struct in_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in_addr + s_addr*: InAddrScalar + + Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in + sin_family*: TSa_Family ## AF_INET. + sin_port*: InPort ## Port number. + sin_addr*: InAddr ## IP address. + + In6Addr* {.importc: "struct in6_addr", pure, final, + header: "<netinet/in.h>".} = object ## struct in6_addr + s6_addr*: array[0..15, char] + + Sockaddr_in6* {.importc: "struct sockaddr_in6", pure, final, + header: "<netinet/in.h>".} = object ## struct sockaddr_in6 + sin6_family*: TSa_Family ## AF_INET6. + sin6_port*: InPort ## Port number. + sin6_flowinfo*: int32 ## IPv6 traffic class and flow information. + sin6_addr*: In6Addr ## IPv6 address. + sin6_scope_id*: int32 ## Set of interfaces for a scope. + + Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, + header: "<netinet/in.h>".} = object ## struct ipv6_mreq + ipv6mr_multiaddr*: In6Addr ## IPv6 multicast address. + ipv6mr_interface*: cuint ## Interface index. + + Hostent* {.importc: "struct hostent", pure, final, + header: "<netdb.h>".} = object ## struct hostent + h_name*: cstring ## Official name of the host. + h_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative host names, terminated by a + ## null pointer. + h_addrtype*: cint ## Address type. + h_length*: cint ## The length, in bytes, of the address. + h_addr_list*: cstringArray ## A pointer to an array of pointers to network + ## addresses (in network byte order) for the + ## host, terminated by a null pointer. + + Tnetent* {.importc: "struct netent", pure, final, + header: "<netdb.h>".} = object ## struct netent + n_name*: cstring ## Official, fully-qualified (including the + ## domain) name of the host. + n_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative network names, terminated by a + ## null pointer. + n_addrtype*: cint ## The address type of the network. + n_net*: InAddrT ## The network number, in host byte order. + + Protoent* {.importc: "struct protoent", pure, final, + header: "<netdb.h>".} = object ## struct protoent + p_name*: cstring ## Official name of the protocol. + p_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative protocol names, terminated by + ## a null pointer. + p_proto*: cint ## The protocol number. + + Servent* {.importc: "struct servent", pure, final, + header: "<netdb.h>".} = object ## struct servent + s_name*: cstring ## Official name of the service. + s_aliases*: cstringArray ## A pointer to an array of pointers to + ## alternative service names, terminated by + ## a null pointer. + s_port*: cint ## The port number at which the service + ## resides, in network byte order. + s_proto*: cstring ## The name of the protocol to use when + ## contacting the service. + + AddrInfo* {.importc: "struct addrinfo", pure, final, + header: "<netdb.h>".} = object ## struct addrinfo + ai_flags*: cint ## Input flags. + ai_family*: cint ## Address family of socket. + ai_socktype*: cint ## Socket type. + ai_protocol*: cint ## Protocol of socket. + ai_addrlen*: SockLen ## Length of socket address. + ai_addr*: ptr SockAddr ## Socket address of socket. + ai_canonname*: cstring ## Canonical name of service location. + ai_next*: ptr AddrInfo ## Pointer to next in list. + + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = culong + +var + errno* {.importc, header: "<errno.h>".}: cint ## error variable + h_errno* {.importc, header: "<netdb.h>".}: cint + daylight* {.importc, header: "<time.h>".}: cint + timezone* {.importc, header: "<time.h>".}: int + +# Regenerate using detect.nim! +include posix_other_consts + +when defined(linux): + var + MAP_POPULATE* {.importc, header: "<sys/mman.h>".}: cint + ## Populate (prefault) page tables for a mapping. +else: + var + MAP_POPULATE*: cint = 0 + +when defined(linux) or defined(nimdoc): + when defined(alpha) or defined(mips) or defined(mipsel) or + defined(mips64) or defined(mips64el) or defined(parisc) or + defined(sparc) or defined(sparc64) or defined(nimdoc): + const SO_REUSEPORT* = cint(0x0200) + ## Multiple binding: load balancing on incoming TCP connections + ## or UDP packets. (Requires Linux kernel > 3.9) + else: + const SO_REUSEPORT* = cint(15) +else: + var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint + +when defined(macosx): + # We can't use the NOSIGNAL flag in the `send` function, it has no effect + # Instead we should use SO_NOSIGPIPE in setsockopt + const + MSG_NOSIGNAL* = 0'i32 + var + SO_NOSIGPIPE* {.importc, header: "<sys/socket.h>".}: cint +elif defined(solaris): + # Solaris doesn't have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0'i32 +else: + var + MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint + ## No SIGPIPE generated when an attempt to send is made on a stream-oriented socket that is no longer connected. + +when defined(haiku): + const + SIGKILLTHR* = 21 ## BeOS specific: Kill just the thread, not team + +when hasSpawnH: + when defined(linux): + # better be safe than sorry; Linux has this flag, macosx doesn't, don't + # know about the other OSes + + # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we + # can't get the magic number from spawn.h + const POSIX_SPAWN_USEVFORK* = cint(0x40) + else: + # macosx lacks this, so we define the constant to be 0 to not affect + # OR'ing of flags: + const POSIX_SPAWN_USEVFORK* = cint(0) + +# <sys/wait.h> +proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Exit code, if WIFEXITED(s) +proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Termination signal, if WIFSIGNALED(s) +proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} + ## Stop signal, if WIFSTOPPED(s) +proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited normally. +proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child exited due to uncaught signal. +proc WIFSTOPPED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child is currently stopped. +proc WIFCONTINUED*(s: cint): bool {.importc, header: "<sys/wait.h>".} + ## True if child has been continued. + +when defined(nimHasStyleChecks): + {.pop.} diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim index bb82d9e5e..8d11c507d 100644 --- a/lib/posix/posix_linux_amd64.nim +++ b/lib/posix/posix_linux_amd64.nim @@ -34,9 +34,6 @@ type type SocketHandle* = distinct cint # The type used to represent socket descriptors -# not detected by detect.nim, guarded by #ifdef __USE_UNIX98 in glibc -const SIG_HOLD* = cast[Sighandler](2) - type Time* {.importc: "time_t", header: "<time.h>".} = distinct clong @@ -50,7 +47,7 @@ type d_ino*: Ino d_off*: Off d_reclen*: cushort - d_type*: int8 # cuchar really! + d_type*: int8 # uint8 really! d_name*: array[256, cchar] Tflock* {.importc: "struct flock", final, pure, @@ -171,7 +168,7 @@ type Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = cuint Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>", pure, final.} = object - abi: array[48 div sizeof(clong), clong] + abi: array[40 div sizeof(clong), clong] Pthread_mutexattr* {.importc: "pthread_mutexattr_t", header: "<sys/types.h>", pure, final.} = object abi: array[4 div sizeof(cint), cint] @@ -228,7 +225,7 @@ type st_mode*: Mode ## Mode of file (see below). st_uid*: Uid ## User ID of file. st_gid*: Gid ## Group ID of file. - pad0: cint + pad0 {.importc: "__pad0".}: cint st_rdev*: Dev ## Device ID (if file is character or block special). st_size*: Off ## For regular files, the file size in bytes. ## For symbolic links, the length in bytes of the @@ -244,8 +241,6 @@ type st_atim*: Timespec ## Time of last access. st_mtim*: Timespec ## Time of last data modification. st_ctim*: Timespec ## Time of last status change. - reserved: array[3, clong] - Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>", final, pure.} = object ## struct statvfs @@ -314,7 +309,7 @@ type sa_mask*: Sigset ## Set of signals to be blocked during execution of ## the signal handling function. sa_flags*: cint ## Special flags. - sa_sigaction*: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} + sa_restorer: proc() {.noconv.} ## not intended for application use. Stack* {.importc: "stack_t", header: "<signal.h>", final, pure.} = object ## stack_t @@ -330,17 +325,23 @@ type SigInfo* {.importc: "siginfo_t", header: "<signal.h>", final, pure.} = object ## siginfo_t si_signo*: cint ## Signal number. - si_code*: cint ## Signal code. si_errno*: cint ## If non-zero, an errno value associated with ## this signal, as defined in <errno.h>. + si_code*: cint ## Signal code. si_pid*: Pid ## Sending process ID. si_uid*: Uid ## Real user ID of sending process. si_addr*: pointer ## Address of faulting instruction. si_status*: cint ## Exit value or signal. si_band*: int ## Band event for SIGPOLL. si_value*: SigVal ## Signal value. - pad {.importc: "_pad"}: array[128 - 56, uint8] + pad {.importc: "_pad".}: array[128 - 56, uint8] +template sa_sigaction*(v: Sigaction): proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.} = + cast[proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.}](v.sa_handler) +proc `sa_sigaction=`*(v: var Sigaction, x: proc (x: cint, y: ptr SigInfo, z: pointer) {.noconv.}) = + v.sa_handler = cast[proc (x: cint) {.noconv.}](x) + +type Nl_item* {.importc: "nl_item", header: "<nl_types.h>".} = cint Nl_catd* {.importc: "nl_catd", header: "<nl_types.h>".} = pointer @@ -433,8 +434,8 @@ type header: "<sys/socket.h>", pure, final.} = object ## struct sockaddr_storage ss_family*: TSa_Family ## Address family. - ss_padding: array[128 - sizeof(cshort) - sizeof(culong), char] - ss_align: clong + ss_padding {.importc: "__ss_padding".}: array[128 - sizeof(cshort) - sizeof(culong), char] + ss_align {.importc: "__ss_align".}: clong Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "<net/if.h>".} = object ## struct if_nameindex @@ -573,8 +574,6 @@ var # Regenerate using detect.nim! include posix_linux_amd64_consts -const POSIX_SPAWN_USEVFORK* = cint(0x40) # needs _GNU_SOURCE! - # <sys/wait.h> proc WEXITSTATUS*(s: cint): cint = (s and 0xff00) shr 8 proc WTERMSIG*(s:cint): cint = s and 0x7f diff --git a/lib/posix/posix_linux_amd64_consts.nim b/lib/posix/posix_linux_amd64_consts.nim index 7352e8e35..fbe8d0666 100644 --- a/lib/posix/posix_linux_amd64_consts.nim +++ b/lib/posix/posix_linux_amd64_consts.nim @@ -100,6 +100,7 @@ const EXDEV* = cint(18) # <fcntl.h> const F_DUPFD* = cint(0) +const F_DUPFD_CLOEXEC* = cint(1030) const F_GETFD* = cint(1) const F_SETFD* = cint(2) const F_GETFL* = cint(3) @@ -126,6 +127,11 @@ const O_ACCMODE* = cint(3) const O_RDONLY* = cint(0) const O_RDWR* = cint(2) const O_WRONLY* = cint(1) +const O_CLOEXEC* = cint(524288) +const O_DIRECT* = cint(16384) +const O_PATH* = cint(2097152) +const O_NOATIME* = cint(262144) +const O_TMPFILE* = cint(4259840) const POSIX_FADV_NORMAL* = cint(0) const POSIX_FADV_SEQUENTIAL* = cint(2) const POSIX_FADV_RANDOM* = cint(1) @@ -172,13 +178,20 @@ const FNM_NOMATCH* = cint(1) const FNM_PATHNAME* = cint(1) const FNM_PERIOD* = cint(4) const FNM_NOESCAPE* = cint(2) +const FNM_NOSYS* = cint(-1) # <ftw.h> const FTW_F* = cint(0) const FTW_D* = cint(1) const FTW_DNR* = cint(2) +const FTW_DP* = cint(5) const FTW_NS* = cint(3) const FTW_SL* = cint(4) +const FTW_SLN* = cint(6) +const FTW_PHYS* = cint(1) +const FTW_MOUNT* = cint(2) +const FTW_DEPTH* = cint(8) +const FTW_CHDIR* = cint(4) # <glob.h> const GLOB_APPEND* = cint(32) @@ -400,6 +413,7 @@ const SS_ONSTACK* = cint(1) const SS_DISABLE* = cint(2) const MINSIGSTKSZ* = cint(2048) const SIGSTKSZ* = cint(8192) +const SIG_HOLD* = cast[Sighandler](2) const SIG_DFL* = cast[Sighandler](0) const SIG_ERR* = cast[Sighandler](-1) const SIG_IGN* = cast[Sighandler](1) @@ -439,6 +453,7 @@ const MAP_POPULATE* = cint(32768) # <sys/resource.h> const RLIMIT_NOFILE* = cint(7) +const RLIMIT_STACK* = cint(3) # <sys/select.h> const FD_SETSIZE* = cint(1024) @@ -450,6 +465,7 @@ const MSG_EOR* = cint(128) const MSG_OOB* = cint(1) const SCM_RIGHTS* = cint(1) const SO_ACCEPTCONN* = cint(30) +const SO_BINDTODEVICE* = cint(25) const SO_BROADCAST* = cint(6) const SO_DEBUG* = cint(1) const SO_DONTROUTE* = cint(5) @@ -469,8 +485,9 @@ const SOCK_DGRAM* = cint(2) const SOCK_RAW* = cint(3) const SOCK_SEQPACKET* = cint(5) const SOCK_STREAM* = cint(1) +const SOCK_CLOEXEC* = cint(524288) const SOL_SOCKET* = cint(1) -const SOMAXCONN* = cint(128) +const SOMAXCONN* = cint(4096) const SO_REUSEPORT* = cint(15) const MSG_NOSIGNAL* = cint(16384) const MSG_PEEK* = cint(2) @@ -528,6 +545,7 @@ const POSIX_SPAWN_SETSCHEDPARAM* = cint(16) const POSIX_SPAWN_SETSCHEDULER* = cint(32) const POSIX_SPAWN_SETSIGDEF* = cint(4) const POSIX_SPAWN_SETSIGMASK* = cint(8) +const POSIX_SPAWN_USEVFORK* = cint(64) # <stdio.h> const IOFBF* = cint(0) diff --git a/lib/posix/posix_macos_amd64.nim b/lib/posix/posix_macos_amd64.nim index 8a20d127f..a4b64ed62 100644 --- a/lib/posix/posix_macos_amd64.nim +++ b/lib/posix/posix_macos_amd64.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated - when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -47,7 +45,7 @@ type d_type*: int8 ## Type of file; not supported by all filesystem types. ## (not POSIX) when defined(linux) or defined(openbsd): - d_off*: Off ## Not an offset. Value that ``telldir()`` would return. + d_off*: Off ## Not an offset. Value that `telldir()` would return. elif defined(haiku): d_pino*: Ino ## Parent inode (only for queries) (not POSIX) d_reclen*: cushort ## Length of this record. (not POSIX) @@ -110,15 +108,6 @@ type p_sign_posn*: char thousands_sep*: cstring - Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object - MqAttr* {.importc: "struct mq_attr", - header: "<mqueue.h>", - final, pure.} = object ## message queue attribute - mq_flags*: int ## Message queue flags. - mq_maxmsg*: int ## Maximum number of messages. - mq_msgsize*: int ## Maximum message size. - mq_curmsgs*: int ## Number of messages currently queued. - Passwd* {.importc: "struct passwd", header: "<pwd.h>", final, pure.} = object ## struct passwd pw_name*: cstring ## User's login name. @@ -133,10 +122,14 @@ type ## used for block sizes Clock* {.importc: "clock_t", header: "<sys/types.h>".} = int ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int - Dev* {.importc: "dev_t", header: "<sys/types.h>".} = int32 + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = ( + when defined(freebsd): + uint32 + else: + int32) Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int - Gid* {.importc: "gid_t", header: "<sys/types.h>".} = int32 + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = uint32 Id* {.importc: "id_t", header: "<sys/types.h>".} = int Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int Key* {.importc: "key_t", header: "<sys/types.h>".} = int @@ -146,7 +139,7 @@ type else: uint16 ) - Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int16 + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = uint16 Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 Pthread_attr* {.importc: "pthread_attr_t", header: "<sys/types.h>".} = int @@ -178,7 +171,7 @@ type Trace_event_set* {.importc: "trace_event_set_t", header: "<sys/types.h>".} = int Trace_id* {.importc: "trace_id_t", header: "<sys/types.h>".} = int - Uid* {.importc: "uid_t", header: "<sys/types.h>".} = int32 + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = uint32 Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = int Utsname* {.importc: "struct utsname", @@ -377,6 +370,18 @@ when hasSpawnH: Tposix_spawn_file_actions* {.importc: "posix_spawn_file_actions_t", header: "<spawn.h>", final, pure.} = object + +when not defined(macos) and not defined(macosx): # freebsd + type + Mqd* {.importc: "mqd_t", header: "<mqueue.h>", final, pure.} = object + MqAttr* {.importc: "struct mq_attr", + header: "<mqueue.h>", + final, pure.} = object ## message queue attribute + mq_flags*: int ## Message queue flags. + mq_maxmsg*: int ## Maximum number of messages. + mq_msgsize*: int ## Maximum message size. + mq_curmsgs*: int ## Number of messages currently queued. + when defined(linux): # from sys/un.h const Sockaddr_un_path_length* = 108 @@ -413,7 +418,7 @@ type IOVec* {.importc: "struct iovec", pure, final, header: "<sys/uio.h>".} = object ## struct iovec iov_base*: pointer ## Base address of a memory region for input or output. - iov_len*: int ## The size of the memory pointed to by iov_base. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. Tmsghdr* {.importc: "struct msghdr", pure, final, header: "<sys/socket.h>".} = object ## struct msghdr @@ -461,9 +466,9 @@ type header: "<netinet/in.h>".} = object ## struct sockaddr_in6 sin6_family*: TSa_Family ## AF_INET6. sin6_port*: InPort ## Port number. - sin6_flowinfo*: int32 ## IPv6 traffic class and flow information. + sin6_flowinfo*: uint32 ## IPv6 traffic class and flow information. sin6_addr*: In6Addr ## IPv6 address. - sin6_scope_id*: int32 ## Set of interfaces for a scope. + sin6_scope_id*: uint32 ## Set of interfaces for a scope. Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, header: "<netinet/in.h>".} = object ## struct ipv6_mreq @@ -490,7 +495,7 @@ type ## alternative network names, terminated by a ## null pointer. n_addrtype*: cint ## The address type of the network. - n_net*: int32 ## The network number, in host byte order. + n_net*: uint32 ## The network number, in host byte order. Protoent* {.importc: "struct protoent", pure, final, header: "<netdb.h>".} = object ## struct protoent @@ -559,8 +564,11 @@ when defined(linux) or defined(nimdoc): else: var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint +when defined(linux) or defined(bsd): + var SOCK_CLOEXEC* {.importc, header: "<sys/socket.h>".}: cint + when defined(macosx): - # We can't use the NOSIGNAL flag in the ``send`` function, it has no effect + # We can't use the NOSIGNAL flag in the `send` function, it has no effect # Instead we should use SO_NOSIGPIPE in setsockopt const MSG_NOSIGNAL* = 0'i32 @@ -594,11 +602,11 @@ when hasSpawnH: # <sys/wait.h> proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Exit code, iff WIFEXITED(s) + ## Exit code, if WIFEXITED(s) proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Termination signal, iff WIFSIGNALED(s) + ## Termination signal, if WIFSIGNALED(s) proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Stop signal, iff WIFSTOPPED(s) + ## Stop signal, if WIFSTOPPED(s) proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} ## True if child exited normally. proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} diff --git a/lib/posix/posix_nintendoswitch.nim b/lib/posix/posix_nintendoswitch.nim index d1cc042ea..b66563695 100644 --- a/lib/posix/posix_nintendoswitch.nim +++ b/lib/posix/posix_nintendoswitch.nim @@ -33,7 +33,7 @@ type Dirent* {.importc: "struct dirent", header: "<dirent.h>", final, pure.} = object ## dirent_t struct d_ino*: Ino - d_type*: int8 # cuchar really! + d_type*: int8 # uint8 really! d_name*: array[256, cchar] Tflock* {.importc: "struct flock", final, pure, @@ -286,7 +286,7 @@ type header: "<signal.h>", final, pure.} = object ## stack_t ss_sp*: pointer ## Stack base or pointer. ss_flags*: cint ## Flags. - ss_size*: csize ## Stack size. + ss_size*: csize_t ## Stack size. SigInfo* {.importc: "siginfo_t", header: "<signal.h>", final, pure.} = object ## siginfo_t @@ -321,7 +321,7 @@ type aio_lio_opcode*: cint ## Operation to be performed. aio_reqprio*: cint ## Request priority offset. aio_buf*: pointer ## Location of buffer. - aio_nbytes*: csize ## Length of transfer. + aio_nbytes*: csize_t ## Length of transfer. aio_sigevent*: SigEvent ## Signal number and value. next_prio: pointer abs_prio: cint @@ -347,20 +347,20 @@ type SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", pure, final.} = object ## struct sockaddr - sa_len: cuchar + sa_len: uint8 sa_family*: TSa_Family ## Address family. sa_data*: array[14, char] ## Socket address (variable-length data). Sockaddr_storage* {.importc: "struct sockaddr_storage", header: "<sys/socket.h>", pure, final.} = object ## struct sockaddr_storage - ss_len: cuchar + ss_len: uint8 ss_family*: TSa_Family ## Address family. - ss_padding1: array[64 - sizeof(cuchar) - sizeof(cshort), char] + ss_padding1: array[64 - sizeof(uint8) - sizeof(cshort), char] ss_align: clonglong ss_padding2: array[ - 128 - sizeof(cuchar) - sizeof(cshort) - - (64 - sizeof(cuchar) - sizeof(cshort)) - 64, char] + 128 - sizeof(uint8) - sizeof(cshort) - + (64 - sizeof(uint8) - sizeof(cshort)) - 64, char] Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "<net/if.h>".} = object ## struct if_nameindex @@ -371,22 +371,22 @@ type IOVec* {.importc: "struct iovec", pure, final, header: "<sys/socket.h>".} = object ## struct iovec iov_base*: pointer ## Base address of a memory region for input or output. - iov_len*: csize ## The size of the memory pointed to by iov_base. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. Tmsghdr* {.importc: "struct msghdr", pure, final, header: "<sys/socket.h>".} = object ## struct msghdr msg_name*: pointer ## Optional address. msg_namelen*: SockLen ## Size of address. msg_iov*: ptr IOVec ## Scatter/gather array. - msg_iovlen*: csize ## Members in msg_iov. + msg_iovlen*: csize_t ## Members in msg_iov. msg_control*: pointer ## Ancillary data; see below. - msg_controllen*: csize ## Ancillary data buffer len. + msg_controllen*: csize_t ## Ancillary data buffer len. msg_flags*: cint ## Flags on received message. Tcmsghdr* {.importc: "struct cmsghdr", pure, final, header: "<sys/socket.h>".} = object ## struct cmsghdr - cmsg_len*: csize ## Data byte count, including the cmsghdr. + cmsg_len*: csize_t ## Data byte count, including the cmsghdr. cmsg_level*: cint ## Originating protocol. cmsg_type*: cint ## Protocol-specific type. diff --git a/lib/posix/posix_openbsd_amd64.nim b/lib/posix/posix_openbsd_amd64.nim index d9ac84a1d..184cd89c0 100644 --- a/lib/posix/posix_openbsd_amd64.nim +++ b/lib/posix/posix_openbsd_amd64.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated - when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -47,7 +45,7 @@ type d_type*: int8 ## Type of file; not supported by all filesystem types. ## (not POSIX) when defined(linux) or defined(openbsd): - d_off*: Off ## Not an offset. Value that ``telldir()`` would return. + d_off*: Off ## Not an offset. Value that `telldir()` would return. elif defined(haiku): d_pino*: Ino ## Parent inode (only for queries) (not POSIX) d_reclen*: cushort ## Length of this record. (not POSIX) @@ -133,15 +131,19 @@ type ## used for block sizes Clock* {.importc: "clock_t", header: "<sys/types.h>".} = int ClockId* {.importc: "clockid_t", header: "<sys/types.h>".} = int - Dev* {.importc: "dev_t", header: "<sys/types.h>".} = int32 + Dev* {.importc: "dev_t", header: "<sys/types.h>".} = ( + when defined(freebsd): + uint32 + else: + int32) Fsblkcnt* {.importc: "fsblkcnt_t", header: "<sys/types.h>".} = int Fsfilcnt* {.importc: "fsfilcnt_t", header: "<sys/types.h>".} = int - Gid* {.importc: "gid_t", header: "<sys/types.h>".} = int32 + Gid* {.importc: "gid_t", header: "<sys/types.h>".} = uint32 Id* {.importc: "id_t", header: "<sys/types.h>".} = int Ino* {.importc: "ino_t", header: "<sys/types.h>".} = int Key* {.importc: "key_t", header: "<sys/types.h>".} = int Mode* {.importc: "mode_t", header: "<sys/types.h>".} = uint32 - Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = int16 + Nlink* {.importc: "nlink_t", header: "<sys/types.h>".} = uint32 Off* {.importc: "off_t", header: "<sys/types.h>".} = int64 Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 Pthread_attr* {.importc: "pthread_attr_t", header: "<pthread.h>".} = int @@ -173,7 +175,7 @@ type Trace_event_set* {.importc: "trace_event_set_t", header: "<sys/types.h>".} = int Trace_id* {.importc: "trace_id_t", header: "<sys/types.h>".} = int - Uid* {.importc: "uid_t", header: "<sys/types.h>".} = int32 + Uid* {.importc: "uid_t", header: "<sys/types.h>".} = uint32 Useconds* {.importc: "useconds_t", header: "<sys/types.h>".} = int Utsname* {.importc: "struct utsname", @@ -400,7 +402,7 @@ type IOVec* {.importc: "struct iovec", pure, final, header: "<sys/uio.h>".} = object ## struct iovec iov_base*: pointer ## Base address of a memory region for input or output. - iov_len*: int ## The size of the memory pointed to by iov_base. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. Tmsghdr* {.importc: "struct msghdr", pure, final, header: "<sys/socket.h>".} = object ## struct msghdr @@ -448,9 +450,9 @@ type header: "<netinet/in.h>".} = object ## struct sockaddr_in6 sin6_family*: TSa_Family ## AF_INET6. sin6_port*: InPort ## Port number. - sin6_flowinfo*: int32 ## IPv6 traffic class and flow information. + sin6_flowinfo*: uint32 ## IPv6 traffic class and flow information. sin6_addr*: In6Addr ## IPv6 address. - sin6_scope_id*: int32 ## Set of interfaces for a scope. + sin6_scope_id*: uint32 ## Set of interfaces for a scope. Tipv6_mreq* {.importc: "struct ipv6_mreq", pure, final, header: "<netinet/in.h>".} = object ## struct ipv6_mreq @@ -477,7 +479,7 @@ type ## alternative network names, terminated by a ## null pointer. n_addrtype*: cint ## The address type of the network. - n_net*: int32 ## The network number, in host byte order. + n_net*: uint32 ## The network number, in host byte order. Protoent* {.importc: "struct protoent", pure, final, header: "<netdb.h>".} = object ## struct protoent @@ -534,6 +536,8 @@ when defined(nimdoc): else: var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint +var SOCK_CLOEXEC* {.importc, header: "<sys/socket.h>".}: cint + var MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint when hasSpawnH: @@ -543,11 +547,11 @@ when hasSpawnH: # <sys/wait.h> proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Exit code, iff WIFEXITED(s) + ## Exit code, if WIFEXITED(s) proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Termination signal, iff WIFSIGNALED(s) + ## Termination signal, if WIFSIGNALED(s) proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Stop signal, iff WIFSTOPPED(s) + ## Stop signal, if WIFSTOPPED(s) proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} ## True if child exited normally. proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index e42092513..ea8731405 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -7,14 +7,17 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated - when defined(nimHasStyleChecks): {.push styleChecks: off.} -const - hasSpawnH = true # should exist for every Posix system nowadays - hasAioH = defined(linux) +when defined(freertos) or defined(zephyr): + const + hasSpawnH = false # should exist for every Posix system nowadays + hasAioH = false +else: + const + hasSpawnH = true # should exist for every Posix system nowadays + hasAioH = defined(linux) when defined(linux) and not defined(android): # On Linux: @@ -34,7 +37,12 @@ type SocketHandle* = distinct cint # The type used to represent socket descriptors type - Time* {.importc: "time_t", header: "<time.h>".} = distinct clong + Time* {.importc: "time_t", header: "<time.h>".} = distinct ( + when defined(nimUse64BitCTime): + int64 + else: + clong + ) Timespec* {.importc: "struct timespec", header: "<time.h>", final, pure.} = object ## struct timespec @@ -56,7 +64,7 @@ type d_type*: int8 ## Type of file; not supported by all filesystem types. ## (not POSIX) when defined(linux) or defined(openbsd): - d_off*: Off ## Not an offset. Value that ``telldir()`` would return. + d_off*: Off ## Not an offset. Value that `telldir()` would return. elif defined(haiku): d_pino*: Ino ## Parent inode (only for queries) (not POSIX) d_reclen*: cushort ## Length of this record. (not POSIX) @@ -383,9 +391,33 @@ when hasSpawnH: header: "<spawn.h>", final, pure.} = object when defined(linux): + const Sockaddr_max_length* = 255 # from sys/un.h const Sockaddr_un_path_length* = 108 +elif defined(zephyr): + when defined(net_ipv6): + const Sockaddr_max_length* = 24 + elif defined(net_raw): + const Sockaddr_max_length* = 20 + elif defined(net_ipv4): + const Sockaddr_max_length* = 8 + else: + const Sockaddr_max_length* = 255 # just for compilation purposes + + const Sockaddr_un_path_length* = Sockaddr_max_length + # Zephyr is heavily customizable so it's easy to get to a state + # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. + when defined(net_ipv4) or defined(net_ipv6) or defined(net_raw): + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} +elif defined(freertos) or defined(lwip): + const Sockaddr_max_length* = 14 + const Sockaddr_un_path_length* = 108 else: + const Sockaddr_max_length* = 255 # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html # this is >=92 const Sockaddr_un_path_length* = 92 @@ -394,21 +426,57 @@ type SockLen* {.importc: "socklen_t", header: "<sys/socket.h>".} = cuint TSa_Family* {.importc: "sa_family_t", header: "<sys/socket.h>".} = cushort - SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", - pure, final.} = object ## struct sockaddr - sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). +when defined(lwip): + type + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_len*: uint8 ## Address family. + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + s2_len*: uint8 ## Address family. + ss_family*: TSa_Family ## Address family. + s2_data1*: array[2, char] ## Address family. + s2_data2*: array[3, uint32] ## Address family. + when defined(lwip6) or defined(net_ipv6): + s2_data3*: array[3, uint32] ## Address family. +elif defined(zephyr): + type + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].} + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].} +else: + type + SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "<sys/socket.h>", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. +type Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>", pure, final.} = object ## struct sockaddr_un sun_family*: TSa_Family ## Address family. - sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "<sys/socket.h>", - pure, final.} = object ## struct sockaddr_storage - ss_family*: TSa_Family ## Address family. +type Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "<net/if.h>".} = object ## struct if_nameindex if_index*: cint ## Numeric index of the interface. @@ -418,7 +486,7 @@ type IOVec* {.importc: "struct iovec", pure, final, header: "<sys/uio.h>".} = object ## struct iovec iov_base*: pointer ## Base address of a memory region for input or output. - iov_len*: int ## The size of the memory pointed to by iov_base. + iov_len*: csize_t ## The size of the memory pointed to by iov_base. Tmsghdr* {.importc: "struct msghdr", pure, final, header: "<sys/socket.h>".} = object ## struct msghdr @@ -452,6 +520,7 @@ type header: "<netinet/in.h>".} = object ## struct in_addr s_addr*: InAddrScalar + # TODO: Fixme for FreeRTOS/LwIP, these are incorrect Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, header: "<netinet/in.h>".} = object ## struct sockaddr_in sin_family*: TSa_Family ## AF_INET. @@ -527,13 +596,20 @@ type ai_canonname*: cstring ## Canonical name of service location. ai_next*: ptr AddrInfo ## Pointer to next in list. - TPollfd* {.importc: "struct pollfd", pure, final, - header: "<poll.h>".} = object ## struct pollfd - fd*: cint ## The following descriptor being polled. - events*: cshort ## The input event flags (see below). - revents*: cshort ## The output event flags (see below). - - Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint +when not defined(lwip): + type + TPollfd* {.importc: "struct pollfd", pure, final, + header: "<poll.h>".} = object ## struct pollfd + fd*: cint ## The following descriptor being polled. + events*: cshort ## The input event flags (see below). + revents*: cshort ## The output event flags (see below). + + when defined(zephyr): + type + Tnfds* = distinct cint + else: + type + Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint var errno* {.importc, header: "<errno.h>".}: cint ## error variable @@ -542,7 +618,10 @@ var timezone* {.importc, header: "<time.h>".}: int # Regenerate using detect.nim! -include posix_other_consts +when defined(lwip): + include posix_freertos_consts +else: + include posix_other_consts when defined(linux): var @@ -561,11 +640,17 @@ when defined(linux) or defined(nimdoc): ## or UDP packets. (Requires Linux kernel > 3.9) else: const SO_REUSEPORT* = cint(15) +elif defined(nuttx): + # Not supported, use SO_REUSEADDR to avoid compilation errors. + var SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "<sys/socket.h>".}: cint else: var SO_REUSEPORT* {.importc, header: "<sys/socket.h>".}: cint +when defined(linux) or defined(bsd) or defined(nuttx): + var SOCK_CLOEXEC* {.importc, header: "<sys/socket.h>".}: cint + when defined(macosx): - # We can't use the NOSIGNAL flag in the ``send`` function, it has no effect + # We can't use the NOSIGNAL flag in the `send` function, it has no effect # Instead we should use SO_NOSIGPIPE in setsockopt const MSG_NOSIGNAL* = 0'i32 @@ -575,6 +660,10 @@ elif defined(solaris): # Solaris doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0'i32 +elif defined(zephyr) or defined(freertos) or defined(lwip): + # LwIP/FreeRTOS doesn't have MSG_NOSIGNAL + const + MSG_NOSIGNAL* = 0x20'i32 else: var MSG_NOSIGNAL* {.importc, header: "<sys/socket.h>".}: cint @@ -586,24 +675,24 @@ when defined(haiku): when hasSpawnH: when defined(linux): - # better be safe than sorry; Linux has this flag, macosx doesn't, don't - # know about the other OSes + # better be safe than sorry; Linux has this flag, macosx and NuttX don't, + # don't know about the other OSes - # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we + # Non-GNU systems like TCC and musl-libc don't define __USE_GNU, so we # can't get the magic number from spawn.h const POSIX_SPAWN_USEVFORK* = cint(0x40) else: - # macosx lacks this, so we define the constant to be 0 to not affect + # macosx and NuttX lack this, so we define the constant to be 0 to not affect # OR'ing of flags: const POSIX_SPAWN_USEVFORK* = cint(0) # <sys/wait.h> proc WEXITSTATUS*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Exit code, iff WIFEXITED(s) + ## Exit code, if WIFEXITED(s) proc WTERMSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Termination signal, iff WIFSIGNALED(s) + ## Termination signal, if WIFSIGNALED(s) proc WSTOPSIG*(s: cint): cint {.importc, header: "<sys/wait.h>".} - ## Stop signal, iff WIFSTOPPED(s) + ## Stop signal, if WIFSTOPPED(s) proc WIFEXITED*(s: cint): bool {.importc, header: "<sys/wait.h>".} ## True if child exited normally. proc WIFSIGNALED*(s: cint): bool {.importc, header: "<sys/wait.h>".} diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 9fcbe425d..d346b4150 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -99,6 +99,10 @@ var EXDEV* {.importc: "EXDEV", header: "<errno.h>".}: cint # <fcntl.h> var F_DUPFD* {.importc: "F_DUPFD", header: "<fcntl.h>".}: cint +when defined(nuttx): + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD_CLOEXEC", header: "<fcntl.h>".}: cint +else: + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "<fcntl.h>".}: cint var F_GETFD* {.importc: "F_GETFD", header: "<fcntl.h>".}: cint var F_SETFD* {.importc: "F_SETFD", header: "<fcntl.h>".}: cint var F_GETFL* {.importc: "F_GETFL", header: "<fcntl.h>".}: cint @@ -125,6 +129,12 @@ var O_ACCMODE* {.importc: "O_ACCMODE", header: "<fcntl.h>".}: cint var O_RDONLY* {.importc: "O_RDONLY", header: "<fcntl.h>".}: cint var O_RDWR* {.importc: "O_RDWR", header: "<fcntl.h>".}: cint var O_WRONLY* {.importc: "O_WRONLY", header: "<fcntl.h>".}: cint +var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "<fcntl.h>".}: cint +when defined(nuttx): + var O_DIRECT* {.importc: "O_DIRECT", header: "<fcntl.h>".}: cint + var O_PATH* {.importc: "O_PATH", header: "<fcntl.h>".}: cint + var O_NOATIME* {.importc: "O_NOATIME", header: "<fcntl.h>".}: cint + var O_TMPFILE* {.importc: "O_TMPFILE", header: "<fcntl.h>".}: cint var POSIX_FADV_NORMAL* {.importc: "POSIX_FADV_NORMAL", header: "<fcntl.h>".}: cint var POSIX_FADV_SEQUENTIAL* {.importc: "POSIX_FADV_SEQUENTIAL", header: "<fcntl.h>".}: cint var POSIX_FADV_RANDOM* {.importc: "POSIX_FADV_RANDOM", header: "<fcntl.h>".}: cint @@ -457,9 +467,13 @@ var POSIX_TYPED_MEM_MAP_ALLOCATABLE* {.importc: "POSIX_TYPED_MEM_MAP_ALLOCATABLE # <sys/resource.h> var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint +var RLIMIT_STACK* {.importc: "RLIMIT_STACK", header: "<sys/resource.h>".}: cint # <sys/select.h> var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint +when defined(zephyr): + # Zephyr specific hardcoded value + var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "<sys/select.h>".}: cint # <sys/socket.h> var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "<sys/socket.h>".}: cint @@ -468,6 +482,7 @@ var MSG_EOR* {.importc: "MSG_EOR", header: "<sys/socket.h>".}: cint var MSG_OOB* {.importc: "MSG_OOB", header: "<sys/socket.h>".}: cint var SCM_RIGHTS* {.importc: "SCM_RIGHTS", header: "<sys/socket.h>".}: cint var SO_ACCEPTCONN* {.importc: "SO_ACCEPTCONN", header: "<sys/socket.h>".}: cint +var SO_BINDTODEVICE* {.importc: "SO_BINDTODEVICE", header: "<sys/socket.h>".}: cint var SO_BROADCAST* {.importc: "SO_BROADCAST", header: "<sys/socket.h>".}: cint var SO_DEBUG* {.importc: "SO_DEBUG", header: "<sys/socket.h>".}: cint var SO_DONTROUTE* {.importc: "SO_DONTROUTE", header: "<sys/socket.h>".}: cint @@ -485,10 +500,15 @@ var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "<sys/socket.h>".}: cint var SO_TYPE* {.importc: "SO_TYPE", header: "<sys/socket.h>".}: cint var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "<sys/socket.h>".}: cint var SOCK_RAW* {.importc: "SOCK_RAW", header: "<sys/socket.h>".}: cint -var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint +when defined(zephyr): + const SOCK_SEQPACKET* = cint(5) + var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "<sys/socket.h>".}: cint +else: + var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint + var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint + var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "<sys/socket.h>".}: cint var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "<sys/socket.h>".}: cint -var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint var MSG_PEEK* {.importc: "MSG_PEEK", header: "<sys/socket.h>".}: cint var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "<sys/socket.h>".}: cint var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "<sys/socket.h>".}: cint @@ -732,3 +752,6 @@ var SEEK_SET* {.importc: "SEEK_SET", header: "<unistd.h>".}: cint var SEEK_CUR* {.importc: "SEEK_CUR", header: "<unistd.h>".}: cint var SEEK_END* {.importc: "SEEK_END", header: "<unistd.h>".}: cint +# <nuttx/config.h> +when defined(nuttx): + var NEPOLL_MAX* {.importc: "CONFIG_FS_NEPOLL_DESCRIPTORS", header: "<nuttx/config.h>".}: cint diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim index 20a71971a..0c668246f 100644 --- a/lib/posix/posix_utils.nim +++ b/lib/posix/posix_utils.nim @@ -7,19 +7,21 @@ # ## A set of helpers for the POSIX module. -## Raw interfaces are in the other posix*.nim files. +## Raw interfaces are in the other ``posix*.nim`` files. # Where possible, contribute OS-independent procs in `os <os.html>`_ instead. -{.deadCodeElim: on.} # dce option deprecated +import std/[posix, parsecfg, os] +import std/private/since -import posix +when defined(nimPreviewSlimSystem): + import std/syncio type Uname* = object sysname*, nodename*, release*, version*, machine*: string template charArrayToString(input: typed): string = - $cstring(addr input) + $cast[cstring](addr input) proc uname*(): Uname = ## Provides system information in a `Uname` struct with sysname, nodename, @@ -32,7 +34,7 @@ proc uname*(): Uname = var u: Utsname if uname(u) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) result.sysname = charArrayToString u.sysname result.nodename = charArrayToString u.nodename @@ -41,61 +43,91 @@ proc uname*(): Uname = result.machine = charArrayToString u.machine proc fsync*(fd: int) = - ## synchronize a file's buffer cache to the storage device - if fsync(fd.cint) != 0: - raise newException(OSError, $strerror(errno)) + ## synchronize a file's buffer cache to the storage device + if fsync(fd.cint) != 0: + raiseOSError(OSErrorCode(errno)) proc stat*(path: string): Stat = ## Returns file status in a `Stat` structure if stat(path.cstring, result) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryLock*(a1: pointer, a2: int) = ## Locks pages starting from a1 for a1 bytes and prevent them from being swapped. if mlock(a1, a2) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryLockAll*(flags: int) = ## Locks all memory for the running process to prevent swapping. ## - ## example:: - ## + ## example: + ## ```nim ## memoryLockAll(MCL_CURRENT or MCL_FUTURE) + ## ``` if mlockall(flags.cint) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryUnlock*(a1: pointer, a2: int) = ## Unlock pages starting from a1 for a1 bytes and allow them to be swapped. if munlock(a1, a2) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc memoryUnlockAll*() = ## Unlocks all memory for the running process to allow swapping. if munlockall() != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc sendSignal*(pid: Pid, signal: int) = ## Sends a signal to a running process by calling `kill`. ## Raise exception in case of failure e.g. process not running. if kill(pid, signal.cint) != 0: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) -proc mkstemp*(prefix: string): (string, File) = - ## Creates a unique temporary file from a prefix string. Adds a six chars suffix. +proc mkstemp*(prefix: string, suffix=""): (string, File) = + ## Creates a unique temporary file from a prefix string. A six-character string + ## will be added. If suffix is provided it will be added to the string ## The file is created with perms 0600. ## Returns the filename and a file opened in r/w mode. - var tmpl = cstring(prefix & "XXXXXX") - let fd = mkstemp(tmpl) + var tmpl = cstring(prefix & "XXXXXX" & suffix) + let fd = + if len(suffix) == 0: + when declared(mkostemp): + mkostemp(tmpl, O_CLOEXEC) + else: + mkstemp(tmpl) + else: + when declared(mkostemps): + mkostemps(tmpl, cint(len(suffix)), O_CLOEXEC) + else: + mkstemps(tmpl, cint(len(suffix))) var f: File if open(f, fd, fmReadWrite): return ($tmpl, f) - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) proc mkdtemp*(prefix: string): string = ## Creates a unique temporary directory from a prefix string. Adds a six chars suffix. ## The directory is created with permissions 0700. Returns the directory name. var tmpl = cstring(prefix & "XXXXXX") if mkdtemp(tmpl) == nil: - raise newException(OSError, $strerror(errno)) + raiseOSError(OSErrorCode(errno)) return $tmpl +proc osReleaseFile*(): Config {.since: (1, 5).} = + ## Gets system identification from `os-release` file and returns it as a `parsecfg.Config`. + ## You also need to import the `parsecfg` module to gain access to this object. + ## The `os-release` file is an official Freedesktop.org open standard. + ## Available in Linux and BSD distributions, except Android and Android-based Linux. + ## `os-release` file is not available on Windows and OS X by design. + ## * https://www.freedesktop.org/software/systemd/man/os-release.html + runnableExamples: + import std/parsecfg + when defined(linux): + let data = osReleaseFile() + echo "OS name: ", data.getSectionValue("", "NAME") ## the data is up to each distro. + + # We do not use a {.strdefine.} because Standard says it *must* be that path. + for osReleaseFile in ["/etc/os-release", "/usr/lib/os-release"]: + if fileExists(osReleaseFile): + return loadConfig(osReleaseFile) + raise newException(IOError, "File not found: /etc/os-release, /usr/lib/os-release") diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index 6a7ffb33e..7fb6bb81c 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -7,8 +7,7 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated -import posix +import std/posix type Speed* = cuint @@ -238,8 +237,11 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow", header: "<termios.h>".} # Get process group ID for session leader for controlling terminal FD. -# Window size ioctl. Should work on on any Unix that xterm has been ported to. -var TIOCGWINSZ*{.importc, header: "<sys/ioctl.h>".}: culong +# Window size ioctl. Solaris based systems have an uncommen place for this. +when defined(solaris) or defined(sunos): + var TIOCGWINSZ*{.importc, header: "<sys/termios.h>".}: culong +else: + var TIOCGWINSZ*{.importc, header: "<sys/ioctl.h>".}: culong when defined(nimHasStyleChecks): {.push styleChecks: off.} diff --git a/lib/prelude.nim b/lib/prelude.nim deleted file mode 100644 index 940864207..000000000 --- a/lib/prelude.nim +++ /dev/null @@ -1,23 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This is an include file that simply imports common modules for your -## convenience: -## -## .. code-block:: nim -## include prelude -## -## Same as: -## -## .. code-block:: nim -## import os, strutils, times, parseutils, parseopt, hashes, tables, sets - -import os, strutils, times, parseutils, parseopt, hashes, tables, sets - - diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index d40b2fd63..b12ed7cdd 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -7,60 +7,64 @@ # distribution, for details about the copyright. # -## This module implements some common generic algorithms. +## This module implements some common generic algorithms on `openArray`s. ## ## Basic usage ## =========== ## -## .. code-block:: -## import algorithm -## -## type People = tuple -## year: int -## name: string -## -## var a: seq[People] -## -## a.add((2000, "John")) -## a.add((2005, "Marie")) -## a.add((2010, "Jane")) -## -## # Sorting with default system.cmp -## a.sort() -## assert a == @[(year: 2000, name: "John"), (year: 2005, name: "Marie"), -## (year: 2010, name: "Jane")] -## -## proc myCmp(x, y: People): int = -## if x.name < y.name: -1 else: 1 -## -## # Sorting with custom proc -## a.sort(myCmp) -## assert a == @[(year: 2010, name: "Jane"), (year: 2000, name: "John"), -## (year: 2005, name: "Marie")] -## -## + +runnableExamples: + type People = tuple + year: int + name: string + + var a: seq[People] + + a.add((2000, "John")) + a.add((2005, "Marie")) + a.add((2010, "Jane")) + + # Sorting with default system.cmp + a.sort() + assert a == @[(year: 2000, name: "John"), (year: 2005, name: "Marie"), + (year: 2010, name: "Jane")] + + proc myCmp(x, y: People): int = + cmp(x.name, y.name) + + # Sorting with custom proc + a.sort(myCmp) + assert a == @[(year: 2010, name: "Jane"), (year: 2000, name: "John"), + (year: 2005, name: "Marie")] + ## See also ## ======== ## * `sequtils module<sequtils.html>`_ for working with the built-in seq type ## * `tables module<tables.html>`_ for sorting tables +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + + type SortOrder* = enum Descending, Ascending proc `*`*(x: int, order: SortOrder): int {.inline.} = - ## Flips ``x`` if ``order == Descending``. - ## If ``order == Ascending`` then ``x`` is returned. + ## Flips the sign of `x` if `order == Descending`. + ## If `order == Ascending` then `x` is returned. ## - ## ``x`` is supposed to be the result of a comparator, i.e. - ## | ``< 0`` for *less than*, - ## | ``== 0`` for *equal*, - ## | ``> 0`` for *greater than*. + ## `x` is supposed to be the result of a comparator, i.e. + ## | `< 0` for *less than*, + ## | `== 0` for *equal*, + ## | `> 0` for *greater than*. runnableExamples: - assert `*`(-123, Descending) == 123 - assert `*`(123, Descending) == -123 - assert `*`(-123, Ascending) == -123 - assert `*`(123, Ascending) == 123 + assert -123 * Descending == 123 + assert 123 * Descending == -123 + assert -123 * Ascending == -123 + assert 123 * Ascending == 123 var y = order.ord - 1 result = (x xor y) - y @@ -71,20 +75,20 @@ template fillImpl[T](a: var openArray[T], first, last: int, value: T) = inc(x) proc fill*[T](a: var openArray[T], first, last: Natural, value: T) = - ## Fills the slice ``a[first..last]`` with ``value``. + ## Assigns `value` to all elements of the slice `a[first..last]`. ## - ## If an invalid range is passed, it raises IndexError. + ## If an invalid range is passed, it raises `IndexDefect`. runnableExamples: var a: array[6, int] a.fill(1, 3, 9) assert a == [0, 9, 9, 9, 0, 0] a.fill(3, 5, 7) assert a == [0, 9, 9, 7, 7, 7] - doAssertRaises(IndexError, a.fill(1, 7, 9)) + doAssertRaises(IndexDefect, a.fill(1, 7, 9)) fillImpl(a, first, last, value) proc fill*[T](a: var openArray[T], value: T) = - ## Fills the container ``a`` with ``value``. + ## Assigns `value` to all elements of the container `a`. runnableExamples: var a: array[6, int] a.fill(9) @@ -95,20 +99,20 @@ proc fill*[T](a: var openArray[T], value: T) = proc reverse*[T](a: var openArray[T], first, last: Natural) = - ## Reverses the slice ``a[first..last]``. + ## Reverses the slice `a[first..last]`. ## - ## If an invalid range is passed, it raises IndexError. + ## If an invalid range is passed, it raises `IndexDefect`. ## ## **See also:** - ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a ``seq[T]`` - ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a ``seq[T]`` + ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a `seq[T]` + ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a `seq[T]` runnableExamples: var a = [1, 2, 3, 4, 5, 6] a.reverse(1, 3) assert a == [1, 4, 3, 2, 5, 6] a.reverse(1, 3) assert a == [1, 2, 3, 4, 5, 6] - doAssertRaises(IndexError, a.reverse(1, 7)) + doAssertRaises(IndexDefect, a.reverse(1, 7)) var x = first var y = last while x < y: @@ -117,74 +121,63 @@ proc reverse*[T](a: var openArray[T], first, last: Natural) = inc(x) proc reverse*[T](a: var openArray[T]) = - ## Reverses the contents of the container ``a``. + ## Reverses the contents of the container `a`. ## ## **See also:** - ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a ``seq[T]`` - ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a ``seq[T]`` + ## * `reversed proc<#reversed,openArray[T],Natural,int>`_ reverse a slice and returns a `seq[T]` + ## * `reversed proc<#reversed,openArray[T]>`_ reverse and returns a `seq[T]` runnableExamples: var a = [1, 2, 3, 4, 5, 6] a.reverse() assert a == [6, 5, 4, 3, 2, 1] a.reverse() assert a == [1, 2, 3, 4, 5, 6] + # the max is needed, since a.high is -1 if a is empty reverse(a, 0, max(0, a.high)) -proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = - ## Returns the reverse of the slice ``a[first..last]``. - ## - ## If an invalid range is passed, it raises IndexError. +proc reversed*[T](a: openArray[T]): seq[T] {.inline.} = + ## Returns the elements of `a` in reverse order. ## ## **See also:** - ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice ## * `reverse proc<#reverse,openArray[T]>`_ runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = a.reversed(1, 3) - assert b == @[4, 3, 2] - assert last >= first-1 - var i = last - first - var x = first.int - result = newSeq[T](i + 1) - while i >= 0: - result[i] = a[x] - dec(i) - inc(x) + assert [10, 11, 12].reversed == @[12, 11, 10] + assert seq[string].default.reversed == @[] + let n = a.len + result.setLen(n) + for i in 0..<n: result[i] = a[n - (i + 1)] -proc reversed*[T](a: openArray[T]): seq[T] = - ## Returns the reverse of the container ``a``. - ## - ## **See also:** - ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice - ## * `reverse proc<#reverse,openArray[T]>`_ - runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = reversed(a) - assert b == @[6, 5, 4, 3, 2, 1] - reversed(a, 0, a.high) +proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] + {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = + reversed(toOpenArray(a, first, last)) + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} proc binarySearch*[T, K](a: openArray[T], key: K, - cmp: proc (x: T, y: K): int {.closure.}): int = - ## Binary search for ``key`` in ``a``. Returns -1 if not found. + cmp: proc (x: T, y: K): int {.closure.}): int {.effectsOf: cmp.} = + ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. + ## Assumes that `a` is sorted according to `cmp`. ## - ## ``cmp`` is the comparator function to use, the expected return values are - ## the same as that of system.cmp. + ## `cmp` is the comparator function to use, the expected return values are + ## the same as those of system.cmp. runnableExamples: assert binarySearch(["a", "b", "c", "d"], "d", system.cmp[string]) == 3 - assert binarySearch(["a", "b", "d", "c"], "d", system.cmp[string]) == 2 - if a.len == 0: - return -1 - + assert binarySearch(["a", "b", "c", "d"], "c", system.cmp[string]) == 2 let len = a.len + if len == 0: + return -1 + if len == 1: if cmp(a[0], key) == 0: return 0 else: return -1 + result = 0 if (len and (len - 1)) == 0: # when `len` is a power of 2, a faster shr can be used. var step = len shr 1 @@ -195,7 +188,7 @@ proc binarySearch*[T, K](a: openArray[T], key: K, if cmpRes == 0: return i - if cmpRes < 1: + if cmpRes < 0: result = i step = step shr 1 if cmp(a[result], key) != 0: result = -1 @@ -215,34 +208,32 @@ proc binarySearch*[T, K](a: openArray[T], key: K, if result >= len or cmp(a[result], key) != 0: result = -1 proc binarySearch*[T](a: openArray[T], key: T): int = - ## Binary search for ``key`` in ``a``. Returns -1 if not found. + ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. + ## Assumes that `a` is sorted. runnableExamples: assert binarySearch([0, 1, 2, 3, 4], 4) == 4 - assert binarySearch([0, 1, 4, 2, 3], 4) == 2 - binarySearch(a, key, cmp[T]) - -proc smartBinarySearch*[T](a: openArray[T], key: T): int {.deprecated: - "Deprecated since v0.18.1; Use 'binarySearch'".} = + assert binarySearch([0, 1, 2, 3, 4], 2) == 2 binarySearch(a, key, cmp[T]) const onlySafeCode = true -proc lowerBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {. - closure.}): int = - ## Returns a position to the first element in the ``a`` that is greater than - ## ``key``, or last if no such element is found. +proc lowerBound*[T, K](a: openArray[T], key: K, + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = + ## Returns the index of the first element in `a` that is not less than + ## (i.e. greater or equal to) `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call - ## ``insert(thing, elm, lowerBound(thing, elm))`` + ## `insert(thing, elm, lowerBound(thing, elm))` ## the sequence will still be sorted. + ## Assumes that `a` is sorted according to `cmp`. ## - ## If an invalid range is passed, it raises IndexError. + ## If an invalid range is passed, it raises `IndexDefect`. ## - ## The version uses ``cmp`` to compare the elements. - ## The expected return values are the same as that of ``system.cmp``. + ## This version uses `cmp` to compare the elements. + ## The expected return values are the same as those of `system.cmp`. ## ## **See also:** - ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order ## * `upperBound proc<#upperBound,openArray[T],T>`_ runnableExamples: var arr = @[1, 2, 3, 5, 6, 7, 8, 9] @@ -264,33 +255,35 @@ proc lowerBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {. count = step proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) - ## Returns a position to the first element in the ``a`` that is greater than - ## ``key``, or last if no such element is found. + ## Returns the index of the first element in `a` that is not less than + ## (i.e. greater or equal to) `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call - ## ``insert(thing, elm, lowerBound(thing, elm))`` + ## `insert(thing, elm, lowerBound(thing, elm))` ## the sequence will still be sorted. + ## Assumes that `a` is sorted. ## - ## The version uses the default comparison function ``cmp``. + ## This version uses the default comparison function `cmp`. ## ## **See also:** - ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `upperBound proc<#upperBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order ## * `upperBound proc<#upperBound,openArray[T],T>`_ -proc upperBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {. - closure.}): int = - ## Returns a position to the first element in the ``a`` that is not less - ## (i.e. greater or equal to) than ``key``, or last if no such element is found. +proc upperBound*[T, K](a: openArray[T], key: K, + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = + ## Returns the index of the first element in `a` that is greater than + ## `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call - ## ``insert(thing, elm, upperBound(thing, elm))`` + ## `insert(thing, elm, upperBound(thing, elm))` ## the sequence will still be sorted. + ## Assumes that `a` is sorted according to `cmp`. ## - ## If an invalid range is passed, it raises IndexError. + ## If an invalid range is passed, it raises `IndexDefect`. ## - ## The version uses ``cmp`` to compare the elements. The expected - ## return values are the same as that of ``system.cmp``. + ## This version uses `cmp` to compare the elements. The expected + ## return values are the same as those of `system.cmp`. ## ## **See also:** - ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order ## * `lowerBound proc<#lowerBound,openArray[T],T>`_ runnableExamples: var arr = @[1, 2, 3, 5, 6, 7, 8, 9] @@ -312,19 +305,20 @@ proc upperBound*[T, K](a: openArray[T], key: K, cmp: proc(x: T, k: K): int {. count = step proc upperBound*[T](a: openArray[T], key: T): int = upperBound(a, key, cmp[T]) - ## Returns a position to the first element in the ``a`` that is not less - ## (i.e. greater or equal to) than ``key``, or last if no such element is found. + ## Returns the index of the first element in `a` that is greater than + ## `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call - ## ``insert(thing, elm, upperBound(thing, elm))`` + ## `insert(thing, elm, upperBound(thing, elm))` ## the sequence will still be sorted. + ## Assumes that `a` is sorted. ## - ## The version uses the default comparison function ``cmp``. + ## This version uses the default comparison function `cmp`. ## ## **See also:** - ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by ``cmp`` in the specified order + ## * `lowerBound proc<#lowerBound,openArray[T],K,proc(T,K)>`_ sorted by `cmp` in the specified order ## * `lowerBound proc<#lowerBound,openArray[T],T>`_ -template `<-` (a, b) = +template `<-`(a, b) = when defined(gcDestructors): a = move b elif onlySafeCode: @@ -332,12 +326,12 @@ template `<-` (a, b) = else: copyMem(addr(a), addr(b), sizeof(T)) -proc merge[T](a, b: var openArray[T], lo, m, hi: int, - cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = - # optimization: If max(left) <= min(right) there is nothing to do! - # 1 2 3 4 ## 5 6 7 8 +proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, + cmp: proc (x, y: T): int {.closure.}, order: SortOrder) {.effectsOf: cmp.} = + # Optimization: If max(left) <= min(right) there is nothing to do! + # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. - # On random data this safes up to 40% of merge calls + # On random data this saves up to 40% of mergeAlt calls. if cmp(a[m], a[m+1]) * order <= 0: return var j = lo # copy a[j..m] into b: @@ -373,37 +367,38 @@ proc merge[T](a, b: var openArray[T], lo, m, hi: int, func sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Default Nim sort (an implementation of merge sort). The sorting - ## is guaranteed to be stable and the worst case is guaranteed to - ## be O(n log n). + ## is guaranteed to be stable (that is, equal elements stay in the same order) + ## and the worst case is guaranteed to be O(n log n). + ## Sorts by `cmp` in the specified `order`. ## ## The current implementation uses an iterative ## mergesort to achieve this. It uses a temporary sequence of - ## length ``a.len div 2``. If you do not wish to provide your own - ## ``cmp``, you may use ``system.cmp`` or instead call the overloaded - ## version of ``sort``, which uses ``system.cmp``. - ## - ## .. code-block:: nim + ## length `a.len div 2`. If you do not wish to provide your own + ## `cmp`, you may use `system.cmp` or instead call the overloaded + ## version of `sort`, which uses `system.cmp`. ## - ## sort(myIntArray, system.cmp[int]) - ## # do not use cmp[string] here as we want to use the specialized - ## # overload: - ## sort(myStrArray, system.cmp) + ## ```nim + ## sort(myIntArray, system.cmp[int]) + ## # do not use cmp[string] here as we want to use the specialized + ## # overload: + ## sort(myStrArray, system.cmp) + ## ``` ## ## You can inline adhoc comparison procs with the `do notation - ## <manual_experimental.html#do-notation>`_. Example: - ## - ## .. code-block:: nim + ## <manual.html#procedures-do-notation>`_. Example: ## + ## ```nim ## people.sort do (x, y: Person) -> int: ## result = cmp(x.surname, y.surname) ## if result == 0: ## result = cmp(x.name, y.name) + ## ``` ## ## **See also:** ## * `sort proc<#sort,openArray[T]>`_ - ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by ``cmp`` in the specified order + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by `cmp` in the specified order ## * `sorted proc<#sorted,openArray[T]>`_ ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ runnableExamples: @@ -414,29 +409,28 @@ func sort*[T](a: var openArray[T], sort(d, myCmp) assert d == ["fo", "qux", "boo", "barr"] var n = a.len - var b: seq[T] - newSeq(b, n div 2) + var b = newSeq[T](n div 2) var s = 1 while s < n: var m = n-1-s while m >= 0: - merge(a, b, max(m-s+1, 0), m, m+s, cmp, order) + mergeAlt(a, b, max(m-s+1, 0), m, m+s, cmp, order) dec(m, s*2) s = s*2 proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, system.cmp[T], order) - ## Shortcut version of ``sort`` that uses ``system.cmp[T]`` as the comparison function. + ## Shortcut version of `sort` that uses `system.cmp[T]` as the comparison function. ## ## **See also:** ## * `sort func<#sort,openArray[T],proc(T,T)>`_ - ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by ``cmp`` in the specified order + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by `cmp` in the specified order ## * `sorted proc<#sorted,openArray[T]>`_ ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): seq[T] = - ## Returns ``a`` sorted by ``cmp`` in the specified ``order``. + order = SortOrder.Ascending): seq[T] {.effectsOf: cmp.} = + ## Returns `a` sorted by `cmp` in the specified `order`. ## ## **See also:** ## * `sort func<#sort,openArray[T],proc(T,T)>`_ @@ -457,7 +451,7 @@ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, sort(result, cmp, order) proc sorted*[T](a: openArray[T], order = SortOrder.Ascending): seq[T] = - ## Shortcut version of ``sorted`` that uses ``system.cmp[T]`` as the comparison function. + ## Shortcut version of `sorted` that uses `system.cmp[T]` as the comparison function. ## ## **See also:** ## * `sort func<#sort,openArray[T],proc(T,T)>`_ @@ -475,18 +469,18 @@ proc sorted*[T](a: openArray[T], order = SortOrder.Ascending): seq[T] = sorted[T](a, system.cmp[T], order) template sortedByIt*(seq1, op: untyped): untyped = - ## Convenience template around the ``sorted`` proc to reduce typing. + ## Convenience template around the `sorted` proc to reduce typing. ## - ## The template injects the ``it`` variable which you can use directly in an + ## The template injects the `it` variable which you can use directly in an ## expression. ## - ## Because the underlying ``cmp()`` is defined for tuples you can do + ## Because the underlying `cmp()` is defined for tuples you can also do ## a nested sort. ## ## **See also:** ## * `sort func<#sort,openArray[T],proc(T,T)>`_ ## * `sort proc<#sort,openArray[T]>`_ - ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by ``cmp`` in the specified order + ## * `sorted proc<#sorted,openArray[T],proc(T,T)>`_ sorted by `cmp` in the specified order ## * `sorted proc<#sorted,openArray[T]>`_ runnableExamples: type Person = tuple[name: string, age: int] @@ -502,7 +496,7 @@ template sortedByIt*(seq1, op: untyped): untyped = # Nested sort assert people.sortedByIt((it.age, it.name)) == @[(name: "p2", age: 20), (name: "p3", age: 30), (name: "p4", age: 30), (name: "p1", age: 60)] - var result = sorted(seq1, proc(x, y: type(seq1[0])): int = + var result = sorted(seq1, proc(x, y: typeof(items(seq1), typeOfIter)): int = var it {.inject.} = x let a = op it = y @@ -512,10 +506,10 @@ template sortedByIt*(seq1, op: untyped): untyped = func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): bool = - ## Checks to see whether ``a`` is already sorted in ``order`` - ## using ``cmp`` for the comparison. Parameters identical - ## to ``sort``. Requires O(n) time. + order = SortOrder.Ascending): bool {.effectsOf: cmp.} = + ## Checks to see whether `a` is already sorted in `order` + ## using `cmp` for the comparison. The parameters are identical + ## to `sort`. Requires O(n) time. ## ## **See also:** ## * `isSorted proc<#isSorted,openArray[T]>`_ @@ -538,7 +532,7 @@ func isSorted*[T](a: openArray[T], return false proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = - ## Shortcut version of ``isSorted`` that uses ``system.cmp[T]`` as the comparison function. + ## Shortcut version of `isSorted` that uses `system.cmp[T]` as the comparison function. ## ## **See also:** ## * `isSorted func<#isSorted,openArray[T],proc(T,T)>`_ @@ -557,47 +551,142 @@ proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = assert isSorted(e) == false isSorted(a, system.cmp[T], order) +proc merge*[T]( + result: var seq[T], + x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} +) {.since: (1, 5, 1), effectsOf: cmp.} = + ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. + ## If you do not wish to provide your own `cmp`, + ## you may use `system.cmp` or instead call the overloaded + ## version of `merge`, which uses `system.cmp`. + ## + ## .. note:: The original data of `result` is not cleared, + ## new data is appended to `result`. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T]>`_ + runnableExamples: + let x = @[1, 3, 6] + let y = @[2, 3, 4] + + block: + var merged = @[7] # new data is appended to merged sequence + merged.merge(x, y, system.cmp[int]) + assert merged == @[7, 1, 2, 3, 3, 4, 6] + + block: + var merged = @[7] # if you only want new data, clear merged sequence first + merged.setLen(0) + merged.merge(x, y, system.cmp[int]) + assert merged.isSorted + assert merged == @[1, 2, 3, 3, 4, 6] + + import std/sugar + + var res: seq[(int, int)] + res.merge([(1, 1)], [(1, 2)], (a, b) => a[0] - b[0]) + assert res == @[(1, 1), (1, 2)] + + assert seq[int].default.dup(merge([1, 3], [2, 4])) == @[1, 2, 3, 4] + + let + sizeX = x.len + sizeY = y.len + oldLen = result.len + + result.setLen(oldLen + sizeX + sizeY) + + var + ix = 0 + iy = 0 + i = oldLen + + while true: + if ix == sizeX: + while iy < sizeY: + result[i] = y[iy] + inc i + inc iy + return + + if iy == sizeY: + while ix < sizeX: + result[i] = x[ix] + inc i + inc ix + return + + let itemX = x[ix] + let itemY = y[iy] + + if cmp(itemX, itemY) > 0: # to have a stable sort + result[i] = itemY + inc iy + else: + result[i] = itemX + inc ix + + inc i + +proc merge*[T](result: var seq[T], x, y: openArray[T]) {.inline, since: (1, 5, 1).} = + ## Shortcut version of `merge` that uses `system.cmp[T]` as the comparison function. + ## + ## **See also:** + ## * `merge proc<#merge,seq[T],openArray[T],openArray[T],proc(T,T)>`_ + runnableExamples: + let x = [5, 10, 15, 20, 25] + let y = [50, 40, 30, 20, 10].sorted + + var merged: seq[int] + merged.merge(x, y) + assert merged.isSorted + assert merged == @[5, 10, 10, 15, 20, 20, 25, 30, 40, 50] + merge(result, x, y, system.cmp) + proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = - ## Produces the Cartesian product of the array. Warning: complexity - ## may explode. + ## Produces the Cartesian product of the array. + ## Every element of the result is a combination of one element from each seq in `x`, + ## with the ith element coming from `x[i]`. + ## + ## .. warning:: complexity may explode. runnableExamples: assert product(@[@[1], @[2]]) == @[@[1, 2]] assert product(@[@["A", "K"], @["Q"]]) == @[@["K", "Q"], @["A", "Q"]] + let xLen = x.len result = newSeq[seq[T]]() - if x.len == 0: + if xLen == 0: return - if x.len == 1: + if xLen == 1: result = @x return var - indexes = newSeq[int](x.len) - initial = newSeq[int](x.len) + indices = newSeq[int](xLen) + initial = newSeq[int](xLen) index = 0 - var next = newSeq[T]() - next.setLen(x.len) - for i in 0..(x.len-1): + var next = newSeq[T](xLen) + for i in 0 ..< xLen: if len(x[i]) == 0: return - initial[i] = len(x[i])-1 - indexes = initial + initial[i] = len(x[i]) - 1 + indices = initial while true: - while indexes[index] == -1: - indexes[index] = initial[index] + while indices[index] == -1: + indices[index] = initial[index] index += 1 - if index == x.len: return - indexes[index] -= 1 - for ni, i in indexes: + if index == xLen: return + indices[index] -= 1 + for ni, i in indices: next[ni] = x[ni][i] result.add(next) index = 0 - indexes[index] -= 1 + indices[index] -= 1 proc nextPermutation*[T](x: var openArray[T]): bool {.discardable.} = - ## Calculates the next lexicographic permutation, directly modifying ``x``. + ## Calculates the next lexicographic permutation, directly modifying `x`. ## The result is whether a permutation happened, otherwise we have reached ## the last-ordered permutation. ## ## If you start with an unsorted array/seq, the repeated permutations - ## will **not** give you all permutations but stop with last. + ## will **not** give you all permutations but stop with the last. ## ## **See also:** ## * `prevPermutation proc<#prevPermutation,openArray[T]>`_ @@ -633,7 +722,7 @@ proc nextPermutation*[T](x: var openArray[T]): bool {.discardable.} = proc prevPermutation*[T](x: var openArray[T]): bool {.discardable.} = ## Calculates the previous lexicographic permutation, directly modifying - ## ``x``. The result is whether a permutation happened, otherwise we have + ## `x`. The result is whether a permutation happened, otherwise we have ## reached the first-ordered permutation. ## ## **See also:** @@ -666,38 +755,9 @@ proc prevPermutation*[T](x: var openArray[T]): bool {.discardable.} = result = true -when isMainModule: - # Tests for lowerBound - var arr = @[1, 2, 3, 5, 6, 7, 8, 9] - assert arr.lowerBound(0) == 0 - assert arr.lowerBound(4) == 3 - assert arr.lowerBound(5) == 3 - assert arr.lowerBound(10) == 8 - arr = @[1, 5, 10] - assert arr.lowerBound(4) == 1 - assert arr.lowerBound(5) == 1 - assert arr.lowerBound(6) == 2 - # Tests for isSorted - var srt1 = [1, 2, 3, 4, 4, 4, 4, 5] - var srt2 = ["iello", "hello"] - var srt3 = [1.0, 1.0, 1.0] - var srt4: seq[int] - assert srt1.isSorted(cmp) == true - assert srt2.isSorted(cmp) == false - assert srt3.isSorted(cmp) == true - assert srt4.isSorted(cmp) == true - var srtseq = newSeq[int]() - assert srtseq.isSorted(cmp) == true - # Tests for reversed - var arr1 = @[0, 1, 2, 3, 4] - assert arr1.reversed() == @[4, 3, 2, 1, 0] - for i in 0 .. high(arr1): - assert arr1.reversed(0, i) == arr1.reversed()[high(arr1) - i .. high(arr1)] - assert arr1.reversed(i, high(arr1)) == arr1.reversed()[0 .. high(arr1) - i] - - proc rotateInternal[T](arg: var openArray[T]; first, middle, last: int): int = - ## A port of std::rotate from c++. Ported from `this reference <http://www.cplusplus.com/reference/algorithm/rotate/>`_. + ## A port of std::rotate from C++. + ## Ported from [this reference](http://www.cplusplus.com/reference/algorithm/rotate/). result = first + last - middle if first == middle or middle == last: @@ -736,7 +796,8 @@ proc rotateInternal[T](arg: var openArray[T]; first, middle, last: int): int = next = mMiddle proc rotatedInternal[T](arg: openArray[T]; first, middle, last: int): seq[T] = - result = newSeq[T](arg.len) + let argLen = arg.len + result = newSeq[T](argLen) for i in 0 ..< first: result[i] = arg[i] let n = last - middle @@ -745,34 +806,34 @@ proc rotatedInternal[T](arg: openArray[T]; first, middle, last: int): seq[T] = result[first+i] = arg[middle+i] for i in 0 ..< m: result[first+n+i] = arg[first+i] - for i in last ..< arg.len: + for i in last ..< argLen: result[i] = arg[i] proc rotateLeft*[T](arg: var openArray[T]; slice: HSlice[int, int]; - dist: int): int {.discardable.} = + dist: int): int {.discardable.} = ## Performs a left rotation on a range of elements. If you want to rotate - ## right, use a negative ``dist``. Specifically, ``rotateLeft`` rotates - ## the elements at ``slice`` by ``dist`` positions. + ## right, use a negative `dist`. Specifically, `rotateLeft` rotates + ## the elements at `slice` by `dist` positions. ## - ## | The element at index ``slice.a + dist`` will be at index ``slice.a``. - ## | The element at index ``slice.b`` will be at ``slice.a + dist -1``. - ## | The element at index ``slice.a`` will be at ``slice.b + 1 - dist``. - ## | The element at index ``slice.a + dist - 1`` will be at ``slice.b``. + ## | The element at index `slice.a + dist` will be at index `slice.a`. + ## | The element at index `slice.b` will be at `slice.a + dist - 1`. + ## | The element at index `slice.a` will be at `slice.b + 1 - dist`. + ## | The element at index `slice.a + dist - 1` will be at `slice.b`. ## - ## Elements outside of ``slice`` will be left unchanged. - ## The time complexity is linear to ``slice.b - slice.a + 1``. - ## If an invalid range (``HSlice``) is passed, it raises IndexError. + ## Elements outside of `slice` will be left unchanged. + ## The time complexity is linear to `slice.b - slice.a + 1`. + ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. ## - ## ``slice`` - ## The indices of the element range that should be rotated. + ## `slice` + ## : The indices of the element range that should be rotated. ## - ## ``dist`` - ## The distance in amount of elements that the data should be rotated. + ## `dist` + ## : The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## ## **See also:** ## * `rotateLeft proc<#rotateLeft,openArray[T],int>`_ for a version which rotates the whole container - ## * `rotatedLeft proc<#rotatedLeft,openArray[T],HSlice[int,int],int>`_ for a version which returns a ``seq[T]`` + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],HSlice[int,int],int>`_ for a version which returns a `seq[T]` runnableExamples: var a = [0, 1, 2, 3, 4, 5] a.rotateLeft(1 .. 4, 3) @@ -781,18 +842,19 @@ proc rotateLeft*[T](arg: var openArray[T]; slice: HSlice[int, int]; assert a == [0, 3, 4, 1, 2, 5] a.rotateLeft(1 .. 4, -3) assert a == [0, 4, 1, 2, 3, 5] - doAssertRaises(IndexError, a.rotateLeft(1 .. 7, 2)) + doAssertRaises(IndexDefect, a.rotateLeft(1 .. 7, 2)) let sliceLen = slice.b + 1 - slice.a let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen - arg.rotateInternal(slice.a, slice.a+distLeft, slice.b + 1) + arg.rotateInternal(slice.a, slice.a + distLeft, slice.b + 1) proc rotateLeft*[T](arg: var openArray[T]; dist: int): int {.discardable.} = - ## Default arguments for slice, so that this procedure operates on the entire - ## ``arg``, and not just on a part of it. + ## Same as `rotateLeft`, but with default arguments for slice, + ## so that this procedure operates on the entire + ## `arg`, and not just on a part of it. ## ## **See also:** ## * `rotateLeft proc<#rotateLeft,openArray[T],HSlice[int,int],int>`_ for a version which rotates a range - ## * `rotatedLeft proc<#rotatedLeft,openArray[T],int>`_ for a version which returns a ``seq[T]`` + ## * `rotatedLeft proc<#rotatedLeft,openArray[T],int>`_ for a version which returns a `seq[T]` runnableExamples: var a = [1, 2, 3, 4, 5] a.rotateLeft(2) @@ -801,23 +863,23 @@ proc rotateLeft*[T](arg: var openArray[T]; dist: int): int {.discardable.} = assert a == [2, 3, 4, 5, 1] a.rotateLeft(-6) assert a == [1, 2, 3, 4, 5] - let arglen = arg.len - let distLeft = ((dist mod arglen) + arglen) mod arglen - arg.rotateInternal(0, distLeft, arglen) + let argLen = arg.len + let distLeft = ((dist mod argLen) + argLen) mod argLen + arg.rotateInternal(0, distLeft, argLen) proc rotatedLeft*[T](arg: openArray[T]; slice: HSlice[int, int], - dist: int): seq[T] = - ## Same as ``rotateLeft``, just with the difference that it does - ## not modify the argument. It creates a new ``seq`` instead. + dist: int): seq[T] = + ## Same as `rotateLeft`, just with the difference that it does + ## not modify the argument. It creates a new `seq` instead. ## - ## Elements outside of ``slice`` will be left unchanged. - ## If an invalid range (``HSlice``) is passed, it raises IndexError. + ## Elements outside of `slice` will be left unchanged. + ## If an invalid range (`HSlice`) is passed, it raises `IndexDefect`. ## - ## ``slice`` - ## The indices of the element range that should be rotated. + ## `slice` + ## : The indices of the element range that should be rotated. ## - ## ``dist`` - ## The distance in amount of elements that the data should be rotated. + ## `dist` + ## : The distance in amount of elements that the data should be rotated. ## Can be negative, can be any number. ## ## **See also:** @@ -833,11 +895,11 @@ proc rotatedLeft*[T](arg: openArray[T]; slice: HSlice[int, int], assert a == @[1, 5, 2, 3, 4] let sliceLen = slice.b + 1 - slice.a let distLeft = ((dist mod sliceLen) + sliceLen) mod sliceLen - arg.rotatedInternal(slice.a, slice.a+distLeft, slice.b+1) + arg.rotatedInternal(slice.a, slice.a + distLeft, slice.b + 1) proc rotatedLeft*[T](arg: openArray[T]; dist: int): seq[T] = - ## Same as ``rotateLeft``, just with the difference that it does - ## not modify the argument. It creates a new ``seq`` instead. + ## Same as `rotateLeft`, just with the difference that it does + ## not modify the argument. It creates a new `seq` instead. ## ## **See also:** ## * `rotateLeft proc<#rotateLeft,openArray[T],int>`_ for the in-place version of this proc @@ -850,74 +912,6 @@ proc rotatedLeft*[T](arg: openArray[T]; dist: int): seq[T] = assert a == @[2, 3, 4, 5, 1] a = rotatedLeft(a, -6) assert a == @[1, 2, 3, 4, 5] - let arglen = arg.len - let distLeft = ((dist mod arglen) + arglen) mod arglen - arg.rotatedInternal(0, distLeft, arg.len) - -when isMainModule: - var list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - let list2 = list.rotatedLeft(1 ..< 9, 3) - let expected = [0, 4, 5, 6, 7, 8, 1, 2, 3, 9, 10] - - doAssert list.rotateLeft(1 ..< 9, 3) == 6 - doAssert list == expected - doAssert list2 == @expected - - var s0, s1, s2, s3, s4, s5 = "xxxabcdefgxxx" - - doAssert s0.rotateLeft(3 ..< 10, 3) == 7 - doAssert s0 == "xxxdefgabcxxx" - doAssert s1.rotateLeft(3 ..< 10, 2) == 8 - doAssert s1 == "xxxcdefgabxxx" - doAssert s2.rotateLeft(3 ..< 10, 4) == 6 - doAssert s2 == "xxxefgabcdxxx" - doAssert s3.rotateLeft(3 ..< 10, -3) == 6 - doAssert s3 == "xxxefgabcdxxx" - doAssert s4.rotateLeft(3 ..< 10, -10) == 6 - doAssert s4 == "xxxefgabcdxxx" - doAssert s5.rotateLeft(3 ..< 10, 11) == 6 - doAssert s5 == "xxxefgabcdxxx" - - block product: - doAssert product(newSeq[seq[int]]()) == newSeq[seq[int]](), "empty input" - doAssert product(@[newSeq[int](), @[], @[]]) == newSeq[seq[int]](), "bit more empty input" - doAssert product(@[@[1, 2]]) == @[@[1, 2]], "a simple case of one element" - doAssert product(@[@[1, 2], @[3, 4]]) == @[@[2, 4], @[1, 4], @[2, 3], @[1, - 3]], "two elements" - doAssert product(@[@[1, 2], @[3, 4], @[5, 6]]) == @[@[2, 4, 6], @[1, 4, 6], - @[2, 3, 6], @[1, 3, 6], @[2, 4, 5], @[1, 4, 5], @[2, 3, 5], @[1, 3, 5]], "three elements" - doAssert product(@[@[1, 2], @[]]) == newSeq[seq[int]](), "two elements, but one empty" - - block lowerBound: - doAssert lowerBound([1, 2, 4], 3, system.cmp[int]) == 2 - doAssert lowerBound([1, 2, 2, 3], 4, system.cmp[int]) == 4 - doAssert lowerBound([1, 2, 3, 10], 11) == 4 - - block upperBound: - doAssert upperBound([1, 2, 4], 3, system.cmp[int]) == 2 - doAssert upperBound([1, 2, 2, 3], 3, system.cmp[int]) == 4 - doAssert upperBound([1, 2, 3, 5], 3) == 3 - - block fillEmptySeq: - var s = newSeq[int]() - s.fill(0) - - block testBinarySearch: - var noData: seq[int] - doAssert binarySearch(noData, 7) == -1 - let oneData = @[1] - doAssert binarySearch(oneData, 1) == 0 - doAssert binarySearch(onedata, 7) == -1 - let someData = @[1, 3, 4, 7] - doAssert binarySearch(someData, 1) == 0 - doAssert binarySearch(somedata, 7) == 3 - doAssert binarySearch(someData, -1) == -1 - doAssert binarySearch(someData, 5) == -1 - doAssert binarySearch(someData, 13) == -1 - let moreData = @[1, 3, 5, 7, 4711] - doAssert binarySearch(moreData, -1) == -1 - doAssert binarySearch(moreData, 1) == 0 - doAssert binarySearch(moreData, 5) == 2 - doAssert binarySearch(moreData, 6) == -1 - doAssert binarySearch(moreData, 4711) == 4 - doAssert binarySearch(moreData, 4712) == -1 + let argLen = arg.len + let distLeft = ((dist mod argLen) + argLen) mod argLen + arg.rotatedInternal(0, distLeft, argLen) diff --git a/lib/pure/async.nim b/lib/pure/async.nim index 97b29f81d..e4d8d41c3 100644 --- a/lib/pure/async.nim +++ b/lib/pure/async.nim @@ -1,6 +1,9 @@ +## Exports [asyncmacro](asyncmacro.html) and [asyncfutures](asyncfutures.html) for native backends, +## and [asyncjs](asyncjs.html) on the JS backend. + when defined(js): - import asyncjs - export asyncjs + import std/asyncjs + export asyncjs else: - import asyncmacro, asyncfutures - export asyncmacro, asyncfutures + import std/[asyncmacro, asyncfutures] + export asyncmacro, asyncfutures diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index d724242d7..126db7a7f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -8,25 +8,25 @@ # ## This module implements asynchronous IO. This includes a dispatcher, -## a ``Future`` type implementation, and an ``async`` macro which allows -## asynchronous code to be written in a synchronous style with the ``await`` +## a `Future` type implementation, and an `async` macro which allows +## asynchronous code to be written in a synchronous style with the `await` ## keyword. ## -## The dispatcher acts as a kind of event loop. You must call ``poll`` on it -## (or a function which does so for you such as ``waitFor`` or ``runForever``) +## The dispatcher acts as a kind of event loop. You must call `poll` on it +## (or a function which does so for you such as `waitFor` or `runForever`) ## in order to poll for any outstanding events. The underlying implementation ## is based on epoll on Linux, IO Completion Ports on Windows and select on ## other operating systems. ## -## The ``poll`` function will not, on its own, return any events. Instead -## an appropriate ``Future`` object will be completed. A ``Future`` is a +## The `poll` function will not, on its own, return any events. Instead +## an appropriate `Future` object will be completed. A `Future` is a ## type which holds a value which is not yet available, but which *may* be ## available in the future. You can check whether a future is finished -## by using the ``finished`` function. When a future is finished it means that +## by using the `finished` function. When a future is finished it means that ## either the value that it holds is now available or it holds an error instead. ## The latter situation occurs when the operation to complete a future fails ## with an exception. You can distinguish between the two situations with the -## ``failed`` function. +## `failed` function. ## ## Future objects can also store a callback procedure which will be called ## automatically once the future completes. @@ -35,34 +35,34 @@ ## pattern. In this ## pattern you make a request for an action, and once that action is fulfilled ## a future is completed with the result of that action. Requests can be -## made by calling the appropriate functions. For example: calling the ``recv`` +## made by calling the appropriate functions. For example: calling the `recv` ## function will create a request for some data to be read from a socket. The -## future which the ``recv`` function returns will then complete once the +## future which the `recv` function returns will then complete once the ## requested amount of data is read **or** an exception occurs. ## ## Code to read some data from a socket may look something like this: +## ```Nim +## var future = socket.recv(100) +## future.addCallback( +## proc () = +## echo(future.read) +## ) +## ``` ## -## .. code-block::nim -## var future = socket.recv(100) -## future.addCallback( -## proc () = -## echo(future.read) -## ) -## -## All asynchronous functions returning a ``Future`` will not block. They +## All asynchronous functions returning a `Future` will not block. They ## will not however return immediately. An asynchronous function will have ## code which will be executed before an asynchronous request is made, in most ## cases this code sets up the request. ## -## In the above example, the ``recv`` function will return a brand new -## ``Future`` instance once the request for data to be read from the socket -## is made. This ``Future`` instance will complete once the requested amount +## In the above example, the `recv` function will return a brand new +## `Future` instance once the request for data to be read from the socket +## is made. This `Future` instance will complete once the requested amount ## of data is read, in this case it is 100 bytes. The second line sets a ## callback on this future which will be called once the future completes. -## All the callback does is write the data stored in the future to ``stdout``. -## The ``read`` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. +## All the callback does is write the data stored in the future to `stdout`. +## The `read` function is used for this and it checks whether the future +## completes with an error for you (if it did, it will simply raise the +## error), if there is no error, however, it returns the value of the future. ## ## Asynchronous procedures ## ======================= @@ -71,14 +71,14 @@ ## this by allowing you to write asynchronous code the same way as you would ## write synchronous code. ## -## An asynchronous procedure is marked using the ``{.async.}`` pragma. -## When marking a procedure with the ``{.async.}`` pragma it must have a -## ``Future[T]`` return type or no return type at all. If you do not specify -## a return type then ``Future[void]`` is assumed. +## An asynchronous procedure is marked using the `{.async.}` pragma. +## When marking a procedure with the `{.async.}` pragma it must have a +## `Future[T]` return type or no return type at all. If you do not specify +## a return type then `Future[void]` is assumed. ## -## Inside asynchronous procedures ``await`` can be used to call any +## Inside asynchronous procedures `await` can be used to call any ## procedures which return a -## ``Future``; this includes asynchronous procedures. When a procedure is +## `Future`; this includes asynchronous procedures. When a procedure is ## "awaited", the asynchronous procedure it is awaited in will ## suspend its execution ## until the awaited procedure's Future completes. At which point the @@ -86,52 +86,83 @@ ## when an asynchronous procedure is suspended other asynchronous procedures ## will be run by the dispatcher. ## -## The ``await`` call may be used in many contexts. It can be used on the right -## hand side of a variable declaration: ``var data = await socket.recv(100)``, +## The `await` call may be used in many contexts. It can be used on the right +## hand side of a variable declaration: `var data = await socket.recv(100)`, ## in which case the variable will be set to the value of the future -## automatically. It can be used to await a ``Future`` object, and it can -## be used to await a procedure returning a ``Future[void]``: -## ``await socket.send("foobar")``. +## automatically. It can be used to await a `Future` object, and it can +## be used to await a procedure returning a `Future[void]`: +## `await socket.send("foobar")`. ## -## If an awaited future completes with an error, then ``await`` will re-raise -## this error. To avoid this, you can use the ``yield`` keyword instead of -## ``await``. The following section shows different ways that you can handle +## If an awaited future completes with an error, then `await` will re-raise +## this error. To avoid this, you can use the `yield` keyword instead of +## `await`. The following section shows different ways that you can handle ## exceptions in async procs. ## +## .. caution:: +## Procedures marked {.async.} do not support mutable parameters such +## as `var int`. References such as `ref int` should be used instead. +## ## Handling Exceptions ## ------------------- ## -## The most reliable way to handle exceptions is to use ``yield`` on a future -## then check the future's ``failed`` property. For example: +## You can handle exceptions in the same way as in ordinary Nim code; +## by using the try statement: ## -## .. code-block:: Nim -## var future = sock.recv(100) -## yield future -## if future.failed: -## # Handle exception +## ```Nim +## try: +## let data = await sock.recv(100) +## echo("Received ", data) +## except: +## # Handle exception +## ``` ## -## The ``async`` procedures also offer limited support for the try statement. +## An alternative approach to handling exceptions is to use `yield` on a future +## then check the future's `failed` property. For example: ## -## .. code-block:: Nim -## try: -## let data = await sock.recv(100) -## echo("Received ", data) -## except: -## # Handle exception -## -## Unfortunately the semantics of the try statement may not always be correct, -## and occasionally the compilation may fail altogether. -## As such it is better to use the former style when possible. +## ```Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception +## ``` ## ## ## Discarding futures ## ================== ## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. Note -## however that this does not wait for completion, and you should use -## ``waitFor`` for that purpose. +## Futures should **never** be discarded directly because they may contain +## errors. If you do not care for the result of a Future then you should use +## the `asyncCheck` procedure instead of the `discard` keyword. Note that this +## does not wait for completion, and you should use `waitFor` or `await` for that purpose. +## +## .. note:: `await` also checks if the future fails, so you can safely discard +## its result. +## +## Handling futures +## ================ +## +## There are many different operations that apply to a future. +## The three primary high-level operations are `asyncCheck`, +## `waitFor`, and `await`. +## +## * `asyncCheck`: Raises an exception if the future fails. It neither waits +## for the future to finish nor returns the result of the future. +## * `waitFor`: Polls the event loop and blocks the current thread until the +## future finishes. This is often used to call an async procedure from a +## synchronous context and should never be used in an `async` proc. +## * `await`: Pauses execution in the current async procedure until the future +## finishes. While the current procedure is paused, other async procedures will +## continue running. Should be used instead of `waitFor` in an async +## procedure. +## +## Here is a handy quick reference chart showing their high-level differences: +## ============== ===================== ======================= +## Procedure Context Blocking +## ============== ===================== ======================= +## `asyncCheck` non-async and async non-blocking +## `waitFor` non-async blocks current thread +## `await` async suspends current proc +## ============== ===================== ======================= ## ## Examples ## ======== @@ -144,14 +175,14 @@ ## ============================= ## ## It's possible to get into a situation where an async proc, or more accurately -## a ``Future[T]`` gets stuck and +## a `Future[T]` gets stuck and ## never completes. This can happen for various reasons and can cause serious ## memory leaks. When this occurs it's hard to identify the procedure that is ## stuck. ## ## Thankfully there is a mechanism which tracks the count of each pending future. -## All you need to do to enable it is compile with ``-d:futureLogging`` and -## use the ``getFuturesInProgress`` procedure to get the list of pending futures +## All you need to do to enable it is compile with `-d:futureLogging` and +## use the `getFuturesInProgress` procedure to get the list of pending futures ## together with the stack traces to the moment of their creation. ## ## You may also find it useful to use this @@ -164,22 +195,50 @@ ## Limitations/Bugs ## ================ ## -## * The effect system (``raises: []``) does not work with async procedures. +## * The effect system (`raises: []`) does not work with async procedures. +## * Mutable parameters are not supported by async procedures. +## +## +## Multiple async backend support +## ============================== +## +## Thanks to its powerful macro support, Nim allows ``async``/``await`` to be +## implemented in libraries with only minimal support from the language - as +## such, multiple ``async`` libraries exist, including ``asyncdispatch`` and +## ``chronos``, and more may come to be developed in the future. +## +## Libraries built on top of async/await may wish to support multiple async +## backends - the best way to do so is to create separate modules for each backend +## that may be imported side-by-side. +## +## An alternative way is to select backend using a global compile flag - this +## method makes it difficult to compose applications that use both backends as may +## happen with transitive dependencies, but may be appropriate in some cases - +## libraries choosing this path should call the flag `asyncBackend`, allowing +## applications to choose the backend with `-d:asyncBackend=<backend_name>`. +## +## Known `async` backends include: +## +## * `-d:asyncBackend=none`: disable `async` support completely +## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html +## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/ +## +## ``none`` can be used when a library supports both a synchronous and +## asynchronous API, to disable the latter. -include "system/inclrtl" +import std/[os, tables, strutils, times, heapqueue, options, asyncstreams] +import std/[math, monotimes] +import std/asyncfutures except callSoon -import os, tables, strutils, times, heapqueue, lists, options, asyncstreams -import options, math, std/monotimes -import asyncfutures except callSoon +import std/[nativesockets, net, deques] -import nativesockets, net, deques +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] export Port, SocketFlag export asyncfutures except callSoon export asyncstreams -#{.injectStmt: newGcInvariant().} - # TODO: Check if yielded future is nil and throw a more meaningful exception type @@ -222,6 +281,8 @@ proc adjustTimeout( result = max(nextTimer.get(), 0) result = min(pollTimeout, result) +proc runOnce(timeout: int): bool {.gcsafe.} + proc callSoon*(cbproc: proc () {.gcsafe.}) {.gcsafe.} ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. @@ -230,8 +291,18 @@ proc initCallSoonProc = if asyncfutures.getCallSoonProc().isNil: asyncfutures.setCallSoonProc(callSoon) +template implementSetInheritable() {.dirty.} = + when declared(setInheritable): + proc setInheritable*(fd: AsyncFD, inheritable: bool): bool = + ## Control whether a file handle can be inherited by child processes. + ## Returns `true` on success. + ## + ## This procedure is not guaranteed to be available for all platforms. + ## Test for availability with `declared() <system.html#declared,untyped>`_. + fd.FileHandle.setInheritable(inheritable) + when defined(windows) or defined(nimdoc): - import winlean, sets, hashes + import std/[winlean, sets, hashes] type CompletionKey = ULONG_PTR @@ -245,7 +316,7 @@ when defined(windows) or defined(nimdoc): PDispatcher* = ref object of PDispatcherBase ioPort: Handle - handles: HashSet[AsyncFD] + handles*: HashSet[AsyncFD] # Export handles so that an external library can register them. CustomObj = object of OVERLAPPED data*: CompletionData @@ -267,7 +338,7 @@ when defined(windows) or defined(nimdoc): pcd: PostCallbackDataPtr AsyncEvent* = ptr AsyncEventImpl - Callback = proc (fd: AsyncFD): bool {.closure, gcsafe.} + Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.} proc hash(x: AsyncFD): Hash {.borrow.} proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} @@ -276,13 +347,13 @@ when defined(windows) or defined(nimdoc): ## Creates a new Dispatcher instance. new result result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) - result.handles = initSet[AsyncFD]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](64) + result.handles = initHashSet[AsyncFD]() + result.timers.clear() + result.callbacks = initDeque[proc () {.closure, gcsafe.}](64) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher - proc setGlobalDispatcher*(disp: owned PDispatcher) = + proc setGlobalDispatcher*(disp: sink PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 gDisp = disp @@ -299,7 +370,7 @@ when defined(windows) or defined(nimdoc): return disp.ioPort proc register*(fd: AsyncFD) = - ## Registers ``fd`` with the dispatcher. + ## Registers `fd` with the dispatcher. let p = getGlobalDispatcher() if createIoCompletionPort(fd.Handle, p.ioPort, @@ -321,7 +392,7 @@ when defined(windows) or defined(nimdoc): let p = getGlobalDispatcher() p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: raise newException(ValueError, @@ -398,7 +469,7 @@ when defined(windows) or defined(nimdoc): addr bytesRet, nil, nil) == 0 proc initAll() = - let dummySock = newNativeSocket() + let dummySock = createNativeSocket() if dummySock == INVALID_SOCKET: raiseOSError(osLastError()) var fun: pointer = nil @@ -422,16 +493,16 @@ when defined(windows) or defined(nimdoc): proc recv*(socket: AsyncFD, size: int, flags = {SocketFlag.SafeDisconn}): owned(Future[string]) = - ## Reads **up to** ``size`` bytes from ``socket``. Returned future will + ## Reads **up to** `size` bytes from `socket`. Returned future will ## complete once all the data requested is read, a part of the data has been ## read, or the socket has disconnected in which case the future will - ## complete with a value of ``""``. + ## complete with a value of `""`. ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + ## .. warning:: The `Peek` socket flag is not supported on Windows. # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very + # * When WSARecv completes immediately then `bytesReceived` is very # unreliable. # * Still need to implement message-oriented socket disconnection, # '\0' in the message currently signifies a socket disconnect. Who @@ -462,7 +533,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dealloc dataBuf.buf dataBuf.buf = nil @@ -480,7 +551,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -495,17 +566,17 @@ when defined(windows) or defined(nimdoc): proc recvInto*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must + ## Reads **up to** `size` bytes from `socket` into `buf`, which must ## at least be of that size. Returned future will complete once all the ## data requested is read, a part of the data has been read, or the socket ## has disconnected in which case the future will complete with a value of - ## ``0``. + ## `0`. ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. + ## .. warning:: The `Peek` socket flag is not supported on Windows. # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very + # * When WSARecv completes immediately then `bytesReceived` is very # unreliable. # * Still need to implement message-oriented socket disconnection, # '\0' in the message currently signifies a socket disconnect. Who @@ -532,7 +603,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(errcode): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if dataBuf.buf != nil: dataBuf.buf = nil ) @@ -548,7 +619,7 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) elif ret == 0: # Request completed immediately. if bytesReceived != 0: @@ -561,11 +632,11 @@ when defined(windows) or defined(nimdoc): proc send*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = - ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future + ## Sends `size` bytes from `buf` to `socket`. The returned future ## will complete once all data has been sent. ## - ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, - ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. + ## .. warning:: Use it with caution. If `buf` refers to GC'ed object, + ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. verifyPresence(socket) var retFuture = newFuture[void]("send") @@ -596,19 +667,19 @@ when defined(windows) or defined(nimdoc): if flags.isDisconnectionError(err): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed + # We don't deallocate `ol` here because even though this completed # immediately poll will still be notified about its completion and it will - # free ``ol``. + # free `ol`. return retFuture proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, saddrLen: SockLen, flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = - ## Sends ``data`` to specified destination ``saddr``, using - ## socket ``socket``. The returned future will complete once all data + ## Sends `data` to specified destination `saddr`, using + ## socket `socket`. The returned future will complete once all data ## has been sent. verifyPresence(socket) var retFuture = newFuture[void]("sendTo") @@ -631,7 +702,7 @@ when defined(windows) or defined(nimdoc): if errcode == OSErrorCode(-1): retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, @@ -641,20 +712,20 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed + # We don't deallocate `ol` here because even though this completed # immediately poll will still be notified about its completion and it will - # free ``ol``. + # free `ol`. return retFuture proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, saddrLen: ptr SockLen, flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = - ## Receives a datagram data from ``socket`` into ``buf``, which must - ## be at least of size ``size``, address of datagram's sender will be - ## stored into ``saddr`` and ``saddrLen``. Returned future will complete + ## Receives a datagram data from `socket` into `buf`, which must + ## be at least of size `size`, address of datagram's sender will be + ## stored into `saddr` and `saddrLen`. Returned future will complete ## once one datagram has been received, and will return size of packet ## received. verifyPresence(socket) @@ -675,7 +746,7 @@ when defined(windows) or defined(nimdoc): else: # datagram sockets don't have disconnection, # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, @@ -686,7 +757,7 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. if bytesReceived != 0: @@ -697,8 +768,9 @@ when defined(windows) or defined(nimdoc): retFuture.complete(bytesReceived) return retFuture - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - owned(Future[tuple[address: string, client: AsyncFD]]) = + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): + owned(Future[tuple[address: string, client: AsyncFD]]) {.gcsafe.} = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. @@ -706,14 +778,17 @@ when defined(windows) or defined(nimdoc): ## The resulting client socket is automatically registered to the ## dispatcher. ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## If `inheritable` is false (the default), the resulting client socket will + ## not be inheritable by child processes. + ## + ## The `accept` call may result in an error if the connecting socket + ## disconnects during the duration of the `accept`. If the `SafeDisconn` ## flag is specified then this error will not be raised and instead ## accept will be called again. verifyPresence(socket) var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - var clientSock = newNativeSocket() + var clientSock = createNativeSocket(inheritable = inheritable) if clientSock == osInvalidSocket: raiseOSError(osLastError()) const lpOutputLen = 1024 @@ -733,7 +808,7 @@ when defined(windows) or defined(nimdoc): else: retFuture.complete(newAcceptFut.read) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) template completeAccept() {.dirty.} = var listenSock = socket @@ -762,7 +837,7 @@ when defined(windows) or defined(nimdoc): var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = if not retFuture.finished: if errcode == OSErrorCode(-1): completeAccept() @@ -784,19 +859,21 @@ when defined(windows) or defined(nimdoc): GC_unref(ol) else: completeAccept() - # We don't deallocate ``ol`` here because even though this completed + # We don't deallocate `ol` here because even though this completed # immediately poll will still be notified about its completion and it will - # free ``ol``. + # free `ol`. return retFuture + implementSetInheritable() + proc closeSocket*(socket: AsyncFD) = ## Closes a socket and ensures that it is unregistered. socket.SocketHandle.close() getGlobalDispatcher().handles.excl(socket) proc unregister*(fd: AsyncFD) = - ## Unregisters ``fd``. + ## Unregisters `fd`. getGlobalDispatcher().handles.excl(fd) proc contains*(disp: PDispatcher, fd: AsyncFD): bool = @@ -896,9 +973,9 @@ when defined(windows) or defined(nimdoc): proc addRead*(fd: AsyncFD, cb: Callback) = ## Start watching the file descriptor for read availability and then call - ## the callback ``cb``. + ## the callback `cb`. ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## This is not `pure` mechanism for Windows Completion Ports (IOCP), ## so if you can avoid it, please do it. Use `addRead` only if really ## need it (main usecase is adaptation of unix-like libraries to be ## asynchronous on Windows). @@ -907,16 +984,16 @@ when defined(windows) or defined(nimdoc): ## or asyncdispatch.accept(), because they are using IOCP, please use ## nativesockets.recv() and nativesockets.accept() instead. ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `read` notifications, and ``false``, if you want to continue + ## Be sure your callback `cb` returns `true`, if you want to remove + ## watch of `read` notifications, and `false`, if you want to continue ## receiving notifications. registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) proc addWrite*(fd: AsyncFD, cb: Callback) = ## Start watching the file descriptor for write availability and then call - ## the callback ``cb``. + ## the callback `cb`. ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), + ## This is not `pure` mechanism for Windows Completion Ports (IOCP), ## so if you can avoid it, please do it. Use `addWrite` only if really ## need it (main usecase is adaptation of unix-like libraries to be ## asynchronous on Windows). @@ -925,8 +1002,8 @@ when defined(windows) or defined(nimdoc): ## or asyncdispatch.connect(), because they are using IOCP, please use ## nativesockets.send() and nativesockets.connect() instead. ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `write` notifications, and ``false``, if you want to continue + ## Be sure your callback `cb` returns `true`, if you want to remove + ## watch of `write` notifications, and `false`, if you want to continue ## receiving notifications. registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) @@ -966,12 +1043,12 @@ when defined(windows) or defined(nimdoc): raiseOSError(osLastError()) proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Registers callback ``cb`` to be called when timer expired. + ## Registers callback `cb` to be called when timer expired. ## ## Parameters: ## - ## * ``timeout`` - timeout value in milliseconds. - ## * ``oneshot`` + ## * `timeout` - timeout value in milliseconds. + ## * `oneshot` ## * `true` - generate only one timeout event ## * `false` - generate timeout events periodically @@ -1000,16 +1077,17 @@ when defined(windows) or defined(nimdoc): registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) proc addProcess*(pid: int, cb: Callback) = - ## Registers callback ``cb`` to be called when process with process ID - ## ``pid`` exited. + ## Registers callback `cb` to be called when process with process ID + ## `pid` exited. + const NULL = Handle(0) let p = getGlobalDispatcher() let procFlags = SYNCHRONIZE var hProcess = openProcess(procFlags, 0, pid.DWORD) - if hProcess == INVALID_HANDLE_VALUE: + if hProcess == NULL: raiseOSError(osLastError()) var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.DWORD + var flags = WT_EXECUTEINWAITTHREAD.DWORD or WT_EXECUTEONLYONCE.DWORD proc proccb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = closeWaitable(hProcess) @@ -1018,10 +1096,10 @@ when defined(windows) or defined(nimdoc): registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) proc newAsyncEvent*(): AsyncEvent = - ## Creates a new thread-safe ``AsyncEvent`` object. + ## Creates a new thread-safe `AsyncEvent` object. ## - ## New ``AsyncEvent`` object is not automatically registered with - ## dispatcher like ``AsyncSocket``. + ## New `AsyncEvent` object is not automatically registered with + ## dispatcher like `AsyncSocket`. var sa = SECURITY_ATTRIBUTES( nLength: sizeof(SECURITY_ATTRIBUTES).cint, bInheritHandle: 1 @@ -1033,12 +1111,12 @@ when defined(windows) or defined(nimdoc): result.hEvent = event proc trigger*(ev: AsyncEvent) = - ## Set event ``ev`` to signaled state. + ## Set event `ev` to signaled state. if setEvent(ev.hEvent) == 0: raiseOSError(osLastError()) proc unregister*(ev: AsyncEvent) = - ## Unregisters event ``ev``. + ## Unregisters event `ev`. doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") let p = getGlobalDispatcher() p.handles.excl(AsyncFD(ev.hEvent)) @@ -1049,14 +1127,14 @@ when defined(windows) or defined(nimdoc): ev.hWaiter = 0 proc close*(ev: AsyncEvent) = - ## Closes event ``ev``. + ## Closes event `ev`. let res = closeHandle(ev.hEvent) deallocShared(cast[pointer](ev)) if res == 0: raiseOSError(osLastError()) proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Registers callback ``cb`` to be called when ``ev`` will be signaled + ## Registers callback `cb` to be called when `ev` will be signaled doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") let p = getGlobalDispatcher() @@ -1088,9 +1166,15 @@ when defined(windows) or defined(nimdoc): initAll() else: - import selectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, + import std/selectors + from std/posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL + when declared(posix.accept4): + from std/posix import accept4, SOCK_CLOEXEC + when defined(genode): + import genode/env # get the implicit Genode env + import genode/signals + const InitCallbackListSize = 4 # initial size of callbacks sequence, # associated with file/socket descriptor. @@ -1098,7 +1182,7 @@ else: # queue. type AsyncFD* = distinct cint - Callback = proc (fd: AsyncFD): bool {.closure, gcsafe.} + Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.} AsyncData = object readList: seq[Callback] @@ -1108,6 +1192,8 @@ else: PDispatcher* = ref object of PDispatcherBase selector: Selector[AsyncData] + when defined(genode): + signalHandler: SignalHandler proc `==`*(x, y: AsyncFD): bool {.borrow.} proc `==`*(x, y: AsyncEvent): bool {.borrow.} @@ -1121,11 +1207,24 @@ else: proc newDispatcher*(): owned(PDispatcher) = new result result.selector = newSelector[AsyncData]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize) + result.timers.clear() + result.callbacks = initDeque[proc () {.closure, gcsafe.}](InitDelayedCallbackListSize) + when defined(genode): + let entrypoint = ep(cast[GenodeEnv](runtimeEnv)) + result.signalHandler = newSignalHandler(entrypoint): + discard runOnce(0) var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + proc setGlobalDispatcher*(disp: owned PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 @@ -1135,6 +1234,8 @@ else: proc getGlobalDispatcher*(): PDispatcher = if gDisp.isNil: setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = @@ -1180,6 +1281,12 @@ else: let p = getGlobalDispatcher() not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 + proc prependSeq(dest: var seq[Callback]; src: sink seq[Callback]) = + var old = move dest + dest = src + for i in 0..high(old): + dest.add(move old[i]) + proc processBasicCallbacks( fd: AsyncFD, event: Event ): tuple[readCbListCount, writeCbListCount: int] = @@ -1199,10 +1306,12 @@ else: withData(selector, fd.int, fdData): case event of Event.Read: - shallowCopy(curList, fdData.readList) + #shallowCopy(curList, fdData.readList) + curList = move fdData.readList fdData.readList = newSeqOfCap[Callback](InitCallbackListSize) of Event.Write: - shallowCopy(curList, fdData.writeList) + #shallowCopy(curList, fdData.writeList) + curList = move fdData.writeList fdData.writeList = newSeqOfCap[Callback](InitCallbackListSize) else: assert false, "Cannot process callbacks for " & $event @@ -1210,22 +1319,24 @@ else: let newLength = max(len(curList), InitCallbackListSize) var newList = newSeqOfCap[Callback](newLength) + var eventsExtinguished = false for cb in curList: - if not cb(fd): + if eventsExtinguished: + newList.add(cb) + elif not cb(fd): # Callback wants to be called again. newList.add(cb) # This callback has returned with EAGAIN, so we don't need to # call any other callbacks as they are all waiting for the same event # on the same fd. - break + # We do need to ensure they are called again though. + eventsExtinguished = true withData(selector, fd.int, fdData) do: # Descriptor is still present in the queue. case event - of Event.Read: - fdData.readList = newList & fdData.readList - of Event.Write: - fdData.writeList = newList & fdData.writeList + of Event.Read: prependSeq(fdData.readList, newList) + of Event.Write: prependSeq(fdData.writeList, newList) else: assert false, "Cannot process callbacks for " & $event @@ -1236,25 +1347,25 @@ else: result.readCbListCount = -1 result.writeCbListCount = -1 - template processCustomCallbacks(ident: untyped) = + proc processCustomCallbacks(p: PDispatcher; fd: AsyncFD) = # Process pending custom event callbacks. Custom events are # {Event.Timer, Event.Signal, Event.Process, Event.Vnode}. # There can be only one callback registered with one descriptor, # so there is no need to iterate over list. var curList: seq[Callback] - withData(p.selector, ident.int, adata) do: - shallowCopy(curList, adata.readList) + withData(p.selector, fd.int, adata) do: + curList = move adata.readList adata.readList = newSeqOfCap[Callback](InitCallbackListSize) let newLength = len(curList) var newList = newSeqOfCap[Callback](newLength) var cb = curList[0] - if not cb(fd.AsyncFD): + if not cb(fd): newList.add(cb) - withData(p.selector, ident.int, adata) do: + withData(p.selector, fd.int, adata) do: # descriptor still present in queue. adata.readList = newList & adata.readList if len(adata.readList) == 0: @@ -1264,6 +1375,8 @@ else: # descriptor was unregistered in callback via `unregister()`. discard + implementSetInheritable() + proc closeSocket*(sock: AsyncFD) = let selector = getGlobalDispatcher().selector if sock.SocketHandle notin selector: @@ -1280,14 +1393,11 @@ else: ValueError, "Expecting async operations to stop when fd has closed." ) - - proc runOnce(timeout = 500): bool = + proc runOnce(timeout: int): bool = let p = getGlobalDispatcher() - when ioselSupportedPlatform: - let customSet = {Event.Timer, Event.Signal, Event.Process, - Event.Vnode} - if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: + when defined(genode): + if timeout == 0: return raise newException(ValueError, "No handles or timers registered in dispatcher.") @@ -1321,9 +1431,11 @@ else: result = true when ioselSupportedPlatform: + const customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode} if (customSet * events) != {}: isCustomEvent = true - processCustomCallbacks(fd) + processCustomCallbacks(p, fd) result = true # because state `data` can be modified in callback we need to update @@ -1351,11 +1463,12 @@ else: flags.toOSFlags()) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: if flags.isDisconnectionError(lastError): retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -1379,11 +1492,12 @@ else: flags.toOSFlags()) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: if flags.isDisconnectionError(lastError): retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1407,7 +1521,9 @@ else: MSG_NOSIGNAL) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if lastError.int32 != EINTR and + lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: if flags.isDisconnectionError(lastError): retFuture.complete() else: @@ -1428,8 +1544,8 @@ else: proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, saddrLen: SockLen, flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = - ## Sends ``data`` of size ``size`` in bytes to specified destination - ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. + ## Sends `data` of size `size` in bytes to specified destination + ## (`saddr` of size `saddrLen` in bytes, using socket `socket`. ## The returned future will complete once all data has been sent. var retFuture = newFuture[void]("sendTo") @@ -1445,8 +1561,9 @@ else: cast[ptr SockAddr](addr(staddr[0])), stalen) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -1458,9 +1575,9 @@ else: proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, saddrLen: ptr SockLen, flags = {SocketFlag.SafeDisconn}): owned(Future[int]) = - ## Receives a datagram data from ``socket`` into ``data``, which must - ## be at least of size ``size`` in bytes, address of datagram's sender - ## will be stored into ``saddr`` and ``saddrLen``. Returned future will + ## Receives a datagram data from `socket` into `data`, which must + ## be at least of size `size` in bytes, address of datagram's sender + ## will be stored into `saddr` and `saddrLen`. Returned future will ## complete once one datagram has been received, and will return size ## of packet received. var retFuture = newFuture[int]("recvFromInto") @@ -1470,8 +1587,9 @@ else: saddr, saddrLen) if res < 0: let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + if lastError.int32 != EINTR and lastError.int32 != EWOULDBLOCK and + lastError.int32 != EAGAIN: + retFuture.fail(newOSError(lastError)) else: result = false else: @@ -1479,26 +1597,40 @@ else: addRead(socket, cb) return retFuture - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): + proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage var addrLen = sizeof(sockAddress).SockLen - var client = accept(sock.SocketHandle, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + var client = + when declared(accept4): + accept4(sock.SocketHandle, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen), if inheritable: 0 else: SOCK_CLOEXEC) + else: + accept(sock.SocketHandle, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + when declared(setInheritable) and not declared(accept4): + if client != osInvalidSocket and not setInheritable(client, inheritable): + # Set failure first because close() itself can fail, + # altering osLastError(). + retFuture.fail(newOSError(osLastError())) + close client + return false + if client == osInvalidSocket: let lastError = osLastError() - assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + assert lastError.int32 != EWOULDBLOCK and lastError.int32 != EAGAIN if lastError.int32 == EINTR: return false else: if flags.isDisconnectionError(lastError): return false else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: try: let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) @@ -1515,45 +1647,45 @@ else: proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = ## Start watching for timeout expiration, and then call the - ## callback ``cb``. - ## ``timeout`` - time in milliseconds, - ## ``oneshot`` - if ``true`` only one event will be dispatched, - ## if ``false`` continuous events every ``timeout`` milliseconds. + ## callback `cb`. + ## `timeout` - time in milliseconds, + ## `oneshot` - if `true` only one event will be dispatched, + ## if `false` continuous events every `timeout` milliseconds. let p = getGlobalDispatcher() var data = newAsyncData() data.readList.add(cb) p.selector.registerTimer(timeout, oneshot, data) proc addSignal*(signal: int, cb: Callback) = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb``. + ## Start watching signal `signal`, and when signal appears, call the + ## callback `cb`. let p = getGlobalDispatcher() var data = newAsyncData() data.readList.add(cb) p.selector.registerSignal(signal, data) proc addProcess*(pid: int, cb: Callback) = - ## Start watching for process exit with pid ``pid``, and then call - ## the callback ``cb``. + ## Start watching for process exit with pid `pid`, and then call + ## the callback `cb`. let p = getGlobalDispatcher() var data = newAsyncData() data.readList.add(cb) p.selector.registerProcess(pid, data) proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent``. + ## Creates new `AsyncEvent`. result = AsyncEvent(newSelectEvent()) proc trigger*(ev: AsyncEvent) = - ## Sets new ``AsyncEvent`` to signaled state. + ## Sets new `AsyncEvent` to signaled state. trigger(SelectEvent(ev)) proc close*(ev: AsyncEvent) = - ## Closes ``AsyncEvent`` + ## Closes `AsyncEvent` close(SelectEvent(ev)) proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Start watching for event ``ev``, and call callback ``cb``, when + ## Start watching for event `ev`, and call callback `cb`, when ## ev will be set to signaled state. let p = getGlobalDispatcher() var data = newAsyncData() @@ -1561,20 +1693,26 @@ else: p.selector.registerEvent(SelectEvent(ev), data) proc drain*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` - ## if there are no pending operations. In contrast to ``poll`` this - ## processes as many events as are available. - if runOnce(timeout) or hasPendingOperations(): - while hasPendingOperations() and runOnce(timeout): discard + ## Waits for completion of **all** events and processes them. Raises `ValueError` + ## if there are no pending operations. In contrast to `poll` this + ## processes as many events as are available until the timeout has elapsed. + var curTimeout = timeout + let start = now() + while hasPendingOperations(): + discard runOnce(curTimeout) + curTimeout -= (now() - start).inMilliseconds.int + if curTimeout < 0: + break proc poll*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` + ## Waits for completion events and processes them. Raises `ValueError` ## if there are no pending operations. This runs the underlying OS ## `epoll`:idx: or `kqueue`:idx: primitive only once. discard runOnce(timeout) -template createAsyncNativeSocketImpl(domain, sockType, protocol) = - let handle = newNativeSocket(domain, sockType, protocol) +template createAsyncNativeSocketImpl(domain, sockType, protocol: untyped, + inheritable = defined(nimInheritHandles)) = + let handle = createNativeSocket(domain, sockType, protocol, inheritable) if handle == osInvalidSocket: return osInvalidSocket.AsyncFD handle.setBlocking(false) @@ -1584,23 +1722,15 @@ template createAsyncNativeSocketImpl(domain, sockType, protocol) = register(result) proc createAsyncNativeSocket*(domain: cint, sockType: cint, - protocol: cint): AsyncFD = - createAsyncNativeSocketImpl(domain, sockType, protocol) + protocol: cint, + inheritable = defined(nimInheritHandles)): AsyncFD = + createAsyncNativeSocketImpl(domain, sockType, protocol, inheritable) proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): AsyncFD = - createAsyncNativeSocketImpl(domain, sockType, protocol) - -proc newAsyncNativeSocket*(domain: cint, sockType: cint, - protocol: cint): AsyncFD {.deprecated: "use createAsyncNativeSocket instead".} = - createAsyncNativeSocketImpl(domain, sockType, protocol) - -proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): AsyncFD - {.deprecated: "use createAsyncNativeSocket instead".} = - createAsyncNativeSocketImpl(domain, sockType, protocol) + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, + inheritable = defined(nimInheritHandles)): AsyncFD = + createAsyncNativeSocketImpl(domain, sockType, protocol, inheritable) when defined(windows) or defined(nimdoc): proc bindToDomain(handle: SocketHandle, domain: Domain) = @@ -1629,9 +1759,11 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): + const SO_UPDATE_CONNECT_CONTEXT = 0x7010 + socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) let ret = connectEx(socket.SocketHandle, addrInfo.ai_addr, @@ -1640,16 +1772,16 @@ when defined(windows) or defined(nimdoc): if ret: # Request to connect completed immediately. retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed + # We don't deallocate `ol` here because even though this completed # immediately poll will still be notified about its completion and it - # will free ``ol``. + # will free `ol`. else: let lastError = osLastError() if lastError.int32 != ERROR_IO_PENDING: - # With ERROR_IO_PENDING ``ol`` will be deallocated in ``poll``, + # With ERROR_IO_PENDING `ol` will be deallocated in `poll`, # and the future will be completed/failed there, too. GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) = let retFuture = newFuture[void]("doConnect") @@ -1666,7 +1798,7 @@ else: # interrupted, keep waiting return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true let ret = connect(socket.SocketHandle, @@ -1680,7 +1812,7 @@ else: if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(socket, cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, protocol: Protocol = IPPROTO_RAW) = @@ -1719,7 +1851,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, curAddrInfo = curAddrInfo.ai_next if curAddrInfo == nil: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds() if lastException != nil: @@ -1735,7 +1867,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, try: curFd = createAsyncNativeSocket(domain, sockType, protocol) except: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) closeUnusedFds() raise getCurrentException() when defined(windows): @@ -1745,7 +1877,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo curAddrInfo = curAddrInfo.ai_next else: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds(ord(domain)) retFuture.complete(curFd) @@ -1756,9 +1888,9 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, proc dial*(address: string, port: Port, protocol: Protocol = IPPROTO_TCP): owned(Future[AsyncFD]) = - ## Establishes connection to the specified ``address``:``port`` pair via the + ## Establishes connection to the specified `address`:`port` pair via the ## specified protocol. The procedure iterates through possible - ## resolutions of the ``address`` until it succeeds, meaning that it + ## resolutions of the `address` until it succeeds, meaning that it ## seamlessly works with both IPv4 and IPv6. ## Returns the async file descriptor, registered in the dispatcher of ## the current thread, ready to send or receive data. @@ -1786,7 +1918,7 @@ proc connect*(socket: AsyncFD, address: string, port: Port, proc sleepAsync*(ms: int | float): owned(Future[void]) = ## Suspends the execution of the current async procedure for the next - ## ``ms`` milliseconds. + ## `ms` milliseconds. var retFuture = newFuture[void]("sleepAsync") let p = getGlobalDispatcher() when ms is int: @@ -1797,11 +1929,11 @@ proc sleepAsync*(ms: int | float): owned(Future[void]) = return retFuture proc withTimeout*[T](fut: Future[T], timeout: int): owned(Future[bool]) = - ## Returns a future which will complete once ``fut`` completes or after - ## ``timeout`` milliseconds has elapsed. + ## Returns a future which will complete once `fut` completes or after + ## `timeout` milliseconds has elapsed. ## - ## If ``fut`` completes first the returned future will hold true, - ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## If `fut` completes first the returned future will hold true, + ## otherwise, if `timeout` milliseconds has elapsed first, the returned ## future will hold false. var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") @@ -1819,12 +1951,17 @@ proc withTimeout*[T](fut: Future[T], timeout: int): owned(Future[bool]) = return retFuture proc accept*(socket: AsyncFD, - flags = {SocketFlag.SafeDisconn}): owned(Future[AsyncFD]) = + flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): owned(Future[AsyncFD]) = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection. + ## + ## If `inheritable` is false (the default), the resulting client socket + ## will not be inheritable by child processes. + ## ## The future will complete when the connection is successfully accepted. var retFut = newFuture[AsyncFD]("accept") - var fut = acceptAddr(socket, flags) + var fut = acceptAddr(socket, flags, inheritable) fut.callback = proc (future: Future[tuple[address: string, client: AsyncFD]]) = assert future.finished @@ -1839,7 +1976,7 @@ proc keepAlive(x: string) = proc send*(socket: AsyncFD, data: string, flags = {SocketFlag.SafeDisconn}): owned(Future[void]) = - ## Sends ``data`` to ``socket``. The returned future will complete once all + ## Sends `data` to `socket`. The returned future will complete once all ## data has been sent. var retFuture = newFuture[void]("send") if data.len > 0: @@ -1857,7 +1994,8 @@ proc send*(socket: AsyncFD, data: string, return retFuture # -- Await Macro -include asyncmacro +import std/asyncmacro +export asyncmacro proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} = ## Returns a future that will complete when all the string data from the @@ -1884,3 +2022,44 @@ proc waitFor*[T](fut: Future[T]): T = poll() fut.read + +proc activeDescriptors*(): int {.inline.} = + ## Returns the current number of active file descriptors for the current + ## event loop. This is a cheap operation that does not involve a system call. + when defined(windows): + result = getGlobalDispatcher().handles.len + elif not defined(nimdoc): + result = getGlobalDispatcher().selector.count + +when defined(posix): + import std/posix + +when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): + proc maxDescriptors*(): int {.raises: OSError.} = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. + when defined(windows): + result = 16_700_000 + elif defined(zephyr) or defined(freertos): + result = FD_MAX + else: + var fdLim: RLimit + if getrlimit(RLIMIT_NOFILE, fdLim) < 0: + raiseOSError(osLastError()) + result = int(fdLim.rlim_cur) - 1 + +when defined(genode): + proc scheduleCallbacks*(): bool {.discardable.} = + ## *Genode only.* + ## Schedule callback processing and return immediately. + ## Returns `false` if there is nothing to schedule. + ## RPC servers should call this to dispatch `callSoon` + ## bodies after retiring an RPC to its client. + ## This is effectively a non-blocking `poll(…)` and is + ## equivalent to scheduling a momentary no-op timeout + ## but faster and with less overhead. + let dis = getGlobalDispatcher() + result = dis.callbacks.len > 0 + if result: submit(dis.signalHandler.cap) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index ad111ec50..0f6504342 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -9,27 +9,33 @@ ## This module implements asynchronous file reading and writing. ## -## .. code-block:: Nim -## import asyncfile, asyncdispatch, os +## ```Nim +## import std/[asyncfile, asyncdispatch, os] ## -## proc main() {.async.} = -## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) -## await file.write("test") -## file.setFilePos(0) -## let data = await file.readAll() -## doAssert data == "test" -## file.close() +## proc main() {.async.} = +## var file = openAsync(getTempDir() / "foobar.txt", fmReadWrite) +## await file.write("test") +## file.setFilePos(0) +## let data = await file.readAll() +## doAssert data == "test" +## file.close() ## -## waitFor main() +## waitFor main() +## ``` -import asyncdispatch, os +import std/[asyncdispatch, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + when defined(windows) or defined(nimdoc): + import std/widestrs # TODO: Fix duplication introduced by PR #4683. when defined(windows) or defined(nimdoc): - import winlean + import std/winlean else: - import posix + import std/posix type AsyncFile* = ref object @@ -90,20 +96,15 @@ proc newAsyncFile*(fd: AsyncFD): AsyncFile = register(fd) proc openAsync*(filename: string, mode = fmRead): AsyncFile = - ## Opens a file specified by the path in ``filename`` using - ## the specified FileMode ``mode`` asynchronously. + ## Opens a file specified by the path in `filename` using + ## the specified FileMode `mode` asynchronously. when defined(windows) or defined(nimdoc): let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - let fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) - else: - let fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) @@ -124,11 +125,11 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = result = newAsyncFile(fd.AsyncFD) proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = - ## Read ``size`` bytes from the specified file asynchronously starting at + ## Read `size` bytes from the specified file asynchronously starting at ## the current position of the file pointer. ## ## If the file pointer is past the end of the file then zero is returned - ## and no bytes are read into ``buf`` + ## and no bytes are read into `buf` var retFuture = newFuture[int]("asyncfile.readBuffer") when defined(windows) or defined(nimdoc): @@ -145,7 +146,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) ol.offset = DWORD(f.offset and 0xffffffff) ol.offsetHigh = DWORD(f.offset shr 32) @@ -161,7 +162,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = # This happens in Windows Server 2003 retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -172,7 +173,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete(0) else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -185,7 +186,7 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -201,11 +202,12 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = return retFuture proc read*(f: AsyncFile, size: int): Future[string] = - ## Read ``size`` bytes from the specified file asynchronously starting at - ## the current position of the file pointer. + ## Read `size` bytes from the specified file asynchronously starting at + ## the current position of the file pointer. `size` should be greater than zero. ## ## If the file pointer is past the end of the file then an empty string is ## returned. + assert size > 0 var retFuture = newFuture[string]("asyncfile.read") when defined(windows) or defined(nimdoc): @@ -226,7 +228,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if errcode.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -249,7 +251,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = # This happens in Windows Server 2003 retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesRead: DWORD @@ -260,7 +262,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if err.int32 == ERROR_HANDLE_EOF: retFuture.complete("") else: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesRead > 0 assert bytesRead <= size @@ -277,7 +279,7 @@ proc read*(f: AsyncFile, size: int): Future[string] = if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. elif res == 0: @@ -299,6 +301,8 @@ proc readLine*(f: AsyncFile): Future[string] {.async.} = result = "" while true: var c = await read(f, 1) + if c.len == 0: + break if c[0] == '\c': c = await read(f, 1) break @@ -332,7 +336,7 @@ proc readAll*(f: AsyncFile): Future[string] {.async.} = result.add data proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = - ## Writes ``size`` bytes from ``buf`` to the file specified asynchronously. + ## Writes `size` bytes from `buf` to the file specified asynchronously. ## ## The returned Future will complete once all data has been written to the ## specified file. @@ -346,7 +350,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = assert bytesCount == size.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) ) # passing -1 here should work according to MSDN, but doesn't. For more # information see @@ -363,14 +367,14 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == size.int32 retFuture.complete() @@ -379,13 +383,13 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = proc cb(fd: AsyncFD): bool = result = true - let remainderSize = size-written + let remainderSize = size - written var cbuf = cast[cstring](buf) let res = write(fd.cint, addr cbuf[written], remainderSize.cint) if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: @@ -401,7 +405,7 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = return retFuture proc write*(f: AsyncFile, data: string): Future[void] = - ## Writes ``data`` to the file specified asynchronously. + ## Writes `data` to the file specified asynchronously. ## ## The returned Future will complete once all data has been written to the ## specified file. @@ -409,7 +413,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = var copy = data when defined(windows) or defined(nimdoc): var buffer = alloc0(data.len) - copyMem(buffer, addr copy[0], data.len) + copyMem(buffer, copy.cstring, data.len) var ol = newCustom() ol.data = CompletionData(fd: f.fd, cb: @@ -419,7 +423,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = assert bytesCount == data.len.int32 retFuture.complete() else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) + retFuture.fail(newOSError(errcode)) if buffer != nil: dealloc buffer buffer = nil @@ -438,14 +442,14 @@ proc write*(f: AsyncFile, data: string): Future[void] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + retFuture.fail(newOSError(err)) else: # Request completed immediately. var bytesWritten: DWORD let overlappedRes = getOverlappedResult(f.fd.Handle, cast[POVERLAPPED](ol), bytesWritten, false.WINBOOL) if not overlappedRes.bool: - retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + retFuture.fail(newOSError(osLastError())) else: assert bytesWritten == data.len.int32 retFuture.complete() @@ -454,12 +458,19 @@ proc write*(f: AsyncFile, data: string): Future[void] = proc cb(fd: AsyncFD): bool = result = true - let remainderSize = data.len-written - let res = write(fd.cint, addr copy[written], remainderSize.cint) + + let remainderSize = data.len - written + + let res = + if data.len == 0: + write(fd.cint, copy.cstring, 0) + else: + write(fd.cint, addr copy[written], remainderSize.cint) + if res < 0: let lastError = osLastError() if lastError.int32 != EAGAIN: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) else: result = false # We still want this callback to be called. else: diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim deleted file mode 100644 index 7bbfc2a35..000000000 --- a/lib/pure/asyncftpclient.nim +++ /dev/null @@ -1,439 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements an asynchronous FTP client. It allows you to connect -## to an FTP server and perform operations on it such as for example: -## -## * The upload of new files. -## * The removal of existing files. -## * Download of files. -## * Changing of files' permissions. -## * Navigation through the FTP server's directories. -## -## Connecting to an FTP server -## =========================== -## -## In order to begin any sort of transfer of files you must first -## connect to an FTP server. You can do so with the ``connect`` procedure. -## -## .. code-block::nim -## import asyncdispatch, asyncftpclient -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## echo("Connected") -## waitFor(main()) -## -## A new ``main`` async procedure must be declared to allow the use of the -## ``await`` keyword. The connection will complete asynchronously and the -## client will be connected after the ``await ftp.connect()`` call. -## -## Uploading a new file -## ==================== -## -## After a connection is made you can use the ``store`` procedure to upload -## a new file to the FTP server. Make sure to check you are in the correct -## working directory before you do so with the ``pwd`` procedure, you can also -## instead specify an absolute path. -## -## .. code-block::nim -## import asyncdispatch, asyncftpclient -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## let currentDir = await ftp.pwd() -## assert currentDir == "/home/user/" -## await ftp.store("file.txt", "file.txt") -## echo("File finished uploading") -## waitFor(main()) -## -## Checking the progress of a file transfer -## ======================================== -## -## The progress of either a file upload or a file download can be checked -## by specifying a ``onProgressChanged`` procedure to the ``store`` or -## ``retrFile`` procedures. -## -## .. code-block::nim -## import asyncdispatch, asyncftpclient -## -## proc onProgressChanged(total, progress: BiggestInt, -## speed: float): Future[void] = -## echo("Uploaded ", progress, " of ", total, " bytes") -## echo("Current speed: ", speed, " kb/s") -## -## proc main() {.async.} = -## var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") -## await ftp.connect() -## await ftp.store("file.txt", "/home/user/file.txt", onProgressChanged) -## echo("File finished uploading") -## waitFor(main()) - - -import asyncdispatch, asyncnet, nativesockets, strutils, parseutils, os, times -from net import BufferSize - -type - AsyncFtpClient* = ref object - csock*: AsyncSocket - dsock*: AsyncSocket - user*, pass*: string - address*: string - port*: Port - jobInProgress*: bool - job*: FtpJob - dsockConnected*: bool - - FtpJobType* = enum - JRetrText, JRetr, JStore - - FtpJob = ref object - prc: proc (ftp: AsyncFtpClient, async: bool): bool {.nimcall, gcsafe.} - case typ*: FtpJobType - of JRetrText: - lines: string - of JRetr, JStore: - file: File - filename: string - total: BiggestInt # In bytes. - progress: BiggestInt # In bytes. - oneSecond: BiggestInt # Bytes transferred in one second. - lastProgressReport: float # Time - toStore: string # Data left to upload (Only used with async) - - FtpEventType* = enum - EvTransferProgress, EvLines, EvRetr, EvStore - - FtpEvent* = object ## Event - filename*: string - case typ*: FtpEventType - of EvLines: - lines*: string ## Lines that have been transferred. - of EvRetr, EvStore: ## Retr/Store operation finished. - nil - of EvTransferProgress: - bytesTotal*: BiggestInt ## Bytes total. - bytesFinished*: BiggestInt ## Bytes transferred. - speed*: BiggestInt ## Speed in bytes/s - currentJob*: FtpJobType ## The current job being performed. - - ReplyError* = object of IOError - - ProgressChangedProc* = - proc (total, progress: BiggestInt, speed: float): - Future[void] {.closure, gcsafe.} - -const multiLineLimit = 10000 - -proc expectReply(ftp: AsyncFtpClient): Future[TaintedString] {.async.} = - var line = await ftp.csock.recvLine() - result = TaintedString(line) - var count = 0 - while line[3] == '-': - ## Multi-line reply. - line = await ftp.csock.recvLine() - string(result).add("\n" & line) - count.inc() - if count >= multiLineLimit: - raise newException(ReplyError, "Reached maximum multi-line reply count.") - -proc send*(ftp: AsyncFtpClient, m: string): Future[TaintedString] {.async.} = - ## Send a message to the server, and wait for a primary reply. - ## ``\c\L`` is added for you. - ## - ## **Note:** The server may return multiple lines of coded replies. - await ftp.csock.send(m & "\c\L") - return await ftp.expectReply() - -proc assertReply(received: TaintedString, expected: varargs[string]) = - for i in items(expected): - if received.string.startsWith(i): return - raise newException(ReplyError, - "Expected reply '$1' got: $2" % - [expected.join("' or '"), received.string]) - -proc pasv(ftp: AsyncFtpClient) {.async.} = - ## Negotiate a data connection. - ftp.dsock = newAsyncSocket() - - var pasvMsg = (await ftp.send("PASV")).string.strip.TaintedString - assertReply(pasvMsg, "227") - var betweenParens = captureBetween(pasvMsg.string, '(', ')') - var nums = betweenParens.split(',') - var ip = nums[0 .. ^3] - var port = nums[^2 .. ^1] - var properPort = port[0].parseInt()*256+port[1].parseInt() - await ftp.dsock.connect(ip.join("."), Port(properPort)) - ftp.dsockConnected = true - -proc normalizePathSep(path: string): string = - return replace(path, '\\', '/') - -proc connect*(ftp: AsyncFtpClient) {.async.} = - ## Connect to the FTP server specified by ``ftp``. - await ftp.csock.connect(ftp.address, ftp.port) - - var reply = await ftp.expectReply() - if string(reply).startsWith("120"): - # 120 Service ready in nnn minutes. - # We wait until we receive 220. - reply = await ftp.expectReply() - - # Handle 220 messages from the server - assertReply(reply, "220") - - if ftp.user != "": - assertReply(await(ftp.send("USER " & ftp.user)), "230", "331") - - if ftp.pass != "": - assertReply(await(ftp.send("PASS " & ftp.pass)), "230") - -proc pwd*(ftp: AsyncFtpClient): Future[TaintedString] {.async.} = - ## Returns the current working directory. - let wd = await ftp.send("PWD") - assertReply wd, "257" - return wd.string.captureBetween('"').TaintedString # " - -proc cd*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Changes the current directory on the remote FTP server to ``dir``. - assertReply(await(ftp.send("CWD " & dir.normalizePathSep)), "250") - -proc cdup*(ftp: AsyncFtpClient) {.async.} = - ## Changes the current directory to the parent of the current directory. - assertReply(await(ftp.send("CDUP")), "200") - -proc getLines(ftp: AsyncFtpClient): Future[string] {.async.} = - ## Downloads text data in ASCII mode - result = "" - assert ftp.dsockConnected - while ftp.dsockConnected: - let r = await ftp.dsock.recvLine() - if r.string == "": - ftp.dsockConnected = false - else: - result.add(r.string & "\n") - - assertReply(await(ftp.expectReply()), "226") - -proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = - ## Returns a list of filenames in the given directory. If ``dir`` is "", - ## the current directory is used. If ``async`` is true, this - ## function will return immediately and it will be your job to - ## use asyncdispatch's ``poll`` to progress this operation. - await ftp.pasv() - - assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) - - result = splitLines(await ftp.getLines()) - -proc existsFile*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = - ## Determines whether ``file`` exists. - var files = await ftp.listDirs() - for f in items(files): - if f.normalizePathSep == file.normalizePathSep: return true - -proc createDir*(ftp: AsyncFtpClient, dir: string, recursive = false){.async.} = - ## Creates a directory ``dir``. If ``recursive`` is true, the topmost - ## subdirectory of ``dir`` will be created first, following the secondmost... - ## etc. this allows you to give a full path as the ``dir`` without worrying - ## about subdirectories not existing. - if not recursive: - assertReply(await(ftp.send("MKD " & dir.normalizePathSep)), "257") - else: - var reply = TaintedString"" - var previousDirs = "" - for p in split(dir, {os.DirSep, os.AltSep}): - if p != "": - previousDirs.add(p) - reply = await ftp.send("MKD " & previousDirs) - previousDirs.add('/') - assertReply reply, "257" - -proc chmod*(ftp: AsyncFtpClient, path: string, - permissions: set[FilePermission]) {.async.} = - ## Changes permission of ``path`` to ``permissions``. - var userOctal = 0 - var groupOctal = 0 - var otherOctal = 0 - for i in items(permissions): - case i - of fpUserExec: userOctal.inc(1) - of fpUserWrite: userOctal.inc(2) - of fpUserRead: userOctal.inc(4) - of fpGroupExec: groupOctal.inc(1) - of fpGroupWrite: groupOctal.inc(2) - of fpGroupRead: groupOctal.inc(4) - of fpOthersExec: otherOctal.inc(1) - of fpOthersWrite: otherOctal.inc(2) - of fpOthersRead: otherOctal.inc(4) - - var perm = $userOctal & $groupOctal & $otherOctal - assertReply(await(ftp.send("SITE CHMOD " & perm & - " " & path.normalizePathSep)), "200") - -proc list*(ftp: AsyncFtpClient, dir = ""): Future[string] {.async.} = - ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current - ## working directory. - await ftp.pasv() - - let reply = await ftp.send("LIST" & " " & dir.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc retrText*(ftp: AsyncFtpClient, file: string): Future[string] {.async.} = - ## Retrieves ``file``. File must be ASCII text. - await ftp.pasv() - let reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply(reply, ["125", "150"]) - - result = await ftp.getLines() - -proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(1000) - var dataFut = ftp.dsock.recv(BufferSize) - while ftp.dsockConnected: - await dataFut or countdownFut - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, - progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(1000) - - if dataFut.finished: - let data = dataFut.read - if data != "": - progress.inc(data.len) - progressInSecond.inc(data.len) - file.write(data) - dataFut = ftp.dsock.recv(BufferSize) - else: - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - -proc defaultOnProgressChanged*(total, progress: BiggestInt, - speed: float): Future[void] {.nimcall, gcsafe, procvar.} = - ## Default FTP ``onProgressChanged`` handler. Does nothing. - result = newFuture[void]() - #echo(total, " ", progress, " ", speed) - result.complete() - -proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Downloads ``file`` and saves it to ``dest``. - ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function - ## when the download is finished. The event's ``filename`` field will be equal - ## to ``file``. - var destFile = open(dest, mode = fmWrite) - await ftp.pasv() - var reply = await ftp.send("RETR " & file.normalizePathSep) - assertReply reply, ["125", "150"] - if {'(', ')'} notin reply.string: - raise newException(ReplyError, "Reply has no file size.") - var fileSize: BiggestInt - if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: - raise newException(ReplyError, "Reply has no file size.") - - await getFile(ftp, destFile, fileSize, onProgressChanged) - destFile.close() - -proc doUpload(ftp: AsyncFtpClient, file: File, - onProgressChanged: ProgressChangedProc) {.async.} = - assert ftp.dsockConnected - - let total = file.getFileSize() - var data = newString(4000) - var progress = 0 - var progressInSecond = 0 - var countdownFut = sleepAsync(1000) - var sendFut: Future[void] = nil - while ftp.dsockConnected: - if sendFut == nil or sendFut.finished: - # TODO: Async file reading. - let len = file.readBuffer(addr(data[0]), 4000) - setLen(data, len) - if len == 0: - # File finished uploading. - ftp.dsock.close() - ftp.dsockConnected = false - - assertReply(await(ftp.expectReply()), "226") - else: - progress.inc(len) - progressInSecond.inc(len) - sendFut = ftp.dsock.send(data) - - if countdownFut.finished: - asyncCheck onProgressChanged(total, progress, progressInSecond.float) - progressInSecond = 0 - countdownFut = sleepAsync(1000) - - await countdownFut or sendFut - -proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = - ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this - ## function asynchronously is recommended to view the progress of - ## the download. - ## The ``EvStore`` event is passed to the specified ``handleEvent`` function - ## when the upload is finished, and the ``filename`` field will be - ## equal to ``file``. - var destFile = open(file) - await ftp.pasv() - - let reply = await ftp.send("STOR " & dest.normalizePathSep) - assertReply reply, ["125", "150"] - - await doUpload(ftp, destFile, onProgressChanged) - -proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} = - ## Rename a file or directory on the remote FTP Server from current name - ## ``name_from`` to new name ``name_to`` - assertReply(await ftp.send("RNFR " & nameFrom), "350") - assertReply(await ftp.send("RNTO " & nameTo), "250") - -proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} = - ## Delete a file ``filename`` on the remote FTP server - assertReply(await ftp.send("DELE " & filename), "250") - -proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} = - ## Delete a directory ``dir`` on the remote FTP server - assertReply(await ftp.send("RMD " & dir), "250") - -proc newAsyncFtpClient*(address: string, port = Port(21), - user, pass = ""): AsyncFtpClient = - ## Creates a new ``AsyncFtpClient`` object. - new result - result.user = user - result.pass = pass - result.address = address - result.port = port - result.dsockConnected = false - result.csock = newAsyncSocket() - -when not defined(testing) and isMainModule: - var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") - proc main(ftp: AsyncFtpClient) {.async.} = - await ftp.connect() - echo await ftp.pwd() - echo await ftp.listDirs() - await ftp.store("payload.jpg", "payload.jpg") - await ftp.retrFile("payload.jpg", "payload2.jpg") - await ftp.rename("payload.jpg", "payload_renamed.jpg") - await ftp.store("payload.jpg", "payload_remove.jpg") - await ftp.removeFile("payload_remove.jpg") - await ftp.createDir("deleteme") - await ftp.removeDir("deleteme") - echo("Finished") - - waitFor main(ftp) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 7aac192d2..29ebf8f89 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -7,7 +7,13 @@ # distribution, for details about the copyright. # -import os, tables, strutils, times, heapqueue, options, deques, cstrutils +import std/[os, tables, strutils, times, heapqueue, options, deques, cstrutils, typetraits] + +import system/stacktraces + +when defined(nimPreviewSlimSystem): + import std/objectdollar # for StackTraceEntry + import std/assertions # TODO: This shouldn't need to be included, but should ideally be exported. type @@ -23,7 +29,7 @@ type finished: bool error*: ref Exception ## Stored exception errorStackTrace*: string - when not defined(release): + when not defined(release) or defined(futureLogging): stackTrace: seq[StackTraceEntry] ## For debugging purposes only. id: int fromProc: string @@ -33,7 +39,7 @@ type FutureVar*[T] = distinct Future[T] - FutureError* = object of Exception + FutureError* = object of Defect cause*: FutureBase when not defined(release): @@ -45,7 +51,7 @@ const NimAsyncContinueSuffix* = "NimAsyncContinue" ## For internal usage. Do not use. when isFutureLoggingEnabled: - import hashes + import std/hashes type FutureInfo* = object stackTrace*: seq[StackTraceEntry] @@ -84,19 +90,19 @@ when isFutureLoggingEnabled: var callSoonProc {.threadvar.}: proc (cbproc: proc ()) {.gcsafe.} proc getCallSoonProc*(): (proc(cbproc: proc ()) {.gcsafe.}) = - ## Get current implementation of ``callSoon``. + ## Get current implementation of `callSoon`. return callSoonProc proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) = - ## Change current implementation of ``callSoon``. This is normally called when dispatcher from ``asyncdispatcher`` is initialized. + ## Change current implementation of `callSoon`. This is normally called when dispatcher from `asyncdispatcher` is initialized. callSoonProc = p -proc callSoon*(cbproc: proc ()) = - ## Call ``cbproc`` "soon". +proc callSoon*(cbproc: proc () {.gcsafe.}) = + ## Call `cbproc` "soon". ## - ## If async dispatcher is running, ``cbproc`` will be executed during next dispatcher tick. + ## If async dispatcher is running, `cbproc` will be executed during next dispatcher tick. ## - ## If async dispatcher is not running, ``cbproc`` will be executed immediately. + ## If async dispatcher is not running, `cbproc` will be executed immediately. if callSoonProc.isNil: # Loop not initialized yet. Call the function directly to allow setup code to use futures. cbproc() @@ -115,29 +121,29 @@ template setupFutureBase(fromProc: string) = proc newFuture*[T](fromProc: string = "unspecified"): owned(Future[T]) = ## Creates a new future. ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## Specifying `fromProc`, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. setupFutureBase(fromProc) when isFutureLoggingEnabled: logFutureStart(result) proc newFutureVar*[T](fromProc = "unspecified"): owned(FutureVar[T]) = - ## Create a new ``FutureVar``. This Future type is ideally suited for + ## Create a new `FutureVar`. This Future type is ideally suited for ## situations where you want to avoid unnecessary allocations of Futures. ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## Specifying `fromProc`, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. let fo = newFuture[T](fromProc) result = typeof(result)(fo) when isFutureLoggingEnabled: logFutureStart(Future[T](result)) proc clean*[T](future: FutureVar[T]) = - ## Resets the ``finished`` status of ``future``. + ## Resets the `finished` status of `future`. Future[T](future).finished = false Future[T](future).error = nil proc checkFinished[T](future: Future[T]) = ## Checks whether `future` is finished. If it is then raises a - ## ``FutureError``. + ## `FutureError`. when not defined(release): if future.finished: var msg = "" @@ -157,33 +163,15 @@ proc checkFinished[T](future: Future[T]) = raise err proc call(callbacks: var CallbackList) = - when not defined(nimV2): - # strictly speaking a little code duplication here, but we strive - # to minimize regressions and I'm not sure I got the 'nimV2' logic - # right: - var current = callbacks - while true: - if not current.function.isNil: - callSoon(current.function) - - if current.next.isNil: - break - else: - current = current.next[] - else: - var currentFunc = unown callbacks.function - var currentNext = unown callbacks.next - - while true: - if not currentFunc.isNil: - callSoon(currentFunc) - - if currentNext.isNil: - break - else: - currentFunc = currentNext.function - currentNext = unown currentNext.next + var current = callbacks + while true: + if not current.function.isNil: + callSoon(current.function) + if current.next.isNil: + break + else: + current = current.next[] # callback will be called only once, let GC collect them now callbacks.next = nil callbacks.function = nil @@ -205,27 +193,25 @@ proc add(callbacks: var CallbackList, function: CallbackFunc) = last = last.next last.next = newCallback -proc complete*[T](future: Future[T], val: T) = - ## Completes ``future`` with value ``val``. +proc completeImpl[T, U](future: Future[T], val: sink U, isVoid: static bool) = #assert(not future.finished, "Future already finished, cannot finish twice.") checkFinished(future) assert(future.error == nil) - future.value = val + when not isVoid: + future.value = val future.finished = true future.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(future) -proc complete*(future: Future[void]) = - ## Completes a void ``future``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - future.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) +proc complete*[T](future: Future[T], val: sink T) = + ## Completes `future` with value `val`. + completeImpl(future, val, false) + +proc complete*(future: Future[void], val = Future[void].default) = + completeImpl(future, (), true) proc complete*[T](future: FutureVar[T]) = - ## Completes a ``FutureVar``. + ## Completes a `FutureVar`. template fut: untyped = Future[T](future) checkFinished(fut) assert(fut.error == nil) @@ -233,8 +219,8 @@ proc complete*[T](future: FutureVar[T]) = fut.callbacks.call() when isFutureLoggingEnabled: logFutureFinish(Future[T](future)) -proc complete*[T](future: FutureVar[T], val: T) = - ## Completes a ``FutureVar`` with value ``val``. +proc complete*[T](future: FutureVar[T], val: sink T) = + ## Completes a `FutureVar` with value `val`. ## ## Any previously stored value will be overwritten. template fut: untyped = Future[T](future) @@ -243,10 +229,10 @@ proc complete*[T](future: FutureVar[T], val: T) = fut.finished = true fut.value = val fut.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) + when isFutureLoggingEnabled: logFutureFinish(fut) proc fail*[T](future: Future[T], error: ref Exception) = - ## Completes ``future`` with ``error``. + ## Completes `future` with `error`. #assert(not future.finished, "Future already finished, cannot finish twice.") checkFinished(future) future.finished = true @@ -263,7 +249,7 @@ proc clearCallbacks*(future: FutureBase) = proc addCallback*(future: FutureBase, cb: proc() {.closure, gcsafe.}) = ## Adds the callbacks proc to be called when the future completes. ## - ## If future has already completed then ``cb`` will be called immediately. + ## If future has already completed then `cb` will be called immediately. assert cb != nil if future.finished: callSoon(cb) @@ -274,7 +260,7 @@ proc addCallback*[T](future: Future[T], cb: proc (future: Future[T]) {.closure, gcsafe.}) = ## Adds the callbacks proc to be called when the future completes. ## - ## If future has already completed then ``cb`` will be called immediately. + ## If future has already completed then `cb` will be called immediately. future.addCallback( proc() = cb(future) @@ -283,9 +269,9 @@ proc addCallback*[T](future: Future[T], proc `callback=`*(future: FutureBase, cb: proc () {.closure, gcsafe.}) = ## Clears the list of callbacks and sets the callback proc to be called when the future completes. ## - ## If future has already completed then ``cb`` will be called immediately. + ## If future has already completed then `cb` will be called immediately. ## - ## It's recommended to use ``addCallback`` or ``then`` instead. + ## It's recommended to use `addCallback` or `then` instead. future.clearCallbacks future.addCallback cb @@ -293,56 +279,71 @@ proc `callback=`*[T](future: Future[T], cb: proc (future: Future[T]) {.closure, gcsafe.}) = ## Sets the callback proc to be called when the future completes. ## - ## If future has already completed then ``cb`` will be called immediately. + ## If future has already completed then `cb` will be called immediately. future.callback = proc () = cb(future) +template getFilenameProcname(entry: StackTraceEntry): (string, string) = + when compiles(entry.filenameStr) and compiles(entry.procnameStr): + # We can't rely on "entry.filename" and "entry.procname" still being valid + # cstring pointers, because the "string.data" buffers they pointed to might + # be already garbage collected (this entry being a non-shallow copy, + # "entry.filename" no longer points to "entry.filenameStr.data", but to the + # buffer of the original object). + (entry.filenameStr, entry.procnameStr) + else: + ($entry.filename, $entry.procname) + proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. + + let (filename, procname) = getFilenameProcname(entry) + result = "" - if entry.procname == cstring"processPendingCallbacks": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + if procname == "processPendingCallbacks": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Executes pending callbacks" - elif entry.procname == cstring"poll": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + elif procname == "poll": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Processes asynchronous completion events" - if entry.procname.endsWith(NimAsyncContinueSuffix): - if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0: + if procname.endsWith(NimAsyncContinueSuffix): + if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0: return "Resumes an async procedure" -proc `$`*(entries: seq[StackTraceEntry]): string = +proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = + when defined(nimStackTraceOverride): + let entries = addDebuggingInfo(stackTraceEntries) + else: + let entries = stackTraceEntries + result = "" # Find longest filename & line number combo for alignment purposes. var longestLeft = 0 for entry in entries: - if entry.procname.isNil: continue + let (filename, procname) = getFilenameProcname(entry) + + if procname == "": continue - let left = $entry.filename & $entry.line - if left.len > longestLeft: - longestLeft = left.len + let leftLen = filename.len + len($entry.line) + if leftLen > longestLeft: + longestLeft = leftLen - var indent = 2 # Format the entries. for entry in entries: - if entry.procname.isNil: - if entry.line == -10: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - else: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue - - let left = "$#($#)" % [$entry.filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ + let (filename, procname) = getFilenameProcname(entry) + + if procname == "" and entry.line == reraisedFromBegin: + break + + let left = "$#($#)" % [filename, $entry.line] + result.add((spaces(2) & "$# $#\n") % [ left, - spaces(longestLeft - left.len + 2), - $entry.procname + procname ]) let hint = getHint(entry) if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + result.add(spaces(4) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): @@ -362,66 +363,73 @@ proc injectStacktrace[T](future: Future[T]) = newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") - newMsg.add("Exception type:") # # For debugging purposes + # newMsg.add("Exception type:") # for entry in getStackTraceEntries(future.error): # newMsg.add "\n" & $entry future.error.msg = newMsg -proc read*[T](future: Future[T] | FutureVar[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - {.push hint[ConvFromXtoItselfNotNeeded]: off.} - let fut = Future[T](future) - {.pop.} +template readImpl(future, T) = + when future is Future[T]: + let fut {.cursor.} = future + else: + let fut {.cursor.} = Future[T](future) if fut.finished: if fut.error != nil: injectStacktrace(fut) raise fut.error when T isnot void: - return fut.value + result = distinctBase(future).value else: # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") +proc read*[T](future: Future[T] | FutureVar[T]): lent T = + ## Retrieves the value of `future`. Future must be finished otherwise + ## this function will fail with a `ValueError` exception. + ## + ## If the result of the future is an error then that error will be raised. + readImpl(future, T) + +proc read*(future: Future[void] | FutureVar[void]) = + readImpl(future, void) + proc readError*[T](future: Future[T]): ref Exception = - ## Retrieves the exception stored in ``future``. + ## Retrieves the exception stored in `future`. ## - ## An ``ValueError`` exception will be thrown if no exception exists + ## An `ValueError` exception will be thrown if no exception exists ## in the specified Future. if future.error != nil: return future.error else: raise newException(ValueError, "No error in future.") proc mget*[T](future: FutureVar[T]): var T = - ## Returns a mutable value stored in ``future``. + ## Returns a mutable value stored in `future`. ## - ## Unlike ``read``, this function will not raise an exception if the + ## Unlike `read`, this function will not raise an exception if the ## Future has not been finished. result = Future[T](future).value proc finished*(future: FutureBase | FutureVar): bool = - ## Determines whether ``future`` has completed. + ## Determines whether `future` has completed. ## - ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + ## `True` may indicate an error or a value. Use `failed` to distinguish. when future is FutureVar: result = (FutureBase(future)).finished else: result = future.finished proc failed*(future: FutureBase): bool = - ## Determines whether ``future`` completed with an error. + ## Determines whether `future` completed with an error. return future.error != nil proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future + ## Sets a callback on `future` which raises an exception if the future ## finished with an error. ## - ## This should be used instead of ``discard`` to discard void futures, - ## or use ``waitFor`` if you need to wait for the future's completion. + ## This should be used instead of `discard` to discard void futures, + ## or use `waitFor` if you need to wait for the future's completion. assert(not future.isNil, "Future is nil") # TODO: We can likely look at the stack trace here and inject the location # where the `asyncCheck` was called to give a better error stack message. @@ -432,7 +440,7 @@ proc asyncCheck*[T](future: Future[T]) = future.callback = asyncCheckCallback proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## Returns a future which will complete once both `fut1` and `fut2` ## complete. var retFuture = newFuture[void]("asyncdispatch.`and`") fut1.callback = @@ -448,7 +456,7 @@ proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = return retFuture proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## Returns a future which will complete once either `fut1` or `fut2` ## complete. var retFuture = newFuture[void]("asyncdispatch.`or`") proc cb[X](fut: Future[X]) = @@ -461,14 +469,14 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = proc all*[T](futs: varargs[Future[T]]): auto = ## Returns a future which will complete once - ## all futures in ``futs`` complete. + ## all futures in `futs` complete. ## If the argument is empty, the returned future completes immediately. ## - ## If the awaited futures are not ``Future[void]``, the returned future + ## If the awaited futures are not `Future[void]`, the returned future ## will hold the values of all awaited futures in a sequence. ## - ## If the awaited futures *are* ``Future[void]``, - ## this proc returns ``Future[void]``. + ## If the awaited futures *are* `Future[void]`, + ## this proc returns `Future[void]`. when T is void: var diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 186f0da41..39e945d5e 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -11,27 +11,41 @@ ## ## This HTTP server has not been designed to be used in production, but ## for testing applications locally. Because of this, when deploying your -## application you should use a reverse proxy (for example nginx) instead of -## allowing users to connect directly to this server. -## -## Basic usage -## =========== -## -## This example will create an HTTP server on port 8080. The server will -## respond to all requests with a ``200 OK`` response code and "Hello World" -## as the response body. -## -## .. code-block::nim -## import asynchttpserver, asyncdispatch -## -## var server = newAsyncHttpServer() -## proc cb(req: Request) {.async.} = -## await req.respond(Http200, "Hello World") -## -## waitFor server.serve(Port(8080), cb) +## application in production you should use a reverse proxy (for example nginx) +## instead of allowing users to connect directly to this server. + +runnableExamples("-r:off"): + # This example will create an HTTP server on an automatically chosen port. + # It will respond to all requests with a `200 OK` response code and "Hello World" + # as the response body. + import std/asyncdispatch + proc main {.async.} = + var server = newAsyncHttpServer() + proc cb(req: Request) {.async.} = + echo (req.reqMethod, req.url, req.headers) + let headers = {"Content-type": "text/plain; charset=utf-8"} + await req.respond(Http200, "Hello World", headers.newHttpHeaders()) + + server.listen(Port(0)) # or Port(8080) to hardcode the standard HTTP port. + let port = server.getPort + echo "test this with: curl localhost:" & $port.uint16 & "/" + while true: + if server.shouldAcceptRequest(): + await server.acceptRequest(cb) + else: + # too many concurrent connections, `maxFDs` exceeded + # wait 500ms for FDs to be closed + await sleepAsync(500) + + waitFor main() -import tables, asyncnet, asyncdispatch, parseutils, uri, strutils -import httpcore +import std/[asyncnet, asyncdispatch, parseutils, uri, strutils] +import std/httpcore +from std/nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions export httpcore except parseHeader @@ -58,14 +72,25 @@ type reuseAddr: bool reusePort: bool maxBody: int ## The maximum content-length that will be read for the body. + maxFDs: int + +proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} = + ## Returns the port `self` was bound to. + ## + ## Useful for identifying what port `self` is bound to, if it + ## was chosen automatically, for example via `listen(Port(0))`. + runnableExamples: + from std/nativesockets import Port + let server = newAsyncHttpServer() + server.listen(Port(0)) + assert server.getPort.uint16 > 0 + server.close() + result = getLocalAddr(self.socket)[1] proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = - ## Creates a new ``AsyncHttpServer`` instance. - new result - result.reuseAddr = reuseAddr - result.reusePort = reusePort - result.maxBody = maxBody + ## Creates a new `AsyncHttpServer` instance. + result = AsyncHttpServer(reuseAddr: reuseAddr, reusePort: reusePort, maxBody: maxBody) proc addHeaders(msg: var string, headers: HttpHeaders) = for k, v in headers: @@ -79,35 +104,40 @@ proc sendHeaders*(req: Request, headers: HttpHeaders): Future[void] = proc respond*(req: Request, code: HttpCode, content: string, headers: HttpHeaders = nil): Future[void] = - ## Responds to the request with the specified ``HttpCode``, headers and + ## Responds to the request with the specified `HttpCode`, headers and ## content. ## ## This procedure will **not** close the client socket. ## ## Example: - ## - ## .. code-block::nim - ## import json - ## proc handler(req: Request) {.async.} = - ## if req.url.path == "/hello-world": - ## let msg = %* {"message": "Hello World"} - ## let headers = newHttpHeaders([("Content-Type","application/json")]) - ## await req.respond(Http200, $msg, headers) - ## else: - ## await req.respond(Http404, "Not Found") + ## ```Nim + ## import std/json + ## proc handler(req: Request) {.async.} = + ## if req.url.path == "/hello-world": + ## let msg = %* {"message": "Hello World"} + ## let headers = newHttpHeaders([("Content-Type","application/json")]) + ## await req.respond(Http200, $msg, headers) + ## else: + ## await req.respond(Http404, "Not Found") + ## ``` var msg = "HTTP/1.1 " & $code & "\c\L" if headers != nil: msg.addHeaders(headers) - msg.add("Content-Length: ") - # this particular way saves allocations: - msg.add content.len - msg.add "\c\L\c\L" + + # If the headers did not contain a Content-Length use our own + if headers.isNil() or not headers.hasKey("Content-Length"): + msg.add("Content-Length: ") + # this particular way saves allocations: + msg.addInt content.len + msg.add "\c\L" + + msg.add "\c\L" msg.add(content) result = req.client.send(msg) proc respondError(req: Request, code: HttpCode): Future[void] = - ## Responds to the request with the specified ``HttpCode``. + ## Responds to the request with the specified `HttpCode`. let content = $code var msg = "HTTP/1.1 " & content & "\c\L" @@ -128,11 +158,22 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") +func hasChunkedEncoding(request: Request): bool = + ## Searches for a chunked transfer encoding + const transferEncoding = "Transfer-Encoding" + + if request.headers.hasKey(transferEncoding): + for encoding in seq[string](request.headers[transferEncoding]): + if "chunked" == encoding.strip: + # Returns true if it is both an HttpPost and has chunked encoding + return request.reqMethod == HttpPost + return false + proc processRequest( server: AsyncHttpServer, req: FutureVar[Request], client: AsyncSocket, - address: string, + address: sink string, lineFut: FutureVar[string], callback: proc (request: Request): Future[void] {.closure, gcsafe.}, ): Future[bool] {.async.} = @@ -146,7 +187,10 @@ proc processRequest( # \n request.headers.clear() request.body = "" - request.hostname.shallowCopy(address) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + request.hostname = address + else: + request.hostname.shallowCopy(address) assert client != nil request.client = client @@ -247,6 +291,43 @@ proc processRequest( if request.body.len != contentLength: await request.respond(Http400, "Bad Request. Content-Length does not match actual.") return true + elif hasChunkedEncoding(request): + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding + var sizeOrData = 0 + var bytesToRead = 0 + request.body = "" + + while true: + lineFut.mget.setLen(0) + lineFut.clean() + + # The encoding format alternates between specifying a number of bytes to read + # and the data to be read, of the previously specified size + if sizeOrData mod 2 == 0: + # Expect a number of chars to read + await client.recvLineInto(lineFut, maxLength = maxLine) + try: + bytesToRead = lineFut.mget.parseHexInt + except ValueError: + # Malformed request + await request.respond(Http411, ("Invalid chunked transfer encoding - " & + "chunk data size must be hex encoded")) + return true + else: + if bytesToRead == 0: + # Done reading chunked data + break + + # Read bytesToRead and add to body + let chunk = await client.recv(bytesToRead) + request.body.add(chunk) + # Skip \r\n (chunk terminating bytes per spec) + let separator = await client.recv(2) + if separator != "\r\n": + await request.respond(Http400, "Bad Request. Encoding separator must be \\r\\n") + return true + + inc sizeOrData elif request.reqMethod == HttpPost: await request.respond(Http411, "Content-Length required.") return true @@ -287,43 +368,73 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string let retry = await processRequest( server, request, client, address, lineFut, callback ) - if not retry: break + if not retry: + client.close() + break + +const + nimMaxDescriptorsFallback* {.intdefine.} = 16_000 ## fallback value for \ + ## when `maxDescriptors` is not available. + ## This can be set on the command line during compilation + ## via `-d:nimMaxDescriptorsFallback=N` + +proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET) = + ## Listen to the given port and address. + when declared(maxDescriptors): + server.maxFDs = try: maxDescriptors() except: nimMaxDescriptorsFallback + else: + server.maxFDs = nimMaxDescriptorsFallback + server.socket = newAsyncSocket(domain) + if server.reuseAddr: + server.socket.setSockOpt(OptReuseAddr, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) + server.socket.bindAddr(port, address) + server.socket.listen() + +proc shouldAcceptRequest*(server: AsyncHttpServer; + assumedDescriptorsPerRequest = 5): bool {.inline.} = + ## Returns true if the process's current number of opened file + ## descriptors is still within the maximum limit and so it's reasonable to + ## accept yet another request. + result = assumedDescriptorsPerRequest < 0 or + (activeDescriptors() + assumedDescriptorsPerRequest < server.maxFDs) + +proc acceptRequest*(server: AsyncHttpServer, + callback: proc (request: Request): Future[void] {.closure, gcsafe.}) {.async.} = + ## Accepts a single request. Write an explicit loop around this proc so that + ## errors can be handled properly. + var (address, client) = await server.socket.acceptAddr() + asyncCheck processClient(server, client, address, callback) proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure, gcsafe.}, - address = "") {.async.} = + address = ""; + assumedDescriptorsPerRequest = -1; + domain = AF_INET) {.async.} = ## Starts the process of listening for incoming HTTP connections on the ## specified address and port. ## ## When a request is made by a client the specified callback will be called. - server.socket = newAsyncSocket() - if server.reuseAddr: - server.socket.setSockOpt(OptReuseAddr, true) - if server.reusePort: - server.socket.setSockOpt(OptReusePort, true) - server.socket.bindAddr(port, address) - server.socket.listen() - + ## + ## If `assumedDescriptorsPerRequest` is 0 or greater the server cares about + ## the process's maximum file descriptor limit. It then ensures that the + ## process still has the resources for `assumedDescriptorsPerRequest` + ## file descriptors before accepting a connection. + ## + ## You should prefer to call `acceptRequest` instead with a custom server + ## loop so that you're in control over the error handling and logging. + listen server, port, address, domain while true: - var (address, client) = await server.socket.acceptAddr() - asyncCheck processClient(server, client, address, callback) + if shouldAcceptRequest(server, assumedDescriptorsPerRequest): + var (address, client) = await server.socket.acceptAddr() + asyncCheck processClient(server, client, address, callback) + else: + poll() #echo(f.isNil) #echo(f.repr) proc close*(server: AsyncHttpServer) = ## Terminates the async http server instance. server.socket.close() - -when not defined(testing) and isMainModule: - proc main = - var server = newAsyncHttpServer() - proc cb(req: Request) {.async.} = - #echo(req.reqMethod, " ", req.url) - #echo(req.headers) - let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", - "Content-type": "text/plain; charset=utf-8"} - await req.respond(Http200, "Hello World", headers.newHttpHeaders()) - - asyncCheck server.serve(Port(5555), cb) - runForever() - main() diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 11eba427b..d4e72c28a 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -7,104 +7,68 @@ # distribution, for details about the copyright. # -## `asyncdispatch` module depends on the `asyncmacro` module to work properly. +## Implements the `async` and `multisync` macros for `asyncdispatch`. -import macros, strutils, asyncfutures +import std/[macros, strutils, asyncfutures] -proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = - # Skips a nest of StmtList's. - result = node - if node[0].kind == nnkStmtList: - result = skipUntilStmtList(node[0]) +type + Context = ref object + inTry: int + hasRet: bool -proc skipStmtList(node: NimNode): NimNode {.compileTime.} = - result = node - if node[0].kind == nnkStmtList: - result = node[0] +# TODO: Ref https://github.com/nim-lang/Nim/issues/5617 +# TODO: Add more line infos +proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimNode]): NimNode = + result = newCall(theProc, args) + result.copyLineInfo(fromNode) template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = bind finished - let retFutUnown = unown retFutureSym - var nameIterVar = iteratorNameSym - proc identName {.closure.} = + proc identName {.closure, stackTrace: off.} = try: if not nameIterVar.finished: - var next = unown nameIterVar() + var next = nameIterVar() # Continue while the yielded future is already finished. while (not next.isNil) and next.finished: - next = unown nameIterVar() + next = nameIterVar() if nameIterVar.finished: break if next == nil: - if not retFutUnown.finished: - let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " & - "`nil` Future?" - raise newException(AssertionError, msg % strName) + if not retFutureSym.finished: + let msg = "Async procedure ($1) yielded `nil`, are you await'ing a `nil` Future?" + raise newException(AssertionDefect, msg % strName) else: {.gcsafe.}: - {.push hint[ConvFromXtoItselfNotNeeded]: off.} - next.callback = (proc() {.closure, gcsafe.})(identName) - {.pop.} + next.addCallback cast[proc() {.closure, gcsafe.}](identName) except: futureVarCompletions - if retFutUnown.finished: + if retFutureSym.finished: # Take a look at tasyncexceptions for the bug which this fixes. # That test explains it better than I can here. raise else: - retFutUnown.fail(getCurrentException()) + retFutureSym.fail(getCurrentException()) identName() -template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, - rootReceiver: untyped, fromNode: NimNode) = - ## Params: - ## futureVarNode: The NimNode which is a symbol identifying the Future[T] - ## variable to yield. - ## fromNode: Used for better debug information (to give context). - ## valueReceiver: The node which defines an expression that retrieves the - ## future's value. - ## - ## rootReceiver: ??? TODO - # -> yield future<x> - result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) - # -> future<x>.read - valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) - result.add rootReceiver - -template createVar(result: var NimNode, futSymName: string, - asyncProc: NimNode, - valueReceiver, rootReceiver: untyped, - fromNode: NimNode) = - result = newNimNode(nnkStmtList, fromNode) - var futSym = genSym(nskVar, "future") - result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - useVar(result, futSym, valueReceiver, rootReceiver, fromNode) - -proc createFutureVarCompletions(futureVarIdents: seq[NimNode], - fromNode: NimNode): NimNode {.compileTime.} = +proc createFutureVarCompletions(futureVarIdents: seq[NimNode], fromNode: NimNode): NimNode = result = newNimNode(nnkStmtList, fromNode) # Add calls to complete each FutureVar parameter. for ident in futureVarIdents: # Only complete them if they have not been completed already by the user. - # TODO: Once https://github.com/nim-lang/Nim/issues/5617 is fixed. - # TODO: Add line info to the complete() call! # In the meantime, this was really useful for debugging :) #result.add(newCall(newIdentNode("echo"), newStrLitNode(fromNode.lineinfo))) result.add newIfStmt( ( newCall(newIdentNode("not"), newDotExpr(ident, newIdentNode("finished"))), - newCall(newIdentNode("complete"), ident) + newCallWithLineInfo(fromNode, newIdentNode("complete"), ident) ) ) -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - futureVarIdents: seq[NimNode]): NimNode {.compileTime.} = - #echo(node.treeRepr) +proc processBody(ctx: Context; node, needsCompletionSym, retFutureSym: NimNode, futureVarIdents: seq[NimNode]): NimNode = result = node case node.kind of nnkReturnStmt: @@ -113,76 +77,57 @@ proc processBody(node, retFutureSym: NimNode, # As I've painfully found out, the order here really DOES matter. result.add createFutureVarCompletions(futureVarIdents, node) + ctx.hasRet = true if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) + if ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, newIdentNode("result")) else: - result.add newCall(newIdentNode("complete"), retFutureSym) + result.add newAssignment(needsCompletionSym, newLit(true)) else: - let x = node[0].processBody(retFutureSym, subTypeIsVoid, - futureVarIdents) + let x = processBody(ctx, node[0], needsCompletionSym, retFutureSym, futureVarIdents) if x.kind == nnkYieldStmt: result.add x + elif ctx.inTry == 0: + result.add newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, x) else: - result.add newCall(newIdentNode("complete"), retFutureSym, x) + result.add newAssignment(newIdentNode("result"), x) + result.add newAssignment(needsCompletionSym, newLit(true)) result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt - of nnkCommand, nnkCall: - if node[0].eqIdent("await"): - case node[1].kind - of nnkIdent, nnkInfix, nnkDotExpr, nnkCall, nnkCommand: - # await x - # await x or y - # await foo(p, x) - # await foo p, x - var futureValue: NimNode - result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, - futureValue, node) - else: - error("Invalid node kind in 'await', got: " & $node[1].kind) - elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].eqIdent("await"): - # foo await x - var newCommand = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], - newCommand, node) - - of nnkVarSection, nnkLetSection: - case node[0][^1].kind - of nnkCommand: - if node[0][^1][0].eqIdent("await"): - # var x = await y - var newVarSection = node # TODO: Should this use copyNimNode? - result.createVar("future" & node[0][0].strVal, node[0][^1][1], - newVarSection[0][^1], newVarSection, node) - else: discard - of nnkAsgn: - case node[1].kind - of nnkCommand: - if node[1][0].eqIdent("await"): - # x = await y - var newAsgn = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], - newAsgn, node) - else: discard - of nnkDiscardStmt: - # discard await x - if node[0].kind == nnkCommand and - node[0][0].eqIdent("await"): - var newDiscard = node - result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], - newDiscard[0], newDiscard, node) of RoutineNodes-{nnkTemplateDef}: # skip all the nested procedure definitions return - else: discard + of nnkTryStmt: + if result[^1].kind == nnkFinally: + inc ctx.inTry + result[0] = processBody(ctx, result[0], needsCompletionSym, retFutureSym, futureVarIdents) + dec ctx.inTry + for i in 1 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + if ctx.inTry == 0 and ctx.hasRet: + let finallyNode = copyNimNode(result[^1]) + let stmtNode = newNimNode(nnkStmtList) + for child in result[^1]: + stmtNode.add child + stmtNode.add newIfStmt( + ( needsCompletionSym, + newCallWithLineInfo(node, newIdentNode("complete"), retFutureSym, + newIdentNode("result") + ) + ) + ) + finallyNode.add stmtNode + result[^1] = finallyNode + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) + else: + for i in 0 ..< result.len: + result[i] = processBody(ctx, result[i], needsCompletionSym, retFutureSym, futureVarIdents) - for i in 0 ..< result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, - futureVarIdents) + # echo result.repr -proc getName(node: NimNode): string {.compileTime.} = +proc getName(node: NimNode): string = case node.kind of nnkPostfix: return node[1].strVal @@ -191,30 +136,68 @@ proc getName(node: NimNode): string {.compileTime.} = of nnkEmpty: return "anonymous" else: - error("Unknown name.") + error("Unknown name.", node) -proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = +proc getFutureVarIdents(params: NimNode): seq[NimNode] = result = @[] for i in 1 ..< len(params): expectKind(params[i], nnkIdentDefs) if params[i][1].kind == nnkBracketExpr and - params[i][1][0].eqIdent("futurevar"): + params[i][1][0].eqIdent(FutureVar.astToStr): + ## eqIdent: first char is case sensitive!!! result.add(params[i][0]) proc isInvalidReturnType(typeName: string): bool = return typeName notin ["Future"] #, "FutureStream"] -proc verifyReturnType(typeName: string) {.compileTime.} = +proc verifyReturnType(typeName: string, node: NimNode = nil) = if typeName.isInvalidReturnType: error("Expected return type of 'Future' got '$1'" % - typeName) + typeName, node) + +template await*(f: typed): untyped {.used.} = + static: + error "await expects Future[T], got " & $typeof(f) -proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = +template await*[T](f: Future[T]): auto {.used.} = + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template yieldFuture {.redefine.} = yield FutureBase() + + when compiles(yieldFuture): + var internalTmpFuture: FutureBase = f + yield internalTmpFuture + (cast[typeof(f)](internalTmpFuture)).read() + else: + macro errorAsync(futureError: Future[T]) = + error( + "Can only 'await' inside a proc marked as 'async'. Use " & + "'waitFor' when calling an 'async' proc in a non-async scope instead", + futureError) + errorAsync(f) + +proc asyncSingleProc(prc: NimNode): NimNode = ## This macro transforms a single procedure into a closure iterator. - ## The ``async`` macro supports a stmtList holding multiple async procedures. + ## The `async` macro supports a stmtList holding multiple async procedures. + if prc.kind == nnkProcTy: + result = prc + if prc[0][0].kind == nnkEmpty: + result[0][0] = quote do: Future[void] + return result + + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform this node kind into an async proc." & - " proc/method definition or lambda node expected.") + " proc/method definition or lambda node expected.", prc) + + if prc[4].kind != nnkEmpty: + for prag in prc[4]: + if prag.eqIdent("discardable"): + error("Cannot make async proc discardable. Futures have to be " & + "checked with `asyncCheck` instead of discarded", prag) let prcName = prc.name.getName @@ -226,29 +209,24 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # Verify that the return type is a Future[T] if returnType.kind == nnkBracketExpr: let fut = repr(returnType[0]) - verifyReturnType(fut) + verifyReturnType(fut, returnType[0]) baseType = returnType[1] elif returnType.kind in nnkCallKinds and returnType[0].eqIdent("[]"): let fut = repr(returnType[1]) - verifyReturnType(fut) + verifyReturnType(fut, returnType[0]) baseType = returnType[2] elif returnType.kind == nnkEmpty: baseType = returnType else: - verifyReturnType(repr(returnType)) - - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].eqIdent("void")) + verifyReturnType(repr(returnType), returnType) let futureVarIdents = getFutureVarIdents(prc.params) - var outerProcBody = newNimNode(nnkStmtList, prc.body) # Extract the documentation comment from the original procedure declaration. # Note that we're not removing it from the body in order not to make this # transformation even more complex. - if prc.body.len > 1 and prc.body[0].kind == nnkCommentStmt: - outerProcBody.add(prc.body[0]) + let body2 = extractDocCommentsAndRunnables(prc.body) # -> var retFuture = newFuture[T]() var retFutureSym = genSym(nskVar, "retFuture") @@ -269,40 +247,42 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> {.pop.} # -> <proc_body> # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") - var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid, - futureVarIdents) + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") + var needsCompletionSym = genSym(nskVar, "needsCompletion") + var ctx = Context() + var procBody = processBody(ctx, prc.body, needsCompletionSym, retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: + # fix #13899, defer should not escape its original scope + let blockStmt = newStmtList(newTree(nnkBlockStmt, newEmptyNode(), procBody)) + procBody = newStmtList() + let resultIdent = ident"result" + procBody.add quote do: + # Check whether there is an implicit return + when typeof(`blockStmt`) is void: + `blockStmt` + else: + `resultIdent` = `blockStmt` procBody.add(createFutureVarCompletions(futureVarIdents, nil)) + procBody.insert(0): quote do: + {.push warning[resultshadowed]: off.} + when `subRetType` isnot void: + var `resultIdent`: `subRetType` + else: + var `resultIdent`: Future[void] + {.pop.} - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc.body).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + var `needsCompletionSym` = false + procBody.add quote do: + complete(`retFutureSym`, `resultIdent`) - var closureIterator = newProc(iteratorNameSym, [parseExpr("owned(FutureBase)")], + var closureIterator = newProc(iteratorNameSym, [quote do: owned(FutureBase)], procBody, nnkIteratorDef) closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom = prc.body) closureIterator.addPragma(newIdentNode("closure")) # If proc has an explicit gcsafe pragma, we add it to iterator as well. - if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == - "gcsafe") != nil: + if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil: closureIterator.addPragma(newIdentNode("gcsafe")) outerProcBody.add(closureIterator) @@ -311,26 +291,26 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # friendlier stack traces: var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prcName), - cbName, - createFutureVarCompletions(futureVarIdents, nil)) + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) outerProcBody.add procCb # -> return retFuture outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) result = prc + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # xxx consider removing `owned`? it's inconsistent with non-void case + result.params[0] = quote do: owned(Future[void]) - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result.params[0] = parseExpr("owned(Future[void])") + # based on the yglukhov's patch to chronos: https://github.com/status-im/nim-chronos/pull/47 if procBody.kind != nnkEmpty: - result.body = outerProcBody - #echo(treeRepr(result)) - #if prcName == "recvLineInto": - # echo(toStrLit(result)) + body2.add quote do: + `outerProcBody` + result.body = body2 macro async*(prc: untyped): untyped = ## Macro which processes async procedures into the appropriate @@ -344,55 +324,11 @@ macro async*(prc: untyped): untyped = when defined(nimDumpAsync): echo repr result - -# Multisync -proc emptyNoop[T](x: T): T = - # The ``await``s are replaced by a call to this for simplicity. - when T isnot void: - return x - -proc stripAwait(node: NimNode): NimNode = - ## Strips out all ``await`` commands from a procedure body, replaces them - ## with ``emptyNoop`` for simplicity. - result = node - - let emptyNoopSym = bindSym("emptyNoop") - - case node.kind - of nnkCommand, nnkCall: - if node[0].eqIdent("await"): - node[0] = emptyNoopSym - elif node.len > 1 and node[1].kind == nnkCommand and node[1][0].eqIdent("await"): - # foo await x - node[1][0] = emptyNoopSym - of nnkVarSection, nnkLetSection: - case node[0][^1].kind - of nnkCommand: - if node[0][^1][0].eqIdent("await"): - # var x = await y - node[0][^1][0] = emptyNoopSym - else: discard - of nnkAsgn: - case node[1].kind - of nnkCommand: - if node[1][0].eqIdent("await"): - # x = await y - node[1][0] = emptyNoopSym - else: discard - of nnkDiscardStmt: - # discard await x - if node[0].kind == nnkCommand and node[0][0].eqIdent("await"): - node[0][0] = emptyNoopSym - else: discard - - for i in 0 ..< result.len: - result[i] = stripAwait(result[i]) - proc splitParamType(paramType: NimNode, async: bool): NimNode = result = paramType if paramType.kind == nnkInfix and paramType[0].strVal in ["|", "or"]: - let firstAsync = "async" in paramType[1].strVal.normalize - let secondAsync = "async" in paramType[2].strVal.normalize + let firstAsync = "async" in paramType[1].toStrLit().strVal.normalize + let secondAsync = "async" in paramType[2].toStrLit().strVal.normalize if firstAsync: result = paramType[if async: 1 else: 2] @@ -404,14 +340,14 @@ proc stripReturnType(returnType: NimNode): NimNode = result = returnType if returnType.kind == nnkBracketExpr: let fut = repr(returnType[0]) - verifyReturnType(fut) + verifyReturnType(fut, returnType) result = returnType[1] proc splitProc(prc: NimNode): (NimNode, NimNode) = ## Takes a procedure definition which takes a generic union of arguments, ## for example: proc (socket: Socket | AsyncSocket). - ## It transforms them so that ``proc (socket: Socket)`` and - ## ``proc (socket: AsyncSocket)`` are returned. + ## It transforms them so that `proc (socket: Socket)` and + ## `proc (socket: AsyncSocket)` are returned. result[0] = prc.copyNimTree() # Retrieve the `T` inside `Future[T]`. @@ -420,8 +356,12 @@ proc splitProc(prc: NimNode): (NimNode, NimNode) = for i in 1 ..< result[0][3].len: # Sync proc (0) -> FormalParams (3) -> IdentDefs, the parameter (i) -> # parameter type (1). - result[0][3][i][1] = splitParamType(result[0][3][i][1], async = false) - result[0][6] = stripAwait(result[0][6]) + result[0][3][i][1] = splitParamType(result[0][3][i][1], async=false) + var multisyncAwait = quote: + template await(value: typed): untyped = + value + + result[0][^1] = nnkStmtList.newTree(multisyncAwait, result[0][^1]) result[1] = prc.copyNimTree() if result[1][3][0].kind == nnkBracketExpr: @@ -435,14 +375,9 @@ macro multisync*(prc: untyped): untyped = ## Macro which processes async procedures into both asynchronous and ## synchronous procedures. ## - ## The generated async procedures use the ``async`` macro, whereas the - ## generated synchronous procedures simply strip off the ``await`` calls. + ## The generated async procedures use the `async` macro, whereas the + ## generated synchronous procedures simply strip off the `await` calls. let (sync, asyncPrc) = splitProc(prc) result = newStmtList() result.add(asyncSingleProc(asyncPrc)) result.add(sync) - -proc await*[T](x: T) = - ## The 'await' keyword is also defined here for technical - ## reasons. (Generic symbol lookup prepass.) - {.error: "Await only available within .async".} diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 88852fb84..ee07e599e 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -8,27 +8,27 @@ # ## This module implements a high-level asynchronous sockets API based on the -## asynchronous dispatcher defined in the ``asyncdispatch`` module. +## asynchronous dispatcher defined in the `asyncdispatch` module. ## ## Asynchronous IO in Nim ## ====================== ## ## Async IO in Nim consists of multiple layers (from highest to lowest): ## -## * ``asyncnet`` module +## * `asyncnet` module ## ## * Async await ## -## * ``asyncdispatch`` module (event loop) +## * `asyncdispatch` module (event loop) ## -## * ``selectors`` module +## * `selectors` module ## ## Each builds on top of the layers below it. The selectors module is an -## abstraction for the various system ``select()`` mechanisms such as epoll or +## abstraction for the various system `select()` mechanisms such as epoll or ## kqueue. If you wish you can use it directly, and some people have done so ## `successfully <http://goran.krampe.se/2014/10/25/nim-socketserver/>`_. ## But you must be aware that on Windows it only supports -## ``select()``. +## `select()`. ## ## The async dispatcher implements the proactor pattern and also has an ## implementation of IOCP. It implements the proactor pattern for other @@ -45,16 +45,16 @@ ## layers interchangeably (as long as you only care about non-Windows ## platforms). ## -## For most applications using ``asyncnet`` is the way to go as it builds +## For most applications using `asyncnet` is the way to go as it builds ## over all the layers, providing some extra features such as buffering. ## ## SSL ## === ## -## SSL can be enabled by compiling with the ``-d:ssl`` flag. +## SSL can be enabled by compiling with the `-d:ssl` flag. ## -## You must create a new SSL context with the ``newContext`` function defined -## in the ``net`` module. You may then call ``wrapSocket`` on your socket using +## You must create a new SSL context with the `newContext` function defined +## in the `net` module. You may then call `wrapSocket` on your socket using ## the newly created SSL context to get an SSL socket. ## ## Examples @@ -65,9 +65,8 @@ ## ## The following example demonstrates a simple chat server. ## -## .. code-block::nim -## -## import asyncnet, asyncdispatch +## ```Nim +## import std/[asyncnet, asyncdispatch] ## ## var clients {.threadvar.}: seq[AsyncSocket] ## @@ -93,21 +92,25 @@ ## ## asyncCheck serve() ## runForever() -## +## ``` + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] -import asyncdispatch -import nativesockets -import net -import os +import std/[asyncdispatch, nativesockets, net, os] export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when defineSsl: - import openssl + import std/openssl type # TODO: I would prefer to just do: @@ -125,25 +128,33 @@ type sslContext: SslContext bioIn: BIO bioOut: BIO + sslNoShutdown: bool domain: Domain sockType: SockType protocol: Protocol AsyncSocket* = ref AsyncSocketDesc proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): owned(AsyncSocket) = - ## Creates a new ``AsyncSocket`` based on the supplied params. + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, + buffered = true, + inheritable = defined(nimInheritHandles)): owned(AsyncSocket) = + ## Creates a new `AsyncSocket` based on the supplied params. ## - ## The supplied ``fd``'s non-blocking state will be enabled implicitly. + ## The supplied `fd`'s non-blocking state will be enabled implicitly. ## - ## **Note**: This procedure will **NOT** register ``fd`` with the global + ## If `inheritable` is false (the default), the supplied `fd` will not + ## be inheritable by child processes. + ## + ## **Note**: This procedure will **NOT** register `fd` with the global ## async dispatcher. You need to do this manually. If you have used - ## ``newAsyncNativeSocket`` to create ``fd`` then it's already registered. + ## `newAsyncNativeSocket` to create `fd` then it's already registered. assert fd != osInvalidSocket.AsyncFD new(result) result.fd = fd.SocketHandle fd.SocketHandle.setBlocking(false) + if not fd.SocketHandle.setInheritable(inheritable): + raiseOSError(osLastError()) result.isBuffered = buffered result.domain = domain result.sockType = sockType @@ -152,15 +163,19 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, result.currPos = 0 proc newAsyncSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): owned(AsyncSocket) = + protocol: Protocol = IPPROTO_TCP, buffered = true, + inheritable = defined(nimInheritHandles)): owned(AsyncSocket) = ## Creates a new asynchronous socket. ## ## This procedure will also create a brand new file descriptor for ## this socket. - let fd = createAsyncNativeSocket(domain, sockType, protocol) + ## + ## If `inheritable` is false (the default), the new file descriptor will not + ## be inheritable by child processes. + let fd = createAsyncNativeSocket(domain, sockType, protocol, inheritable) if fd.SocketHandle == osInvalidSocket: raiseOSError(osLastError()) - result = newAsyncSocket(fd, domain, sockType, protocol, buffered) + result = newAsyncSocket(fd, domain, sockType, protocol, buffered, inheritable) proc getLocalAddr*(socket: AsyncSocket): (string, Port) = ## Get the socket's local address and port number. @@ -168,28 +183,34 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: AsyncSocket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc newAsyncSocket*(domain, sockType, protocol: cint, - buffered = true): owned(AsyncSocket) = + buffered = true, + inheritable = defined(nimInheritHandles)): owned(AsyncSocket) = ## Creates a new asynchronous socket. ## ## This procedure will also create a brand new file descriptor for ## this socket. - let fd = createAsyncNativeSocket(domain, sockType, protocol) + ## + ## If `inheritable` is false (the default), the new file descriptor will not + ## be inheritable by child processes. + let fd = createAsyncNativeSocket(domain, sockType, protocol, inheritable) if fd.SocketHandle == osInvalidSocket: raiseOSError(osLastError()) result = newAsyncSocket(fd, Domain(domain), SockType(sockType), - Protocol(protocol), buffered) + Protocol(protocol), buffered, inheritable) when defineSsl: - proc getSslError(handle: SslPtr, err: cint): cint = + proc getSslError(socket: AsyncSocket, err: cint): cint = + assert socket.isSsl assert err < 0 - var ret = SSL_get_error(handle, err.cint) + var ret = SSL_get_error(socket.sslHandle, err.cint) case ret of SSL_ERROR_ZERO_RETURN: raiseSSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") @@ -200,6 +221,7 @@ when defineSsl: of SSL_ERROR_WANT_X509_LOOKUP: raiseSSLError("Function for x509 lookup has been called.") of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + socket.sslNoShutdown = true raiseSSLError() else: raiseSSLError("Unknown Error") @@ -208,7 +230,7 @@ when defineSsl: let len = bioCtrlPending(socket.bioOut) if len > 0: var data = newString(len) - let read = bioRead(socket.bioOut, addr data[0], len) + let read = bioRead(socket.bioOut, cast[cstring](addr data[0]), len) assert read != 0 if read < 0: raiseSSLError() @@ -217,7 +239,7 @@ when defineSsl: proc appeaseSsl(socket: AsyncSocket, flags: set[SocketFlag], sslError: cint): owned(Future[bool]) {.async.} = - ## Returns ``true`` if ``socket`` is still connected, otherwise ``false``. + ## Returns `true` if `socket` is still connected, otherwise `false`. result = true case sslError of SSL_ERROR_WANT_WRITE: @@ -226,7 +248,7 @@ when defineSsl: var data = await recv(socket.fd.AsyncFD, BufferSize, flags) let length = len(data) if length > 0: - let ret = bioWrite(socket.bioIn, addr data[0], length.cint) + let ret = bioWrite(socket.bioIn, cast[cstring](addr data[0]), length.cint) if ret < 0: raiseSSLError() elif length == 0: @@ -240,18 +262,20 @@ when defineSsl: op: untyped) = var opResult {.inject.} = -1.cint while opResult < 0: + ErrClearError() # Call the desired operation. opResult = op - # Bit hackish here. - # TODO: Introduce an async template transformation pragma? - + let err = + if opResult < 0: + getSslError(socket, opResult.cint) + else: + SSL_ERROR_NONE # Send any remaining pending SSL data. - yield sendPendingSslData(socket, flags) + await sendPendingSslData(socket, flags) # If the operation failed, try to see if SSL has some data to read # or write. if opResult < 0: - let err = getSslError(socket.sslHandle, opResult.cint) let fut = appeaseSsl(socket, flags, err.cint) yield fut if not fut.read(): @@ -264,9 +288,9 @@ when defineSsl: proc dial*(address: string, port: Port, protocol = IPPROTO_TCP, buffered = true): owned(Future[AsyncSocket]) {.async.} = - ## Establishes connection to the specified ``address``:``port`` pair via the + ## Establishes connection to the specified `address`:`port` pair via the ## specified protocol. The procedure iterates through possible - ## resolutions of the ``address`` until it succeeds, meaning that it + ## resolutions of the `address` until it succeeds, meaning that it ## seamlessly works with both IPv4 and IPv6. ## Returns AsyncSocket ready to send or receive data. let asyncFd = await asyncdispatch.dial(address, port, protocol) @@ -275,9 +299,9 @@ proc dial*(address: string, port: Port, protocol = IPPROTO_TCP, result = newAsyncSocket(asyncFd, domain, sockType, protocol, buffered) proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = - ## Connects ``socket`` to server at ``address:port``. + ## Connects `socket` to server at `address:port`. ## - ## Returns a ``Future`` which will complete when the connection succeeds + ## Returns a `Future` which will complete when the connection succeeds ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: @@ -293,7 +317,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = template readInto(buf: pointer, size: int, socket: AsyncSocket, flags: set[SocketFlag]): int = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that + ## Reads **up to** `size` bytes from `socket` into `buf`. Note that ## this is a template and not a proc. assert(not socket.closed, "Cannot `recv` on a closed socket") var res = 0 @@ -304,10 +328,8 @@ template readInto(buf: pointer, size: int, socket: AsyncSocket, sslRead(socket.sslHandle, cast[cstring](buf), size.cint)) res = opResult else: - var recvIntoFut = asyncdispatch.recvInto(socket.fd.AsyncFD, buf, size, flags) - yield recvIntoFut # Not in SSL mode. - res = recvIntoFut.read() + res = await asyncdispatch.recvInto(socket.fd.AsyncFD, buf, size, flags) res template readIntoBuf(socket: AsyncSocket, @@ -319,10 +341,10 @@ template readIntoBuf(socket: AsyncSocket, proc recvInto*(socket: AsyncSocket, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): owned(Future[int]) {.async.} = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. + ## Reads **up to** `size` bytes from `socket` into `buf`. ## ## For buffered sockets this function will attempt to read all the requested - ## data. It will read this data in ``BufferSize`` chunks. + ## data. It will read this data in `BufferSize` chunks. ## ## For unbuffered sockets this function makes no effort to read ## all the data requested. It will return as much data as the operating system @@ -333,7 +355,7 @@ proc recvInto*(socket: AsyncSocket, buf: pointer, size: int, ## requested data. ## ## If socket is disconnected and no data is available - ## to be read then the future will complete with a value of ``0``. + ## to be read then the future will complete with a value of `0`. if socket.isBuffered: let originalBufPos = socket.currPos @@ -367,10 +389,10 @@ proc recvInto*(socket: AsyncSocket, buf: pointer, size: int, proc recv*(socket: AsyncSocket, size: int, flags = {SocketFlag.SafeDisconn}): owned(Future[string]) {.async.} = - ## Reads **up to** ``size`` bytes from ``socket``. + ## Reads **up to** `size` bytes from `socket`. ## ## For buffered sockets this function will attempt to read all the requested - ## data. It will read this data in ``BufferSize`` chunks. + ## data. It will read this data in `BufferSize` chunks. ## ## For unbuffered sockets this function makes no effort to read ## all the data requested. It will return as much data as the operating system @@ -381,10 +403,11 @@ proc recv*(socket: AsyncSocket, size: int, ## requested data. ## ## If socket is disconnected and no data is available - ## to be read then the future will complete with a value of ``""``. + ## to be read then the future will complete with a value of `""`. if socket.isBuffered: result = newString(size) - shallow(result) + when not defined(nimSeqsV2): + shallow(result) let originalBufPos = socket.currPos if socket.bufLen == 0: @@ -419,7 +442,7 @@ proc recv*(socket: AsyncSocket, size: int, proc send*(socket: AsyncSocket, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}) {.async.} = - ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all + ## Sends `size` bytes from `buf` to `socket`. The returned future will complete once all ## data has been sent. assert socket != nil assert(not socket.closed, "Cannot `send` on a closed socket") @@ -433,25 +456,30 @@ proc send*(socket: AsyncSocket, buf: pointer, size: int, proc send*(socket: AsyncSocket, data: string, flags = {SocketFlag.SafeDisconn}) {.async.} = - ## Sends ``data`` to ``socket``. The returned future will complete once all + ## Sends `data` to `socket`. The returned future will complete once all ## data has been sent. assert socket != nil if socket.isSsl: when defineSsl: var copy = data sslLoop(socket, flags, - sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) + sslWrite(socket.sslHandle, cast[cstring](addr copy[0]), copy.len.cint)) await sendPendingSslData(socket, flags) else: await send(socket.fd.AsyncFD, data, flags) -proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}): +proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)): owned(Future[tuple[address: string, client: AsyncSocket]]) = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. + ## + ## If `inheritable` is false (the default), the resulting client socket will + ## not be inheritable by child processes. + ## ## The future will complete when the connection is successfully accepted. var retFuture = newFuture[tuple[address: string, client: AsyncSocket]]("asyncnet.acceptAddr") - var fut = acceptAddr(socket.fd.AsyncFD, flags) + var fut = acceptAddr(socket.fd.AsyncFD, flags, inheritable) fut.callback = proc (future: Future[tuple[address: string, client: AsyncFD]]) = assert future.finished @@ -460,7 +488,7 @@ proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}): else: let resultTup = (future.read.address, newAsyncSocket(future.read.client, socket.domain, - socket.sockType, socket.protocol, socket.isBuffered)) + socket.sockType, socket.protocol, socket.isBuffered, inheritable)) retFuture.complete(resultTup) return retFuture @@ -468,6 +496,8 @@ proc accept*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}): owned(Future[AsyncSocket]) = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection. + ## If `inheritable` is false (the default), the resulting client socket will + ## not be inheritable by child processes. ## The future will complete when the connection is successfully accepted. var retFut = newFuture[AsyncSocket]("asyncnet.accept") var fut = acceptAddr(socket, flags) @@ -482,25 +512,24 @@ proc accept*(socket: AsyncSocket, proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {.async.} = - ## Reads a line of data from ``socket`` into ``resString``. + ## Reads a line of data from `socket` into `resString`. ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## If a full line is read `\r\L` is not + ## added to `line`, however if solely `\r\L` is read then `line` ## will be set to it. ## - ## If the socket is disconnected, ``line`` will be set to ``""``. + ## If the socket is disconnected, `line` will be set to `""`. ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. + ## If the socket is disconnected in the middle of a line (before `\r\L` + ## is read) then line will be set to `""`. ## The partial line **will be lost**. ## - ## The ``maxLength`` parameter determines the maximum amount of characters - ## that can be read. ``resString`` will be truncated after that. + ## The `maxLength` parameter determines the maximum amount of characters + ## that can be read. `resString` will be truncated after that. ## - ## **Warning**: The ``Peek`` flag is not yet implemented. + ## .. warning:: The `Peek` flag is not yet implemented. ## - ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the - ## protocol uses ``\r\L`` to delimit a new line. + ## .. warning:: `recvLineInto` on unbuffered sockets assumes that the protocol uses `\r\L` to delimit a new line. assert SocketFlag.Peek notin flags ## TODO: result = newFuture[void]("asyncnet.recvLineInto") @@ -575,26 +604,25 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], proc recvLine*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength): owned(Future[string]) {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once + ## Reads a line of data from `socket`. Returned future will complete once ## a full line is read or an error occurs. ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## If a full line is read `\r\L` is not + ## added to `line`, however if solely `\r\L` is read then `line` ## will be set to it. ## - ## If the socket is disconnected, ``line`` will be set to ``""``. + ## If the socket is disconnected, `line` will be set to `""`. ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. + ## If the socket is disconnected in the middle of a line (before `\r\L` + ## is read) then line will be set to `""`. ## The partial line **will be lost**. ## - ## The ``maxLength`` parameter determines the maximum amount of characters + ## The `maxLength` parameter determines the maximum amount of characters ## that can be read. The result is truncated after that. ## - ## **Warning**: The ``Peek`` flag is not yet implemented. + ## .. warning:: The `Peek` flag is not yet implemented. ## - ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol - ## uses ``\r\L`` to delimit a new line. + ## .. warning:: `recvLine` on unbuffered sockets assumes that the protocol uses `\r\L` to delimit a new line. assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this @@ -605,8 +633,8 @@ proc recvLine*(socket: AsyncSocket, proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the + ## Marks `socket` as accepting connections. + ## `Backlog` specifies the maximum length of the ## queue of pending connections. ## ## Raises an OSError error upon failure. @@ -614,9 +642,9 @@ proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. tags: [ReadIOEffect].} = - ## Binds ``address``:``port`` to the socket. + ## Binds `address`:`port` to the socket. ## - ## If ``address`` is "" then ADDR_ANY will be bound. + ## If `address` is "" then ADDR_ANY will be bound. var realaddr = address if realaddr == "": case socket.domain @@ -628,11 +656,16 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) -when defined(posix): +proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = + ## Determines whether an AsyncSocket has data buffered. + # xxx dedup with std/net + s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen + +when defined(posix) and not useNimNetLite: proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = ## Binds Unix socket to `path`. @@ -649,12 +682,12 @@ when defined(posix): elif ret == EINTR: return false else: - retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret)))) + retFuture.fail(newOSError(OSErrorCode(ret))) return true var socketAddr = makeUnixAddr(path) let ret = socket.fd.connect(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) if ret == 0: # Request to connect completed immediately. retFuture.complete() @@ -663,7 +696,7 @@ when defined(posix): if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: addWrite(AsyncFD(socket.fd), cb) else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) + retFuture.fail(newOSError(lastError)) proc bindUnix*(socket: AsyncSocket, path: string) {. tags: [ReadIOEffect].} = @@ -672,7 +705,7 @@ when defined(posix): when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) elif defined(nimdoc): @@ -689,22 +722,38 @@ elif defined(nimdoc): proc close*(socket: AsyncSocket) = ## Closes the socket. + if socket.closed: return + defer: socket.fd.AsyncFD.closeSocket() + socket.closed = true # TODO: Add extra debugging checks for this. + when defineSsl: if socket.isSsl: - let res = SSL_shutdown(socket.sslHandle) + let res = + # Don't call SSL_shutdown if the connection has not been fully + # established, see: + # https://github.com/openssl/openssl/issues/710#issuecomment-253897666 + if not socket.sslNoShutdown and SSL_in_init(socket.sslHandle) == 0: + ErrClearError() + SSL_shutdown(socket.sslHandle) + else: + 0 SSL_free(socket.sslHandle) if res == 0: discard elif res != 1: raiseSSLError() - socket.closed = true # TODO: Add extra debugging checks for this. when defineSsl: + proc sslHandle*(self: AsyncSocket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns - ## ``socket`` into an SSL socket. + ## `socket` into an SSL socket. ## ## **Disclaimer**: This code is not well tested, may be very unsafe and ## prone to security vulnerabilities. @@ -718,12 +767,14 @@ when defineSsl: socket.bioOut = bioNew(bioSMem()) sslSetBio(socket.sslHandle, socket.bioIn, socket.bioOut) + socket.sslNoShutdown = true + proc wrapConnectedSocket*(ctx: SslContext, socket: AsyncSocket, handshake: SslHandshakeType, hostname: string = "") = ## Wraps a connected socket in an SSL context. This function effectively - ## turns ``socket`` into an SSL socket. - ## ``hostname`` should be specified so that the client knows which hostname + ## turns `socket` into an SSL socket. + ## `hostname` should be specified so that the client knows which hostname ## the server certificate should be validated against. ## ## This should be called on a connected socket, and will perform @@ -743,20 +794,31 @@ when defineSsl: of handshakeAsServer: sslSetAcceptState(socket.sslHandle) + proc getPeerCertificates*(socket: AsyncSocket): seq[Certificate] {.since: (1, 1).} = + ## Returns the certificate chain received by the peer we are connected to + ## through the given socket. + ## The handshake must have been completed and the certificate chain must + ## have been verified successfully or else an empty sequence is returned. + ## The chain is ordered from leaf certificate to root certificate. + if not socket.isSsl: + result = newSeq[Certificate]() + else: + result = getPeerCertificates(socket.sslHandle) + proc getSockOpt*(socket: AsyncSocket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = - ## Retrieves option ``opt`` as a boolean value. + ## Retrieves option `opt` as a boolean value. var res = getSockOptInt(socket.fd, cint(level), toCInt(opt)) result = res != 0 proc setSockOpt*(socket: AsyncSocket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = - ## Sets option ``opt`` to a boolean value specified by ``value``. + ## Sets option `opt` to a boolean value specified by `value`. var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) proc isSsl*(socket: AsyncSocket): bool = - ## Determines whether ``socket`` is a SSL socket. + ## Determines whether `socket` is a SSL socket. socket.isSsl proc getFd*(socket: AsyncSocket): SocketHandle = @@ -767,6 +829,130 @@ proc isClosed*(socket: AsyncSocket): bool = ## Determines whether the socket has been closed. return socket.closed +proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, + flags = {SocketFlag.SafeDisconn}): owned(Future[void]) + {.async, since: (1, 3).} = + ## This proc sends `data` to the specified `address`, which may be an IP + ## address or a hostname. If a hostname is specified this function will try + ## each IP of that hostname. The returned future will complete once all data + ## has been sent. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + assert(socket.protocol != IPPROTO_TCP, + "Cannot `sendTo` on a TCP socket. Use `send` instead") + assert(not socket.closed, "Cannot `sendTo` on a closed socket") + + let aiList = getAddrInfo(address, port, socket.domain, socket.sockType, + socket.protocol) + + var + it = aiList + success = false + lastException: ref Exception + + while it != nil: + let fut = sendTo(socket.fd.AsyncFD, cstring(data), len(data), it.ai_addr, + it.ai_addrlen.SockLen, flags) + + yield fut + + if not fut.failed: + success = true + + break + + lastException = fut.readError() + + it = it.ai_next + + freeAddrInfo(aiList) + + if not success: + if lastException != nil: + raise lastException + else: + raise newException(IOError, "Couldn't resolve address: " & address) + +proc recvFrom*(socket: AsyncSocket, data: FutureVar[string], size: int, + address: FutureVar[string], port: FutureVar[Port], + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) + {.async, since: (1, 3).} = + ## Receives a datagram data from `socket` into `data`, which must be at + ## least of size `size`. The address and port of datagram's sender will be + ## stored into `address` and `port`, respectively. Returned future will + ## complete once one datagram has been received, and will return size of + ## packet received. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + ## + ## **Notes** + ## * `data` must be initialized to the length of `size`. + ## * `address` must be initialized to 46 in length. + template adaptRecvFromToDomain(domain: Domain) = + var lAddr = sizeof(sAddr).SockLen + + result = await recvFromInto(AsyncFD(getFd(socket)), cstring(data.mget()), size, + cast[ptr SockAddr](addr sAddr), addr lAddr, + flags) + + data.mget().setLen(result) + data.complete() + + getAddrString(cast[ptr SockAddr](addr sAddr), address.mget()) + + address.complete() + + when domain == AF_INET6: + port.complete(ntohs(sAddr.sin6_port).Port) + else: + port.complete(ntohs(sAddr.sin_port).Port) + + assert(socket.protocol != IPPROTO_TCP, + "Cannot `recvFrom` on a TCP socket. Use `recv` or `recvInto` instead") + assert(not socket.closed, "Cannot `recvFrom` on a closed socket") + assert(size == len(data.mget()), + "`date` was not initialized correctly. `size` != `len(data.mget())`") + assert(46 == len(address.mget()), + "`address` was not initialized correctly. 46 != `len(address.mget())`") + + case socket.domain + of AF_INET6: + var sAddr: Sockaddr_in6 + adaptRecvFromToDomain(AF_INET6) + of AF_INET: + var sAddr: Sockaddr_in + adaptRecvFromToDomain(AF_INET) + else: + raise newException(ValueError, "Unknown socket address family") + +proc recvFrom*(socket: AsyncSocket, size: int, + flags = {SocketFlag.SafeDisconn}): + owned(Future[tuple[data: string, address: string, port: Port]]) + {.async, since: (1, 3).} = + ## Receives a datagram data from `socket`, which must be at least of size + ## `size`. Returned future will complete once one datagram has been received + ## and will return tuple with: data of packet received; and address and port + ## of datagram's sender. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + var + data = newFutureVar[string]() + address = newFutureVar[string]() + port = newFutureVar[Port]() + + data.mget().setLen(size) + address.mget().setLen(46) + + let read = await recvFrom(socket, data, size, address, port, flags) + + result = (data.mget(), address.mget(), port.mget()) + when not defined(testing) and isMainModule: type TestCases = enum diff --git a/lib/pure/asyncstreams.nim b/lib/pure/asyncstreams.nim index 44e73003e..c97b98d55 100644 --- a/lib/pure/asyncstreams.nim +++ b/lib/pure/asyncstreams.nim @@ -9,9 +9,12 @@ ## Unstable API. -import asyncfutures +import std/asyncfutures -import deques +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/deques type FutureStream*[T] = ref object ## Special future that acts as @@ -21,16 +24,17 @@ type queue: Deque[T] finished: bool cb: proc () {.closure, gcsafe.} + error*: ref Exception proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] = - ## Create a new ``FutureStream``. This future's callback is activated when + ## Create a new `FutureStream`. This future's callback is activated when ## two events occur: ## ## * New data is written into the future stream. ## * The future stream is completed (this means that no more data will be ## written). ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## Specifying `fromProc`, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. ## ## **Note:** The API of FutureStream is still new and so has a higher @@ -39,8 +43,17 @@ proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] = result.queue = initDeque[T]() proc complete*[T](future: FutureStream[T]) = - ## Completes a ``FutureStream`` signalling the end of data. + ## Completes a `FutureStream` signalling the end of data. + assert(future.error == nil, "Trying to complete failed stream") + future.finished = true + if not future.cb.isNil: + future.cb() + +proc fail*[T](future: FutureStream[T], error: ref Exception) = + ## Completes `future` with `error`. + assert(not future.finished) future.finished = true + future.error = error if not future.cb.isNil: future.cb() @@ -50,24 +63,29 @@ proc `callback=`*[T](future: FutureStream[T], ## future stream. ## ## The callback is also called when the future is completed. So you should - ## use ``finished`` to check whether data is available. + ## use `finished` to check whether data is available. ## - ## If the future stream already has data or is finished then ``cb`` will be + ## If the future stream already has data or is finished then `cb` will be ## called immediately. - future.cb = proc () = cb(future) + proc named() = cb(future) + future.cb = named if future.queue.len > 0 or future.finished: callSoon(future.cb) proc finished*[T](future: FutureStream[T]): bool = - ## Check if a ``FutureStream`` is finished. ``true`` value means that + ## Check if a `FutureStream` is finished. `true` value means that ## no more data will be placed inside the stream *and* that there is ## no data waiting to be retrieved. result = future.finished and future.queue.len == 0 +proc failed*[T](future: FutureStream[T]): bool = + ## Determines whether `future` completed with an error. + return future.error != nil + proc write*[T](future: FutureStream[T], value: T): Future[void] = ## Writes the specified value inside the specified future stream. ## - ## This will raise ``ValueError`` if ``future`` is finished. + ## This will raise `ValueError` if `future` is finished. result = newFuture[void]("FutureStream.put") if future.finished: let msg = "FutureStream is finished and so no longer accepts new data." @@ -80,37 +98,43 @@ proc write*[T](future: FutureStream[T], value: T): Future[void] = result.complete() proc read*[T](future: FutureStream[T]): owned(Future[(bool, T)]) = - ## Returns a future that will complete when the ``FutureStream`` has data + ## Returns a future that will complete when the `FutureStream` has data ## placed into it. The future will be completed with the oldest ## value stored inside the stream. The return value will also determine - ## whether data was retrieved, ``false`` means that the future stream was + ## whether data was retrieved, `false` means that the future stream was ## completed and no data was retrieved. ## ## This function will remove the data that was returned from the underlying - ## ``FutureStream``. + ## `FutureStream`. var resFut = newFuture[(bool, T)]("FutureStream.take") let savedCb = future.cb - var newCb = - proc (fs: FutureStream[T]) = - # Exit early if `resFut` is already complete. (See #8994). - if resFut.finished: return - - # We don't want this callback called again. - future.cb = nil - - # The return value depends on whether the FutureStream has finished. - var res: (bool, T) - if finished(fs): - # Remember, this callback is called when the FutureStream is completed. - res[0] = false - else: - res[0] = true - res[1] = fs.queue.popFirst() - + proc newCb(fs: FutureStream[T]) = + # Exit early if `resFut` is already complete. (See #8994). + if resFut.finished: return + + # We don't want this callback called again. + #future.cb = nil + + # The return value depends on whether the FutureStream has finished. + var res: (bool, T) + if finished(fs): + # Remember, this callback is called when the FutureStream is completed. + res[0] = false + else: + res[0] = true + res[1] = fs.queue.popFirst() + + if fs.failed: + resFut.fail(fs.error) + else: resFut.complete(res) - # If the saved callback isn't nil then let's call it. - if not savedCb.isNil: savedCb() + # If the saved callback isn't nil then let's call it. + if not savedCb.isNil: + if fs.queue.len > 0: + savedCb() + else: + future.cb = savedCb if future.queue.len > 0 or future.finished: newCb(future) diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 067f21c5f..591d22cc0 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -16,59 +16,77 @@ ## Each Base64 digit represents exactly 6 bits of data. Three 8-bit ## bytes (i.e., a total of 24 bits) can therefore be represented by ## four 6-bit Base64 digits. -## -## Basic usage -## =========== -## + +##[ +# Basic usage ## Encoding data -## ------------- -## -## .. code-block::nim -## import base64 -## let encoded = encode("Hello World") -## assert encoded == "SGVsbG8gV29ybGQ=" +]## + +runnableExamples: + let encoded = encode("Hello World") + assert encoded == "SGVsbG8gV29ybGQ=" + ## ## Apart from strings you can also encode lists of integers or characters: ## -## .. code-block::nim -## import base64 -## let encodedInts = encode([1,2,3]) -## assert encodedInts == "AQID" -## let encodedChars = encode(['h','e','y']) -## assert encodedChars == "aGV5" -## -## + +runnableExamples: + let encodedInts = encode([1'u8,2,3]) + assert encodedInts == "AQID" + let encodedChars = encode(['h','e','y']) + assert encodedChars == "aGV5" + +##[ ## Decoding data -## ------------- -## -## .. code-block::nim -## import base64 -## let decoded = decode("SGVsbG8gV29ybGQ=") -## assert decoded == "Hello World" -## -## +]## + +runnableExamples: + let decoded = decode("SGVsbG8gV29ybGQ=") + assert decoded == "Hello World" + +##[ +## URL Safe Base64 +]## + +runnableExamples: + assert encode("c\xf7>", safe = true) == "Y_c-" + assert encode("c\xf7>", safe = false) == "Y/c+" + ## See also ## ======== ## ## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types -## * `md5 module<md5.html>`_ implements the MD5 checksum algorithm -## * `sha1 module<sha1.html>`_ implements a sha1 encoder and decoder +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm +## * `sha1 module<sha1.html>`_ for the SHA-1 checksum algorithm + +template cbBase(a, b): untyped = [ + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b] + +const + cb64 = cbBase('+', '/') + cb64safe = cbBase('-', '_') const - cb64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" invalidChar = 255 -template encodeInternal(s: typed): untyped = +template encodeSize(size: int): int = (size * 4 div 3) + 6 + +template encodeInternal(s, alphabet: typed): untyped = ## encodes `s` into base64 representation. - proc encodeSize(size: int): int = - return (size * 4 div 3) + 6 result.setLen(encodeSize(s.len)) + let + padding = s.len mod 3 + inputEnds = s.len - padding + var inputIndex = 0 outputIndex = 0 - inputEnds = s.len - s.len mod 3 n: uint32 b: uint32 @@ -77,8 +95,8 @@ template encodeInternal(s: typed): untyped = n = exp inc inputIndex - template outputChar(x: untyped) = - result[outputIndex] = cb64[x and 63] + template outputChar(x: typed) = + result[outputIndex] = alphabet[x and 63] inc outputIndex template outputChar(c: char) = @@ -94,7 +112,6 @@ template encodeInternal(s: typed): untyped = outputChar(n shr 6) outputChar(n shr 0) - var padding = s.len mod 3 if padding == 1: inputByte(b shl 16) outputChar(n shr 18) @@ -112,48 +129,71 @@ template encodeInternal(s: typed): untyped = result.setLen(outputIndex) -proc encode*[T: SomeInteger|char](s: openArray[T]): string = +template encodeImpl() {.dirty.} = + if safe: + encodeInternal(s, cb64safe) + else: + encodeInternal(s, cb64) + +proc encode*[T: byte|char](s: openArray[T], safe = false): string = ## Encodes `s` into base64 representation. ## - ## This procedure encodes an openarray (array or sequence) of either integers - ## or characters. + ## If `safe` is `true` then it will encode using the + ## URL-Safe and Filesystem-safe standard alphabet characters, + ## which substitutes `-` instead of `+` and `_` instead of `/`. + ## * https://en.wikipedia.org/wiki/Base64#URL_applications + ## * https://tools.ietf.org/html/rfc4648#page-7 ## ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: + assert encode("Hello World") == "SGVsbG8gV29ybGQ=" assert encode(['n', 'i', 'm']) == "bmlt" assert encode(@['n', 'i', 'm']) == "bmlt" - assert encode([1, 2, 3, 4, 5]) == "AQIDBAU=" - encodeInternal(s) + assert encode([1'u8, 2, 3, 4, 5]) == "AQIDBAU=" + encodeImpl() -proc encode*(s: string): string = - ## Encodes ``s`` into base64 representation. - ## - ## This procedure encodes a string. - ## - ## **See also:** - ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `decode proc<#decode,string>`_ for decoding a string - runnableExamples: - assert encode("Hello World") == "SGVsbG8gV29ybGQ=" - encodeInternal(s) +proc encode*[T: SomeInteger and not byte](s: openArray[T], safe = false): string + {.deprecated: "use `byte` or `char` instead".} = + encodeImpl() -proc encodeMIME*(s: string, lineLen = 75, newLine = "\r\n"): string = - ## Encodes ``s`` into base64 representation as lines. - ## Used in email MIME forma, use ``lineLen`` and ``newline``. +proc encodeMime*(s: string, lineLen = 75.Positive, newLine = "\r\n", + safe = false): string = + ## Encodes `s` into base64 representation as lines. + ## Used in email MIME format, use `lineLen` and `newline`. ## ## This procedure encodes a string according to MIME spec. ## + ## If `safe` is `true` then it will encode using the + ## URL-Safe and Filesystem-safe standard alphabet characters, + ## which substitutes `-` instead of `+` and `_` instead of `/`. + ## * https://en.wikipedia.org/wiki/Base64#URL_applications + ## * https://tools.ietf.org/html/rfc4648#page-7 + ## ## **See also:** - ## * `encode proc<#encode,string>`_ for encoding a string + ## * `encode proc<#encode,openArray[T]>`_ for encoding an openArray ## * `decode proc<#decode,string>`_ for decoding a string runnableExamples: - assert encodeMIME("Hello World", 4, "\n") == "SGVs\nbG8g\nV29y\nbGQ=" - for i, c in encode(s): - if i != 0 and (i mod lineLen == 0): - result.add(newLine) - result.add(c) + assert encodeMime("Hello World", 4, "\n") == "SGVs\nbG8g\nV29y\nbGQ=" + template cpy(l, src, idx) = + b = l + while i < b: + result[i] = src[idx] + inc i + inc idx + + if s.len == 0: return + let e = encode(s, safe) + if e.len <= lineLen or newLine.len == 0: + return e + result = newString(e.len + newLine.len * ((e.len div lineLen) - int(e.len mod lineLen == 0))) + var i, j, k, b: int + let nd = e.len - lineLen + while j < nd: + cpy(i + lineLen, e, j) + cpy(i + newLine.len, newLine, k) + k = 0 + cpy(result.len, e, j) proc initDecodeTable*(): array[256, char] = # computes a decode table at compile time @@ -171,12 +211,11 @@ const decodeTable = initDecodeTable() proc decode*(s: string): string = - ## Decodes string ``s`` in base64 representation back into its original form. + ## Decodes string `s` in base64 representation back into its original form. ## The initial whitespace is skipped. ## ## **See also:** ## * `encode proc<#encode,openArray[T]>`_ for encoding an openarray - ## * `encode proc<#encode,string>`_ for encoding a string runnableExamples: assert decode("SGVsbG8gV29ybGQ=") == "Hello World" assert decode(" SGVsbG8gV29ybGQ=") == "Hello World" @@ -187,11 +226,11 @@ proc decode*(s: string): string = template inputChar(x: untyped) = let x = int decodeTable[ord(s[inputIndex])] - inc inputIndex if x == invalidChar: raise newException(ValueError, "Invalid base64 format character `" & s[inputIndex] & "` (ord " & $s[inputIndex].ord & ") at location " & $inputIndex & ".") + inc inputIndex template outputChar(x: untyped) = result[outputIndex] = char(x and 255) @@ -205,7 +244,7 @@ proc decode*(s: string): string = inputLen = s.len inputEnds = 0 # strip trailing characters - while s[inputLen - 1] in {'\n', '\r', ' ', '='}: + while inputLen > 0 and s[inputLen - 1] in {'\n', '\r', ' ', '='}: dec inputLen # hot loop: read 4 characters at at time inputEnds = inputLen - 4 @@ -232,6 +271,3 @@ proc decode*(s: string): string = outputChar(a shl 2 or b shr 4) outputChar(b shl 4 or c shr 2) result.setLen(outputIndex) - - - diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 9ebdabb7b..0d3351ee5 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -8,140 +8,380 @@ # ## This module implements a series of low level methods for bit manipulation. - -## By default, this module use compiler intrinsics where possible to improve performance -## on supported compilers: ``GCC``, ``LLVM_GCC``, ``CLANG``, ``VCC``, ``ICC``. ## -## The module will fallback to pure nim procs incase the backend is not supported. +## By default, compiler intrinsics are used where possible to improve performance +## on supported compilers: `GCC`, `LLVM_GCC`, `CLANG`, `VCC`, `ICC`. +## +## The module will fallback to pure nim procs in case the backend is not supported. ## You can also use the flag `noIntrinsicsBitOpts` to disable compiler intrinsics. ## -## This module is also compatible with other backends: ``Javascript``, ``Nimscript`` -## as well as the ``compiletime VM``. +## This module is also compatible with other backends: `JavaScript`, `NimScript` +## as well as the `compiletime VM`. ## -## As a result of using optimized function/intrinsics some functions can return +## As a result of using optimized functions/intrinsics, some functions can return ## undefined results if the input is invalid. You can use the flag `noUndefinedBitOpts` ## to force predictable behaviour for all input, causing a small performance hit. ## -## At this time only `fastLog2`, `firstSetBit, `countLeadingZeroBits`, `countTrailingZeroBits` -## may return undefined and/or platform dependent value if given invalid input. +## At this time only `fastLog2`, `firstSetBit`, `countLeadingZeroBits` and `countTrailingZeroBits` +## may return undefined and/or platform dependent values if given invalid input. -proc bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI", noSideEffect.} +import std/macros +import std/private/since +from std/private/bitops_utils import forwardImpl, castToUnsigned + +func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. -proc bitand*[T: SomeInteger](x, y: T): T {.magic: "BitandI", noSideEffect.} - ## Computes the `bitwise and` of numbers `x` and `y`. - -proc bitor*[T: SomeInteger](x, y: T): T {.magic: "BitorI", noSideEffect.} - ## Computes the `bitwise or` of numbers `x` and `y`. - -proc bitxor*[T: SomeInteger](x, y: T): T {.magic: "BitxorI", noSideEffect.} - ## Computes the `bitwise xor` of numbers `x` and `y`. - -const useBuiltins = not defined(noIntrinsicsBitOpts) -const noUndefined = defined(noUndefinedBitOpts) -const useGCC_builtins = (defined(gcc) or defined(llvm_gcc) or - defined(clang)) and useBuiltins -const useICC_builtins = defined(icc) and useBuiltins -const useVCC_builtins = defined(vcc) and useBuiltins -const arch64 = sizeof(int) == 8 - -template toUnsigned(x: int8): uint8 = cast[uint8](x) -template toUnsigned(x: int16): uint16 = cast[uint16](x) -template toUnsigned(x: int32): uint32 = cast[uint32](x) -template toUnsigned(x: int64): uint64 = cast[uint64](x) -template toUnsigned(x: int): uint = cast[uint](x) - -template forwardImpl(impl, arg) {.dirty.} = - when sizeof(x) <= 4: - when x is SomeSignedInt: - impl(cast[uint32](x.int32)) - else: - impl(x.uint32) +func internalBitand[T: SomeInteger](x, y: T): T {.magic: "BitandI".} + +func internalBitor[T: SomeInteger](x, y: T): T {.magic: "BitorI".} + +func internalBitxor[T: SomeInteger](x, y: T): T {.magic: "BitxorI".} + +macro bitand*[T: SomeInteger](x, y: T; z: varargs[T]): T = + ## Computes the `bitwise and` of all arguments collectively. + let fn = bindSym("internalBitand") + result = newCall(fn, x, y) + for extra in z: + result = newCall(fn, result, extra) + +macro bitor*[T: SomeInteger](x, y: T; z: varargs[T]): T = + ## Computes the `bitwise or` of all arguments collectively. + let fn = bindSym("internalBitor") + result = newCall(fn, x, y) + for extra in z: + result = newCall(fn, result, extra) + +macro bitxor*[T: SomeInteger](x, y: T; z: varargs[T]): T = + ## Computes the `bitwise xor` of all arguments collectively. + let fn = bindSym("internalBitxor") + result = newCall(fn, x, y) + for extra in z: + result = newCall(fn, result, extra) + + +type BitsRange*[T] = range[0..sizeof(T)*8-1] + ## A range with all bit positions for type `T`. + +template typeMasked[T: SomeInteger](x: T): T = + when defined(js): + T(x and ((0xffffffff_ffffffff'u shr (64 - sizeof(T) * 8)))) else: - when x is SomeSignedInt: - impl(cast[uint64](x.int64)) - else: - impl(x.uint64) + x + +func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns an extracted (and shifted) slice of bits from `v`. + runnableExamples: + doAssert 0b10111.bitsliced(2 .. 4) == 0b101 + doAssert 0b11100.bitsliced(0 .. 2) == 0b100 + doAssert 0b11100.bitsliced(0 ..< 3) == 0b100 + + let + upmost = sizeof(T) * 8 - 1 + uv = v.castToUnsigned + ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T + +proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v` into an extracted (and shifted) slice of bits from `v`. + runnableExamples: + var x = 0b101110 + x.bitslice(2 .. 4) + doAssert x == 0b011 + + let + upmost = sizeof(T) * 8 - 1 + uv = v.castToUnsigned + v = ((uv shl (upmost - slice.b)).typeMasked shr (upmost - slice.b + slice.a)).T + +func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Creates a bitmask based on a slice of bits. + runnableExamples: + doAssert toMask[int32](1 .. 3) == 0b1110'i32 + doAssert toMask[int32](0 .. 3) == 0b1111'i32 + + let + upmost = sizeof(T) * 8 - 1 + bitmask = bitnot(0.T).castToUnsigned + ((bitmask shl (upmost - slice.b + slice.a)).typeMasked shr (upmost - slice.b)).T + +proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with only the `1` bits from `mask` matching those of + ## `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.masked(0b0000_1010'u8) == 0b0000_0010'u8 + + bitand(v, mask) + +func masked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with only the `1` bits in the range of `slice` + ## matching those of `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_1011'u8 + doAssert v.masked(1 .. 3) == 0b0000_1010'u8 + + bitand(v, toMask[T](slice)) + +proc mask*[T: SomeInteger](v: var T; mask: T) {.inline, since: (1, 3).} = + ## Mutates `v`, with only the `1` bits from `mask` matching those of + ## `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.mask(0b0000_1010'u8) + doAssert v == 0b0000_0010'u8 + + v = bitand(v, mask) + +proc mask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with only the `1` bits in the range of `slice` + ## matching those of `v` set to 1. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_1011'u8 + v.mask(1 .. 3) + doAssert v == 0b0000_1010'u8 + + v = bitand(v, toMask[T](slice)) + +func setMasked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits from `mask` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.setMasked(0b0000_1010'u8) == 0b0000_1011'u8 + + bitor(v, mask) + +func setMasked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits in the range of `slice` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.setMasked(2 .. 3) == 0b0000_1111'u8 + + bitor(v, toMask[T](slice)) + +proc setMask*[T: SomeInteger](v: var T; mask: T) {.inline.} = + ## Mutates `v`, with all the `1` bits from `mask` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.setMask(0b0000_1010'u8) + doAssert v == 0b0000_1011'u8 + + v = bitor(v, mask) + +proc setMask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with all the `1` bits in the range of `slice` set to 1. + ## + ## Effectively maps to a `bitor <#bitor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.setMask(2 .. 3) + doAssert v == 0b0000_1111'u8 + + v = bitor(v, toMask[T](slice)) + +func clearMasked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits from `mask` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.clearMasked(0b0000_1010'u8) == 0b0000_0001'u8 + + bitand(v, bitnot(mask)) + +func clearMasked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits in the range of `slice` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.clearMasked(1 .. 3) == 0b0000_0001'u8 + + bitand(v, bitnot(toMask[T](slice))) + +proc clearMask*[T: SomeInteger](v: var T; mask: T) {.inline.} = + ## Mutates `v`, with all the `1` bits from `mask` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + var v = 0b0000_0011'u8 + v.clearMask(0b0000_1010'u8) + doAssert v == 0b0000_0001'u8 + + v = bitand(v, bitnot(mask)) + +proc clearMask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with all the `1` bits in the range of `slice` set to 0. + ## + ## Effectively maps to a `bitand <#bitand.m,T,T,varargs[T]>`_ operation + ## with an *inverted mask*. + runnableExamples: + var v = 0b0000_0011'u8 + v.clearMask(1 .. 3) + doAssert v == 0b0000_0001'u8 + + v = bitand(v, bitnot(toMask[T](slice))) -when defined(nimHasalignOf): +func flipMasked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits from `mask` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.flipMasked(0b0000_1010'u8) == 0b0000_1001'u8 + + bitxor(v, mask) + +func flipMasked*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, 3).} = + ## Returns `v`, with all the `1` bits in the range of `slice` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + let v = 0b0000_0011'u8 + doAssert v.flipMasked(1 .. 3) == 0b0000_1101'u8 + + bitxor(v, toMask[T](slice)) + +proc flipMask*[T: SomeInteger](v: var T; mask: T) {.inline.} = + ## Mutates `v`, with all the `1` bits from `mask` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.flipMask(0b0000_1010'u8) + doAssert v == 0b0000_1001'u8 + + v = bitxor(v, mask) + +proc flipMask*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = + ## Mutates `v`, with all the `1` bits in the range of `slice` flipped. + ## + ## Effectively maps to a `bitxor <#bitxor.m,T,T,varargs[T]>`_ operation. + runnableExamples: + var v = 0b0000_0011'u8 + v.flipMask(1 .. 3) + doAssert v == 0b0000_1101'u8 + + v = bitxor(v, toMask[T](slice)) + +proc setBit*[T: SomeInteger](v: var T; bit: BitsRange[T]) {.inline.} = + ## Mutates `v`, with the bit at position `bit` set to 1. + runnableExamples: + var v = 0b0000_0011'u8 + v.setBit(5'u8) + doAssert v == 0b0010_0011'u8 + + v.setMask(1.T shl bit) - import macros +proc clearBit*[T: SomeInteger](v: var T; bit: BitsRange[T]) {.inline.} = + ## Mutates `v`, with the bit at position `bit` set to 0. + runnableExamples: + var v = 0b0000_0011'u8 + v.clearBit(1'u8) + doAssert v == 0b0000_0001'u8 - type BitsRange*[T] = range[0..sizeof(T)*8-1] - ## Returns a range with all bit positions for type ``T`` + v.clearMask(1.T shl bit) - proc setMask*[T: SomeInteger](v: var T, mask: T) {.inline.} = - ## Returns ``v``, with all the ``1`` bits from ``mask`` set to 1 - v = v or mask +proc flipBit*[T: SomeInteger](v: var T; bit: BitsRange[T]) {.inline.} = + ## Mutates `v`, with the bit at position `bit` flipped. + runnableExamples: + var v = 0b0000_0011'u8 + v.flipBit(1'u8) + doAssert v == 0b0000_0001'u8 - proc clearMask*[T: SomeInteger](v: var T, mask: T) {.inline.} = - ## Returns ``v``, with all the ``1`` bits from ``mask`` set to 0 - v = v and not mask + v = 0b0000_0011'u8 + v.flipBit(2'u8) + doAssert v == 0b0000_0111'u8 - proc flipMask*[T: SomeInteger](v: var T, mask: T) {.inline.} = - ## Returns ``v``, with all the ``1`` bits from ``mask`` flipped - v = v xor mask + v.flipMask(1.T shl bit) - proc setBit*[T: SomeInteger](v: var T, bit: BitsRange[T]) {.inline.} = - ## Returns ``v``, with the bit at position ``bit`` set to 1 - v.setMask(1.T shl bit) +macro setBits*(v: typed; bits: varargs[typed]): untyped = + ## Mutates `v`, with the bits at positions `bits` set to 1. + runnableExamples: + var v = 0b0000_0011'u8 + v.setBits(3, 5, 7) + doAssert v == 0b1010_1011'u8 - proc clearBit*[T: SomeInteger](v: var T, bit: BitsRange[T]) {.inline.} = - ## Returns ``v``, with the bit at position ``bit`` set to 0 - v.clearMask(1.T shl bit) + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("setBit", v, bit) - proc flipBit*[T: SomeInteger](v: var T, bit: BitsRange[T]) {.inline.} = - ## Returns ``v``, with the bit at position ``bit`` flipped - v.flipMask(1.T shl bit) +macro clearBits*(v: typed; bits: varargs[typed]): untyped = + ## Mutates `v`, with the bits at positions `bits` set to 0. + runnableExamples: + var v = 0b1111_1111'u8 + v.clearBits(1, 3, 5, 7) + doAssert v == 0b0101_0101'u8 + + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("clearBit", v, bit) + +macro flipBits*(v: typed; bits: varargs[typed]): untyped = + ## Mutates `v`, with the bits at positions `bits` set to 0. + runnableExamples: + var v = 0b0000_1111'u8 + v.flipBits(1, 3, 5, 7) + doAssert v == 0b1010_0101'u8 - macro setBits*(v: typed, bits: varargs[typed]): untyped = - ## Returns ``v``, with the bits at positions ``bits`` set to 1 - bits.expectKind(nnkBracket) - result = newStmtList() - for bit in bits: - result.add newCall("setBit", v, bit) + bits.expectKind(nnkBracket) + result = newStmtList() + for bit in bits: + result.add newCall("flipBit", v, bit) - macro clearBits*(v: typed, bits: varargs[typed]): untyped = - ## Returns ``v``, with the bits at positions ``bits`` set to 0 - bits.expectKind(nnkBracket) - result = newStmtList() - for bit in bits: - result.add newCall("clearBit", v, bit) - macro flipBits*(v: typed, bits: varargs[typed]): untyped = - ## Returns ``v``, with the bits at positions ``bits`` set to 0 - bits.expectKind(nnkBracket) - result = newStmtList() - for bit in bits: - result.add newCall("flipBit", v, bit) +proc testBit*[T: SomeInteger](v: T; bit: BitsRange[T]): bool {.inline.} = + ## Returns true if the bit in `v` at positions `bit` is set to 1. + runnableExamples: + let v = 0b0000_1111'u8 + doAssert v.testBit(0) + doAssert not v.testBit(7) - proc testBit*[T: SomeInteger](v: T, bit: BitsRange[T]): bool {.inline.} = - ## Returns true if the bit in ``v`` at positions ``bit`` is set to 1 - let mask = 1.T shl bit - return (v and mask) == mask + let mask = 1.T shl bit + return (v and mask) == mask # #### Pure Nim version #### -proc firstSetBitNim(x: uint32): int {.inline, noSideEffect.} = +func firstSetBitNim(x: uint32): int {.inline.} = ## Returns the 1-based index of the least significant set bit of x, or if x is zero, returns zero. # https://graphics.stanford.edu/%7Eseander/bithacks.html#ZerosOnRightMultLookup const lookup: array[32, uint8] = [0'u8, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9] - var v = x.uint32 - var k = not v + 1 # get two's complement # cast[uint32](-cast[int32](v)) + let v = x.uint32 + let k = not v + 1 # get two's complement # cast[uint32](-cast[int32](v)) result = 1 + lookup[uint32((v and k) * 0x077CB531'u32) shr 27].int -proc firstSetBitNim(x: uint64): int {.inline, noSideEffect.} = +func firstSetBitNim(x: uint64): int {.inline.} = ## Returns the 1-based index of the least significant set bit of x, or if x is zero, returns zero. # https://graphics.stanford.edu/%7Eseander/bithacks.html#ZerosOnRightMultLookup - var v = uint64(x) + let v = uint64(x) var k = uint32(v and 0xFFFFFFFF'u32) if k == 0: k = uint32(v shr 32'u32) and 0xFFFFFFFF'u32 result = 32 + else: + result = 0 result += firstSetBitNim(k) -proc fastlog2Nim(x: uint32): int {.inline, noSideEffect.} = +func fastlog2Nim(x: uint32): int {.inline.} = ## Quickly find the log base 2 of a 32-bit or less integer. # https://graphics.stanford.edu/%7Eseander/bithacks.html#IntegerLogDeBruijn # https://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers @@ -155,7 +395,7 @@ proc fastlog2Nim(x: uint32): int {.inline, noSideEffect.} = v = v or v shr 16 result = lookup[uint32(v * 0x07C4ACDD'u32) shr 27].int -proc fastlog2Nim(x: uint64): int {.inline, noSideEffect.} = +func fastlog2Nim(x: uint64): int {.inline.} = ## Quickly find the log base 2 of a 64-bit integer. # https://graphics.stanford.edu/%7Eseander/bithacks.html#IntegerLogDeBruijn # https://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers @@ -172,13 +412,11 @@ proc fastlog2Nim(x: uint64): int {.inline, noSideEffect.} = v = v or v shr 32 result = lookup[(v * 0x03F6EAF2CD271461'u64) shr 58].int -# sets.nim cannot import bitops, but bitops can use include -# system/sets to eliminate code duplication. sets.nim defines -# countBits32 and countBits64. -include system/sets +import system/countbits_impl -template countSetBitsNim(n: uint32): int = countBits32(n) -template countSetBitsNim(n: uint64): int = countBits64(n) +const useBuiltinsRotate = (defined(amd64) or defined(i386)) and + (defined(gcc) or defined(clang) or defined(vcc) or + (defined(icl) and not defined(cpp))) and useBuiltins template parityImpl[T](value: T): int = # formula id from: https://graphics.stanford.edu/%7Eseander/bithacks.html#ParityParallel @@ -195,11 +433,6 @@ template parityImpl[T](value: T): int = when useGCC_builtins: - # Returns the number of set 1-bits in value. - proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} - proc builtin_popcountll(x: culonglong): cint {. - importc: "__builtin_popcountll", cdecl.} - # Returns the bit parity in value proc builtin_parity(x: cuint): cint {.importc: "__builtin_parity", cdecl.} proc builtin_parityll(x: culonglong): cint {.importc: "__builtin_parityll", cdecl.} @@ -217,97 +450,65 @@ when useGCC_builtins: proc builtin_ctzll(x: culonglong): cint {.importc: "__builtin_ctzll", cdecl.} elif useVCC_builtins: - # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. - proc builtin_popcnt16(a2: uint16): uint16 {. - importc: "__popcnt16"header: "<intrin.h>", noSideEffect.} - proc builtin_popcnt32(a2: uint32): uint32 {. - importc: "__popcnt"header: "<intrin.h>", noSideEffect.} - proc builtin_popcnt64(a2: uint64): uint64 {. - importc: "__popcnt64"header: "<intrin.h>", noSideEffect.} - # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1). - proc bitScanReverse(index: ptr culong, mask: culong): cuchar {. - importc: "_BitScanReverse", header: "<intrin.h>", noSideEffect.} - proc bitScanReverse64(index: ptr culong, mask: uint64): cuchar {. - importc: "_BitScanReverse64", header: "<intrin.h>", noSideEffect.} + func bitScanReverse(index: ptr culong, mask: culong): uint8 {. + importc: "_BitScanReverse", header: "<intrin.h>".} + func bitScanReverse64(index: ptr culong, mask: uint64): uint8 {. + importc: "_BitScanReverse64", header: "<intrin.h>".} # Search the mask data from least significant bit (LSB) to the most significant bit (MSB) for a set bit (1). - proc bitScanForward(index: ptr culong, mask: culong): cuchar {. - importc: "_BitScanForward", header: "<intrin.h>", noSideEffect.} - proc bitScanForward64(index: ptr culong, mask: uint64): cuchar {. - importc: "_BitScanForward64", header: "<intrin.h>", noSideEffect.} + func bitScanForward(index: ptr culong, mask: culong): uint8 {. + importc: "_BitScanForward", header: "<intrin.h>".} + func bitScanForward64(index: ptr culong, mask: uint64): uint8 {. + importc: "_BitScanForward64", header: "<intrin.h>".} template vcc_scan_impl(fnc: untyped; v: untyped): int = - var index: culong + var index {.inject.}: culong = 0 discard fnc(index.addr, v) index.int elif useICC_builtins: - - # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm - # see also: https://software.intel.com/en-us/node/523362 - # Count the number of bits set to 1 in an integer a, and return that count in dst. - proc builtin_popcnt32(a: cint): cint {. - importc: "_popcnt"header: "<immintrin.h>", noSideEffect.} - proc builtin_popcnt64(a: uint64): cint {. - importc: "_popcnt64"header: "<immintrin.h>", noSideEffect.} - # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined. - proc bitScanForward(p: ptr uint32, b: uint32): cuchar {. - importc: "_BitScanForward", header: "<immintrin.h>", noSideEffect.} - proc bitScanForward64(p: ptr uint32, b: uint64): cuchar {. - importc: "_BitScanForward64", header: "<immintrin.h>", noSideEffect.} + func bitScanForward(p: ptr uint32, b: uint32): uint8 {. + importc: "_BitScanForward", header: "<immintrin.h>".} + func bitScanForward64(p: ptr uint32, b: uint64): uint8 {. + importc: "_BitScanForward64", header: "<immintrin.h>".} # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined. - proc bitScanReverse(p: ptr uint32, b: uint32): cuchar {. - importc: "_BitScanReverse", header: "<immintrin.h>", noSideEffect.} - proc bitScanReverse64(p: ptr uint32, b: uint64): cuchar {. - importc: "_BitScanReverse64", header: "<immintrin.h>", noSideEffect.} + func bitScanReverse(p: ptr uint32, b: uint32): uint8 {. + importc: "_BitScanReverse", header: "<immintrin.h>".} + func bitScanReverse64(p: ptr uint32, b: uint64): uint8 {. + importc: "_BitScanReverse64", header: "<immintrin.h>".} template icc_scan_impl(fnc: untyped; v: untyped): int = var index: uint32 discard fnc(index.addr, v) index.int +func countSetBits*(x: SomeInteger): int {.inline.} = + ## Counts the set bits in an integer (also called `Hamming weight`:idx:). + runnableExamples: + doAssert countSetBits(0b0000_0011'u8) == 2 + doAssert countSetBits(0b1010_1010'u8) == 4 -proc countSetBits*(x: SomeInteger): int {.inline, noSideEffect.} = - ## Counts the set bits in integer. (also called `Hamming weight`:idx:.) - # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. - # like GCC and MSVC - when x is SomeSignedInt: - let x = x.toUnsigned - when nimvm: - result = forwardImpl(countSetBitsNim, x) - else: - when useGCC_builtins: - when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int - else: result = builtin_popcountll(x.culonglong).int - elif useVCC_builtins: - when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int - elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int - elif arch64: result = builtin_popcnt64(x.uint64).int - else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + - builtin_popcnt32((x.uint64 shr 32'u64).uint32).int - elif useICC_builtins: - when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int - elif arch64: result = builtin_popcnt64(x.uint64).int - else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + - builtin_popcnt32((x.uint64 shr 32'u64).cint).int - else: - when sizeof(x) <= 4: result = countSetBitsNim(x.uint32) - else: result = countSetBitsNim(x.uint64) + result = countSetBitsImpl(x) -proc popcount*(x: SomeInteger): int {.inline, noSideEffect.} = - ## Alias for for countSetBits (Hamming weight.) +func popcount*(x: SomeInteger): int {.inline.} = + ## Alias for `countSetBits <#countSetBits,SomeInteger>`_ (Hamming weight). result = countSetBits(x) -proc parityBits*(x: SomeInteger): int {.inline, noSideEffect.} = - ## Calculate the bit parity in integer. If number of 1-bit - ## is odd parity is 1, otherwise 0. +func parityBits*(x: SomeInteger): int {.inline.} = + ## Calculate the bit parity in an integer. If the number of 1-bits + ## is odd, the parity is 1, otherwise 0. + runnableExamples: + doAssert parityBits(0b0000_0000'u8) == 0 + doAssert parityBits(0b0101_0001'u8) == 1 + doAssert parityBits(0b0110_1001'u8) == 0 + doAssert parityBits(0b0111_1111'u8) == 1 + # Can be used a base if creating ASM version. # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(parityImpl, x) else: @@ -318,13 +519,19 @@ proc parityBits*(x: SomeInteger): int {.inline, noSideEffect.} = when sizeof(x) <= 4: result = parityImpl(x.uint32) else: result = parityImpl(x.uint64) -proc firstSetBit*(x: SomeInteger): int {.inline, noSideEffect.} = - ## Returns the 1-based index of the least significant set bit of x. - ## If `x` is zero, when ``noUndefinedBitOpts`` is set, result is 0, - ## otherwise result is undefined. +func firstSetBit*(x: SomeInteger): int {.inline.} = + ## Returns the 1-based index of the least significant set bit of `x`. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is 0, + ## otherwise the result is undefined. + runnableExamples: + doAssert firstSetBit(0b0000_0001'u8) == 1 + doAssert firstSetBit(0b0000_0010'u8) == 2 + doAssert firstSetBit(0b0000_0100'u8) == 3 + doAssert firstSetBit(0b0000_1000'u8) == 4 + doAssert firstSetBit(0b0000_1111'u8) == 1 + # GCC builtin 'builtin_ffs' already handle zero input. - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: when noUndefined: if x == 0: @@ -355,12 +562,18 @@ proc firstSetBit*(x: SomeInteger): int {.inline, noSideEffect.} = when sizeof(x) <= 4: result = firstSetBitNim(x.uint32) else: result = firstSetBitNim(x.uint64) -proc fastLog2*(x: SomeInteger): int {.inline, noSideEffect.} = +func fastLog2*(x: SomeInteger): int {.inline.} = ## Quickly find the log base 2 of an integer. - ## If `x` is zero, when ``noUndefinedBitOpts`` is set, result is -1, - ## otherwise result is undefined. - when x is SomeSignedInt: - let x = x.toUnsigned + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is -1, + ## otherwise the result is undefined. + runnableExamples: + doAssert fastLog2(0b0000_0001'u8) == 0 + doAssert fastLog2(0b0000_0010'u8) == 1 + doAssert fastLog2(0b0000_0100'u8) == 2 + doAssert fastLog2(0b0000_1000'u8) == 3 + doAssert fastLog2(0b0000_1111'u8) == 3 + + let x = x.castToUnsigned when noUndefined: if x == 0: return -1 @@ -388,12 +601,21 @@ proc fastLog2*(x: SomeInteger): int {.inline, noSideEffect.} = when sizeof(x) <= 4: result = fastlog2Nim(x.uint32) else: result = fastlog2Nim(x.uint64) -proc countLeadingZeroBits*(x: SomeInteger): int {.inline, noSideEffect.} = - ## Returns the number of leading zero bits in integer. - ## If `x` is zero, when ``noUndefinedBitOpts`` is set, result is 0, - ## otherwise result is undefined. - when x is SomeSignedInt: - let x = x.toUnsigned +func countLeadingZeroBits*(x: SomeInteger): int {.inline.} = + ## Returns the number of leading zero bits in an integer. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is 0, + ## otherwise the result is undefined. + ## + ## **See also:** + ## * `countTrailingZeroBits proc <#countTrailingZeroBits,SomeInteger>`_ + runnableExamples: + doAssert countLeadingZeroBits(0b0000_0001'u8) == 7 + doAssert countLeadingZeroBits(0b0000_0010'u8) == 6 + doAssert countLeadingZeroBits(0b0000_0100'u8) == 5 + doAssert countLeadingZeroBits(0b0000_1000'u8) == 4 + doAssert countLeadingZeroBits(0b0000_1111'u8) == 4 + + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -407,12 +629,21 @@ proc countLeadingZeroBits*(x: SomeInteger): int {.inline, noSideEffect.} = when sizeof(x) <= 4: result = sizeof(x)*8 - 1 - fastlog2Nim(x.uint32) else: result = sizeof(x)*8 - 1 - fastlog2Nim(x.uint64) -proc countTrailingZeroBits*(x: SomeInteger): int {.inline, noSideEffect.} = - ## Returns the number of trailing zeros in integer. - ## If `x` is zero, when ``noUndefinedBitOpts`` is set, result is 0, - ## otherwise result is undefined. - when x is SomeSignedInt: - let x = x.toUnsigned +func countTrailingZeroBits*(x: SomeInteger): int {.inline.} = + ## Returns the number of trailing zeros in an integer. + ## If `x` is zero, when `noUndefinedBitOpts` is set, the result is 0, + ## otherwise the result is undefined. + ## + ## **See also:** + ## * `countLeadingZeroBits proc <#countLeadingZeroBits,SomeInteger>`_ + runnableExamples: + doAssert countTrailingZeroBits(0b0000_0001'u8) == 0 + doAssert countTrailingZeroBits(0b0000_0010'u8) == 1 + doAssert countTrailingZeroBits(0b0000_0100'u8) == 2 + doAssert countTrailingZeroBits(0b0000_1000'u8) == 3 + doAssert countTrailingZeroBits(0b0000_1111'u8) == 0 + + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -425,69 +656,196 @@ proc countTrailingZeroBits*(x: SomeInteger): int {.inline, noSideEffect.} = else: result = firstSetBit(x) - 1 +when useBuiltinsRotate: + when defined(gcc): + # GCC was tested until version 4.8.1 and intrinsics were present. Not tested + # in previous versions. + func builtin_rotl8(value: uint8, shift: cint): uint8 + {.importc: "__rolb", header: "<x86intrin.h>".} + func builtin_rotl16(value: cushort, shift: cint): cushort + {.importc: "__rolw", header: "<x86intrin.h>".} + func builtin_rotl32(value: cuint, shift: cint): cuint + {.importc: "__rold", header: "<x86intrin.h>".} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: cint): culonglong + {.importc: "__rolq", header: "<x86intrin.h>".} + + func builtin_rotr8(value: uint8, shift: cint): uint8 + {.importc: "__rorb", header: "<x86intrin.h>".} + func builtin_rotr16(value: cushort, shift: cint): cushort + {.importc: "__rorw", header: "<x86intrin.h>".} + func builtin_rotr32(value: cuint, shift: cint): cuint + {.importc: "__rord", header: "<x86intrin.h>".} + when defined(amd64): + func builtin_rotr64(value: culonglong, shift: cint): culonglong + {.importc: "__rorq", header: "<x86intrin.h>".} + elif defined(clang): + # In CLANG, builtins have been present since version 8.0.0 and intrinsics + # since version 9.0.0. This implementation chose the builtins, as they have + # been around for longer. + # https://releases.llvm.org/8.0.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release + # https://releases.llvm.org/8.0.0/tools/clang/docs/LanguageExtensions.html#builtin-rotateleft + # source for correct declarations: https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/Builtins.def + func builtin_rotl8(value: uint8, shift: uint8): uint8 + {.importc: "__builtin_rotateleft8", nodecl.} + func builtin_rotl16(value: cushort, shift: cushort): cushort + {.importc: "__builtin_rotateleft16", nodecl.} + func builtin_rotl32(value: cuint, shift: cuint): cuint + {.importc: "__builtin_rotateleft32", nodecl.} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: culonglong): culonglong + {.importc: "__builtin_rotateleft64", nodecl.} + + func builtin_rotr8(value: uint8, shift: uint8): uint8 + {.importc: "__builtin_rotateright8", nodecl.} + func builtin_rotr16(value: cushort, shift: cushort): cushort + {.importc: "__builtin_rotateright16", nodecl.} + func builtin_rotr32(value: cuint, shift: cuint): cuint + {.importc: "__builtin_rotateright32", nodecl.} + when defined(amd64): + # shift is unsigned, refs https://github.com/llvm-mirror/clang/commit/892de415b7fde609dafc4e6c1643b7eaa0150a4d + func builtin_rotr64(value: culonglong, shift: culonglong): culonglong + {.importc: "__builtin_rotateright64", nodecl.} + elif defined(vcc): + # Tested on Microsoft (R) C/C++ Optimizing Compiler 19.28.29335 x64 and x86. + # Not tested in previous versions. + # https://docs.microsoft.com/en-us/cpp/intrinsics/rotl8-rotl16?view=msvc-160 + # https://docs.microsoft.com/en-us/cpp/intrinsics/rotr8-rotr16?view=msvc-160 + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/rotl-rotl64-rotr-rotr64?view=msvc-160 + func builtin_rotl8(value: uint8, shift: uint8): uint8 + {.importc: "_rotl8", header: "<intrin.h>".} + func builtin_rotl16(value: cushort, shift: uint8): cushort + {.importc: "_rotl16", header: "<intrin.h>".} + func builtin_rotl32(value: cuint, shift: cint): cuint + {.importc: "_rotl", header: "<stdlib.h>".} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: cint): culonglong + {.importc: "_rotl64", header: "<stdlib.h>".} + + func builtin_rotr8(value: uint8, shift: uint8): uint8 + {.importc: "_rotr8", header: "<intrin.h>".} + func builtin_rotr16(value: cushort, shift: uint8): cushort + {.importc: "_rotr16", header: "<intrin.h>".} + func builtin_rotr32(value: cuint, shift: cint): cuint + {.importc: "_rotr", header: "<stdlib.h>".} + when defined(amd64): + func builtin_rotr64(value: culonglong, shift: cint): culonglong + {.importc: "_rotr64", header: "<stdlib.h>".} + elif defined(icl): + # Tested on Intel(R) C++ Intel(R) 64 Compiler Classic Version 2021.1.2 Build + # 20201208_000000 x64 and x86. Not tested in previous versions. + func builtin_rotl8(value: uint8, shift: cint): uint8 + {.importc: "__rolb", header: "<immintrin.h>".} + func builtin_rotl16(value: cushort, shift: cint): cushort + {.importc: "__rolw", header: "<immintrin.h>".} + func builtin_rotl32(value: cuint, shift: cint): cuint + {.importc: "__rold", header: "<immintrin.h>".} + when defined(amd64): + func builtin_rotl64(value: culonglong, shift: cint): culonglong + {.importc: "__rolq", header: "<immintrin.h>".} + + func builtin_rotr8(value: uint8, shift: cint): uint8 + {.importc: "__rorb", header: "<immintrin.h>".} + func builtin_rotr16(value: cushort, shift: cint): cushort + {.importc: "__rorw", header: "<immintrin.h>".} + func builtin_rotr32(value: cuint, shift: cint): cuint + {.importc: "__rord", header: "<immintrin.h>".} + when defined(amd64): + func builtin_rotr64(value: culonglong, shift: cint): culonglong + {.importc: "__rorq", header: "<immintrin.h>".} + +func rotl[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} = + ## Left-rotate bits in a `value`. + # https://stackoverflow.com/a/776523 + const mask = 8 * sizeof(value) - 1 + let rot = rot and mask + (value shl rot) or (value shr ((-rot) and mask)) + +func rotr[T: SomeUnsignedInt](value: T, rot: int32): T {.inline.} = + ## Right-rotate bits in a `value`. + const mask = 8 * sizeof(value) - 1 + let rot = rot and mask + (value shr rot) or (value shl ((-rot) and mask)) + +func shiftTypeTo(size: static int, shift: int): auto {.inline.} = + ## Returns the `shift` for the rotation according to the compiler and the + ## `size`. + when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl): + cint(shift) + elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1): + uint8(shift) + elif defined(clang): + when size == 2: + cushort(shift) + elif size == 4: + cuint(shift) + elif size == 8: + culonglong(shift) + +func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Left-rotate bits in a `value`. + runnableExamples: + doAssert rotateLeftBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 + doAssert rotateLeftBits(0b00111100_11000011'u16, 8) == + 0b11000011_00111100'u16 + doAssert rotateLeftBits(0b0000111111110000_1111000000001111'u32, 16) == + 0b1111000000001111_0000111111110000'u32 + doAssert rotateLeftBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == + 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 + when nimvm: + rotl(value, shift.int32) + else: + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotl8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotl32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotl64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotl(value, shift.int32) + else: + rotl(value, shift.int32) + +func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) * 8)]): T {.inline.} = + ## Right-rotate bits in a `value`. + runnableExamples: + doAssert rotateRightBits(0b0110_1001'u8, 4) == 0b1001_0110'u8 + doAssert rotateRightBits(0b00111100_11000011'u16, 8) == + 0b11000011_00111100'u16 + doAssert rotateRightBits(0b0000111111110000_1111000000001111'u32, 16) == + 0b1111000000001111_0000111111110000'u32 + doAssert rotateRightBits(0b00000000111111111111111100000000_11111111000000000000000011111111'u64, 32) == + 0b11111111000000000000000011111111_00000000111111111111111100000000'u64 + when nimvm: + rotr(value, shift.int32) + else: + when useBuiltinsRotate: + const size = sizeof(T) + when size == 1: + builtin_rotr8(value.uint8, shiftTypeTo(size, shift)).T + elif size == 2: + builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T + elif size == 4: + builtin_rotr32(value.cuint, shiftTypeTo(size, shift)).T + elif size == 8 and arch64: + builtin_rotr64(value.culonglong, shiftTypeTo(size, shift)).T + else: + rotr(value, shift.int32) + else: + rotr(value, shift.int32) -proc rotateLeftBits*(value: uint8; - amount: range[0..8]): uint8 {.inline, noSideEffect.} = - ## Left-rotate bits in a 8-bits value. - # using this form instead of the one below should handle any value - # out of range as well as negative values. - # result = (value shl amount) or (value shr (8 - amount)) - # taken from: https://en.wikipedia.org/wiki/Circular_shift#Implementing_circular_shifts - let amount = amount and 7 - result = (value shl amount) or (value shr ( (-amount) and 7)) - -proc rotateLeftBits*(value: uint16; - amount: range[0..16]): uint16 {.inline, noSideEffect.} = - ## Left-rotate bits in a 16-bits value. - let amount = amount and 15 - result = (value shl amount) or (value shr ( (-amount) and 15)) - -proc rotateLeftBits*(value: uint32; - amount: range[0..32]): uint32 {.inline, noSideEffect.} = - ## Left-rotate bits in a 32-bits value. - let amount = amount and 31 - result = (value shl amount) or (value shr ( (-amount) and 31)) - -proc rotateLeftBits*(value: uint64; - amount: range[0..64]): uint64 {.inline, noSideEffect.} = - ## Left-rotate bits in a 64-bits value. - let amount = amount and 63 - result = (value shl amount) or (value shr ( (-amount) and 63)) - - -proc rotateRightBits*(value: uint8; - amount: range[0..8]): uint8 {.inline, noSideEffect.} = - ## Right-rotate bits in a 8-bits value. - let amount = amount and 7 - result = (value shr amount) or (value shl ( (-amount) and 7)) - -proc rotateRightBits*(value: uint16; - amount: range[0..16]): uint16 {.inline, noSideEffect.} = - ## Right-rotate bits in a 16-bits value. - let amount = amount and 15 - result = (value shr amount) or (value shl ( (-amount) and 15)) - -proc rotateRightBits*(value: uint32; - amount: range[0..32]): uint32 {.inline, noSideEffect.} = - ## Right-rotate bits in a 32-bits value. - let amount = amount and 31 - result = (value shr amount) or (value shl ( (-amount) and 31)) - -proc rotateRightBits*(value: uint64; - amount: range[0..64]): uint64 {.inline, noSideEffect.} = - ## Right-rotate bits in a 64-bits value. - let amount = amount and 63 - result = (value shr amount) or (value shl ( (-amount) and 63)) - -proc repeatBits[T: SomeUnsignedInt](x: SomeUnsignedInt; retType: type[T]): T {. - noSideEffect.} = +func repeatBits[T: SomeUnsignedInt](x: SomeUnsignedInt; retType: type[T]): T = result = x var i = 1 while i != (sizeof(T) div sizeof(x)): result = (result shl (sizeof(x)*8*i)) or result i *= 2 -proc reverseBits*[T: SomeUnsignedInt](x: T): T {.noSideEffect.} = +func reverseBits*[T: SomeUnsignedInt](x: T): T = ## Return the bit reversal of x. runnableExamples: doAssert reverseBits(0b10100100'u8) == 0b00100101'u8 diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index 3f2045c3b..59e2078df 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -12,35 +12,101 @@ ## ## Unstable API. -import strutils +import std/private/since # used by the deprecated `openDefaultBrowser()` + +import std/strutils + +when defined(nimPreviewSlimSystem): + import std/assertions when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + from std/os import absolutePath else: - import os, osproc + import std/os + when not defined(osx): + import std/osproc + +const osOpenCmd* = + when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \ + ## Alias for the operating system specific *"open"* command, + ## `"open"` on OSX, MacOS and Windows, `"xdg-open"` on Linux, BSD, etc. + +proc prepare(s: string): string = + if s.contains("://"): + result = s + else: + result = "file://" & absolutePath(s) + +proc openDefaultBrowserRaw(url: string) = + ## note the url argument should be alreadly prepared, i.e. the url is passed "AS IS" -proc openDefaultBrowser*(url: string) = - ## opens `url` with the user's default browser. This does not block. - ## - ## Under Windows, ``ShellExecute`` is used. Under Mac OS X the ``open`` - ## command is used. Under Unix, it is checked if ``xdg-open`` exists and - ## used if it does. Otherwise the environment variable ``BROWSER`` is - ## used to determine the default browser to use. - ## - ## This proc doesn't raise an exception on error, beware. when defined(windows): - var o = newWideCString("open") + var o = newWideCString(osOpenCmd) var u = newWideCString(url) discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL) elif defined(macosx): - discard execShellCmd("open " & quoteShell(url)) + discard execShellCmd(osOpenCmd & " " & quoteShell(url)) else: var u = quoteShell(url) - if execShellCmd("xdg-open " & u) == 0: return - for b in getEnv("BROWSER").string.split(PathSep): + if execShellCmd(osOpenCmd & " " & u) == 0: return + for b in getEnv("BROWSER").split(PathSep): try: - # we use ``startProcess`` here because we don't want to block! + # we use `startProcess` here because we don't want to block! discard startProcess(command = b, args = [url], options = {poUsePath}) return except OSError: discard + +proc openDefaultBrowser*(url: string) = + ## Opens `url` with the user's default browser. This does not block. + ## The URL must not be empty string, to open on a blank page see `openDefaultBrowser()`. + ## + ## Under Windows, `ShellExecute` is used. Under Mac OS X the `open` + ## command is used. Under Unix, it is checked if `xdg-open` exists and + ## used if it does. Otherwise the environment variable `BROWSER` is + ## used to determine the default browser to use. + ## + ## This proc doesn't raise an exception on error, beware. + ## + ## ```nim + ## block: openDefaultBrowser("https://nim-lang.org") + ## ``` + doAssert url.len > 0, "URL must not be empty string" + openDefaultBrowserRaw(url) + +proc openDefaultBrowser*() {.since: (1, 1), deprecated: + "not implemented, please open with a specific url instead".} = + ## Intends to open the user's default browser without any `url` (blank page). + ## This does not block. + ## Intends to implement IETF RFC-6694 Section 3, + ## ("about:blank" is reserved for a blank page). + ## + ## Beware that this intended behavior is **not** implemented and + ## considered not worthy to implement here. + ## + ## The following describes the behavior of current implementation: + ## + ## - Under Windows, this will only cause a pop-up dialog \ + ## asking the assocated application with `about` \ + ## (as Windows simply treats `about:` as a protocol like `http`). + ## - Under Mac OS X the `open "about:blank"` command is used. + ## - Under Unix, it is checked if `xdg-open` exists and used \ + ## if it does and open the application assocated with `text/html` mime \ + ## (not `x-scheme-handler/http`, so maybe html-viewer \ + ## other than your default browser is opened). \ + ## Otherwise the environment variable `BROWSER` is used \ + ## to determine the default browser to use. + ## + ## This proc doesn't raise an exception on error, beware. + ## + ## ```nim + ## block: openDefaultBrowser() + ## ``` + ## + ## **See also:** + ## + ## * https://tools.ietf.org/html/rfc6694#section-3 + openDefaultBrowserRaw("about:blank") # See IETF RFC-6694 Section 3. diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index dd7c4d477..034f224ac 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -9,35 +9,32 @@ ## This module implements helper procs for CGI applications. Example: ## -## .. code-block:: Nim +## ```Nim +## import std/[strtabs, cgi] ## -## import strtabs, cgi -## -## # Fill the values when debugging: -## when debug: -## setTestData("name", "Klaus", "password", "123456") -## # read the data into `myData` -## var myData = readData() -## # check that the data's variable names are "name" or "password" -## validateData(myData, "name", "password") -## # start generating content: -## writeContentType() -## # generate content: -## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") -## write(stdout, "<html><head><title>Test</title></head><body>\n") -## writeLine(stdout, "your name: " & myData["name"]) -## writeLine(stdout, "your password: " & myData["password"]) -## writeLine(stdout, "</body></html>") - -import strutils, os, strtabs, cookies, uri +## # Fill the values when debugging: +## when debug: +## setTestData("name", "Klaus", "password", "123456") +## # read the data into `myData` +## var myData = readData() +## # check that the data's variable names are "name" or "password" +## validateData(myData, "name", "password") +## # start generating content: +## writeContentType() +## # generate content: +## write(stdout, "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n") +## write(stdout, "<html><head><title>Test</title></head><body>\n") +## writeLine(stdout, "your name: " & myData["name"]) +## writeLine(stdout, "your password: " & myData["password"]) +## writeLine(stdout, "</body></html>") +## ``` + +import std/[strutils, os, strtabs, cookies, uri] export uri.encodeUrl, uri.decodeUrl -proc handleHexChar(c: char, x: var int) {.inline.} = - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: assert(false) +when defined(nimPreviewSlimSystem): + import std/syncio + proc addXmlChar(dest: var string, c: char) {.inline.} = case c @@ -49,34 +46,31 @@ proc addXmlChar(dest: var string, c: char) {.inline.} = proc xmlEncode*(s: string): string = ## Encodes a value to be XML safe: - ## * ``"`` is replaced by ``"`` - ## * ``<`` is replaced by ``<`` - ## * ``>`` is replaced by ``>`` - ## * ``&`` is replaced by ``&`` + ## * `"` is replaced by `"` + ## * `<` is replaced by `<` + ## * `>` is replaced by `>` + ## * `&` is replaced by `&` ## * every other character is carried over. result = newStringOfCap(s.len + s.len shr 2) for i in 0..len(s)-1: addXmlChar(result, s[i]) type - CgiError* = object of IOError ## exception that is raised if a CGI error occurs - RequestMethod* = enum ## the used request method + CgiError* = object of IOError ## Exception that is raised if a CGI error occurs. + RequestMethod* = enum ## The used request method. methodNone, ## no REQUEST_METHOD environment variable methodPost, ## query uses the POST method methodGet ## query uses the GET method proc cgiError*(msg: string) {.noreturn.} = - ## raises an ECgi exception with message `msg`. - var e: ref CgiError - new(e) - e.msg = msg - raise e + ## Raises a `CgiError` exception with message `msg`. + raise newException(CgiError, msg) proc getEncodedData(allowedMethods: set[RequestMethod]): string = - case getEnv("REQUEST_METHOD").string + case getEnv("REQUEST_METHOD") of "POST": if methodPost notin allowedMethods: cgiError("'REQUEST_METHOD' 'POST' is not supported") - var L = parseInt(getEnv("CONTENT_LENGTH").string) + var L = parseInt(getEnv("CONTENT_LENGTH")) if L == 0: return "" result = newString(L) @@ -85,218 +79,182 @@ proc getEncodedData(allowedMethods: set[RequestMethod]): string = of "GET": if methodGet notin allowedMethods: cgiError("'REQUEST_METHOD' 'GET' is not supported") - result = getEnv("QUERY_STRING").string + result = getEnv("QUERY_STRING") else: if methodNone notin allowedMethods: cgiError("'REQUEST_METHOD' must be 'POST' or 'GET'") -iterator decodeData*(data: string): tuple[key, value: TaintedString] = +iterator decodeData*(data: string): tuple[key, value: string] = ## Reads and decodes CGI data and yields the (name, value) pairs the ## data consists of. - var i = 0 - var name = "" - var value = "" - # decode everything in one pass: - while i < data.len: - setLen(name, 0) # reuse memory - while i < data.len: - case data[i] - of '%': - var x = 0 - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) - inc(i, 2) - add(name, chr(x)) - of '+': add(name, ' ') - of '=', '&': break - else: add(name, data[i]) - inc(i) - if i >= data.len or data[i] != '=': cgiError("'=' expected") - inc(i) # skip '=' - setLen(value, 0) # reuse memory - while i < data.len: - case data[i] - of '%': - var x = 0 - if i+2 < data.len: - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) - inc(i, 2) - add(value, chr(x)) - of '+': add(value, ' ') - of '&', '\0': break - else: add(value, data[i]) - inc(i) - yield (name.TaintedString, value.TaintedString) - if i < data.len: - if data[i] == '&': inc(i) - else: cgiError("'&' expected") + for (key, value) in uri.decodeQuery(data): + yield (key, value) iterator decodeData*(allowedMethods: set[RequestMethod] = - {methodNone, methodPost, methodGet}): tuple[key, value: TaintedString] = + {methodNone, methodPost, methodGet}): tuple[key, value: string] = ## Reads and decodes CGI data and yields the (name, value) pairs the ## data consists of. If the client does not use a method listed in the - ## `allowedMethods` set, an `ECgi` exception is raised. + ## `allowedMethods` set, a `CgiError` exception is raised. let data = getEncodedData(allowedMethods) - for key, value in decodeData(data): + for (key, value) in uri.decodeQuery(data): yield (key, value) proc readData*(allowedMethods: set[RequestMethod] = {methodNone, methodPost, methodGet}): StringTableRef = - ## Read CGI data. If the client does not use a method listed in the - ## `allowedMethods` set, an `ECgi` exception is raised. + ## Reads CGI data. If the client does not use a method listed in the + ## `allowedMethods` set, a `CgiError` exception is raised. result = newStringTable() for name, value in decodeData(allowedMethods): - result[name.string] = value.string + result[name] = value proc readData*(data: string): StringTableRef = - ## Read CGI data from a string. + ## Reads CGI data from a string. result = newStringTable() for name, value in decodeData(data): - result[name.string] = value.string + result[name] = value proc validateData*(data: StringTableRef, validKeys: varargs[string]) = - ## validates data; raises `ECgi` if this fails. This checks that each variable + ## Validates data; raises `CgiError` if this fails. This checks that each variable ## name of the CGI `data` occurs in the `validKeys` array. for key, val in pairs(data): if find(validKeys, key) < 0: cgiError("unknown variable name: " & key) proc getContentLength*(): string = - ## returns contents of the ``CONTENT_LENGTH`` environment variable - return getEnv("CONTENT_LENGTH").string + ## Returns contents of the `CONTENT_LENGTH` environment variable. + return getEnv("CONTENT_LENGTH") proc getContentType*(): string = - ## returns contents of the ``CONTENT_TYPE`` environment variable - return getEnv("CONTENT_Type").string + ## Returns contents of the `CONTENT_TYPE` environment variable. + return getEnv("CONTENT_Type") proc getDocumentRoot*(): string = - ## returns contents of the ``DOCUMENT_ROOT`` environment variable - return getEnv("DOCUMENT_ROOT").string + ## Returns contents of the `DOCUMENT_ROOT` environment variable. + return getEnv("DOCUMENT_ROOT") proc getGatewayInterface*(): string = - ## returns contents of the ``GATEWAY_INTERFACE`` environment variable - return getEnv("GATEWAY_INTERFACE").string + ## Returns contents of the `GATEWAY_INTERFACE` environment variable. + return getEnv("GATEWAY_INTERFACE") proc getHttpAccept*(): string = - ## returns contents of the ``HTTP_ACCEPT`` environment variable - return getEnv("HTTP_ACCEPT").string + ## Returns contents of the `HTTP_ACCEPT` environment variable. + return getEnv("HTTP_ACCEPT") proc getHttpAcceptCharset*(): string = - ## returns contents of the ``HTTP_ACCEPT_CHARSET`` environment variable - return getEnv("HTTP_ACCEPT_CHARSET").string + ## Returns contents of the `HTTP_ACCEPT_CHARSET` environment variable. + return getEnv("HTTP_ACCEPT_CHARSET") proc getHttpAcceptEncoding*(): string = - ## returns contents of the ``HTTP_ACCEPT_ENCODING`` environment variable - return getEnv("HTTP_ACCEPT_ENCODING").string + ## Returns contents of the `HTTP_ACCEPT_ENCODING` environment variable. + return getEnv("HTTP_ACCEPT_ENCODING") proc getHttpAcceptLanguage*(): string = - ## returns contents of the ``HTTP_ACCEPT_LANGUAGE`` environment variable - return getEnv("HTTP_ACCEPT_LANGUAGE").string + ## Returns contents of the `HTTP_ACCEPT_LANGUAGE` environment variable. + return getEnv("HTTP_ACCEPT_LANGUAGE") proc getHttpConnection*(): string = - ## returns contents of the ``HTTP_CONNECTION`` environment variable - return getEnv("HTTP_CONNECTION").string + ## Returns contents of the `HTTP_CONNECTION` environment variable. + return getEnv("HTTP_CONNECTION") proc getHttpCookie*(): string = - ## returns contents of the ``HTTP_COOKIE`` environment variable - return getEnv("HTTP_COOKIE").string + ## Returns contents of the `HTTP_COOKIE` environment variable. + return getEnv("HTTP_COOKIE") proc getHttpHost*(): string = - ## returns contents of the ``HTTP_HOST`` environment variable - return getEnv("HTTP_HOST").string + ## Returns contents of the `HTTP_HOST` environment variable. + return getEnv("HTTP_HOST") proc getHttpReferer*(): string = - ## returns contents of the ``HTTP_REFERER`` environment variable - return getEnv("HTTP_REFERER").string + ## Returns contents of the `HTTP_REFERER` environment variable. + return getEnv("HTTP_REFERER") proc getHttpUserAgent*(): string = - ## returns contents of the ``HTTP_USER_AGENT`` environment variable - return getEnv("HTTP_USER_AGENT").string + ## Returns contents of the `HTTP_USER_AGENT` environment variable. + return getEnv("HTTP_USER_AGENT") proc getPathInfo*(): string = - ## returns contents of the ``PATH_INFO`` environment variable - return getEnv("PATH_INFO").string + ## Returns contents of the `PATH_INFO` environment variable. + return getEnv("PATH_INFO") proc getPathTranslated*(): string = - ## returns contents of the ``PATH_TRANSLATED`` environment variable - return getEnv("PATH_TRANSLATED").string + ## Returns contents of the `PATH_TRANSLATED` environment variable. + return getEnv("PATH_TRANSLATED") proc getQueryString*(): string = - ## returns contents of the ``QUERY_STRING`` environment variable - return getEnv("QUERY_STRING").string + ## Returns contents of the `QUERY_STRING` environment variable. + return getEnv("QUERY_STRING") proc getRemoteAddr*(): string = - ## returns contents of the ``REMOTE_ADDR`` environment variable - return getEnv("REMOTE_ADDR").string + ## Returns contents of the `REMOTE_ADDR` environment variable. + return getEnv("REMOTE_ADDR") proc getRemoteHost*(): string = - ## returns contents of the ``REMOTE_HOST`` environment variable - return getEnv("REMOTE_HOST").string + ## Returns contents of the `REMOTE_HOST` environment variable. + return getEnv("REMOTE_HOST") proc getRemoteIdent*(): string = - ## returns contents of the ``REMOTE_IDENT`` environment variable - return getEnv("REMOTE_IDENT").string + ## Returns contents of the `REMOTE_IDENT` environment variable. + return getEnv("REMOTE_IDENT") proc getRemotePort*(): string = - ## returns contents of the ``REMOTE_PORT`` environment variable - return getEnv("REMOTE_PORT").string + ## Returns contents of the `REMOTE_PORT` environment variable. + return getEnv("REMOTE_PORT") proc getRemoteUser*(): string = - ## returns contents of the ``REMOTE_USER`` environment variable - return getEnv("REMOTE_USER").string + ## Returns contents of the `REMOTE_USER` environment variable. + return getEnv("REMOTE_USER") proc getRequestMethod*(): string = - ## returns contents of the ``REQUEST_METHOD`` environment variable - return getEnv("REQUEST_METHOD").string + ## Returns contents of the `REQUEST_METHOD` environment variable. + return getEnv("REQUEST_METHOD") proc getRequestURI*(): string = - ## returns contents of the ``REQUEST_URI`` environment variable - return getEnv("REQUEST_URI").string + ## Returns contents of the `REQUEST_URI` environment variable. + return getEnv("REQUEST_URI") proc getScriptFilename*(): string = - ## returns contents of the ``SCRIPT_FILENAME`` environment variable - return getEnv("SCRIPT_FILENAME").string + ## Returns contents of the `SCRIPT_FILENAME` environment variable. + return getEnv("SCRIPT_FILENAME") proc getScriptName*(): string = - ## returns contents of the ``SCRIPT_NAME`` environment variable - return getEnv("SCRIPT_NAME").string + ## Returns contents of the `SCRIPT_NAME` environment variable. + return getEnv("SCRIPT_NAME") proc getServerAddr*(): string = - ## returns contents of the ``SERVER_ADDR`` environment variable - return getEnv("SERVER_ADDR").string + ## Returns contents of the `SERVER_ADDR` environment variable. + return getEnv("SERVER_ADDR") proc getServerAdmin*(): string = - ## returns contents of the ``SERVER_ADMIN`` environment variable - return getEnv("SERVER_ADMIN").string + ## Returns contents of the `SERVER_ADMIN` environment variable. + return getEnv("SERVER_ADMIN") proc getServerName*(): string = - ## returns contents of the ``SERVER_NAME`` environment variable - return getEnv("SERVER_NAME").string + ## Returns contents of the `SERVER_NAME` environment variable. + return getEnv("SERVER_NAME") proc getServerPort*(): string = - ## returns contents of the ``SERVER_PORT`` environment variable - return getEnv("SERVER_PORT").string + ## Returns contents of the `SERVER_PORT` environment variable. + return getEnv("SERVER_PORT") proc getServerProtocol*(): string = - ## returns contents of the ``SERVER_PROTOCOL`` environment variable - return getEnv("SERVER_PROTOCOL").string + ## Returns contents of the `SERVER_PROTOCOL` environment variable. + return getEnv("SERVER_PROTOCOL") proc getServerSignature*(): string = - ## returns contents of the ``SERVER_SIGNATURE`` environment variable - return getEnv("SERVER_SIGNATURE").string + ## Returns contents of the `SERVER_SIGNATURE` environment variable. + return getEnv("SERVER_SIGNATURE") proc getServerSoftware*(): string = - ## returns contents of the ``SERVER_SOFTWARE`` environment variable - return getEnv("SERVER_SOFTWARE").string + ## Returns contents of the `SERVER_SOFTWARE` environment variable. + return getEnv("SERVER_SOFTWARE") proc setTestData*(keysvalues: varargs[string]) = - ## fills the appropriate environment variables to test your CGI application. + ## Fills the appropriate environment variables to test your CGI application. ## This can only simulate the 'GET' request method. `keysvalues` should ## provide embedded (name, value)-pairs. Example: - ## - ## .. code-block:: Nim - ## setTestData("name", "Hanz", "password", "12345") + ## ```Nim + ## setTestData("name", "Hanz", "password", "12345") + ## ``` putEnv("REQUEST_METHOD", "GET") var i = 0 var query = "" @@ -309,11 +267,11 @@ proc setTestData*(keysvalues: varargs[string]) = putEnv("QUERY_STRING", query) proc writeContentType*() = - ## call this before starting to send your HTML data to `stdout`. This + ## Calls this before starting to send your HTML data to `stdout`. This ## implements this part of the CGI protocol: - ## - ## .. code-block:: Nim - ## write(stdout, "Content-type: text/html\n\n") + ## ```Nim + ## write(stdout, "Content-type: text/html\n\n") + ## ``` write(stdout, "Content-type: text/html\n\n") proc resetForStacktrace() = @@ -346,10 +304,10 @@ proc setCookie*(name, value: string) = var gcookies {.threadvar.}: StringTableRef -proc getCookie*(name: string): TaintedString = +proc getCookie*(name: string): string = ## Gets a cookie. If no cookie of `name` exists, "" is returned. if gcookies == nil: gcookies = parseCookies(getHttpCookie()) - result = TaintedString(gcookies.getOrDefault(name)) + result = gcookies.getOrDefault(name) proc existsCookie*(name: string): bool = ## Checks if a cookie of `name` exists. diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index a28bb447c..24257dacb 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -8,11 +8,36 @@ # ## This module implements a `crit bit tree`:idx: which is an efficient -## container for a sorted set of strings, or for a sorted mapping of strings. Based on the excellent paper -## by Adam Langley. +## container for a sorted set of strings, or for a sorted mapping of strings. Based on the +## [excellent paper by Adam Langley](https://www.imperialviolet.org/binary/critbit.pdf). ## (A crit bit tree is a form of `radix tree`:idx: or `patricia trie`:idx:.) -include "system/inclrtl" +runnableExamples: + from std/sequtils import toSeq + + var critbitAsSet: CritBitTree[void] = ["kitten", "puppy"].toCritBitTree + doAssert critbitAsSet.len == 2 + critbitAsSet.incl("") + doAssert "" in critbitAsSet + critbitAsSet.excl("") + doAssert "" notin critbitAsSet + doAssert toSeq(critbitAsSet.items) == @["kitten", "puppy"] + let same = ["puppy", "kitten", "puppy"].toCritBitTree + doAssert toSeq(same.keys) == toSeq(critbitAsSet.keys) + + var critbitAsDict: CritBitTree[int] = {"key1": 42}.toCritBitTree + doAssert critbitAsDict.len == 1 + critbitAsDict["key2"] = 0 + doAssert "key2" in critbitAsDict + doAssert critbitAsDict["key2"] == 0 + critbitAsDict.excl("key1") + doAssert "key1" notin critbitAsDict + doAssert toSeq(critbitAsDict.pairs) == @[("key2", 0)] + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions type NodeObj[T] {.acyclic.} = object @@ -28,13 +53,17 @@ type Node[T] = ref NodeObj[T] CritBitTree*[T] = object ## The crit bit tree can either be used ## as a mapping from strings to - ## some type ``T`` or as a set of - ## strings if ``T`` is void. + ## some type `T` or as a set of + ## strings if `T` is `void`. root: Node[T] count: int -proc len*[T](c: CritBitTree[T]): int = - ## returns the number of elements in `c` in O(1). +func len*[T](c: CritBitTree[T]): int {.inline.} = + ## Returns the number of elements in `c` in O(1). + runnableExamples: + let c = ["key1", "key2"].toCritBitTree + doAssert c.len == 2 + result = c.count proc rawGet[T](c: CritBitTree[T], key: string): Node[T] = @@ -47,12 +76,17 @@ proc rawGet[T](c: CritBitTree[T], key: string): Node[T] = else: return if it.key == key: it else: nil -proc contains*[T](c: CritBitTree[T], key: string): bool {.inline.} = - ## returns true iff `c` contains the given `key`. +func contains*[T](c: CritBitTree[T], key: string): bool {.inline.} = + ## Returns true if `c` contains the given `key`. + runnableExamples: + var c: CritBitTree[void] + incl(c, "key") + doAssert c.contains("key") + result = rawGet(c, key) != nil -proc hasKey*[T](c: CritBitTree[T], key: string): bool {.inline.} = - ## alias for `contains`. +func hasKey*[T](c: CritBitTree[T], key: string): bool {.inline.} = + ## Alias for `contains <#contains,CritBitTree[T],string>`_. result = rawGet(c, key) != nil proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = @@ -105,7 +139,7 @@ proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] = wherep[] = inner inc c.count -proc exclImpl[T](c: var CritBitTree[T], key: string): int = +func exclImpl[T](c: var CritBitTree[T], key: string): int = var p = c.root var wherep = addr(c.root) var whereq: ptr Node[T] = nil @@ -130,20 +164,62 @@ proc exclImpl[T](c: var CritBitTree[T], key: string): int = return c.count proc excl*[T](c: var CritBitTree[T], key: string) = - ## removes `key` (and its associated value) from the set `c`. + ## Removes `key` (and its associated value) from the set `c`. ## If the `key` does not exist, nothing happens. + ## + ## **See also:** + ## * `incl proc <#incl,CritBitTree[void],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + runnableExamples: + var c: CritBitTree[void] + incl(c, "key") + excl(c, "key") + doAssert not c.contains("key") + discard exclImpl(c, key) proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = - ## Returns true iff `c` does not contain the given `key`. If the key - ## does exist, c.excl(key) is performed. + ## Returns true if `c` does not contain the given `key`. If the key + ## does exist, `c.excl(key)` is performed. + ## + ## **See also:** + ## * `excl proc <#excl,CritBitTree[T],string>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[T],string,T>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[void],string>`_ + runnableExamples: + block: + var c: CritBitTree[void] + doAssert c.missingOrExcl("key") + block: + var c: CritBitTree[void] + incl(c, "key") + doAssert not c.missingOrExcl("key") + doAssert not c.contains("key") + let oldCount = c.count discard exclImpl(c, key) result = c.count == oldCount -proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = - ## returns true iff `c` contains the given `key`. If the key does not exist - ## ``c[key] = val`` is performed. +proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: sink T): bool = + ## Returns true if `c` contains the given `key`. If the key does not exist, + ## `c[key] = val` is performed. + ## + ## **See also:** + ## * `incl proc <#incl,CritBitTree[void],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[void],string>`_ + ## * `missingOrExcl proc <#missingOrExcl,CritBitTree[T],string>`_ + runnableExamples: + block: + var c: CritBitTree[int] + doAssert not c.containsOrIncl("key", 42) + doAssert c.contains("key") + block: + var c: CritBitTree[int] + incl(c, "key", 21) + doAssert c.containsOrIncl("key", 42) + doAssert c["key"] == 21 + let oldCount = c.count var n = rawInsert(c, key) result = c.count == oldCount @@ -151,50 +227,99 @@ proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = if not result: n.val = val proc containsOrIncl*(c: var CritBitTree[void], key: string): bool = - ## returns true iff `c` contains the given `key`. If the key does not exist + ## Returns true if `c` contains the given `key`. If the key does not exist, ## it is inserted into `c`. + ## + ## **See also:** + ## * `incl proc <#incl,CritBitTree[void],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + ## * `containsOrIncl proc <#containsOrIncl,CritBitTree[T],string,T>`_ + ## * `missingOrExcl proc <#missingOrExcl,CritBitTree[T],string>`_ + runnableExamples: + block: + var c: CritBitTree[void] + doAssert not c.containsOrIncl("key") + doAssert c.contains("key") + block: + var c: CritBitTree[void] + incl(c, "key") + doAssert c.containsOrIncl("key") + let oldCount = c.count discard rawInsert(c, key) result = c.count == oldCount proc inc*(c: var CritBitTree[int]; key: string, val: int = 1) = - ## increments `c[key]` by `val`. + ## Increments `c[key]` by `val`. + runnableExamples: + var c: CritBitTree[int] + c["key"] = 1 + inc(c, "key") + doAssert c["key"] == 2 + var n = rawInsert(c, key) inc n.val, val proc incl*(c: var CritBitTree[void], key: string) = - ## includes `key` in `c`. + ## Includes `key` in `c`. + ## + ## **See also:** + ## * `excl proc <#excl,CritBitTree[T],string>`_ + ## * `incl proc <#incl,CritBitTree[T],string,T>`_ + runnableExamples: + var c: CritBitTree[void] + incl(c, "key") + doAssert c.hasKey("key") + discard rawInsert(c, key) -proc incl*[T](c: var CritBitTree[T], key: string, val: T) = - ## inserts `key` with value `val` into `c`. +proc incl*[T](c: var CritBitTree[T], key: string, val: sink T) = + ## Inserts `key` with value `val` into `c`. + ## + ## **See also:** + ## * `excl proc <#excl,CritBitTree[T],string>`_ + ## * `incl proc <#incl,CritBitTree[void],string>`_ + runnableExamples: + var c: CritBitTree[int] + incl(c, "key", 42) + doAssert c["key"] == 42 + var n = rawInsert(c, key) n.val = val -proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = - ## puts a (key, value)-pair into `t`. +proc `[]=`*[T](c: var CritBitTree[T], key: string, val: sink T) = + ## Alias for `incl <#incl,CritBitTree[T],string,T>`_. + ## + ## **See also:** + ## * `[] proc <#[],CritBitTree[T],string>`_ + ## * `[] proc <#[],CritBitTree[T],string_2>`_ var n = rawInsert(c, key) n.val = val template get[T](c: CritBitTree[T], key: string): T = let n = rawGet(c, key) if n == nil: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raise newException(KeyError, "key not found: " & key) n.val -proc `[]`*[T](c: CritBitTree[T], key: string): T {.inline.} = - ## retrieves the value at ``c[key]``. If `key` is not in `t`, the - ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether +func `[]`*[T](c: CritBitTree[T], key: string): lent T {.inline.} = + ## Retrieves the value at `c[key]`. If `key` is not in `t`, the + ## `KeyError` exception is raised. One can check with `hasKey` whether ## the key exists. + ## + ## **See also:** + ## * `[] proc <#[],CritBitTree[T],string_2>`_ + ## * `[]= proc <#[]=,CritBitTree[T],string,T>`_ get(c, key) -proc `[]`*[T](c: var CritBitTree[T], key: string): var T {.inline.} = - ## retrieves the value at ``c[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. +func `[]`*[T](c: var CritBitTree[T], key: string): var T {.inline.} = + ## Retrieves the value at `c[key]`. The value can be modified. + ## If `key` is not in `t`, the `KeyError` exception is raised. + ## + ## **See also:** + ## * `[] proc <#[],CritBitTree[T],string>`_ + ## * `[]= proc <#[]=,CritBitTree[T],string,T>`_ get(c, key) iterator leaves[T](n: Node[T]): Node[T] = @@ -211,33 +336,64 @@ iterator leaves[T](n: Node[T]): Node[T] = yield it iterator keys*[T](c: CritBitTree[T]): string = - ## yields all keys in lexicographical order. + ## Yields all keys in lexicographical order. + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 1, "key2": 2}.toCritBitTree + doAssert toSeq(c.keys) == @["key1", "key2"] + for x in leaves(c.root): yield x.key -iterator values*[T](c: CritBitTree[T]): T = - ## yields all values of `c` in the lexicographical order of the +iterator values*[T](c: CritBitTree[T]): lent T = + ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. + ## + ## **See also:** + ## * `mvalues iterator <#mvalues.i,CritBitTree[T]>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 1, "key2": 2}.toCritBitTree + doAssert toSeq(c.values) == @[1, 2] + for x in leaves(c.root): yield x.val iterator mvalues*[T](c: var CritBitTree[T]): var T = - ## yields all values of `c` in the lexicographical order of the + ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. The values can be modified. + ## + ## **See also:** + ## * `values iterator <#values.i,CritBitTree[T]>`_ for x in leaves(c.root): yield x.val iterator items*[T](c: CritBitTree[T]): string = - ## yields all keys in lexicographical order. + ## Alias for `keys <#keys.i,CritBitTree[T]>`_. for x in leaves(c.root): yield x.key iterator pairs*[T](c: CritBitTree[T]): tuple[key: string, val: T] = - ## yields all (key, value)-pairs of `c`. + ## Yields all `(key, value)`-pairs of `c` in the lexicographical order of the + ## corresponding keys. + ## + ## **See also:** + ## * `mpairs iterator <#mpairs.i,CritBitTree[T]>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 1, "key2": 2}.toCritBitTree + doAssert toSeq(c.pairs) == @[(key: "key1", val: 1), (key: "key2", val: 2)] + for x in leaves(c.root): yield (x.key, x.val) iterator mpairs*[T](c: var CritBitTree[T]): tuple[key: string, val: var T] = - ## yields all (key, value)-pairs of `c`. The yielded values can be modified. + ## Yields all `(key, value)`-pairs of `c` in the lexicographical order of the + ## corresponding keys. The yielded values can be modified. + ## + ## **See also:** + ## * `pairs iterator <#pairs.i,CritBitTree[T]>`_ for x in leaves(c.root): yield (x.key, x.val) -proc allprefixedAux[T](c: CritBitTree[T], key: string; - longestMatch: bool): Node[T] = +proc allprefixedAux[T](c: CritBitTree[T], key: string): Node[T] = var p = c.root var top = p if p != nil: @@ -247,58 +403,83 @@ proc allprefixedAux[T](c: CritBitTree[T], key: string; let dir = (1 + (ch.ord or p.otherBits.ord)) shr 8 p = p.child[dir] if q.byte < key.len: top = p - if not longestMatch: - for i in 0 ..< key.len: - if i >= p.key.len or p.key[i] != key[i]: return + for i in 0 ..< key.len: + if i >= p.key.len or p.key[i] != key[i]: return result = top -iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string; - longestMatch = false): string = - ## yields all keys starting with `prefix`. If `longestMatch` is true, - ## the longest match is returned, it doesn't have to be a complete match then. - let top = allprefixedAux(c, prefix, longestMatch) - for x in leaves(top): yield x.key +iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = + ## Yields all keys starting with `prefix`. + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 42, "key2": 43}.toCritBitTree + doAssert toSeq(c.keysWithPrefix("key")) == @["key1", "key2"] -iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string; - longestMatch = false): string = - ## yields all keys starting with `prefix`. - let top = allprefixedAux(c, prefix, longestMatch) + let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string; - longestMatch = false): T = - ## yields all values of `c` starting with `prefix` of the +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): lent T = + ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. - let top = allprefixedAux(c, prefix, longestMatch) + ## + ## **See also:** + ## * `mvaluesWithPrefix iterator <#mvaluesWithPrefix.i,CritBitTree[T],string>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 42, "key2": 43}.toCritBitTree + doAssert toSeq(c.valuesWithPrefix("key")) == @[42, 43] + + let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.val -iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string; - longestMatch = false): var T = - ## yields all values of `c` starting with `prefix` of the +iterator mvaluesWithPrefix*[T](c: var CritBitTree[T], prefix: string): var T = + ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. The values can be modified. - let top = allprefixedAux(c, prefix, longestMatch) + ## + ## **See also:** + ## * `valuesWithPrefix iterator <#valuesWithPrefix.i,CritBitTree[T],string>`_ + let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.val +iterator itemsWithPrefix*[T](c: CritBitTree[T], prefix: string): string = + ## Alias for `keysWithPrefix <#keysWithPrefix.i,CritBitTree[T],string>`_. + let top = allprefixedAux(c, prefix) + for x in leaves(top): yield x.key + iterator pairsWithPrefix*[T](c: CritBitTree[T], - prefix: string; - longestMatch = false): tuple[key: string, val: T] = - ## yields all (key, value)-pairs of `c` starting with `prefix`. - let top = allprefixedAux(c, prefix, longestMatch) + prefix: string): tuple[key: string, val: T] = + ## Yields all (key, value)-pairs of `c` starting with `prefix`. + ## + ## **See also:** + ## * `mpairsWithPrefix iterator <#mpairsWithPrefix.i,CritBitTree[T],string>`_ + runnableExamples: + from std/sequtils import toSeq + + let c = {"key1": 42, "key2": 43}.toCritBitTree + doAssert toSeq(c.pairsWithPrefix("key")) == @[(key: "key1", val: 42), (key: "key2", val: 43)] + + let top = allprefixedAux(c, prefix) for x in leaves(top): yield (x.key, x.val) iterator mpairsWithPrefix*[T](c: var CritBitTree[T], - prefix: string; - longestMatch = false): tuple[key: string, val: var T] = - ## yields all (key, value)-pairs of `c` starting with `prefix`. + prefix: string): tuple[key: string, val: var T] = + ## Yields all (key, value)-pairs of `c` starting with `prefix`. ## The yielded values can be modified. - let top = allprefixedAux(c, prefix, longestMatch) + ## + ## **See also:** + ## * `pairsWithPrefix iterator <#pairsWithPrefix.i,CritBitTree[T],string>`_ + let top = allprefixedAux(c, prefix) for x in leaves(top): yield (x.key, x.val) -proc `$`*[T](c: CritBitTree[T]): string = - ## turns `c` into a string representation. Example outputs: - ## ``{keyA: value, keyB: value}``, ``{:}`` - ## If `T` is void the outputs look like: - ## ``{keyA, keyB}``, ``{}``. +func `$`*[T](c: CritBitTree[T]): string = + ## Turns `c` into a string representation. + runnableExamples: + doAssert $CritBitTree[int].default == "{:}" + doAssert $toCritBitTree({"key1": 1, "key2": 2}) == """{"key1": 1, "key2": 2}""" + doAssert $CritBitTree[void].default == "{}" + doAssert $toCritBitTree(["key1", "key2"]) == """{"key1", "key2"}""" + if c.len == 0: when T is void: result = "{}" @@ -324,64 +505,33 @@ proc `$`*[T](c: CritBitTree[T]): string = result.addQuoted(val) result.add("}") -when isMainModule: - import sequtils - - var r: CritBitTree[void] - r.incl "abc" - r.incl "xyz" - r.incl "def" - r.incl "definition" - r.incl "prefix" - r.incl "foo" - - doAssert r.contains"def" - - r.excl "def" - assert r.missingOrExcl("foo") == false - assert "foo" notin toSeq(r.items) - - assert r.missingOrExcl("foo") == true - - assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"] - - assert toSeq(r.itemsWithPrefix("de")) == @["definition"] - var c = CritBitTree[int]() - - c.inc("a") - assert c["a"] == 1 - - c.inc("a", 4) - assert c["a"] == 5 - - c.inc("a", -5) - assert c["a"] == 0 - - c.inc("b", 2) - assert c["b"] == 2 - - c.inc("c", 3) - assert c["c"] == 3 - - c.inc("a", 1) - assert c["a"] == 1 - - var cf = CritBitTree[float]() - - cf.incl("a", 1.0) - assert cf["a"] == 1.0 - - cf.incl("b", 2.0) - assert cf["b"] == 2.0 - - cf.incl("c", 3.0) - assert cf["c"] == 3.0 - - assert cf.len == 3 - cf.excl("c") - assert cf.len == 2 - - var cb: CritBitTree[string] - cb.incl("help", "help") - for k in cb.keysWithPrefix("helpp"): - doAssert false, "there is no prefix helpp" +func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} = + ## Returns the length of the longest common prefix of all keys in `c`. + ## If `c` is empty, returns 0. + runnableExamples: + var c: CritBitTree[void] + doAssert c.commonPrefixLen == 0 + incl(c, "key1") + doAssert c.commonPrefixLen == 4 + incl(c, "key2") + doAssert c.commonPrefixLen == 3 + + if c.root != nil: + if c.root.isLeaf: len(c.root.key) + else: c.root.byte + else: 0 + +proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = + ## Creates a new `CritBitTree` that contains the given `pairs`. + runnableExamples: + doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string] + doAssert {"a": 0, "b": 1, "c": 2}.toCritBitTree is CritBitTree[int] + + for item in pairs: result.incl item[0], item[1] + +proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} = + ## Creates a new `CritBitTree` that contains the given `items`. + runnableExamples: + doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void] + + for item in items: result.incl item diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 1aa525bd6..d2b0099f2 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -7,186 +7,179 @@ # distribution, for details about the copyright. # -## Implementation of a `deque`:idx: (double-ended queue). -## The underlying implementation uses a ``seq``. +## An implementation of a `deque`:idx: (double-ended queue). +## The underlying implementation uses a `seq`. ## -## None of the procs that get an individual value from the deque can be used -## on an empty deque. -## If compiled with `boundChecks` option, those procs will raise an `IndexError` -## on such access. This should not be relied upon, as `-d:release` will -## disable those checks and may return garbage or crash the program. +## .. note:: None of the procs that get an individual value from the deque should be used +## on an empty deque. +## +## If compiled with the `boundChecks` option, those procs will raise an `IndexDefect` +## on such access. This should not be relied upon, as `-d:danger` or `--checks:off` will +## disable those checks and then the procs may return garbage or crash the program. ## ## As such, a check to see if the deque is empty is needed before any ## access, unless your program logic guarantees it indirectly. -## -## .. code-block:: Nim -## import deques -## -## var a = initDeque[int]() -## -## doAssertRaises(IndexError, echo a[0]) -## -## for i in 1 .. 5: -## a.addLast(10*i) -## assert $a == "[10, 20, 30, 40, 50]" -## -## assert a.peekFirst == 10 -## assert a.peekLast == 50 -## assert len(a) == 5 -## -## assert a.popFirst == 10 -## assert a.popLast == 50 -## assert len(a) == 3 -## -## a.addFirst(11) -## a.addFirst(22) -## a.addFirst(33) -## assert $a == "[33, 22, 11, 20, 30, 40]" -## -## a.shrink(fromFirst = 1, fromLast = 2) -## assert $a == "[22, 11, 20]" -## -## -## **See also:** + +runnableExamples: + var a = [10, 20, 30, 40].toDeque + + doAssertRaises(IndexDefect, echo a[4]) + + a.addLast(50) + assert $a == "[10, 20, 30, 40, 50]" + + assert a.peekFirst == 10 + assert a.peekLast == 50 + assert len(a) == 5 + + assert a.popFirst == 10 + assert a.popLast == 50 + assert len(a) == 3 + + a.addFirst(11) + a.addFirst(22) + a.addFirst(33) + assert $a == "[33, 22, 11, 20, 30, 40]" + + a.shrink(fromFirst = 1, fromLast = 2) + assert $a == "[22, 11, 20]" + +## See also +## ======== ## * `lists module <lists.html>`_ for singly and doubly linked lists and rings -## * `channels module <channels.html>`_ for inter-thread communication +import std/private/since -import math +import std/[assertions, hashes, math] type Deque*[T] = object - ## A double-ended queue backed with a ringed seq buffer. + ## A double-ended queue backed with a ringed `seq` buffer. ## - ## To initialize an empty deque use `initDeque proc <#initDeque,int>`_. + ## To initialize an empty deque, + ## use the `initDeque proc <#initDeque,int>`_. data: seq[T] - head, tail, count, mask: int + + # `head` and `tail` are masked only when accessing an element of `data` + # so that `tail - head == data.len` when the deque is full. + # They are uint so that incrementing/decrementing them doesn't cause + # over/underflow. You can get a number of items with `tail - head` + # even if `tail` or `head` is wraps around and `tail < head`, because + # `tail - head == (uint.high + 1 + tail) - head` when `tail < head`. + head, tail: uint const defaultInitialSize* = 4 template initImpl(result: typed, initialSize: int) = - assert isPowerOfTwo(initialSize) - result.mask = initialSize-1 - newSeq(result.data, initialSize) - + let correctSize = nextPowerOfTwo(initialSize) + newSeq(result.data, correctSize) + template checkIfInitialized(deq: typed) = - when compiles(defaultInitialSize): - if deq.mask == 0: - initImpl(deq, defaultInitialSize) + if deq.data.len == 0: + initImpl(deq, defaultInitialSize) + +func mask[T](deq: Deque[T]): uint {.inline.} = + uint(deq.data.len) - 1 -proc initDeque*[T](initialSize: int = 4): Deque[T] = - ## Create a new empty deque. +proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = + ## Creates a new empty deque. ## ## Optionally, the initial capacity can be reserved via `initialSize` - ## as a performance optimization. + ## as a performance optimization + ## (default: `defaultInitialSize <#defaultInitialSize>`_). ## The length of a newly created deque will still be 0. ## - ## ``initialSize`` must be a power of two (default: 4). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_. + ## **See also:** + ## * `toDeque proc <#toDeque,openArray[T]>`_ result.initImpl(initialSize) -proc len*[T](deq: Deque[T]): int {.inline.} = - ## Return the number of elements of `deq`. - result = deq.count +func len*[T](deq: Deque[T]): int {.inline.} = + ## Returns the number of elements of `deq`. + int(deq.tail - deq.head) template emptyCheck(deq) = # Bounds check for the regular deque access. when compileOption("boundChecks"): - if unlikely(deq.count < 1): - raise newException(IndexError, "Empty deque.") + if unlikely(deq.len < 1): + raise newException(IndexDefect, "Empty deque.") template xBoundsCheck(deq, i) = # Bounds check for the array like accesses. - when compileOption("boundChecks"): # d:release should disable this. - if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter - raise newException(IndexError, - "Out of bounds: " & $i & " > " & $(deq.count - 1)) + when compileOption("boundChecks"): # `-d:danger` or `--checks:off` should disable this. + if unlikely(i >= deq.len): # x < deq.low is taken care by the Natural parameter + raise newException(IndexDefect, + "Out of bounds: " & $i & " > " & $(deq.len - 1)) if unlikely(i < 0): # when used with BackwardsIndex - raise newException(IndexError, + raise newException(IndexDefect, "Out of bounds: " & $i & " < 0") -proc `[]`*[T](deq: Deque[T], i: Natural): T {.inline.} = - ## Access the i-th element of `deq`. +proc `[]`*[T](deq: Deque[T], i: Natural): lent T {.inline.} = + ## Accesses the `i`-th element of `deq`. runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + let a = [10, 20, 30, 40, 50].toDeque assert a[0] == 10 assert a[3] == 40 - doAssertRaises(IndexError, echo a[8]) + doAssertRaises(IndexDefect, echo a[8]) xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = - ## Access the i-th element of `deq` and return a mutable + ## Accesses the `i`-th element of `deq` and returns a mutable ## reference to it. runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) - assert a[0] == 10 - assert a[3] == 40 - doAssertRaises(IndexError, echo a[8]) + var a = [10, 20, 30, 40, 50].toDeque + inc(a[0]) + assert a[0] == 11 xBoundsCheck(deq, i) - return deq.data[(deq.head + i) and deq.mask] + return deq.data[(deq.head + i.uint) and deq.mask] -proc `[]=`*[T](deq: var Deque[T], i: Natural, val: T) {.inline.} = - ## Change the i-th element of `deq`. +proc `[]=`*[T](deq: var Deque[T], i: Natural, val: sink T) {.inline.} = + ## Sets the `i`-th element of `deq` to `val`. runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + var a = [10, 20, 30, 40, 50].toDeque a[0] = 99 a[3] = 66 assert $a == "[99, 20, 30, 66, 50]" checkIfInitialized(deq) xBoundsCheck(deq, i) - deq.data[(deq.head + i) and deq.mask] = val + deq.data[(deq.head + i.uint) and deq.mask] = val -proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): T {.inline.} = - ## Access the backwards indexed i-th element. +proc `[]`*[T](deq: Deque[T], i: BackwardsIndex): lent T {.inline.} = + ## Accesses the backwards indexed `i`-th element. ## ## `deq[^1]` is the last element. runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + let a = [10, 20, 30, 40, 50].toDeque assert a[^1] == 50 assert a[^4] == 20 - doAssertRaises(IndexError, echo a[^9]) + doAssertRaises(IndexDefect, echo a[^9]) xBoundsCheck(deq, deq.len - int(i)) return deq[deq.len - int(i)] proc `[]`*[T](deq: var Deque[T], i: BackwardsIndex): var T {.inline.} = - ## Access the backwards indexed i-th element. + ## Accesses the backwards indexed `i`-th element and returns a mutable + ## reference to it. ## ## `deq[^1]` is the last element. runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) - assert a[^1] == 50 - assert a[^4] == 20 - doAssertRaises(IndexError, echo a[^9]) + var a = [10, 20, 30, 40, 50].toDeque + inc(a[^1]) + assert a[^1] == 51 xBoundsCheck(deq, deq.len - int(i)) return deq[deq.len - int(i)] -proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = - ## Change the backwards indexed i-th element. +proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: sink T) {.inline.} = + ## Sets the backwards indexed `i`-th element of `deq` to `x`. ## ## `deq[^1]` is the last element. runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + var a = [10, 20, 30, 40, 50].toDeque a[^1] = 99 a[^3] = 77 assert $a == "[10, 20, 77, 40, 99]" @@ -195,166 +188,143 @@ proc `[]=`*[T](deq: var Deque[T], i: BackwardsIndex, x: T) {.inline.} = xBoundsCheck(deq, deq.len - int(i)) deq[deq.len - int(i)] = x -iterator items*[T](deq: Deque[T]): T = - ## Yield every element of `deq`. - ## - ## **Examples:** +iterator items*[T](deq: Deque[T]): lent T = + ## Yields every element of `deq`. ## - ## .. code-block:: - ## var a = initDeque[int]() - ## for i in 1 .. 3: - ## a.addLast(10*i) - ## - ## for x in a: # the same as: for x in items(a): - ## echo x - ## - ## # 10 - ## # 20 - ## # 30 - ## - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + ## **See also:** + ## * `mitems iterator <#mitems.i,Deque[T]>`_ + runnableExamples: + from std/sequtils import toSeq + + let a = [10, 20, 30, 40, 50].toDeque + assert toSeq(a.items) == @[10, 20, 30, 40, 50] + + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator mitems*[T](deq: var Deque[T]): var T = - ## Yield every element of `deq`, which can be modified. + ## Yields every element of `deq`, which can be modified. + ## + ## **See also:** + ## * `items iterator <#items.i,Deque[T]>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + var a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" for x in mitems(a): - x = 5*x - 1 + x = 5 * x - 1 assert $a == "[49, 99, 149, 199, 249]" - var i = deq.head - for c in 0 ..< deq.count: - yield deq.data[i] - i = (i + 1) and deq.mask + for c in 0 ..< deq.len: + yield deq.data[(deq.head + c.uint) and deq.mask] iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = - ## Yield every (position, value) of `deq`. - ## - ## **Examples:** - ## - ## .. code-block:: - ## var a = initDeque[int]() - ## for i in 1 .. 3: - ## a.addLast(10*i) - ## - ## for k, v in pairs(a): - ## echo "key: ", k, ", value: ", v - ## - ## # key: 0, value: 10 - ## # key: 1, value: 20 - ## # key: 2, value: 30 - ## - var i = deq.head - for c in 0 ..< deq.count: - yield (c, deq.data[i]) - i = (i + 1) and deq.mask + ## Yields every `(position, value)`-pair of `deq`. + runnableExamples: + from std/sequtils import toSeq + + let a = [10, 20, 30].toDeque + assert toSeq(a.pairs) == @[(0, 10), (1, 20), (2, 30)] + + for c in 0 ..< deq.len: + yield (c, deq.data[(deq.head + c.uint) and deq.mask]) proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = - ## Return true if `item` is in `deq` or false if not found. - ## - ## Usually used via the ``in`` operator. - ## It is the equivalent of ``deq.find(item) >= 0``. + ## Returns true if `item` is in `deq` or false if not found. ## - ## .. code-block:: Nim - ## if x in q: - ## assert q.contains(x) + ## Usually used via the `in` operator. + ## It is the equivalent of `deq.find(item) >= 0`. + runnableExamples: + let q = [7, 9].toDeque + assert 7 in q + assert q.contains(7) + assert 8 notin q + for e in deq: if e == item: return true return false proc expandIfNeeded[T](deq: var Deque[T]) = checkIfInitialized(deq) - var cap = deq.mask + 1 - if unlikely(deq.count >= cap): + let cap = deq.data.len + assert deq.len <= cap + if unlikely(deq.len == cap): var n = newSeq[T](cap * 2) var i = 0 - for x in mitems(deq): # don't use copyMem because of the GC and because it's slower. - n[i] = move(x) + for x in mitems(deq): + when nimvm: n[i] = x # workaround for VM bug + else: n[i] = move(x) inc i deq.data = move(n) - deq.mask = cap * 2 - 1 - deq.tail = deq.count + deq.tail = cap.uint deq.head = 0 -proc addFirst*[T](deq: var Deque[T], item: T) = - ## Add an `item` to the beginning of the `deq`. +proc addFirst*[T](deq: var Deque[T], item: sink T) = + ## Adds an `item` to the beginning of `deq`. ## - ## See also: - ## * `addLast proc <#addLast,Deque[T],T>`_ - ## * `peekFirst proc <#peekFirst,Deque[T]>`_ - ## * `peekLast proc <#peekLast,Deque[T]>`_ - ## * `popFirst proc <#popFirst,Deque[T]>`_ - ## * `popLast proc <#popLast,Deque[T]>`_ + ## **See also:** + ## * `addLast proc <#addLast,Deque[T],sinkT>`_ runnableExamples: var a = initDeque[int]() for i in 1 .. 5: - a.addFirst(10*i) + a.addFirst(10 * i) assert $a == "[50, 40, 30, 20, 10]" expandIfNeeded(deq) - inc deq.count - deq.head = (deq.head - 1) and deq.mask - deq.data[deq.head] = item + dec deq.head + deq.data[deq.head and deq.mask] = item -proc addLast*[T](deq: var Deque[T], item: T) = - ## Add an `item` to the end of the `deq`. +proc addLast*[T](deq: var Deque[T], item: sink T) = + ## Adds an `item` to the end of `deq`. ## - ## See also: - ## * `addFirst proc <#addFirst,Deque[T],T>`_ - ## * `peekFirst proc <#peekFirst,Deque[T]>`_ - ## * `peekLast proc <#peekLast,Deque[T]>`_ - ## * `popFirst proc <#popFirst,Deque[T]>`_ - ## * `popLast proc <#popLast,Deque[T]>`_ + ## **See also:** + ## * `addFirst proc <#addFirst,Deque[T],sinkT>`_ runnableExamples: var a = initDeque[int]() for i in 1 .. 5: - a.addLast(10*i) + a.addLast(10 * i) assert $a == "[10, 20, 30, 40, 50]" expandIfNeeded(deq) - inc deq.count - deq.data[deq.tail] = item - deq.tail = (deq.tail + 1) and deq.mask + deq.data[deq.tail and deq.mask] = item + inc deq.tail + +proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = + ## Creates a new deque that contains the elements of `x` (in the same order). + ## + ## **See also:** + ## * `initDeque proc <#initDeque,int>`_ + runnableExamples: + let a = toDeque([7, 8, 9]) + assert len(a) == 3 + assert $a == "[7, 8, 9]" + + result.initImpl(x.len) + for item in items(x): + result.addLast(item) -proc peekFirst*[T](deq: Deque[T]): T {.inline.} = +proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the first element of `deq`, but does not remove it from the deque. ## - ## See also: - ## * `addFirst proc <#addFirst,Deque[T],T>`_ - ## * `addLast proc <#addLast,Deque[T],T>`_ + ## **See also:** + ## * `peekFirst proc <#peekFirst,Deque[T]_2>`_ which returns a mutable reference ## * `peekLast proc <#peekLast,Deque[T]>`_ - ## * `popFirst proc <#popFirst,Deque[T]>`_ - ## * `popLast proc <#popLast,Deque[T]>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + let a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" assert a.peekFirst == 10 assert len(a) == 5 emptyCheck(deq) - result = deq.data[deq.head] + result = deq.data[deq.head and deq.mask] -proc peekLast*[T](deq: Deque[T]): T {.inline.} = +proc peekLast*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the last element of `deq`, but does not remove it from the deque. ## - ## See also: - ## * `addFirst proc <#addFirst,Deque[T],T>`_ - ## * `addLast proc <#addLast,Deque[T],T>`_ + ## **See also:** + ## * `peekLast proc <#peekLast,Deque[T]_2>`_ which returns a mutable reference ## * `peekFirst proc <#peekFirst,Deque[T]>`_ - ## * `popFirst proc <#popFirst,Deque[T]>`_ - ## * `popLast proc <#popLast,Deque[T]>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + let a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" assert a.peekLast == 50 assert len(a) == 5 @@ -362,210 +332,149 @@ proc peekLast*[T](deq: Deque[T]): T {.inline.} = emptyCheck(deq) result = deq.data[(deq.tail - 1) and deq.mask] +proc peekFirst*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = + ## Returns a mutable reference to the first element of `deq`, + ## but does not remove it from the deque. + ## + ## **See also:** + ## * `peekFirst proc <#peekFirst,Deque[T]>`_ + ## * `peekLast proc <#peekLast,Deque[T]_2>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + a.peekFirst() = 99 + assert $a == "[99, 20, 30, 40, 50]" + + emptyCheck(deq) + result = deq.data[deq.head and deq.mask] + +proc peekLast*[T](deq: var Deque[T]): var T {.inline, since: (1, 3).} = + ## Returns a mutable reference to the last element of `deq`, + ## but does not remove it from the deque. + ## + ## **See also:** + ## * `peekFirst proc <#peekFirst,Deque[T]_2>`_ + ## * `peekLast proc <#peekLast,Deque[T]>`_ + runnableExamples: + var a = [10, 20, 30, 40, 50].toDeque + a.peekLast() = 99 + assert $a == "[10, 20, 30, 40, 99]" + + emptyCheck(deq) + result = deq.data[(deq.tail - 1) and deq.mask] + template destroy(x: untyped) = reset(x) proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = - ## Remove and returns the first element of the `deq`. + ## Removes and returns the first element of the `deq`. ## ## See also: - ## * `addFirst proc <#addFirst,Deque[T],T>`_ - ## * `addLast proc <#addLast,Deque[T],T>`_ - ## * `peekFirst proc <#peekFirst,Deque[T]>`_ - ## * `peekLast proc <#peekLast,Deque[T]>`_ ## * `popLast proc <#popLast,Deque[T]>`_ - ## * `clear proc <#clear,Deque[T]>`_ ## * `shrink proc <#shrink,Deque[T],int,int>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + var a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" assert a.popFirst == 10 assert $a == "[20, 30, 40, 50]" emptyCheck(deq) - dec deq.count - result = deq.data[deq.head] - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + result = move deq.data[deq.head and deq.mask] + inc deq.head proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = - ## Remove and returns the last element of the `deq`. + ## Removes and returns the last element of the `deq`. ## - ## See also: - ## * `addFirst proc <#addFirst,Deque[T],T>`_ - ## * `addLast proc <#addLast,Deque[T],T>`_ - ## * `peekFirst proc <#peekFirst,Deque[T]>`_ - ## * `peekLast proc <#peekLast,Deque[T]>`_ + ## **See also:** ## * `popFirst proc <#popFirst,Deque[T]>`_ - ## * `clear proc <#clear,Deque[T]>`_ ## * `shrink proc <#shrink,Deque[T],int,int>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addLast(10*i) + var a = [10, 20, 30, 40, 50].toDeque assert $a == "[10, 20, 30, 40, 50]" assert a.popLast == 50 assert $a == "[10, 20, 30, 40]" emptyCheck(deq) - dec deq.count - deq.tail = (deq.tail - 1) and deq.mask - result = deq.data[deq.tail] - destroy(deq.data[deq.tail]) + dec deq.tail + result = move deq.data[deq.tail and deq.mask] proc clear*[T](deq: var Deque[T]) {.inline.} = ## Resets the deque so that it is empty. ## - ## See also: - ## * `clear proc <#clear,Deque[T]>`_ + ## **See also:** ## * `shrink proc <#shrink,Deque[T],int,int>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addFirst(10*i) - assert $a == "[50, 40, 30, 20, 10]" + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" clear(a) assert len(a) == 0 for el in mitems(deq): destroy(el) - deq.count = 0 deq.tail = deq.head proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = - ## Remove `fromFirst` elements from the front of the deque and + ## Removes `fromFirst` elements from the front of the deque and ## `fromLast` elements from the back. ## ## If the supplied number of elements exceeds the total number of elements ## in the deque, the deque will remain empty. ## - ## See also: + ## **See also:** ## * `clear proc <#clear,Deque[T]>`_ + ## * `popFirst proc <#popFirst,Deque[T]>`_ + ## * `popLast proc <#popLast,Deque[T]>`_ runnableExamples: - var a = initDeque[int]() - for i in 1 .. 5: - a.addFirst(10*i) - assert $a == "[50, 40, 30, 20, 10]" + var a = [10, 20, 30, 40, 50].toDeque + assert $a == "[10, 20, 30, 40, 50]" a.shrink(fromFirst = 2, fromLast = 1) - assert $a == "[30, 20]" + assert $a == "[30, 40]" - if fromFirst + fromLast > deq.count: + if fromFirst + fromLast > deq.len: clear(deq) return for i in 0 ..< fromFirst: - destroy(deq.data[deq.head]) - deq.head = (deq.head + 1) and deq.mask + destroy(deq.data[deq.head and deq.mask]) + inc deq.head for i in 0 ..< fromLast: - destroy(deq.data[deq.tail]) - deq.tail = (deq.tail - 1) and deq.mask - - dec deq.count, fromFirst + fromLast + destroy(deq.data[(deq.tail - 1) and deq.mask]) + dec deq.tail proc `$`*[T](deq: Deque[T]): string = - ## Turn a deque into its string representation. + ## Turns a deque into its string representation. + runnableExamples: + let a = [10, 20, 30].toDeque + assert $a == "[10, 20, 30]" + result = "[" for x in deq: if result.len > 1: result.add(", ") result.addQuoted(x) result.add("]") +func `==`*[T](deq1, deq2: Deque[T]): bool = + ## The `==` operator for Deque. + ## Returns `true` if both deques contains the same values in the same order. + runnableExamples: + var a, b = initDeque[int]() + a.addFirst(2) + a.addFirst(1) + b.addLast(1) + b.addLast(2) + doAssert a == b + if deq1.len != deq2.len: + return false -when isMainModule: - var deq = initDeque[int](1) - deq.addLast(4) - deq.addFirst(9) - deq.addFirst(123) - var first = deq.popFirst() - deq.addLast(56) - assert(deq.peekLast() == 56) - deq.addLast(6) - assert(deq.peekLast() == 6) - var second = deq.popFirst() - deq.addLast(789) - assert(deq.peekLast() == 789) - - assert first == 123 - assert second == 9 - assert($deq == "[4, 56, 6, 789]") - - assert deq[0] == deq.peekFirst and deq.peekFirst == 4 - #assert deq[^1] == deq.peekLast and deq.peekLast == 789 - deq[0] = 42 - deq[deq.len - 1] = 7 - - assert 6 in deq and 789 notin deq - assert deq.find(6) >= 0 - assert deq.find(789) < 0 - - block: - var d = initDeque[int](1) - d.addLast 7 - d.addLast 8 - d.addLast 10 - d.addFirst 5 - d.addFirst 2 - d.addFirst 1 - d.addLast 20 - d.shrink(fromLast = 2) - doAssert($d == "[1, 2, 5, 7, 8]") - d.shrink(2, 1) - doAssert($d == "[5, 7]") - d.shrink(2, 2) - doAssert d.len == 0 - - for i in -2 .. 10: - if i in deq: - assert deq.contains(i) and deq.find(i) >= 0 - else: - assert(not deq.contains(i) and deq.find(i) < 0) + for i in 0 ..< deq1.len: + if deq1.data[(deq1.head + i.uint) and deq1.mask] != deq2.data[(deq2.head + i.uint) and deq2.mask]: + return false - when compileOption("boundChecks"): - try: - echo deq[99] - assert false - except IndexError: - discard - - try: - assert deq.len == 4 - for i in 0 ..< 5: deq.popFirst() - assert false - except IndexError: - discard - - # grabs some types of resize error. - deq = initDeque[int]() - for i in 1 .. 4: deq.addLast i - deq.popFirst() - deq.popLast() - for i in 5 .. 8: deq.addFirst i - assert $deq == "[8, 7, 6, 5, 2, 3]" - - # Similar to proc from the documentation example - proc foo(a, b: Positive) = # assume random positive values for `a` and `b`. - var deq = initDeque[int]() - assert deq.len == 0 - for i in 1 .. a: deq.addLast i - - if b < deq.len: # checking before indexed access. - assert deq[b] == b + 1 - - # The following two lines don't need any checking on access due to the logic - # of the program, but that would not be the case if `a` could be 0. - assert deq.peekFirst == 1 - assert deq.peekLast == a - - while deq.len > 0: # checking if the deque is empty - assert deq.popFirst() > 0 - - #foo(0,0) - foo(8, 5) - foo(10, 9) - foo(1, 1) - foo(2, 1) - foo(1, 5) - foo(3, 2) + true + +func hash*[T](deq: Deque[T]): Hash = + ## Hashing of Deque. + var h: Hash = 0 + for x in deq: + h = h !& hash(x) + !$h diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index d9a914afc..17785c8c7 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -7,17 +7,17 @@ # distribution, for details about the copyright. # -# An ``include`` file which contains common code for +# An `include` file which contains common code for # hash sets and tables. +when defined(nimPreviewSlimSystem): + import std/assertions + +import std / outparams + const growthFactor = 2 -when not defined(nimHasDefault): - template default[T](t: typedesc[T]): T = - var v: T - v - # hcode for real keys cannot be zero. hcode==0 signifies an empty slot. These # two procs retain clarity of that encoding without the space cost of an enum. proc isEmpty(hcode: Hash): bool {.inline.} = @@ -29,9 +29,14 @@ proc isFilled(hcode: Hash): bool {.inline.} = proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = (h + 1) and maxHash -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) +proc mustRehash[T](t: T): bool {.inline.} = + # If this is changed, make sure to synchronize it with `slotsNeeded` below + assert(t.dataLen > t.counter) + result = (t.dataLen * 2 < t.counter * 3) or (t.dataLen - t.counter < 4) + +proc slotsNeeded(count: Natural): int {.inline.} = + # Make sure to synchronize with `mustRehash` above + result = nextPowerOfTwo(count * 3 div 2 + 4) template rawGetKnownHCImpl() {.dirty.} = if t.dataLen == 0: @@ -53,7 +58,10 @@ proc rawGetKnownHC[X, A](t: X, key: A, hc: Hash): int {.inline.} = template genHashImpl(key, hc: typed) = hc = hash(key) if hc == 0: # This almost never taken branch should be very predictable. - hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + when sizeof(int) < 4: + hc = 31415 # Value doesn't matter; Any non-zero favorite is fine <= 16-bit. + else: + hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. template genHash(key: typed): Hash = var res: Hash @@ -64,5 +72,5 @@ template rawGetImpl() {.dirty.} = genHashImpl(key, hc) rawGetKnownHCImpl() -proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGet[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetImpl() diff --git a/lib/pure/collections/heapqueue.nim b/lib/pure/collections/heapqueue.nim index f958a5b0a..96f9b4430 100644 --- a/lib/pure/collections/heapqueue.nim +++ b/lib/pure/collections/heapqueue.nim @@ -6,76 +6,91 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -##[ - The `heapqueue` module implements a - `heap data structure<https://en.wikipedia.org/wiki/Heap_(data_structure)>`_ - that can be used as a - `priority queue<https://en.wikipedia.org/wiki/Priority_queue>`_. - Heaps are arrays for which `a[k] <= a[2*k+1]` and `a[k] <= a[2*k+2]` for - all `k`, counting elements from 0. The interesting property of a heap is that - `a[0]` is always its smallest element. - - Basic usage - ----------- - .. code-block:: Nim - import heapqueue - - var heap = initHeapQueue[int]() - heap.push(8) - heap.push(2) - heap.push(5) - # The first element is the lowest element - assert heap[0] == 2 - # Remove and return the lowest element - assert heap.pop() == 2 - # The lowest element remaining is 5 - assert heap[0] == 5 - - Usage with custom object - ------------------------ - To use a `HeapQueue` with a custom object, the `<` operator must be - implemented. - - .. code-block:: Nim - import heapqueue - - type Job = object - priority: int - proc `<`(a, b: Job): bool = a.priority < b.priority - - var jobs = initHeapQueue[Job]() - jobs.push(Job(priority: 1)) - jobs.push(Job(priority: 2)) - - assert jobs[0].priority == 1 -]## +## The `heapqueue` module implements a +## `binary heap data structure<https://en.wikipedia.org/wiki/Binary_heap>`_ +## that can be used as a `priority queue<https://en.wikipedia.org/wiki/Priority_queue>`_. +## They are represented as arrays for which `a[k] <= a[2*k+1]` and `a[k] <= a[2*k+2]` +## for all indices `k` (counting elements from 0). The interesting property of a heap is that +## `a[0]` is always its smallest element. +## +## Basic usage +## ----------- +## +runnableExamples: + var heap = [8, 2].toHeapQueue + heap.push(5) + # the first element is the lowest element + assert heap[0] == 2 + # remove and return the lowest element + assert heap.pop() == 2 + # the lowest element remaining is 5 + assert heap[0] == 5 + +## Usage with custom objects +## ------------------------- +## To use a `HeapQueue` with a custom object, the `<` operator must be +## implemented. + +runnableExamples: + type Job = object + priority: int + + proc `<`(a, b: Job): bool = a.priority < b.priority + + var jobs = initHeapQueue[Job]() + jobs.push(Job(priority: 1)) + jobs.push(Job(priority: 2)) + + assert jobs[0].priority == 1 + + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions type HeapQueue*[T] = object ## A heap queue, commonly known as a priority queue. data: seq[T] proc initHeapQueue*[T](): HeapQueue[T] = - ## Create a new empty heap. - discard + ## Creates a new empty heap. + ## + ## Heaps are initialized by default, so it is not necessary to call + ## this function explicitly. + ## + ## **See also:** + ## * `toHeapQueue proc <#toHeapQueue,openArray[T]>`_ + result = default(HeapQueue[T]) proc len*[T](heap: HeapQueue[T]): int {.inline.} = - ## Return the number of elements of `heap`. + ## Returns the number of elements of `heap`. + runnableExamples: + let heap = [9, 5, 8].toHeapQueue + assert heap.len == 3 + heap.data.len -proc `[]`*[T](heap: HeapQueue[T], i: Natural): T {.inline.} = - ## Access the i-th element of `heap`. +proc `[]`*[T](heap: HeapQueue[T], i: Natural): lent T {.inline.} = + ## Accesses the i-th element of `heap`. heap.data[i] -proc heapCmp[T](x, y: T): bool {.inline.} = - return (x < y) +iterator items*[T](heap: HeapQueue[T]): lent T {.inline, since: (2, 1, 1).} = + ## Iterates over each item of `heap`. + let L = len(heap) + for i in 0 .. high(heap.data): + yield heap.data[i] + assert(len(heap) == L, "the length of the HeapQueue changed while iterating over it") -proc siftdown[T](heap: var HeapQueue[T], startpos, p: int) = - ## 'heap' is a heap at all indices >= startpos, except possibly for pos. pos - ## is the index of a leaf with a possibly out-of-order value. Restore the +proc heapCmp[T](x, y: T): bool {.inline.} = x < y + +proc siftup[T](heap: var HeapQueue[T], startpos, p: int) = + ## `heap` is a heap at all indices >= `startpos`, except possibly for `p`. `p` + ## is the index of a leaf with a possibly out-of-order value. Restores the ## heap invariant. var pos = p - var newitem = heap[pos] + let newitem = heap[pos] # Follow the path to the root, moving parents down until finding a place # newitem fits. while pos > startpos: @@ -88,13 +103,14 @@ proc siftdown[T](heap: var HeapQueue[T], startpos, p: int) = break heap.data[pos] = newitem -proc siftup[T](heap: var HeapQueue[T], p: int) = +proc siftdownToBottom[T](heap: var HeapQueue[T], p: int) = + # This is faster when the element should be close to the bottom. let endpos = len(heap) var pos = p let startpos = pos let newitem = heap[pos] # Bubble up the smaller child until hitting a leaf. - var childpos = 2*pos + 1 # leftmost child position + var childpos = 2 * pos + 1 # leftmost child position while childpos < endpos: # Set childpos to index of smaller child. let rightpos = childpos + 1 @@ -103,132 +119,148 @@ proc siftup[T](heap: var HeapQueue[T], p: int) = # Move the smaller child up. heap.data[pos] = heap[childpos] pos = childpos - childpos = 2*pos + 1 - # The leaf at pos is empty now. Put newitem there, and bubble it up + childpos = 2 * pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up # to its final resting place (by sifting its parents down). heap.data[pos] = newitem - siftdown(heap, startpos, pos) + siftup(heap, startpos, pos) + +proc siftdown[T](heap: var HeapQueue[T], p: int) = + let endpos = len(heap) + var pos = p + let newitem = heap[pos] + var childpos = 2 * pos + 1 + while childpos < endpos: + let rightpos = childpos + 1 + if rightpos < endpos and not heapCmp(heap[childpos], heap[rightpos]): + childpos = rightpos + if not heapCmp(heap[childpos], newitem): + break + heap.data[pos] = heap[childpos] + pos = childpos + childpos = 2 * pos + 1 + heap.data[pos] = newitem -proc push*[T](heap: var HeapQueue[T], item: T) = - ## Push `item` onto heap, maintaining the heap invariant. +proc push*[T](heap: var HeapQueue[T], item: sink T) = + ## Pushes `item` onto `heap`, maintaining the heap invariant. heap.data.add(item) - siftdown(heap, 0, len(heap)-1) + siftup(heap, 0, len(heap) - 1) + +proc toHeapQueue*[T](x: openArray[T]): HeapQueue[T] {.since: (1, 3).} = + ## Creates a new HeapQueue that contains the elements of `x`. + ## + ## **See also:** + ## * `initHeapQueue proc <#initHeapQueue>`_ + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + assert heap.pop() == 5 + assert heap[0] == 8 + + # see https://en.wikipedia.org/wiki/Binary_heap#Building_a_heap + result.data = @x + for i in countdown(x.len div 2 - 1, 0): + siftdown(result, i) proc pop*[T](heap: var HeapQueue[T]): T = - ## Pop and return the smallest item from `heap`, + ## Pops and returns the smallest item from `heap`, ## maintaining the heap invariant. + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + assert heap.pop() == 5 + let lastelt = heap.data.pop() if heap.len > 0: result = heap[0] heap.data[0] = lastelt - siftup(heap, 0) + siftdownToBottom(heap, 0) else: result = lastelt +proc find*[T](heap: HeapQueue[T], x: T): int {.since: (1, 3).} = + ## Linear scan to find the index of the item `x` or -1 if not found. + runnableExamples: + let heap = [9, 5, 8].toHeapQueue + assert heap.find(5) == 0 + assert heap.find(9) == 1 + assert heap.find(777) == -1 + + result = -1 + for i in 0 ..< heap.len: + if heap[i] == x: return i + +proc contains*[T](heap: HeapQueue[T], x: T): bool {.since: (2, 1, 1).} = + ## Returns true if `x` is in `heap` or false if not found. This is a shortcut + ## for `find(heap, x) >= 0`. + result = find(heap, x) >= 0 + proc del*[T](heap: var HeapQueue[T], index: Natural) = ## Removes the element at `index` from `heap`, maintaining the heap invariant. + runnableExamples: + var heap = [9, 5, 8].toHeapQueue + heap.del(1) + assert heap[0] == 5 + assert heap[1] == 8 + swap(heap.data[^1], heap.data[index]) let newLen = heap.len - 1 heap.data.setLen(newLen) if index < newLen: - heap.siftup(index) + siftdownToBottom(heap, index) -proc replace*[T](heap: var HeapQueue[T], item: T): T = - ## Pop and return the current smallest value, and add the new item. - ## This is more efficient than pop() followed by push(), and can be +proc replace*[T](heap: var HeapQueue[T], item: sink T): T = + ## Pops and returns the current smallest value, and add the new item. + ## This is more efficient than `pop()` followed by `push()`, and can be ## more appropriate when using a fixed-size heap. Note that the value - ## returned may be larger than item! That constrains reasonable uses of - ## this routine unless written as part of a conditional replacement: + ## returned may be larger than `item`! That constrains reasonable uses of + ## this routine unless written as part of a conditional replacement. ## - ## .. code-block:: nim - ## if item > heap[0]: - ## item = replace(heap, item) + ## **See also:** + ## * `pushpop proc <#pushpop,HeapQueue[T],sinkT>`_ + runnableExamples: + var heap = [5, 12].toHeapQueue + assert heap.replace(6) == 5 + assert heap.len == 2 + assert heap[0] == 6 + assert heap.replace(4) == 6 + result = heap[0] heap.data[0] = item - siftup(heap, 0) + siftdown(heap, 0) -proc pushpop*[T](heap: var HeapQueue[T], item: T): T = - ## Fast version of a push followed by a pop. - if heap.len > 0 and heapCmp(heap[0], item): - swap(item, heap[0]) - siftup(heap, 0) - return item +proc pushpop*[T](heap: var HeapQueue[T], item: sink T): T = + ## Fast version of a `push()` followed by a `pop()`. + ## + ## **See also:** + ## * `replace proc <#replace,HeapQueue[T],sinkT>`_ + runnableExamples: + var heap = [5, 12].toHeapQueue + assert heap.pushpop(6) == 5 + assert heap.len == 2 + assert heap[0] == 6 + assert heap.pushpop(4) == 4 + + result = item + if heap.len > 0 and heapCmp(heap.data[0], result): + swap(result, heap.data[0]) + siftdown(heap, 0) proc clear*[T](heap: var HeapQueue[T]) = - ## Remove all elements from `heap`, making it empty. + ## Removes all elements from `heap`, making it empty. runnableExamples: - var heap = initHeapQueue[int]() - heap.push(1) + var heap = [9, 5, 8].toHeapQueue heap.clear() assert heap.len == 0 + heap.data.setLen(0) proc `$`*[T](heap: HeapQueue[T]): string = - ## Turn a heap into its string representation. + ## Turns a heap into its string representation. runnableExamples: - var heap = initHeapQueue[int]() - heap.push(1) - heap.push(2) + let heap = [1, 2].toHeapQueue assert $heap == "[1, 2]" + result = "[" for x in heap.data: if result.len > 1: result.add(", ") result.addQuoted(x) result.add("]") - -proc newHeapQueue*[T](): HeapQueue[T] {.deprecated: - "Deprecated since v0.20.0: use 'initHeapQueue' instead.".} = - initHeapQueue[T]() - -proc newHeapQueue*[T](heap: var HeapQueue[T]) {.deprecated: - "Deprecated since v0.20.0: use 'clear' instead.".} = - heap.clear() - -when isMainModule: - proc toSortedSeq[T](h: HeapQueue[T]): seq[T] = - var tmp = h - result = @[] - while tmp.len > 0: - result.add(pop(tmp)) - - block: # Simple sanity test - var heap = initHeapQueue[int]() - let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] - for item in data: - push(heap, item) - doAssert(heap[0] == 0) - doAssert(heap.toSortedSeq == @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - - block: # Test del - var heap = initHeapQueue[int]() - let data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] - for item in data: push(heap, item) - - heap.del(0) - doAssert(heap[0] == 1) - - heap.del(heap.data.find(7)) - doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 5, 6, 8, 9]) - - heap.del(heap.data.find(5)) - doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 6, 8, 9]) - - heap.del(heap.data.find(6)) - doAssert(heap.toSortedSeq == @[1, 2, 3, 4, 8, 9]) - - heap.del(heap.data.find(2)) - doAssert(heap.toSortedSeq == @[1, 3, 4, 8, 9]) - - block: # Test del last - var heap = initHeapQueue[int]() - let data = [1, 2, 3] - for item in data: push(heap, item) - - heap.del(2) - doAssert(heap.toSortedSeq == @[1, 2]) - - heap.del(1) - doAssert(heap.toSortedSeq == @[1]) - - heap.del(0) - doAssert(heap.toSortedSeq == @[]) diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim index 8338ceaca..765a23e97 100644 --- a/lib/pure/collections/intsets.nim +++ b/lib/pure/collections/intsets.nim @@ -7,666 +7,17 @@ # distribution, for details about the copyright. # -## The ``intsets`` module implements an efficient `int` set implemented as a -## `sparse bit set`:idx:. -## -## **Note**: Currently the assignment operator ``=`` for ``IntSet`` -## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use `assign proc -## <#assign,IntSet,IntSet>`_ to get a deep copy. -## -## **See also:** -## * `sets module <sets.html>`_ for more general hash sets +## Specialization of the generic `packedsets module <packedsets.html>`_ +## (see its documentation for more examples) for ordinal sparse sets. - -import - hashes - -type - BitScalar = uint - -const - InitIntSetSize = 8 # must be a power of two! - TrunkShift = 9 - BitsPerTrunk = 1 shl TrunkShift # needs to be a power of 2 and - # divisible by 64 - TrunkMask = BitsPerTrunk - 1 - IntsPerTrunk = BitsPerTrunk div (sizeof(BitScalar) * 8) - IntShift = 5 + ord(sizeof(BitScalar) == 8) # 5 or 6, depending on int width - IntMask = 1 shl IntShift - 1 +import std/private/since +import std/packedsets +export packedsets type - PTrunk = ref Trunk - Trunk = object - next: PTrunk # all nodes are connected with this pointer - key: int # start address at bit 0 - bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector - - TrunkSeq = seq[PTrunk] - IntSet* = object ## An efficient set of `int` implemented as a sparse bit set. - elems: int # only valid for small numbers - counter, max: int - head: PTrunk - data: TrunkSeq - a: array[0..33, int] # profiling shows that 34 elements are enough - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash - -proc intSetGet(t: IntSet, key: int): PTrunk = - var h = key and t.max - while t.data[h] != nil: - if t.data[h].key == key: - return t.data[h] - h = nextTry(h, t.max) - result = nil - -proc intSetRawInsert(t: IntSet, data: var TrunkSeq, desc: PTrunk) = - var h = desc.key and t.max - while data[h] != nil: - assert(data[h] != desc) - h = nextTry(h, t.max) - assert(data[h] == nil) - data[h] = desc - -proc intSetEnlarge(t: var IntSet) = - var n: TrunkSeq - var oldMax = t.max - t.max = ((t.max + 1) * 2) - 1 - newSeq(n, t.max + 1) - for i in countup(0, oldMax): - if t.data[i] != nil: intSetRawInsert(t, n, t.data[i]) - swap(t.data, n) - -proc intSetPut(t: var IntSet, key: int): PTrunk = - var h = key and t.max - while t.data[h] != nil: - if t.data[h].key == key: - return t.data[h] - h = nextTry(h, t.max) - if mustRehash(t.max + 1, t.counter): intSetEnlarge(t) - inc(t.counter) - h = key and t.max - while t.data[h] != nil: h = nextTry(h, t.max) - assert(t.data[h] == nil) - new(result) - result.next = t.head - result.key = key - t.head = result - t.data[h] = result - -proc bitincl(s: var IntSet, key: int) {.inline.} = - var t = intSetPut(s, `shr`(key, TrunkShift)) - var u = key and TrunkMask - t.bits[u shr IntShift] = t.bits[u shr IntShift] or - (BitScalar(1) shl (u and IntMask)) - -proc exclImpl(s: var IntSet, key: int) = - if s.elems <= s.a.len: - for i in 0..<s.elems: - if s.a[i] == key: - s.a[i] = s.a[s.elems-1] - dec s.elems - return - else: - var t = intSetGet(s, key shr TrunkShift) - if t != nil: - var u = key and TrunkMask - t.bits[u shr IntShift] = t.bits[u shr IntShift] and - not(BitScalar(1) shl (u and IntMask)) - -template dollarImpl(): untyped = - result = "{" - for key in items(s): - if result.len > 1: result.add(", ") - result.add($key) - result.add("}") - - -iterator items*(s: IntSet): int {.inline.} = - ## Iterates over any included element of `s`. - if s.elems <= s.a.len: - for i in 0..<s.elems: - yield s.a[i] - else: - var r = s.head - while r != nil: - var i = 0 - while i <= high(r.bits): - var w: uint = r.bits[i] - # taking a copy of r.bits[i] here is correct, because - # modifying operations are not allowed during traversation - var j = 0 - while w != 0: # test all remaining bits for zero - if (w and 1) != 0: # the bit is set! - yield (r.key shl TrunkShift) or (i shl IntShift +% j) - inc(j) - w = w shr 1 - inc(i) - r = r.next - - -proc initIntSet*: IntSet = - ## Returns an empty IntSet. - runnableExamples: - var a = initIntSet() - assert len(a) == 0 - - # newSeq(result.data, InitIntSetSize) - # result.max = InitIntSetSize-1 - result = IntSet( - elems: 0, - counter: 0, - max: 0, - head: nil, - data: when defined(nimNoNilSeqs): @[] else: nil) - # a: array[0..33, int] # profiling shows that 34 elements are enough - -proc contains*(s: IntSet, key: int): bool = - ## Returns true if `key` is in `s`. - ## - ## This allows the usage of `in` operator. - runnableExamples: - var a = initIntSet() - for x in [1, 3, 5]: - a.incl(x) - assert a.contains(3) - assert 3 in a - assert(not a.contains(8)) - assert 8 notin a - - if s.elems <= s.a.len: - for i in 0..<s.elems: - if s.a[i] == key: return true - else: - var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: - var u = key and TrunkMask - result = (t.bits[u shr IntShift] and - (BitScalar(1) shl (u and IntMask))) != 0 - else: - result = false - -proc incl*(s: var IntSet, key: int) = - ## Includes an element `key` in `s`. - ## - ## This doesn't do anything if `key` is already in `s`. - ## - ## See also: - ## * `excl proc <#excl,IntSet,int>`_ for excluding an element - ## * `incl proc <#incl,IntSet,IntSet>`_ for including other set - ## * `containsOrIncl proc <#containsOrIncl,IntSet,int>`_ - runnableExamples: - var a = initIntSet() - a.incl(3) - a.incl(3) - assert len(a) == 1 - - if s.elems <= s.a.len: - for i in 0..<s.elems: - if s.a[i] == key: return - if s.elems < s.a.len: - s.a[s.elems] = key - inc s.elems - return - newSeq(s.data, InitIntSetSize) - s.max = InitIntSetSize-1 - for i in 0..<s.elems: - bitincl(s, s.a[i]) - s.elems = s.a.len + 1 - # fall through: - bitincl(s, key) - -proc incl*(s: var IntSet, other: IntSet) = - ## Includes all elements from `other` into `s`. - ## - ## This is the in-place version of `s + other <#+,IntSet,IntSet>`_. - ## - ## See also: - ## * `excl proc <#excl,IntSet,IntSet>`_ for excluding other set - ## * `incl proc <#incl,IntSet,int>`_ for including an element - ## * `containsOrIncl proc <#containsOrIncl,IntSet,int>`_ - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1) - b.incl(5) - a.incl(b) - assert len(a) == 2 - assert 5 in a - - for item in other: incl(s, item) - -proc containsOrIncl*(s: var IntSet, key: int): bool = - ## Includes `key` in the set `s` and tells if `key` was already in `s`. - ## - ## The difference with regards to the `incl proc <#incl,IntSet,int>`_ is - ## that this proc returns `true` if `s` already contained `key`. The - ## proc will return `false` if `key` was added as a new value to `s` during - ## this call. - ## - ## See also: - ## * `incl proc <#incl,IntSet,int>`_ for including an element - ## * `missingOrExcl proc <#missingOrExcl,IntSet,int>`_ - runnableExamples: - var a = initIntSet() - assert a.containsOrIncl(3) == false - assert a.containsOrIncl(3) == true - assert a.containsOrIncl(4) == false - - if s.elems <= s.a.len: - for i in 0..<s.elems: - if s.a[i] == key: - return true - incl(s, key) - result = false - else: - var t = intSetGet(s, `shr`(key, TrunkShift)) - if t != nil: - var u = key and TrunkMask - result = (t.bits[u shr IntShift] and BitScalar(1) shl (u and IntMask)) != 0 - if not result: - t.bits[u shr IntShift] = t.bits[u shr IntShift] or - (BitScalar(1) shl (u and IntMask)) - else: - incl(s, key) - result = false - -proc excl*(s: var IntSet, key: int) = - ## Excludes `key` from the set `s`. - ## - ## This doesn't do anything if `key` is not found in `s`. - ## - ## See also: - ## * `incl proc <#incl,IntSet,int>`_ for including an element - ## * `excl proc <#excl,IntSet,IntSet>`_ for excluding other set - ## * `missingOrExcl proc <#missingOrExcl,IntSet,int>`_ - runnableExamples: - var a = initIntSet() - a.incl(3) - a.excl(3) - a.excl(3) - a.excl(99) - assert len(a) == 0 - exclImpl(s, key) - -proc excl*(s: var IntSet, other: IntSet) = - ## Excludes all elements from `other` from `s`. - ## - ## This is the in-place version of `s - other <#-,IntSet,IntSet>`_. - ## - ## See also: - ## * `incl proc <#incl,IntSet,IntSet>`_ for including other set - ## * `excl proc <#excl,IntSet,int>`_ for excluding an element - ## * `missingOrExcl proc <#missingOrExcl,IntSet,int>`_ - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1) - a.incl(5) - b.incl(5) - a.excl(b) - assert len(a) == 1 - assert 5 notin a - - for item in other: excl(s, item) - -proc missingOrExcl*(s: var IntSet, key: int): bool = - ## Excludes `key` in the set `s` and tells if `key` was already missing from `s`. - ## - ## The difference with regards to the `excl proc <#excl,IntSet,int>`_ is - ## that this proc returns `true` if `key` was missing from `s`. - ## The proc will return `false` if `key` was in `s` and it was removed - ## during this call. - ## - ## See also: - ## * `excl proc <#excl,IntSet,int>`_ for excluding an element - ## * `excl proc <#excl,IntSet,IntSet>`_ for excluding other set - ## * `containsOrIncl proc <#containsOrIncl,IntSet,int>`_ - runnableExamples: - var a = initIntSet() - a.incl(5) - assert a.missingOrExcl(5) == false - assert a.missingOrExcl(5) == true - - var count = s.elems - exclImpl(s, key) - result = count == s.elems - -proc clear*(result: var IntSet) = - ## Clears the IntSet back to an empty state. - runnableExamples: - var a = initIntSet() - a.incl(5) - a.incl(7) - clear(a) - assert len(a) == 0 - - # setLen(result.data, InitIntSetSize) - # for i in 0..InitIntSetSize-1: result.data[i] = nil - # result.max = InitIntSetSize-1 - when defined(nimNoNilSeqs): - result.data = @[] - else: - result.data = nil - result.max = 0 - result.counter = 0 - result.head = nil - result.elems = 0 - -proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil and x.elems == 0 - -proc assign*(dest: var IntSet, src: IntSet) = - ## Copies `src` to `dest`. - ## `dest` does not need to be initialized by `initIntSet proc <#initIntSet>`_. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - b.incl(5) - b.incl(7) - a.assign(b) - assert len(a) == 2 - - if src.elems <= src.a.len: - when defined(nimNoNilSeqs): - dest.data = @[] - else: - dest.data = nil - dest.max = 0 - dest.counter = src.counter - dest.head = nil - dest.elems = src.elems - dest.a = src.a - else: - dest.counter = src.counter - dest.max = src.max - dest.elems = src.elems - newSeq(dest.data, src.data.len) - - var it = src.head - while it != nil: - var h = it.key and dest.max - while dest.data[h] != nil: h = nextTry(h, dest.max) - assert(dest.data[h] == nil) - var n: PTrunk - new(n) - n.next = dest.head - n.key = it.key - n.bits = it.bits - dest.head = n - dest.data[h] = n - it = it.next - -proc union*(s1, s2: IntSet): IntSet = - ## Returns the union of the sets `s1` and `s2`. - ## - ## The same as `s1 + s2 <#+,IntSet,IntSet>`_. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1); a.incl(2); a.incl(3) - b.incl(3); b.incl(4); b.incl(5) - assert union(a, b).len == 5 - ## {1, 2, 3, 4, 5} - - result.assign(s1) - incl(result, s2) - -proc intersection*(s1, s2: IntSet): IntSet = - ## Returns the intersection of the sets `s1` and `s2`. - ## - ## The same as `s1 * s2 <#*,IntSet,IntSet>`_. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1); a.incl(2); a.incl(3) - b.incl(3); b.incl(4); b.incl(5) - assert intersection(a, b).len == 1 - ## {3} - - result = initIntSet() - for item in s1: - if contains(s2, item): - incl(result, item) - -proc difference*(s1, s2: IntSet): IntSet = - ## Returns the difference of the sets `s1` and `s2`. - ## - ## The same as `s1 - s2 <#-,IntSet,IntSet>`_. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1); a.incl(2); a.incl(3) - b.incl(3); b.incl(4); b.incl(5) - assert difference(a, b).len == 2 - ## {1, 2} - - result = initIntSet() - for item in s1: - if not contains(s2, item): - incl(result, item) - -proc symmetricDifference*(s1, s2: IntSet): IntSet = - ## Returns the symmetric difference of the sets `s1` and `s2`. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1); a.incl(2); a.incl(3) - b.incl(3); b.incl(4); b.incl(5) - assert symmetricDifference(a, b).len == 4 - ## {1, 2, 4, 5} - - result.assign(s1) - for item in s2: - if containsOrIncl(result, item): excl(result, item) - -proc `+`*(s1, s2: IntSet): IntSet {.inline.} = - ## Alias for `union(s1, s2) <#union,IntSet,IntSet>`_. - result = union(s1, s2) - -proc `*`*(s1, s2: IntSet): IntSet {.inline.} = - ## Alias for `intersection(s1, s2) <#intersection,IntSet,IntSet>`_. - result = intersection(s1, s2) - -proc `-`*(s1, s2: IntSet): IntSet {.inline.} = - ## Alias for `difference(s1, s2) <#difference,IntSet,IntSet>`_. - result = difference(s1, s2) - -proc disjoint*(s1, s2: IntSet): bool = - ## Returns true if the sets `s1` and `s2` have no items in common. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1); a.incl(2) - b.incl(2); b.incl(3) - assert disjoint(a, b) == false - b.excl(2) - assert disjoint(a, b) == true - - for item in s1: - if contains(s2, item): - return false - return true - -proc len*(s: IntSet): int {.inline.} = - ## Returns the number of elements in `s`. - if s.elems < s.a.len: - result = s.elems - else: - result = 0 - for _ in s: - inc(result) - -proc card*(s: IntSet): int {.inline.} = - ## Alias for `len() <#len,IntSet>`_. - result = s.len() - -proc `<=`*(s1, s2: IntSet): bool = - ## Returns true if `s1` is subset of `s2`. - ## - ## A subset `s1` has all of its elements in `s2`, and `s2` doesn't necessarily - ## have more elements than `s1`. That is, `s1` can be equal to `s2`. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1) - b.incl(1); b.incl(2) - assert a <= b - a.incl(2) - assert a <= b - a.incl(3) - assert(not (a <= b)) - - for item in s1: - if not s2.contains(item): - return false - return true - -proc `<`*(s1, s2: IntSet): bool = - ## Returns true if `s1` is proper subset of `s2`. - ## - ## A strict or proper subset `s1` has all of its elements in `s2`, but `s2` has - ## more elements than `s1`. - runnableExamples: - var - a = initIntSet() - b = initIntSet() - a.incl(1) - b.incl(1); b.incl(2) - assert a < b - a.incl(2) - assert(not (a < b)) - return s1 <= s2 and not (s2 <= s1) - -proc `==`*(s1, s2: IntSet): bool = - ## Returns true if both `s1` and `s2` have the same elements and set size. - return s1 <= s2 and s2 <= s1 - -proc `$`*(s: IntSet): string = - ## The `$` operator for int sets. - ## - ## Converts the set `s` to a string, mostly for logging and printing purposes. - dollarImpl() - - - -when isMainModule: - import sequtils, algorithm - - var x = initIntSet() - x.incl(1) - x.incl(2) - x.incl(7) - x.incl(1056) - - x.incl(1044) - x.excl(1044) - - assert x.containsOrIncl(888) == false - assert 888 in x - assert x.containsOrIncl(888) == true - - assert x.missingOrExcl(888) == false - assert 888 notin x - assert x.missingOrExcl(888) == true - - var xs = toSeq(items(x)) - xs.sort(cmp[int]) - assert xs == @[1, 2, 7, 1056] - - var y: IntSet - assign(y, x) - var ys = toSeq(items(y)) - ys.sort(cmp[int]) - assert ys == @[1, 2, 7, 1056] - - assert x == y - - var z: IntSet - for i in 0..1000: - incl z, i - assert z.len() == i+1 - for i in 0..1000: - assert z.contains(i) - - var w = initIntSet() - w.incl(1) - w.incl(4) - w.incl(50) - w.incl(1001) - w.incl(1056) - - var xuw = x.union(w) - var xuws = toSeq(items(xuw)) - xuws.sort(cmp[int]) - assert xuws == @[1, 2, 4, 7, 50, 1001, 1056] - - var xiw = x.intersection(w) - var xiws = toSeq(items(xiw)) - xiws.sort(cmp[int]) - assert xiws == @[1, 1056] - - var xdw = x.difference(w) - var xdws = toSeq(items(xdw)) - xdws.sort(cmp[int]) - assert xdws == @[2, 7] - - var xsw = x.symmetricDifference(w) - var xsws = toSeq(items(xsw)) - xsws.sort(cmp[int]) - assert xsws == @[2, 4, 7, 50, 1001] - - x.incl(w) - xs = toSeq(items(x)) - xs.sort(cmp[int]) - assert xs == @[1, 2, 4, 7, 50, 1001, 1056] - - assert w <= x - - assert w < x - - assert(not disjoint(w, x)) - - var u = initIntSet() - u.incl(3) - u.incl(5) - u.incl(500) - assert disjoint(u, x) - - var v = initIntSet() - v.incl(2) - v.incl(50) - - x.excl(v) - xs = toSeq(items(x)) - xs.sort(cmp[int]) - assert xs == @[1, 4, 7, 1001, 1056] - - proc bug12366 = - var - x = initIntSet() - y = initIntSet() - n = 3584 + IntSet* = PackedSet[int] - for i in 0..n: - x.incl(i) - y.incl(i) +proc toIntSet*(x: openArray[int]): IntSet {.since: (1, 3), inline.} = toPackedSet[int](x) - let z = symmetricDifference(x, y) - doAssert z.len == 0 - doAssert $z == "{}" +proc initIntSet*(): IntSet {.inline.} = initPackedSet[int]() - bug12366() diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index bd22949bf..6b88747ef 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -13,119 +13,90 @@ ## * `singly linked rings <#SinglyLinkedRing>`_ (circular lists) ## * `doubly linked rings <#DoublyLinkedRing>`_ (circular lists) ## -## -## Basic Usage -## =========== -## +## # Basic Usage ## Because it makes no sense to do otherwise, the `next` and `prev` pointers ## are not hidden from you and can be manipulated directly for efficiency. ## -## Lists -## ----- -## -## .. code-block:: -## import lists -## -## var -## l = initDoublyLinkedList[int]() -## a = newDoublyLinkedNode[int](3) -## b = newDoublyLinkedNode[int](7) -## c = newDoublyLinkedNode[int](9) -## -## l.append(a) -## l.append(b) -## l.prepend(c) -## -## assert a.next == b -## assert a.prev == c -## assert c.next == a -## assert c.next.next == b -## assert c.prev == nil -## assert b.next == nil -## -## -## Rings -## ----- -## -## .. code-block:: -## import lists -## -## var -## l = initSinglyLinkedRing[int]() -## a = newSinglyLinkedNode[int](3) -## b = newSinglyLinkedNode[int](7) -## c = newSinglyLinkedNode[int](9) -## -## l.append(a) -## l.append(b) -## l.prepend(c) -## -## assert c.next == a -## assert a.next == b -## assert c.next.next == b -## assert b.next == c -## assert c.next.next.next == c -## -## See also -## ======== -## +## ## Lists +runnableExamples: + var list = initDoublyLinkedList[int]() + let + a = newDoublyLinkedNode[int](3) + b = newDoublyLinkedNode[int](7) + c = newDoublyLinkedNode[int](9) + + list.add(a) + list.add(b) + list.prepend(c) + + assert a.next == b + assert a.prev == c + assert c.next == a + assert c.next.next == b + assert c.prev == nil + assert b.next == nil + +## ## Rings +runnableExamples: + var ring = initSinglyLinkedRing[int]() + let + a = newSinglyLinkedNode[int](3) + b = newSinglyLinkedNode[int](7) + c = newSinglyLinkedNode[int](9) + + ring.add(a) + ring.add(b) + ring.prepend(c) + + assert c.next == a + assert a.next == b + assert c.next.next == b + assert b.next == c + assert c.next.next.next == c + +## # See also ## * `deques module <deques.html>`_ for double-ended queues -## * `sharedlist module <sharedlist.html>`_ for shared singly-linked lists - -when not defined(nimhygiene): - {.pragma: dirty.} +import std/private/since -when not defined(nimHasCursor): - {.pragma: cursor.} +when defined(nimPreviewSlimSystem): + import std/assertions type - DoublyLinkedNodeObj*[T] = object ## \ - ## A node a doubly linked list consists of. + DoublyLinkedNodeObj*[T] = object + ## A node of a doubly linked list. ## ## It consists of a `value` field, and pointers to `next` and `prev`. - next*: <//>(ref DoublyLinkedNodeObj[T]) - prev* {.cursor.}: ref DoublyLinkedNodeObj[T] + next*: DoublyLinkedNode[T] + prev* {.cursor.}: DoublyLinkedNode[T] value*: T DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T] - SinglyLinkedNodeObj*[T] = object ## \ - ## A node a singly linked list consists of. + SinglyLinkedNodeObj*[T] = object + ## A node of a singly linked list. ## ## It consists of a `value` field, and a pointer to `next`. - next*: <//>(ref SinglyLinkedNodeObj[T]) + next*: SinglyLinkedNode[T] value*: T SinglyLinkedNode*[T] = ref SinglyLinkedNodeObj[T] - SinglyLinkedList*[T] = object ## \ + SinglyLinkedList*[T] = object ## A singly linked list. - ## - ## Use `initSinglyLinkedList proc <#initSinglyLinkedList>`_ to create - ## a new empty list. - head*: <//>(SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] - DoublyLinkedList*[T] = object ## \ + DoublyLinkedList*[T] = object ## A doubly linked list. - ## - ## Use `initDoublyLinkedList proc <#initDoublyLinkedList>`_ to create - ## a new empty list. - head*: <//>(DoublyLinkedNode[T]) + head*: DoublyLinkedNode[T] tail* {.cursor.}: DoublyLinkedNode[T] - SinglyLinkedRing*[T] = object ## \ + SinglyLinkedRing*[T] = object ## A singly linked ring. - ## - ## Use `initSinglyLinkedRing proc <#initSinglyLinkedRing>`_ to create - ## a new empty ring. - head*: <//>(SinglyLinkedNode[T]) + head*: SinglyLinkedNode[T] tail* {.cursor.}: SinglyLinkedNode[T] - DoublyLinkedRing*[T] = object ## \ + DoublyLinkedRing*[T] = object ## A doubly linked ring. - ## - ## Use `initDoublyLinkedRing proc <#initDoublyLinkedRing>`_ to create - ## a new empty ring. head*: DoublyLinkedNode[T] SomeLinkedList*[T] = SinglyLinkedList[T] | DoublyLinkedList[T] @@ -138,54 +109,70 @@ type proc initSinglyLinkedList*[T](): SinglyLinkedList[T] = ## Creates a new singly linked list that is empty. + ## + ## Singly linked lists are initialized by default, so it is not necessary to + ## call this function explicitly. runnableExamples: - var a = initSinglyLinkedList[int]() + let a = initSinglyLinkedList[int]() + discard proc initDoublyLinkedList*[T](): DoublyLinkedList[T] = ## Creates a new doubly linked list that is empty. + ## + ## Doubly linked lists are initialized by default, so it is not necessary to + ## call this function explicitly. runnableExamples: - var a = initDoublyLinkedList[int]() + let a = initDoublyLinkedList[int]() + discard proc initSinglyLinkedRing*[T](): SinglyLinkedRing[T] = ## Creates a new singly linked ring that is empty. + ## + ## Singly linked rings are initialized by default, so it is not necessary to + ## call this function explicitly. runnableExamples: - var a = initSinglyLinkedRing[int]() + let a = initSinglyLinkedRing[int]() + discard proc initDoublyLinkedRing*[T](): DoublyLinkedRing[T] = ## Creates a new doubly linked ring that is empty. + ## + ## Doubly linked rings are initialized by default, so it is not necessary to + ## call this function explicitly. runnableExamples: - var a = initDoublyLinkedRing[int]() + let a = initDoublyLinkedRing[int]() + discard -proc newDoublyLinkedNode*[T](value: T): <//>(DoublyLinkedNode[T]) = +proc newDoublyLinkedNode*[T](value: T): DoublyLinkedNode[T] = ## Creates a new doubly linked node with the given `value`. runnableExamples: - var n = newDoublyLinkedNode[int](5) + let n = newDoublyLinkedNode[int](5) assert n.value == 5 new(result) result.value = value -proc newSinglyLinkedNode*[T](value: T): <//>(SinglyLinkedNode[T]) = +proc newSinglyLinkedNode*[T](value: T): SinglyLinkedNode[T] = ## Creates a new singly linked node with the given `value`. runnableExamples: - var n = newSinglyLinkedNode[int](5) + let n = newSinglyLinkedNode[int](5) assert n.value == 5 new(result) result.value = value template itemsListImpl() {.dirty.} = - var it = L.head + var it {.cursor.} = L.head while it != nil: yield it.value it = it.next template itemsRingImpl() {.dirty.} = - var it = L.head + var it {.cursor.} = L.head if it != nil: while true: yield it.value @@ -195,101 +182,91 @@ template itemsRingImpl() {.dirty.} = iterator items*[T](L: SomeLinkedList[T]): T = ## Yields every value of `L`. ## - ## See also: + ## **See also:** ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ ## * `nodes iterator <#nodes.i,SomeLinkedList[T]>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## var a = initSinglyLinkedList[int]() - ## for i in 1 .. 3: - ## a.append(10*i) - ## - ## for x in a: # the same as: for x in items(a): - ## echo x - ## - ## # 10 - ## # 20 - ## # 30 + runnableExamples: + from std/sugar import collect + from std/sequtils import toSeq + let a = collect(initSinglyLinkedList): + for i in 1..3: 10 * i + assert toSeq(items(a)) == toSeq(a) + assert toSeq(a) == @[10, 20, 30] + itemsListImpl() iterator items*[T](L: SomeLinkedRing[T]): T = ## Yields every value of `L`. ## - ## See also: + ## **See also:** ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ ## * `nodes iterator <#nodes.i,SomeLinkedRing[T]>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## var a = initSinglyLinkedRing[int]() - ## for i in 1 .. 3: - ## a.append(10*i) - ## - ## for x in a: # the same as: for x in items(a): - ## echo x - ## - ## # 10 - ## # 20 - ## # 30 + runnableExamples: + from std/sugar import collect + from std/sequtils import toSeq + let a = collect(initSinglyLinkedRing): + for i in 1..3: 10 * i + assert toSeq(items(a)) == toSeq(a) + assert toSeq(a) == @[10, 20, 30] + itemsRingImpl() iterator mitems*[T](L: var SomeLinkedList[T]): var T = ## Yields every value of `L` so that you can modify it. ## - ## See also: + ## **See also:** ## * `items iterator <#items.i,SomeLinkedList[T]>`_ ## * `nodes iterator <#nodes.i,SomeLinkedList[T]>`_ runnableExamples: var a = initSinglyLinkedList[int]() - for i in 1 .. 5: - a.append(10*i) + for i in 1..5: + a.add(10 * i) assert $a == "[10, 20, 30, 40, 50]" for x in mitems(a): - x = 5*x - 1 + x = 5 * x - 1 assert $a == "[49, 99, 149, 199, 249]" + itemsListImpl() iterator mitems*[T](L: var SomeLinkedRing[T]): var T = ## Yields every value of `L` so that you can modify it. ## - ## See also: + ## **See also:** ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ ## * `nodes iterator <#nodes.i,SomeLinkedRing[T]>`_ runnableExamples: var a = initSinglyLinkedRing[int]() - for i in 1 .. 5: - a.append(10*i) + for i in 1..5: + a.add(10 * i) assert $a == "[10, 20, 30, 40, 50]" for x in mitems(a): - x = 5*x - 1 + x = 5 * x - 1 assert $a == "[49, 99, 149, 199, 249]" + itemsRingImpl() iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] = ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. ## - ## See also: + ## **See also:** ## * `items iterator <#items.i,SomeLinkedList[T]>`_ ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_ runnableExamples: var a = initDoublyLinkedList[int]() - for i in 1 .. 5: - a.append(10*i) + for i in 1..5: + a.add(10 * i) assert $a == "[10, 20, 30, 40, 50]" for x in nodes(a): if x.value == 30: a.remove(x) else: - x.value = 5*x.value - 1 + x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = L.head + var it {.cursor.} = L.head while it != nil: - var nxt = it.next + let nxt = it.next yield it it = nxt @@ -297,31 +274,35 @@ iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] = ## Iterates over every node of `x`. Removing the current node from the ## list during traversal is supported. ## - ## See also: + ## **See also:** ## * `items iterator <#items.i,SomeLinkedRing[T]>`_ ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_ runnableExamples: var a = initDoublyLinkedRing[int]() - for i in 1 .. 5: - a.append(10*i) + for i in 1..5: + a.add(10 * i) assert $a == "[10, 20, 30, 40, 50]" for x in nodes(a): if x.value == 30: a.remove(x) else: - x.value = 5*x.value - 1 + x.value = 5 * x.value - 1 assert $a == "[49, 99, 199, 249]" - var it = L.head + var it {.cursor.} = L.head if it != nil: while true: - var nxt = it.next + let nxt = it.next yield it it = nxt if it == L.head: break proc `$`*[T](L: SomeLinkedCollection[T]): string = ## Turns a list into its string representation for logging and printing. + runnableExamples: + let a = [1, 2, 3, 4].toSinglyLinkedList + assert $a == "[1, 2, 3, 4]" + result = "[" for x in nodes(L): if result.len > 1: result.add(", ") @@ -332,12 +313,10 @@ proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = ## Searches in the list for a value. Returns `nil` if the value does not ## exist. ## - ## See also: + ## **See also:** ## * `contains proc <#contains,SomeLinkedCollection[T],T>`_ runnableExamples: - var a = initSinglyLinkedList[int]() - a.append(9) - a.append(8) + let a = [9, 8].toSinglyLinkedList assert a.find(9).value == 9 assert a.find(1) == nil @@ -346,14 +325,13 @@ proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] = proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = ## Searches in the list for a value. Returns `false` if the value does not - ## exist, `true` otherwise. + ## exist, `true` otherwise. This allows the usage of the `in` and `notin` + ## operators. ## - ## See also: + ## **See also:** ## * `find proc <#find,SomeLinkedCollection[T],T>`_ runnableExamples: - var a = initSinglyLinkedList[int]() - a.append(9) - a.append(8) + let a = [9, 8].toSinglyLinkedList assert a.contains(9) assert 8 in a assert(not a.contains(1)) @@ -361,20 +339,65 @@ proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} = result = find(L, value) != nil -proc append*[T](L: var SinglyLinkedList[T], - n: SinglyLinkedNode[T]) {.inline.} = +proc prepend*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = + ## Prepends a shallow copy of `b` to the beginning of `a`. + ## + ## **See also:** + ## * `prependMoved proc <#prependMoved,T,T>`_ + ## for moving the second list instead of copying + runnableExamples: + from std/sequtils import toSeq + var a = [4, 5].toSinglyLinkedList + let b = [1, 2, 3].toSinglyLinkedList + a.prepend(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [1, 2, 3] + a.prepend(a) + assert a.toSeq == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] + + var tmp = b.copy + tmp.addMoved(a) + a = tmp + +proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = + ## Moves `b` before the head of `a`. Efficiency: O(1). + ## Note that `b` becomes empty after the operation unless it has the same address as `a`. + ## Self-prepending results in a cycle. + ## + ## **See also:** + ## * `prepend proc <#prepend,T,T>`_ + ## for prepending a copy of a list + runnableExamples: + import std/[sequtils, enumerate, sugar] + var + a = [4, 5].toSinglyLinkedList + b = [1, 2, 3].toSinglyLinkedList + c = [0, 1].toSinglyLinkedList + a.prependMoved(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [] + c.prependMoved(c) + let s = collect: + for i, ci in enumerate(c): + if i == 6: break + ci + assert s == [0, 1, 0, 1, 0, 1] + + b.addMoved(a) + swap a, b + +proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ ## for prepending a node ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value runnableExamples: - var - a = initSinglyLinkedList[int]() - n = newSinglyLinkedNode[int](9) - a.append(n) + var a = initSinglyLinkedList[int]() + let n = newSinglyLinkedNode[int](9) + a.add(n) assert a.contains(9) n.next = nil @@ -384,34 +407,34 @@ proc append*[T](L: var SinglyLinkedList[T], L.tail = n if L.head == nil: L.head = n -proc append*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = +proc add*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ ## for prepending a node ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value runnableExamples: var a = initSinglyLinkedList[int]() - a.append(9) - a.append(8) + a.add(9) + a.add(8) assert a.contains(9) - append(L, newSinglyLinkedNode(value)) + + add(L, newSinglyLinkedNode(value)) proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} = ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedList[T],T>`_ for prepending a value runnableExamples: - var - a = initSinglyLinkedList[int]() - n = newSinglyLinkedNode[int](9) + var a = initSinglyLinkedList[int]() + let n = newSinglyLinkedNode[int](9) a.prepend(n) assert a.contains(9) @@ -422,10 +445,10 @@ proc prepend*[T](L: var SinglyLinkedList[T], proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,SinglyLinkedList[T],T>`_ for appending a value + ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ ## for prepending a node runnableExamples: @@ -433,25 +456,80 @@ proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} = a.prepend(9) a.prepend(8) assert a.contains(9) - prepend(L, newSinglyLinkedNode(value)) + prepend(L, newSinglyLinkedNode(value)) +func copy*[T](a: SinglyLinkedList[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a shallow copy of `a`. + runnableExamples: + from std/sequtils import toSeq + type Foo = ref object + x: int + var + f = Foo(x: 1) + a = [f].toSinglyLinkedList + let b = a.copy + a.add([f].toSinglyLinkedList) + assert a.toSeq == [f, f] + assert b.toSeq == [f] # b isn't modified... + f.x = 42 + assert a.head.value.x == 42 + assert b.head.value.x == 42 # ... but the elements are not deep copied + + let c = [1, 2, 3].toSinglyLinkedList + assert $c == $c.copy + + result = initSinglyLinkedList[T]() + for x in a.items: + result.add(x) + +proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} = + ## Moves `b` to the end of `a`. Efficiency: O(1). + ## Note that `b` becomes empty after the operation unless it has the same address as `a`. + ## Self-adding results in a cycle. + ## + ## **See also:** + ## * `add proc <#add,T,T>`_ for adding a copy of a list + runnableExamples: + import std/[sequtils, enumerate, sugar] + var + a = [1, 2, 3].toSinglyLinkedList + b = [4, 5].toSinglyLinkedList + c = [0, 1].toSinglyLinkedList + a.addMoved(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [] + c.addMoved(c) + let s = collect: + for i, ci in enumerate(c): + if i == 6: break + ci + assert s == [0, 1, 0, 1, 0, 1] + + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + a.tail.next = b.head + a.tail = b.tail + if a.addr != b.addr: + b.head = nil + b.tail = nil -proc append*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = +proc add*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedList[T],T>`_ for appending a value + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for prepending a node ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for removing a node runnableExamples: - var - a = initDoublyLinkedList[int]() - n = newDoublyLinkedNode[int](9) - a.append(n) + var a = initDoublyLinkedList[int]() + let n = newDoublyLinkedNode[int](9) + a.add(n) assert a.contains(9) n.next = nil @@ -462,11 +540,11 @@ proc append*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = L.tail = n if L.head == nil: L.head = n -proc append*[T](L: var DoublyLinkedList[T], value: T) = +proc add*[T](L: var DoublyLinkedList[T], value: T) = ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for appending a node ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for prepending a node @@ -475,25 +553,25 @@ proc append*[T](L: var DoublyLinkedList[T], value: T) = ## for removing a node runnableExamples: var a = initDoublyLinkedList[int]() - a.append(9) - a.append(8) + a.add(9) + a.add(8) assert a.contains(9) - append(L, newDoublyLinkedNode(value)) + + add(L, newDoublyLinkedNode(value)) proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,DoublyLinkedList[T],T>`_ for appending a value + ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,DoublyLinkedList[T],T>`_ for prepending a value ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for removing a node runnableExamples: - var - a = initDoublyLinkedList[int]() - n = newDoublyLinkedNode[int](9) + var a = initDoublyLinkedList[int]() + let n = newDoublyLinkedNode[int](9) a.prepend(n) assert a.contains(9) @@ -508,10 +586,10 @@ proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = proc prepend*[T](L: var DoublyLinkedList[T], value: T) = ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,DoublyLinkedList[T],T>`_ for appending a value + ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value ## * `prepend proc <#prepend,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ ## for prepending a node ## * `remove proc <#remove,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ @@ -521,18 +599,147 @@ proc prepend*[T](L: var DoublyLinkedList[T], value: T) = a.prepend(9) a.prepend(8) assert a.contains(9) + prepend(L, newDoublyLinkedNode(value)) +func copy*[T](a: DoublyLinkedList[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a shallow copy of `a`. + runnableExamples: + from std/sequtils import toSeq + type Foo = ref object + x: int + var + f = Foo(x: 1) + a = [f].toDoublyLinkedList + let b = a.copy + a.add([f].toDoublyLinkedList) + assert a.toSeq == [f, f] + assert b.toSeq == [f] # b isn't modified... + f.x = 42 + assert a.head.value.x == 42 + assert b.head.value.x == 42 # ... but the elements are not deep copied + + let c = [1, 2, 3].toDoublyLinkedList + assert $c == $c.copy + + result = initDoublyLinkedList[T]() + for x in a.items: + result.add(x) + +proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} = + ## Moves `b` to the end of `a`. Efficiency: O(1). + ## Note that `b` becomes empty after the operation unless it has the same address as `a`. + ## Self-adding results in a cycle. + ## + ## **See also:** + ## * `add proc <#add,T,T>`_ + ## for adding a copy of a list + runnableExamples: + import std/[sequtils, enumerate, sugar] + var + a = [1, 2, 3].toDoublyLinkedList + b = [4, 5].toDoublyLinkedList + c = [0, 1].toDoublyLinkedList + a.addMoved(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [] + c.addMoved(c) + let s = collect: + for i, ci in enumerate(c): + if i == 6: break + ci + assert s == [0, 1, 0, 1, 0, 1] + + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + b.head.prev = a.tail + a.tail.next = b.head + a.tail = b.tail + if a.addr != b.addr: + b.head = nil + b.tail = nil + +proc add*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} = + ## Appends a shallow copy of `b` to the end of `a`. + ## + ## **See also:** + ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ + ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ + ## for moving the second list instead of copying + runnableExamples: + from std/sequtils import toSeq + var a = [1, 2, 3].toSinglyLinkedList + let b = [4, 5].toSinglyLinkedList + a.add(b) + assert a.toSeq == [1, 2, 3, 4, 5] + assert b.toSeq == [4, 5] + a.add(a) + assert a.toSeq == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] + + var tmp = b.copy + a.addMoved(tmp) + +proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} = + ## Removes a node `n` from `L`. + ## Returns `true` if `n` was found in `L`. + ## Efficiency: O(n); the list is traversed until `n` is found. + ## Attempting to remove an element not contained in the list is a no-op. + ## When the list is cyclic, the cycle is preserved after removal. + runnableExamples: + import std/[sequtils, enumerate, sugar] + var a = [0, 1, 2].toSinglyLinkedList + let n = a.head.next + assert n.value == 1 + assert a.remove(n) == true + assert a.toSeq == [0, 2] + assert a.remove(n) == false + assert a.toSeq == [0, 2] + a.addMoved(a) # cycle: [0, 2, 0, 2, ...] + a.remove(a.head) + let s = collect: + for i, ai in enumerate(a): + if i == 4: break + ai + assert s == [2, 2, 2, 2] + + if n == L.head: + L.head = n.next + if L.tail.next == n: + L.tail.next = L.head # restore cycle + else: + var prev {.cursor.} = L.head + while prev.next != n and prev.next != nil: + prev = prev.next + if prev.next == nil: + return false + prev.next = n.next + if L.tail == n: + L.tail = prev # update tail if we removed the last node + true + proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = ## Removes a node `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, + ## otherwise the effects are undefined. + ## When the list is cyclic, the cycle is preserved after removal. runnableExamples: - var - a = initDoublyLinkedList[int]() - n = newDoublyLinkedNode[int](5) - a.append(n) - assert 5 in a + import std/[sequtils, enumerate, sugar] + var a = [0, 1, 2].toSinglyLinkedList + let n = a.head.next + assert n.value == 1 a.remove(n) - assert 5 notin a + assert a.toSeq == [0, 2] + a.remove(n) + assert a.toSeq == [0, 2] + a.addMoved(a) # cycle: [0, 2, 0, 2, ...] + a.remove(a.head) + let s = collect: + for i, ai in enumerate(a): + if i == 4: break + ai + assert s == [2, 2, 2, 2] if n == L.tail: L.tail = n.prev if n == L.head: L.head = n.next @@ -541,59 +748,57 @@ proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = -proc append*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = +proc add*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedRing[T],T>`_ for appending a value + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ ## for prepending a node ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value runnableExamples: - var - a = initSinglyLinkedRing[int]() - n = newSinglyLinkedNode[int](9) - a.append(n) + var a = initSinglyLinkedRing[int]() + let n = newSinglyLinkedNode[int](9) + a.add(n) assert a.contains(9) if L.head != nil: n.next = L.head assert(L.tail != nil) L.tail.next = n - L.tail = n else: n.next = n L.head = n - L.tail = n + L.tail = n -proc append*[T](L: var SinglyLinkedRing[T], value: T) = +proc add*[T](L: var SinglyLinkedRing[T], value: T) = ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ ## for appending a node ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ ## for prepending a node ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value runnableExamples: var a = initSinglyLinkedRing[int]() - a.append(9) - a.append(8) + a.add(9) + a.add(8) assert a.contains(9) - append(L, newSinglyLinkedNode(value)) + + add(L, newSinglyLinkedNode(value)) proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,SinglyLinkedRing[T],T>`_ for appending a value + ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedRing[T],T>`_ for prepending a value runnableExamples: - var - a = initSinglyLinkedRing[int]() - n = newSinglyLinkedNode[int](9) + var a = initSinglyLinkedRing[int]() + let n = newSinglyLinkedNode[int](9) a.prepend(n) assert a.contains(9) @@ -609,10 +814,10 @@ proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) = proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,SinglyLinkedRing[T],T>`_ for appending a value + ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value ## * `prepend proc <#prepend,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_ ## for prepending a node runnableExamples: @@ -620,25 +825,25 @@ proc prepend*[T](L: var SinglyLinkedRing[T], value: T) = a.prepend(9) a.prepend(8) assert a.contains(9) + prepend(L, newSinglyLinkedNode(value)) -proc append*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = +proc add*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedRing[T],T>`_ for appending a value + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for prepending a node ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for removing a node runnableExamples: - var - a = initDoublyLinkedRing[int]() - n = newDoublyLinkedNode[int](9) - a.append(n) + var a = initDoublyLinkedRing[int]() + let n = newDoublyLinkedNode[int](9) + a.add(n) assert a.contains(9) if L.head != nil: @@ -651,11 +856,11 @@ proc append*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = n.next = n L.head = n -proc append*[T](L: var DoublyLinkedRing[T], value: T) = +proc add*[T](L: var DoublyLinkedRing[T], value: T) = ## Appends (adds to the end) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for appending a node ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for prepending a node @@ -664,25 +869,25 @@ proc append*[T](L: var DoublyLinkedRing[T], value: T) = ## for removing a node runnableExamples: var a = initDoublyLinkedRing[int]() - a.append(9) - a.append(8) + a.add(9) + a.add(8) assert a.contains(9) - append(L, newDoublyLinkedNode(value)) + + add(L, newDoublyLinkedNode(value)) proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,DoublyLinkedRing[T],T>`_ for appending a value + ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value ## * `prepend proc <#prepend,DoublyLinkedRing[T],T>`_ for prepending a value ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for removing a node runnableExamples: - var - a = initDoublyLinkedRing[int]() - n = newDoublyLinkedNode[int](9) + var a = initDoublyLinkedRing[int]() + let n = newDoublyLinkedNode[int](9) a.prepend(n) assert a.contains(9) @@ -699,10 +904,10 @@ proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = proc prepend*[T](L: var DoublyLinkedRing[T], value: T) = ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1). ## - ## See also: - ## * `append proc <#append,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ + ## **See also:** + ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for appending a node - ## * `append proc <#append,DoublyLinkedRing[T],T>`_ for appending a value + ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value ## * `prepend proc <#prepend,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ ## for prepending a node ## * `remove proc <#remove,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_ @@ -712,15 +917,17 @@ proc prepend*[T](L: var DoublyLinkedRing[T], value: T) = a.prepend(9) a.prepend(8) assert a.contains(9) + prepend(L, newDoublyLinkedNode(value)) proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = ## Removes `n` from `L`. Efficiency: O(1). + ## This function assumes, for the sake of efficiency, that `n` is contained in `L`, + ## otherwise the effects are undefined. runnableExamples: - var - a = initDoublyLinkedRing[int]() - n = newDoublyLinkedNode[int](5) - a.append(n) + var a = initDoublyLinkedRing[int]() + let n = newDoublyLinkedNode[int](5) + a.add(n) assert 5 in a a.remove(n) assert 5 notin a @@ -728,9 +935,81 @@ proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) = n.next.prev = n.prev n.prev.next = n.next if n == L.head: - var p = L.head.prev + let p = L.head.prev if p == L.head: # only one element left: L.head = nil else: - L.head = L.head.prev + L.head = p + +proc append*[T](a: var (SinglyLinkedList[T] | SinglyLinkedRing[T]), + b: SinglyLinkedList[T] | SinglyLinkedNode[T] | T) = + ## Alias for `a.add(b)`. + ## + ## **See also:** + ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_ + ## * `add proc <#add,SinglyLinkedList[T],T>`_ + ## * `add proc <#add,T,T>`_ + a.add(b) + +proc append*[T](a: var (DoublyLinkedList[T] | DoublyLinkedRing[T]), + b: DoublyLinkedList[T] | DoublyLinkedNode[T] | T) = + ## Alias for `a.add(b)`. + ## + ## **See also:** + ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_ + ## * `add proc <#add,DoublyLinkedList[T],T>`_ + ## * `add proc <#add,T,T>`_ + a.add(b) + +proc appendMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} = + ## Alias for `a.addMoved(b)`. + ## + ## **See also:** + ## * `addMoved proc <#addMoved,SinglyLinkedList[T],SinglyLinkedList[T]>`_ + ## * `addMoved proc <#addMoved,DoublyLinkedList[T],DoublyLinkedList[T]>`_ + a.addMoved(b) + +func toSinglyLinkedList*[T](elems: openArray[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `SinglyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toSinglyLinkedRing*[T](elems: openArray[T]): SinglyLinkedRing[T] = + ## Creates a new `SinglyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toSinglyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initSinglyLinkedRing[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} = + ## Creates a new `DoublyLinkedList` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedList + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedList[T]() + for elem in elems.items: + result.add(elem) + +func toDoublyLinkedRing*[T](elems: openArray[T]): DoublyLinkedRing[T] = + ## Creates a new `DoublyLinkedRing` from the members of `elems`. + runnableExamples: + from std/sequtils import toSeq + let a = [1, 2, 3, 4, 5].toDoublyLinkedRing + assert a.toSeq == [1, 2, 3, 4, 5] + + result = initDoublyLinkedRing[T]() + for elem in elems.items: + result.add(elem) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index d56742ba9..3c0d8dc0e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -7,19 +7,19 @@ # distribution, for details about the copyright. # -## Although this module has ``seq`` in its name, it implements operations -## not only for `seq`:idx: type, but for three built-in container types under -## the ``openArray`` umbrella: +## Although this module has `seq` in its name, it implements operations +## not only for the `seq`:idx: type, but for three built-in container types +## under the `openArray` umbrella: ## * sequences ## * strings ## * array ## -## The system module defines several common functions, such as: -## * ``newSeq[T]`` for creating new sequences of type ``T`` -## * ``@`` for converting arrays and strings to sequences -## * ``add`` for adding new elements to strings and sequences -## * ``&`` for string and seq concatenation -## * ``in`` (alias for ``contains``) and ``notin`` for checking if an item is +## The `system` module defines several common functions, such as: +## * `newSeq[T]` for creating new sequences of type `T` +## * `@` for converting arrays and strings to sequences +## * `add` for adding new elements to strings and sequences +## * `&` for string and seq concatenation +## * `in` (alias for `contains`) and `notin` for checking if an item is ## in a container ## ## This module builds upon that, providing additional functionality in form of @@ -27,45 +27,52 @@ ## languages. ## ## For functional style programming you have different options at your disposal: -## * pass `anonymous proc<manual.html#procedures-anonymous-procs>`_ -## * import `sugar module<sugar.html>`_ and use -## `=> macro<sugar.html#%3D>.m,untyped,untyped>`_ +## * the `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ +## * pass an `anonymous proc<manual.html#procedures-anonymous-procs>`_ +## * import the `sugar module<sugar.html>`_ and use +## the `=> macro<sugar.html#%3D>.m,untyped,untyped>`_ ## * use `...It templates<#18>`_ ## (`mapIt<#mapIt.t,typed,untyped>`_, ## `filterIt<#filterIt.t,untyped,untyped>`_, etc.) ## -## The chaining of functions is possible thanks to the +## Chaining of functions is possible thanks to the ## `method call syntax<manual.html#procedures-method-call-syntax>`_. -## -## .. code-block:: -## import sequtils, sugar -## -## # Creating a sequence from 1 to 10, multiplying each member by 2, -## # keeping only the members which are not divisible by 6. -## let -## foo = toSeq(1..10).map(x => x*2).filter(x => x mod 6 != 0) -## bar = toSeq(1..10).mapIt(it*2).filterIt(it mod 6 != 0) -## -## doAssert foo == bar -## echo foo # @[2, 4, 8, 10, 14, 16, 20] -## -## echo foo.any(x => x > 17) # true -## echo bar.allIt(it < 20) # false -## echo foo.foldl(a + b) # 74; sum of all members -## -## .. code-block:: -## import sequtils -## from strutils import join -## -## let -## vowels = @"aeiou" # creates a sequence @['a', 'e', 'i', 'o', 'u'] -## foo = "sequtils is an awesome module" -## -## echo foo.filterIt(it notin vowels).join # "sqtls s n wsm mdl" -## -## ---- -## -## **See also**: + +runnableExamples: + import std/sugar + + # Creating a sequence from 1 to 10, multiplying each member by 2, + # keeping only the members which are not divisible by 6. + let + foo = toSeq(1..10).map(x => x * 2).filter(x => x mod 6 != 0) + bar = toSeq(1..10).mapIt(it * 2).filterIt(it mod 6 != 0) + baz = collect: + for i in 1..10: + let j = 2 * i + if j mod 6 != 0: + j + + doAssert foo == bar + doAssert foo == baz + doAssert foo == @[2, 4, 8, 10, 14, 16, 20] + + doAssert foo.any(x => x > 17) + doAssert not bar.allIt(it < 20) + doAssert foo.foldl(a + b) == 74 # sum of all members + + +runnableExamples: + from std/strutils import join + + let + vowels = @"aeiou" + foo = "sequtils is an awesome module" + + doAssert (vowels is seq[char]) and (vowels == @['a', 'e', 'i', 'o', 'u']) + doAssert foo.filterIt(it notin vowels).join == "sqtls s n wsm mdl" + +## See also +## ======== ## * `strutils module<strutils.html>`_ for common string functions ## * `sugar module<sugar.html>`_ for syntactic sugar macros ## * `algorithm module<algorithm.html>`_ for common generic algorithms @@ -73,22 +80,28 @@ ## heterogeneous members -include "system/inclrtl" +import std/private/since + +import std/macros +from std/typetraits import supportsCopyMem -import macros +when defined(nimPreviewSlimSystem): + import std/assertions -when not defined(nimhygiene): - {.pragma: dirty.} +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped = - ## Injects ``expAlias`` in caller scope, to avoid bugs involving multiple - ## substitution in macro arguments such as - ## https://github.com/nim-lang/Nim/issues/7187 - ## ``evalOnceAs(myAlias, myExp)`` will behave as ``let myAlias = myExp`` - ## except when ``letAssigneable`` is false (e.g. to handle openArray) where - ## it just forwards ``exp`` unchanged + ## Injects `expAlias` in caller scope, to avoid bugs involving multiple + ## substitution in macro arguments such as + ## https://github.com/nim-lang/Nim/issues/7187. + ## `evalOnceAs(myAlias, myExp)` will behave as `let myAlias = myExp` + ## except when `letAssigneable` is false (e.g. to handle openArray) where + ## it just forwards `exp` unchanged. expectKind(expAlias, nnkIdent) var val = exp @@ -103,12 +116,12 @@ macro evalOnceAs(expAlias, exp: untyped, newProc(name = genSym(nskTemplate, $expAlias), params = [getType(untyped)], body = val, procType = nnkTemplateDef)) -proc concat*[T](seqs: varargs[seq[T]]): seq[T] = +func concat*[T](seqs: varargs[seq[T]]): seq[T] = ## Takes several sequences' items and returns them inside a new sequence. ## All sequences must be of the same type. ## - ## See also: - ## * `distribute proc<#distribute,seq[T],Positive>`_ for a reverse + ## **See also:** + ## * `distribute func<#distribute,seq[T],Positive>`_ for a reverse ## operation ## runnableExamples: @@ -128,7 +141,23 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = result[i] = itm inc(i) -proc count*[T](s: openArray[T], x: T): int = +func addUnique*[T](s: var seq[T], x: sink T) = + ## Adds `x` to the container `s` if it is not already present. + ## Uses `==` to check if the item is already present. + runnableExamples: + var a = @[1, 2, 3] + a.addUnique(4) + a.addUnique(4) + assert a == @[1, 2, 3, 4] + + for i in 0..high(s): + if s[i] == x: return + when declared(ensureMove): + s.add ensureMove(x) + else: + s.add x + +func count*[T](s: openArray[T], x: T): int = ## Returns the number of occurrences of the item `x` in the container `s`. ## runnableExamples: @@ -143,7 +172,7 @@ proc count*[T](s: openArray[T], x: T): int = if itm == x: inc result -proc cycle*[T](s: openArray[T], n: Natural): seq[T] = +func cycle*[T](s: openArray[T], n: Natural): seq[T] = ## Returns a new sequence with the items of the container `s` repeated ## `n` times. ## `n` must be a non-negative number (zero or more). @@ -174,10 +203,10 @@ proc repeat*[T](x: T, n: Natural): seq[T] = for i in 0 ..< n: result[i] = x -proc deduplicate*[T](s: openArray[T], isSorted: bool = false): seq[T] = +func deduplicate*[T](s: openArray[T], isSorted: bool = false): seq[T] = ## Returns a new sequence without duplicates. ## - ## Setting the optional argument ``isSorted`` to ``true`` (default: false) + ## Setting the optional argument `isSorted` to true (default: false) ## uses a faster algorithm for deduplication. ## runnableExamples: @@ -202,9 +231,9 @@ proc deduplicate*[T](s: openArray[T], isSorted: bool = false): seq[T] = for itm in items(s): if not result.contains(itm): result.add(itm) -proc minIndex*[T](s: openArray[T]): int {.since: (1, 1).} = +func minIndex*[T](s: openArray[T]): int {.since: (1, 1).} = ## Returns the index of the minimum value of `s`. - ## ``T`` needs to have a ``<`` operator. + ## `T` needs to have a `<` operator. runnableExamples: let a = @[1, 2, 3, 4] @@ -219,9 +248,9 @@ proc minIndex*[T](s: openArray[T]): int {.since: (1, 1).} = for i in 1..high(s): if s[i] < s[result]: result = i -proc maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = +func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = ## Returns the index of the maximum value of `s`. - ## ``T`` needs to have a ``<`` operator. + ## `T` needs to have a `<` operator. runnableExamples: let a = @[1, 2, 3, 4] @@ -236,6 +265,15 @@ proc maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = for i in 1..high(s): if s[i] > s[result]: result = i +func minmax*[T](x: openArray[T]): (T, T) = + ## The minimum and maximum values of `x`. `T` needs to have a `<` operator. + var l = x[0] + var h = x[0] + for i in 1..high(x): + if x[i] < l: l = x[i] + if h < x[i]: h = x[i] + result = (l, h) + template zipImpl(s1, s2, retType: untyped): untyped = proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = @@ -245,9 +283,9 @@ template zipImpl(s1, s2, retType: untyped): untyped = ## If one container is shorter, the remaining items in the longer container ## are discarded. ## - ## **Note**: For Nim 1.0.x and older version, ``zip`` returned a seq of - ## named tuple with fields ``a`` and ``b``. For Nim versions 1.1.x and newer, - ## ``zip`` returns a seq of unnamed tuples. + ## **Note**: For Nim 1.0.x and older version, `zip` returned a seq of + ## named tuples with fields `a` and `b`. For Nim versions 1.1.x and newer, + ## `zip` returns a seq of unnamed tuples. runnableExamples: let short = @[1, 2, 3] @@ -281,19 +319,33 @@ when (NimMajor, NimMinor) <= (1, 0): else: zipImpl(s1, s2, seq[(S, T)]) -proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = +proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = + ## Returns a tuple of two sequences split out from a sequence of 2-field tuples. + runnableExamples: + let + zipped = @[(1, 'a'), (2, 'b'), (3, 'c')] + unzipped1 = @[1, 2, 3] + unzipped2 = @['a', 'b', 'c'] + assert zipped.unzip() == (unzipped1, unzipped2) + assert zip(unzipped1, unzipped2).unzip() == (unzipped1, unzipped2) + result = (newSeq[S](s.len), newSeq[T](s.len)) + for i in 0..<s.len: + result[0][i] = s[i][0] + result[1][i] = s[i][1] + +func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = ## Splits and distributes a sequence `s` into `num` sub-sequences. ## ## Returns a sequence of `num` sequences. For *some* input values this is the - ## inverse of the `concat <#concat,varargs[seq[T]]>`_ proc. + ## inverse of the `concat <#concat,varargs[seq[T]]>`_ func. ## The input sequence `s` can be empty, which will produce ## `num` empty sequences. ## ## If `spread` is false and the length of `s` is not a multiple of `num`, the - ## proc will max out the first sub-sequence with ``1 + len(s) div num`` + ## func will max out the first sub-sequence with `1 + len(s) div num` ## entries, leaving the remainder of elements to the last sequence. ## - ## On the other hand, if `spread` is true, the proc will distribute evenly + ## On the other hand, if `spread` is true, the func will distribute evenly ## the remainder of the division across all sequences, which makes the result ## more suited to multithreading where you are passing equal sized work units ## to a thread pool and want to maximize core usage. @@ -308,7 +360,6 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = if num < 2: result = @[s] return - let num = int(num) # XXX probably only needed because of .. bug # Create the result and calculate the stride size and the remainder if any. result = newSeq[seq[T]](num) @@ -339,14 +390,18 @@ proc distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = first = last proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): - seq[S]{.inline.} = - ## Returns a new sequence with the results of `op` proc applied to every + seq[S] {.inline, effectsOf: op.} = + ## Returns a new sequence with the results of the `op` proc applied to every ## item in the container `s`. ## - ## Since the input is not modified you can use it to + ## Since the input is not modified, you can use it to ## transform the type of the elements in the input container. ## - ## See also: + ## Instead of using `map` and `filter`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ ## * `mapIt template<#mapIt.t,typed,untyped>`_ ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ for the in-place version ## @@ -361,15 +416,14 @@ proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): result[i] = op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) - {.inline.} = - ## Applies `op` to every item in `s` modifying it directly. + {.inline, effectsOf: op.} = + ## Applies `op` to every item in `s`, modifying it directly. ## - ## Note that container `s` must be declared as a ``var`` - ## and it is required for your input and output types to - ## be the same, since `s` is modified in-place. - ## The parameter function takes a ``var T`` type parameter. + ## Note that the container `s` must be declared as a `var`, + ## since `s` is modified in-place. + ## The parameter function takes a `var T` type parameter. ## - ## See also: + ## **See also:** ## * `applyIt template<#applyIt.t,untyped,untyped>`_ ## * `map proc<#map,openArray[T],proc(T)>`_ ## @@ -381,15 +435,15 @@ proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) for i in 0 ..< s.len: op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s` modifying it directly. ## - ## Note that container `s` must be declared as a ``var`` + ## Note that the container `s` must be declared as a `var` ## and it is required for your input and output types to ## be the same, since `s` is modified in-place. - ## The parameter function takes and returns a ``T`` type variable. + ## The parameter function takes and returns a `T` type variable. ## - ## See also: + ## **See also:** ## * `applyIt template<#applyIt.t,untyped,untyped>`_ ## * `map proc<#map,openArray[T],proc(T)>`_ ## @@ -400,12 +454,25 @@ proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) for i in 0 ..< s.len: s[i] = op(s[i]) -iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = +proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3), effectsOf: op.} = + ## Same as `apply` but for a proc that does not return anything + ## and does not mutate `s` directly. + runnableExamples: + var message: string + apply([0, 1, 2, 3, 4], proc(item: int) = message.addInt item) + assert message == "01234" + for i in 0 ..< s.len: op(s[i]) + +iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T {.effectsOf: pred.} = ## Iterates through a container `s` and yields every item that fulfills the - ## predicate `pred` (function that returns a `bool`). + ## predicate `pred` (a function that returns a `bool`). + ## + ## Instead of using `map` and `filter`, consider using the `collect` macro + ## from the `sugar` module. ## - ## See also: - ## * `fliter proc<#filter,openArray[T],proc(T)>`_ + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `filter proc<#filter,openArray[T],proc(T)>`_ ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## runnableExamples: @@ -420,11 +487,15 @@ iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = yield s[i] proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] - {.inline.} = - ## Returns a new sequence with all the items of `s` that fulfilled the - ## predicate `pred` (function that returns a `bool`). + {.inline, effectsOf: pred.} = + ## Returns a new sequence with all the items of `s` that fulfill the + ## predicate `pred` (a function that returns a `bool`). ## - ## See also: + ## Instead of using `map` and `filter`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ for the in-place version @@ -443,16 +514,16 @@ proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] result.add(s[i]) proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) - {.inline.} = - ## Keeps the items in the passed sequence `s` if they fulfilled the - ## predicate `pred` (function that returns a `bool`). + {.inline, effectsOf: pred.} = + ## Keeps the items in the passed sequence `s` if they fulfill the + ## predicate `pred` (a function that returns a `bool`). ## - ## Note that `s` must be declared as a ``var``. + ## Note that `s` must be declared as a `var`. ## ## Similar to the `filter proc<#filter,openArray[T],proc(T)>`_, ## but modifies the sequence directly. ## - ## See also: + ## **See also:** ## * `keepItIf template<#keepItIf.t,seq,untyped>`_ ## * `filter proc<#filter,openArray[T],proc(T)>`_ ## @@ -472,12 +543,51 @@ proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) inc(pos) setLen(s, pos) -proc delete*[T](s: var seq[T]; first, last: Natural) = - ## Deletes in the items of a sequence `s` at positions ``first..last`` - ## (including both ends of a range). - ## This modifies `s` itself, it does not return a copy. +func delete*[T](s: var seq[T]; slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. ## + ## This operation moves all elements after `s[slice]` in linear time. runnableExamples: + var a = @[10, 11, 12, 13, 14] + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == @[10, 11, 12, 13, 14] + a.delete(4..4) + assert a == @[10, 11, 12, 13] + a.delete(1..2) + assert a == @[10, 13] + a.delete(1..<1) # empty slice + assert a == @[10, 13] + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + template defaultImpl = + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + while i < newLen: + when defined(gcDestructors): + s[i] = move(s[j]) + else: + s[i].shallowCopy(s[j]) + inc(i) + inc(j) + setLen(s, newLen) + when nimvm: defaultImpl() + else: + when defined(js): + let n = slice.b - slice.a + 1 + let first = slice.a + {.emit: "`s`.splice(`first`, `n`);".} + else: + defaultImpl() + +func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete(s, first..last)`".} = + ## Deletes the items of a sequence `s` at positions `first..last` + ## (including both ends of the range). + ## This modifies `s` itself, it does not return a copy. + runnableExamples("--warning:deprecated:off"): let outcome = @[1, 1, 1, 1, 1, 1, 1, 1] var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] dest.delete(3, 8) @@ -486,8 +596,8 @@ proc delete*[T](s: var seq[T]; first, last: Natural) = if first >= s.len: return var i = first - var j = min(len(s), last+1) - var newLen = len(s)-j+i + var j = min(len(s), last + 1) + var newLen = len(s) - j + i while i < newLen: when defined(gcDestructors): s[i] = move(s[j]) @@ -497,11 +607,11 @@ proc delete*[T](s: var seq[T]; first, last: Natural) = inc(j) setLen(s, newLen) -proc insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = +func insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = ## Inserts items from `src` into `dest` at position `pos`. This modifies ## `dest` itself, it does not return a copy. ## - ## Notice that `src` and `dest` must be of the same type. + ## Note that the elements of `src` and `dest` must be of the same type. ## runnableExamples: var dest = @[1, 1, 1, 1, 1, 1, 1, 1] @@ -512,7 +622,8 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = assert dest == outcome var j = len(dest) - 1 - var i = len(dest) + len(src) - 1 + var i = j + len(src) + if i == j: return dest.setLen(i + 1) # Move items after `pos` to the end of the sequence. @@ -531,16 +642,20 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = template filterIt*(s, pred: untyped): untyped = - ## Returns a new sequence with all the items of `s` that fulfilled the + ## Returns a new sequence with all the items of `s` that fulfill the ## predicate `pred`. ## ## Unlike the `filter proc<#filter,openArray[T],proc(T)>`_ and ## `filter iterator<#filter.i,openArray[T],proc(T)>`_, - ## the predicate needs to be an expression using the ``it`` variable - ## for testing, like: ``filterIt("abcxyz", it == 'x')``. + ## the predicate needs to be an expression using the `it` variable + ## for testing, like: `filterIt("abcxyz", it == 'x')`. + ## + ## Instead of using `mapIt` and `filterIt`, consider using the `collect` macro + ## from the `sugar` module. ## - ## See also: - ## * `fliter proc<#filter,openArray[T],proc(T)>`_ + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ + ## * `filter proc<#filter,openArray[T],proc(T)>`_ ## * `filter iterator<#filter.i,openArray[T],proc(T)>`_ ## runnableExamples: @@ -551,20 +666,20 @@ template filterIt*(s, pred: untyped): untyped = assert acceptable == @[-2.0, 24.5, 44.31] assert notAcceptable == @[-272.15, 99.9, -113.44] - var result = newSeq[type(s[0])]() + var result = newSeq[typeof(s[0])]() for it {.inject.} in items(s): if pred: result.add(it) result template keepItIf*(varSeq: seq, pred: untyped) = - ## Keeps the items in the passed sequence (must be declared as a ``var``) - ## if they fulfilled the predicate. + ## Keeps the items in the passed sequence (must be declared as a `var`) + ## if they fulfill the predicate. ## ## Unlike the `keepIf proc<#keepIf,seq[T],proc(T)>`_, ## the predicate needs to be an expression using - ## the ``it`` variable for testing, like: ``keepItIf("abcxyz", it == 'x')``. + ## the `it` variable for testing, like: `keepItIf("abcxyz", it == 'x')`. ## - ## See also: + ## **See also:** ## * `keepIf proc<#keepIf,seq[T],proc(T)>`_ ## * `filterIt template<#filterIt.t,untyped,untyped>`_ ## @@ -585,23 +700,42 @@ template keepItIf*(varSeq: seq, pred: untyped) = inc(pos) setLen(varSeq, pos) -proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +since (1, 1): + template countIt*(s, pred: untyped): int = + ## Returns a count of all the items that fulfill the predicate. + ## + ## The predicate needs to be an expression using + ## the `it` variable for testing, like: `countIt(@[1, 2, 3], it > 2)`. + ## + runnableExamples: + let numbers = @[-3, -2, -1, 0, 1, 2, 3, 4, 5, 6] + iterator iota(n: int): int = + for i in 0..<n: yield i + assert numbers.countIt(it < 0) == 3 + assert countIt(iota(10), it < 2) == 2 + + var result = 0 + for it {.inject.} in s: + if pred: result += 1 + result + +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if every item fulfills the ## predicate. ## - ## See also: + ## **See also:** ## * `allIt template<#allIt.t,untyped,untyped>`_ ## * `any proc<#any,openArray[T],proc(T)>`_ ## runnableExamples: let numbers = @[1, 4, 5, 8, 9, 7, 4] - assert all(numbers, proc (x: int): bool = return x < 10) == true - assert all(numbers, proc (x: int): bool = return x < 9) == false + assert all(numbers, proc (x: int): bool = x < 10) == true + assert all(numbers, proc (x: int): bool = x < 9) == false for i in s: if not pred(i): return false - return true + true template allIt*(s, pred: untyped): bool = ## Iterates through a container and checks if every item fulfills the @@ -609,9 +743,9 @@ template allIt*(s, pred: untyped): bool = ## ## Unlike the `all proc<#all,openArray[T],proc(T)>`_, ## the predicate needs to be an expression using - ## the ``it`` variable for testing, like: ``allIt("abba", it == 'a')``. + ## the `it` variable for testing, like: `allIt("abba", it == 'a')`. ## - ## See also: + ## **See also:** ## * `all proc<#all,openArray[T],proc(T)>`_ ## * `anyIt template<#anyIt.t,untyped,untyped>`_ ## @@ -627,33 +761,33 @@ template allIt*(s, pred: untyped): bool = break result -proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = - ## Iterates through a container and checks if some item fulfills the - ## predicate. +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = + ## Iterates through a container and checks if at least one item + ## fulfills the predicate. ## - ## See also: + ## **See also:** ## * `anyIt template<#anyIt.t,untyped,untyped>`_ ## * `all proc<#all,openArray[T],proc(T)>`_ ## runnableExamples: let numbers = @[1, 4, 5, 8, 9, 7, 4] - assert any(numbers, proc (x: int): bool = return x > 8) == true - assert any(numbers, proc (x: int): bool = return x > 9) == false + assert any(numbers, proc (x: int): bool = x > 8) == true + assert any(numbers, proc (x: int): bool = x > 9) == false for i in s: if pred(i): return true - return false + false template anyIt*(s, pred: untyped): bool = - ## Iterates through a container and checks if some item fulfills the - ## predicate. + ## Iterates through a container and checks if at least one item + ## fulfills the predicate. ## ## Unlike the `any proc<#any,openArray[T],proc(T)>`_, ## the predicate needs to be an expression using - ## the ``it`` variable for testing, like: ``anyIt("abba", it == 'a')``. + ## the `it` variable for testing, like: `anyIt("abba", it == 'a')`. ## - ## See also: + ## **See also:** ## * `any proc<#any,openArray[T],proc(T)>`_ ## * `allIt template<#allIt.t,untyped,untyped>`_ ## @@ -671,7 +805,7 @@ template anyIt*(s, pred: untyped): bool = template toSeq1(s: not iterator): untyped = # overload for typed but not iterator - type OutType = type(items(s)) + type OutType = typeof(items(s)) when compiles(s.len): block: evalOnceAs(s2, s, compiles((let _ = s))) @@ -682,7 +816,7 @@ template toSeq1(s: not iterator): untyped = i += 1 result else: - var result: seq[OutType] = @[] + var result: seq[OutType]# = @[] for it in s: result.add(it) result @@ -692,14 +826,14 @@ template toSeq2(iter: iterator): untyped = evalOnceAs(iter2, iter(), false) when compiles(iter2.len): var i = 0 - var result = newSeq[type(iter2)](iter2.len) + var result = newSeq[typeof(iter2)](iter2.len) for x in iter2: result[i] = x inc i result else: - type OutType = type(iter2()) - var result: seq[OutType] = @[] + type OutType = typeof(iter2()) + var result: seq[OutType]# = @[] when compiles(iter2()): evalOnceAs(iter4, iter, false) let iter3 = iter4() @@ -718,8 +852,8 @@ template toSeq*(iter: untyped): untyped = let myRange = 1..5 mySet: set[int8] = {5'i8, 3, 1} - assert type(myRange) is HSlice[system.int, system.int] - assert type(mySet) is set[int8] + assert typeof(myRange) is HSlice[system.int, system.int] + assert typeof(mySet) is set[int8] let mySeq1 = toSeq(myRange) @@ -736,14 +870,14 @@ template toSeq*(iter: untyped): untyped = when compiles(iter.len): block: evalOnceAs(iter2, iter, true) - var result = newSeq[type(iter)](iter2.len) + var result = newSeq[typeof(iter)](iter2.len) var i = 0 for x in iter2: result[i] = x inc i result else: - var result: seq[type(iter)] = @[] + var result: seq[typeof(iter)] = @[] for x in iter: result.add(x) result @@ -754,15 +888,15 @@ template foldl*(sequence, operation: untyped): untyped = ## The sequence is required to have at least a single element. Debug versions ## of your program will assert in this situation but release versions will ## happily go ahead. If the sequence has a single element it will be returned - ## without applying ``operation``. + ## without applying `operation`. ## - ## The ``operation`` parameter should be an expression which uses the - ## variables ``a`` and ``b`` for each step of the fold. Since this is a left + ## The `operation` parameter should be an expression which uses the + ## variables `a` and `b` for each step of the fold. Since this is a left ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (((1) - 2) - ## 3). ## - ## See also: + ## **See also:** ## * `foldl template<#foldl.t,,,>`_ with a starting parameter ## * `foldr template<#foldr.t,untyped,untyped>`_ ## @@ -774,14 +908,21 @@ template foldl*(sequence, operation: untyped): untyped = multiplication = foldl(numbers, a * b) words = @["nim", "is", "cool"] concatenation = foldl(words, a & b) + procs = @["proc", "Is", "Also", "Fine"] + + + func foo(acc, cur: string): string = + result = acc & cur + assert addition == 25, "Addition is (((5)+9)+11)" assert subtraction == -15, "Subtraction is (((5)-9)-11)" assert multiplication == 495, "Multiplication is (((5)*9)*11)" assert concatenation == "nimiscool" + assert foldl(procs, foo(a, b)) == "procIsAlsoFine" let s = sequence assert s.len > 0, "Can't fold empty sequences" - var result: type(s[0]) + var result: typeof(s[0]) result = s[0] for i in 1..<s.len: let @@ -793,14 +934,14 @@ template foldl*(sequence, operation: untyped): untyped = template foldl*(sequence, operation, first): untyped = ## Template to fold a sequence from left to right, returning the accumulation. ## - ## This version of ``foldl`` gets a **starting parameter**. This makes it possible + ## This version of `foldl` gets a **starting parameter**. This makes it possible ## to accumulate the sequence into a different type than the sequence elements. ## - ## The ``operation`` parameter should be an expression which uses the variables - ## ``a`` and ``b`` for each step of the fold. The ``first`` parameter is the - ## start value (the first ``a``) and therefor defines the type of the result. + ## The `operation` parameter should be an expression which uses the variables + ## `a` and `b` for each step of the fold. The `first` parameter is the + ## start value (the first `a`) and therefore defines the type of the result. ## - ## See also: + ## **See also:** ## * `foldr template<#foldr.t,untyped,untyped>`_ ## runnableExamples: @@ -809,8 +950,7 @@ template foldl*(sequence, operation, first): untyped = digits = foldl(numbers, a & (chr(b + ord('0'))), "") assert digits == "0815" - var result: type(first) - result = first + var result: typeof(first) = first for x in items(sequence): let a {.inject.} = result @@ -824,15 +964,15 @@ template foldr*(sequence, operation: untyped): untyped = ## The sequence is required to have at least a single element. Debug versions ## of your program will assert in this situation but release versions will ## happily go ahead. If the sequence has a single element it will be returned - ## without applying ``operation``. + ## without applying `operation`. ## - ## The ``operation`` parameter should be an expression which uses the - ## variables ``a`` and ``b`` for each step of the fold. Since this is a right + ## The `operation` parameter should be an expression which uses the + ## variables `a` and `b` for each step of the fold. Since this is a right ## fold, for non associative binary operations like subtraction think that ## the sequence of numbers 1, 2 and 3 will be parenthesized as (1 - (2 - ## (3))). ## - ## See also: + ## **See also:** ## * `foldl template<#foldl.t,untyped,untyped>`_ ## * `foldl template<#foldl.t,,,>`_ with a starting parameter ## @@ -849,11 +989,11 @@ template foldr*(sequence, operation: untyped): untyped = assert multiplication == 495, "Multiplication is (5*(9*(11)))" assert concatenation == "nimiscool" - let s = sequence - assert s.len > 0, "Can't fold empty sequences" - var result: type(s[0]) - result = sequence[s.len - 1] - for i in countdown(s.len - 2, 0): + let s = sequence # xxx inefficient, use {.evalonce.} pending #13750 + let n = s.len + assert n > 0, "Can't fold empty sequences" + var result = s[n - 1] + for i in countdown(n - 2, 0): let a {.inject.} = s[i] b {.inject.} = result @@ -861,16 +1001,20 @@ template foldr*(sequence, operation: untyped): untyped = result template mapIt*(s: typed, op: untyped): untyped = - ## Returns a new sequence with the results of `op` proc applied to every + ## Returns a new sequence with the results of the `op` proc applied to every ## item in the container `s`. ## ## Since the input is not modified you can use it to ## transform the type of the elements in the input container. ## - ## The template injects the ``it`` variable which you can use directly in an + ## The template injects the `it` variable which you can use directly in an ## expression. ## - ## See also: + ## Instead of using `mapIt` and `filterIt`, consider using the `collect` macro + ## from the `sugar` module. + ## + ## **See also:** + ## * `sugar.collect macro<sugar.html#collect.m%2Cuntyped%2Cuntyped>`_ ## * `map proc<#map,openArray[T],proc(T)>`_ ## * `applyIt template<#applyIt.t,untyped,untyped>`_ for the in-place version ## @@ -880,43 +1024,55 @@ template mapIt*(s: typed, op: untyped): untyped = strings = nums.mapIt($(4 * it)) assert strings == @["4", "8", "12", "16"] - when defined(nimHasTypeof): - type OutType = typeof(( - block: - var it{.inject.}: typeof(items(s), typeOfIter); - op), typeOfProc) - else: - type OutType = type(( - block: - var it{.inject.}: type(items(s)); - op)) - when compiles(s.len): - block: # using a block avoids https://github.com/nim-lang/Nim/issues/8580 - - # BUG: `evalOnceAs(s2, s, false)` would lead to C compile errors - # (`error: use of undeclared identifier`) instead of Nim compile errors - evalOnceAs(s2, s, compiles((let _ = s))) + type OutType = typeof(( + block: + var it{.inject.}: typeof(items(s), typeOfIter); + op), typeOfProc) + when OutType is not (proc): + # Here, we avoid to create closures in loops. + # This avoids https://github.com/nim-lang/Nim/issues/12625 + when compiles(s.len): + block: # using a block avoids https://github.com/nim-lang/Nim/issues/8580 + + # BUG: `evalOnceAs(s2, s, false)` would lead to C compile errors + # (`error: use of undeclared identifier`) instead of Nim compile errors + evalOnceAs(s2, s, compiles((let _ = s))) - var i = 0 - var result = newSeq[OutType](s2.len) - for it {.inject.} in s2: - result[i] = op - i += 1 + var i = 0 + var result = newSeq[OutType](s2.len) + for it {.inject.} in s2: + result[i] = op + i += 1 + result + else: + var result: seq[OutType]# = @[] + # use `items` to avoid https://github.com/nim-lang/Nim/issues/12639 + for it {.inject.} in items(s): + result.add(op) result else: - var result: seq[OutType] = @[] - for it {.inject.} in s: - result.add(op) - result + # `op` is going to create closures in loops, let's fallback to `map`. + # NOTE: Without this fallback, developers have to define a helper function and + # call `map`: + # [1, 2].map((it) => ((x: int) => it + x)) + # With this fallback, above code can be simplified to: + # [1, 2].mapIt((x: int) => it + x) + # In this case, `mapIt` is just syntax sugar for `map`. + type InType = typeof(items(s), typeOfIter) + # Use a help proc `f` to create closures for each element in `s` + let f = proc (x: InType): OutType = + let it {.inject.} = x + op + map(s, f) template applyIt*(varSeq, op: untyped) = - ## Convenience template around the mutable ``apply`` proc to reduce typing. + ## Convenience template around the mutable `apply` proc to reduce typing. ## - ## The template injects the ``it`` variable which you can use directly in an - ## expression. The expression has to return the same type as the sequence you - ## are mutating. + ## The template injects the `it` variable which you can use directly in an + ## expression. The expression has to return the same type as the elements + ## of the sequence you are mutating. ## - ## See also: + ## **See also:** ## * `apply proc<#apply,openArray[T],proc(T)_2>`_ ## * `mapIt template<#mapIt.t,typed,untyped>`_ ## @@ -931,29 +1087,33 @@ template applyIt*(varSeq, op: untyped) = template newSeqWith*(len: int, init: untyped): untyped = - ## Creates a new sequence of length `len`, calling `init` to initialize - ## each value of the sequence. - ## - ## Useful for creating "2D" sequences - sequences containing other sequences - ## or to populate fields of the created sequence. + ## Creates a new `seq` of length `len`, calling `init` to initialize + ## each value of the seq. ## + ## Useful for creating "2D" seqs - seqs containing other seqs + ## or to populate fields of the created seq. runnableExamples: - ## Creates a sequence containing 5 bool sequences, each of length of 3. + ## Creates a seq containing 5 bool seqs, each of length of 3. var seq2D = newSeqWith(5, newSeq[bool](3)) assert seq2D.len == 5 assert seq2D[0].len == 3 assert seq2D[4][2] == false - ## Creates a sequence of 20 random numbers from 1 to 10 - import random - var seqRand = newSeqWith(20, rand(10)) - - var result = newSeq[type(init)](len) - for i in 0 ..< len: + ## Creates a seq with random numbers + import std/random + var seqRand = newSeqWith(20, rand(1.0)) + assert seqRand[0] != seqRand[1] + type T = typeof(init) + let newLen = len + when supportsCopyMem(T) and declared(newSeqUninit): + var result = newSeqUninit[T](newLen) + else: # TODO: use `newSeqUnsafe` when that's available + var result = newSeq[T](newLen) + for i in 0 ..< newLen: result[i] = init - result + move(result) # refs bug #7295 -proc mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; +func mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; filter = nnkLiterals): NimNode = if constructor.kind in filter: result = newNimNode(nnkCall, lineInfoFrom = constructor) @@ -969,504 +1129,34 @@ proc mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; macro mapLiterals*(constructor, op: untyped; nested = true): untyped = - ## Applies ``op`` to each of the **atomic** literals like ``3`` - ## or ``"abc"`` in the specified ``constructor`` AST. This can + ## Applies `op` to each of the **atomic** literals like `3` + ## or `"abc"` in the specified `constructor` AST. This can ## be used to map every array element to some target type: - ## - ## Example: - ## - ## .. code-block:: - ## let x = mapLiterals([0.1, 1.2, 2.3, 3.4], int) - ## doAssert x is array[4, int] - ## - ## Short notation for: - ## - ## .. code-block:: - ## let x = [int(0.1), int(1.2), int(2.3), int(3.4)] - ## - ## If ``nested`` is true (which is the default), the literals are replaced - ## everywhere in the ``constructor`` AST, otherwise only the first level + runnableExamples: + let x = mapLiterals([0.1, 1.2, 2.3, 3.4], int) + doAssert x is array[4, int] + doAssert x == [int(0.1), int(1.2), int(2.3), int(3.4)] + ## If `nested` is true (which is the default), the literals are replaced + ## everywhere in the `constructor` AST, otherwise only the first level ## is considered: - ## - ## .. code-block:: - ## let a = mapLiterals((1.2, (2.3, 3.4), 4.8), int) - ## let b = mapLiterals((1.2, (2.3, 3.4), 4.8), int, nested=false) - ## assert a == (1, (2, 3), 4) - ## assert b == (1, (2.3, 3.4), 4) - ## - ## let c = mapLiterals((1, (2, 3), 4, (5, 6)), `$`) - ## let d = mapLiterals((1, (2, 3), 4, (5, 6)), `$`, nested=false) - ## assert c == ("1", ("2", "3"), "4", ("5", "6")) - ## assert d == ("1", (2, 3), "4", (5, 6)) - ## - ## There are no constraints for the ``constructor`` AST, it + runnableExamples: + let a = mapLiterals((1.2, (2.3, 3.4), 4.8), int) + let b = mapLiterals((1.2, (2.3, 3.4), 4.8), int, nested=false) + assert a == (1, (2, 3), 4) + assert b == (1, (2.3, 3.4), 4) + + let c = mapLiterals((1, (2, 3), 4, (5, 6)), `$`) + let d = mapLiterals((1, (2, 3), 4, (5, 6)), `$`, nested=false) + assert c == ("1", ("2", "3"), "4", ("5", "6")) + assert d == ("1", (2, 3), "4", (5, 6)) + ## There are no constraints for the `constructor` AST, it ## works for nested tuples of arrays of sets etc. result = mapLitsImpl(constructor, op, nested.boolVal) iterator items*[T](xs: iterator: T): T = - ## iterates over each element yielded by a closure iterator. This may + ## Iterates over each element yielded by a closure iterator. This may ## not seem particularly useful on its own, but this allows closure - ## iterators to be used by the the mapIt, filterIt, allIt, anyIt, etc. + ## iterators to be used by the mapIt, filterIt, allIt, anyIt, etc. ## templates. for x in xs(): yield x - -when isMainModule: - import strutils - from algorithm import sorted - - # helper for testing double substitution side effects which are handled - # by `evalOnceAs` - var counter = 0 - proc identity[T](a: T): auto = - counter.inc - a - - block: # concat test - let - s1 = @[1, 2, 3] - s2 = @[4, 5] - s3 = @[6, 7] - total = concat(s1, s2, s3) - assert total == @[1, 2, 3, 4, 5, 6, 7] - - block: # count test - let - s1 = @[1, 2, 3, 2] - s2 = @['a', 'b', 'x', 'a'] - a1 = [1, 2, 3, 2] - a2 = ['a', 'b', 'x', 'a'] - r0 = count(s1, 0) - r1 = count(s1, 1) - r2 = count(s1, 2) - r3 = count(s2, 'y') - r4 = count(s2, 'x') - r5 = count(s2, 'a') - ar0 = count(a1, 0) - ar1 = count(a1, 1) - ar2 = count(a1, 2) - ar3 = count(a2, 'y') - ar4 = count(a2, 'x') - ar5 = count(a2, 'a') - assert r0 == 0 - assert r1 == 1 - assert r2 == 2 - assert r3 == 0 - assert r4 == 1 - assert r5 == 2 - assert ar0 == 0 - assert ar1 == 1 - assert ar2 == 2 - assert ar3 == 0 - assert ar4 == 1 - assert ar5 == 2 - - block: # cycle tests - let - a = @[1, 2, 3] - b: seq[int] = @[] - c = [1, 2, 3] - - doAssert a.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] - doAssert a.cycle(0) == @[] - #doAssert a.cycle(-1) == @[] # will not compile! - doAssert b.cycle(3) == @[] - doAssert c.cycle(3) == @[1, 2, 3, 1, 2, 3, 1, 2, 3] - doAssert c.cycle(0) == @[] - - block: # repeat tests - assert repeat(10, 5) == @[10, 10, 10, 10, 10] - assert repeat(@[1, 2, 3], 2) == @[@[1, 2, 3], @[1, 2, 3]] - assert repeat([1, 2, 3], 2) == @[[1, 2, 3], [1, 2, 3]] - - block: # deduplicates test - let - dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] - dup2 = @["a", "a", "c", "d", "d"] - dup3 = [1, 1, 3, 4, 2, 2, 8, 1, 4] - dup4 = ["a", "a", "c", "d", "d"] - unique1 = deduplicate(dup1) - unique2 = deduplicate(dup2) - unique3 = deduplicate(dup3) - unique4 = deduplicate(dup4) - unique5 = deduplicate(dup1.sorted, true) - unique6 = deduplicate(dup2, true) - unique7 = deduplicate(dup3.sorted, true) - unique8 = deduplicate(dup4, true) - assert unique1 == @[1, 3, 4, 2, 8] - assert unique2 == @["a", "c", "d"] - assert unique3 == @[1, 3, 4, 2, 8] - assert unique4 == @["a", "c", "d"] - assert unique5 == @[1, 2, 3, 4, 8] - assert unique6 == @["a", "c", "d"] - assert unique7 == @[1, 2, 3, 4, 8] - assert unique8 == @["a", "c", "d"] - - block: # zip test - let - short = @[1, 2, 3] - long = @[6, 5, 4, 3, 2, 1] - words = @["one", "two", "three"] - ashort = [1, 2, 3] - along = [6, 5, 4, 3, 2, 1] - awords = ["one", "two", "three"] - zip1 = zip(short, long) - zip2 = zip(short, words) - zip3 = zip(ashort, along) - assert zip1 == @[(1, 6), (2, 5), (3, 4)] - assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] - assert zip3 == @[(1, 6), (2, 5), (3, 4)] - assert zip1[2][1] == 4 - assert zip2[2][1] == "three" - assert zip3[2][1] == 4 - when (NimMajor, NimMinor) <= (1, 0): - let - # In Nim 1.0.x and older, zip returned a seq of tuple strictly - # with fields named "a" and "b". - zipAb = zip(ashort, awords) - assert zipAb == @[(a: 1, b: "one"), (2, "two"), (3, "three")] - assert zipAb[2].b == "three" - else: - let - # As zip returns seq of anonymous tuples, they can be assigned - # to any variable that's a sequence of named tuples too. - zipXy: seq[tuple[x: int, y: string]] = zip(ashort, awords) - zipMn: seq[tuple[m: int, n: string]] = zip(ashort, words) - assert zipXy == @[(x: 1, y: "one"), (2, "two"), (3, "three")] - assert zipMn == @[(m: 1, n: "one"), (2, "two"), (3, "three")] - assert zipXy[2].y == "three" - assert zipMn[2].n == "three" - - block: # distribute tests - let numbers = @[1, 2, 3, 4, 5, 6, 7] - doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert numbers.distribute(6)[0] == @[1, 2] - doAssert numbers.distribute(6)[5] == @[7] - let a = @[1, 2, 3, 4, 5, 6, 7] - doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] - doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] - doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] - doAssert a.distribute(6, false) == @[ - @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] - doAssert a.distribute(8, false) == a.distribute(8, true) - doAssert a.distribute(90, false) == a.distribute(90, true) - var b = @[0] - for f in 1 .. 25: b.add(f) - doAssert b.distribute(5, true)[4].len == 5 - doAssert b.distribute(5, false)[4].len == 2 - - block: # map test - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - anumbers = [1, 4, 5, 8, 9, 7, 4] - m1 = map(numbers, proc(x: int): int = 2*x) - m2 = map(anumbers, proc(x: int): int = 2*x) - assert m1 == @[2, 8, 10, 16, 18, 14, 8] - assert m2 == @[2, 8, 10, 16, 18, 14, 8] - - block: # apply test - var a = @["1", "2", "3", "4"] - apply(a, proc(x: var string) = x &= "42") - assert a == @["142", "242", "342", "442"] - - block: # filter proc test - let - colors = @["red", "yellow", "black"] - acolors = ["red", "yellow", "black"] - f1 = filter(colors, proc(x: string): bool = x.len < 6) - f2 = filter(colors) do (x: string) -> bool: x.len > 5 - f3 = filter(acolors, proc(x: string): bool = x.len < 6) - f4 = filter(acolors) do (x: string) -> bool: x.len > 5 - assert f1 == @["red", "black"] - assert f2 == @["yellow"] - assert f3 == @["red", "black"] - assert f4 == @["yellow"] - - block: # filter iterator test - let numbers = @[1, 4, 5, 8, 9, 7, 4] - let anumbers = [1, 4, 5, 8, 9, 7, 4] - assert toSeq(filter(numbers, proc (x: int): bool = x mod 2 == 0)) == - @[4, 8, 4] - assert toSeq(filter(anumbers, proc (x: int): bool = x mod 2 == 0)) == - @[4, 8, 4] - - block: # keepIf test - var floats = @[13.0, 12.5, 5.8, 2.0, 6.1, 9.9, 10.1] - keepIf(floats, proc(x: float): bool = x > 10) - assert floats == @[13.0, 12.5, 10.1] - - block: # delete tests - let outcome = @[1, 1, 1, 1, 1, 1, 1, 1] - var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] - dest.delete(3, 8) - assert outcome == dest, """\ - Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1] - is [1,1,1,1,1,1,1,1]""" - var x = @[1, 2, 3] - x.delete(100, 100) - assert x == @[1, 2, 3] - - block: # insert tests - var dest = @[1, 1, 1, 1, 1, 1, 1, 1] - let - src = @[2, 2, 2, 2, 2, 2] - outcome = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1] - dest.insert(src, 3) - assert dest == outcome, """\ - Inserting [2,2,2,2,2,2] into [1,1,1,1,1,1,1,1] - at 3 is [1,1,1,2,2,2,2,2,2,1,1,1,1,1]""" - - block: # filterIt test - let - temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] - acceptable = filterIt(temperatures, it < 50 and it > -10) - notAcceptable = filterIt(temperatures, it > 50 or it < -10) - assert acceptable == @[-2.0, 24.5, 44.31] - assert notAcceptable == @[-272.15, 99.9, -113.44] - - block: # keepItIf test - var candidates = @["foo", "bar", "baz", "foobar"] - keepItIf(candidates, it.len == 3 and it[0] == 'b') - assert candidates == @["bar", "baz"] - - block: # all - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - anumbers = [1, 4, 5, 8, 9, 7, 4] - len0seq: seq[int] = @[] - assert all(numbers, proc (x: int): bool = return x < 10) == true - assert all(numbers, proc (x: int): bool = return x < 9) == false - assert all(len0seq, proc (x: int): bool = return false) == true - assert all(anumbers, proc (x: int): bool = return x < 10) == true - assert all(anumbers, proc (x: int): bool = return x < 9) == false - - block: # allIt - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - anumbers = [1, 4, 5, 8, 9, 7, 4] - len0seq: seq[int] = @[] - assert allIt(numbers, it < 10) == true - assert allIt(numbers, it < 9) == false - assert allIt(len0seq, false) == true - assert allIt(anumbers, it < 10) == true - assert allIt(anumbers, it < 9) == false - - block: # any - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - anumbers = [1, 4, 5, 8, 9, 7, 4] - len0seq: seq[int] = @[] - assert any(numbers, proc (x: int): bool = return x > 8) == true - assert any(numbers, proc (x: int): bool = return x > 9) == false - assert any(len0seq, proc (x: int): bool = return true) == false - assert any(anumbers, proc (x: int): bool = return x > 8) == true - assert any(anumbers, proc (x: int): bool = return x > 9) == false - - block: # anyIt - let - numbers = @[1, 4, 5, 8, 9, 7, 4] - anumbers = [1, 4, 5, 8, 9, 7, 4] - len0seq: seq[int] = @[] - assert anyIt(numbers, it > 8) == true - assert anyIt(numbers, it > 9) == false - assert anyIt(len0seq, true) == false - assert anyIt(anumbers, it > 8) == true - assert anyIt(anumbers, it > 9) == false - - block: # toSeq test - block: - let - numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9] - oddNumbers = toSeq(filter(numeric) do (x: int) -> bool: - if x mod 2 == 1: - result = true) - assert oddNumbers == @[1, 3, 5, 7, 9] - - block: - doAssert [1, 2].toSeq == @[1, 2] - doAssert @[1, 2].toSeq == @[1, 2] - - doAssert @[1, 2].toSeq == @[1, 2] - doAssert toSeq(@[1, 2]) == @[1, 2] - - block: - iterator myIter(seed: int): auto = - for i in 0..<seed: - yield i - doAssert toSeq(myIter(2)) == @[0, 1] - - block: - iterator myIter(): auto {.inline.} = - yield 1 - yield 2 - - doAssert myIter.toSeq == @[1, 2] - doAssert toSeq(myIter) == @[1, 2] - - block: - iterator myIter(): int {.closure.} = - yield 1 - yield 2 - - doAssert myIter.toSeq == @[1, 2] - doAssert toSeq(myIter) == @[1, 2] - - block: - proc myIter(): auto = - iterator ret(): int {.closure.} = - yield 1 - yield 2 - result = ret - - doAssert myIter().toSeq == @[1, 2] - doAssert toSeq(myIter()) == @[1, 2] - - block: - proc myIter(n: int): auto = - var counter = 0 - iterator ret(): int {.closure.} = - while counter < n: - yield counter - counter.inc - result = ret - - block: - let myIter3 = myIter(3) - doAssert myIter3.toSeq == @[0, 1, 2] - block: - let myIter3 = myIter(3) - doAssert toSeq(myIter3) == @[0, 1, 2] - block: - # makes sure this does not hang forever - doAssert myIter(3).toSeq == @[0, 1, 2] - doAssert toSeq(myIter(3)) == @[0, 1, 2] - - block: - # tests https://github.com/nim-lang/Nim/issues/7187 - counter = 0 - let ret = toSeq(@[1, 2, 3].identity().filter(proc (x: int): bool = x < 3)) - doAssert ret == @[1, 2] - doAssert counter == 1 - block: # foldl tests - let - numbers = @[5, 9, 11] - addition = foldl(numbers, a + b) - subtraction = foldl(numbers, a - b) - multiplication = foldl(numbers, a * b) - words = @["nim", "is", "cool"] - concatenation = foldl(words, a & b) - assert addition == 25, "Addition is (((5)+9)+11)" - assert subtraction == -15, "Subtraction is (((5)-9)-11)" - assert multiplication == 495, "Multiplication is (((5)*9)*11)" - assert concatenation == "nimiscool" - - block: # foldr tests - let - numbers = @[5, 9, 11] - addition = foldr(numbers, a + b) - subtraction = foldr(numbers, a - b) - multiplication = foldr(numbers, a * b) - words = @["nim", "is", "cool"] - concatenation = foldr(words, a & b) - assert addition == 25, "Addition is (5+(9+(11)))" - assert subtraction == 7, "Subtraction is (5-(9-(11)))" - assert multiplication == 495, "Multiplication is (5*(9*(11)))" - assert concatenation == "nimiscool" - - block: # mapIt + applyIt test - counter = 0 - var - nums = @[1, 2, 3, 4] - strings = nums.identity.mapIt($(4 * it)) - doAssert counter == 1 - nums.applyIt(it * 3) - assert nums[0] + nums[3] == 15 - assert strings[2] == "12" - - block: # newSeqWith tests - var seq2D = newSeqWith(4, newSeq[bool](2)) - seq2D[0][0] = true - seq2D[1][0] = true - seq2D[0][1] = true - doAssert seq2D == @[@[true, true], @[true, false], @[false, false], @[false, false]] - - block: # mapLiterals tests - let x = mapLiterals([0.1, 1.2, 2.3, 3.4], int) - doAssert x is array[4, int] - doAssert mapLiterals((1, ("abc"), 2), float, nested = false) == - (float(1), "abc", float(2)) - doAssert mapLiterals(([1], ("abc"), 2), `$`, nested = true) == - (["1"], "abc", "2") - - block: # mapIt with openArray - counter = 0 - proc foo(x: openArray[int]): seq[int] = x.mapIt(it * 10) - doAssert foo([identity(1), identity(2)]) == @[10, 20] - doAssert counter == 2 - - block: # mapIt with direct openArray - proc foo1(x: openArray[int]): seq[int] = x.mapIt(it * 10) - counter = 0 - doAssert foo1(openArray[int]([identity(1), identity(2)])) == @[10, 20] - doAssert counter == 2 - - # Corner cases (openArray literals should not be common) - template foo2(x: openArray[int]): seq[int] = x.mapIt(it * 10) - counter = 0 - doAssert foo2(openArray[int]([identity(1), identity(2)])) == @[10, 20] - # TODO: this fails; not sure how to fix this case - # doAssert counter == 2 - - counter = 0 - doAssert openArray[int]([identity(1), identity(2)]).mapIt(it) == @[1, 2] - # ditto - # doAssert counter == 2 - - block: # mapIt empty test, see https://github.com/nim-lang/Nim/pull/8584#pullrequestreview-144723468 - # NOTE: `[].mapIt(it)` is illegal, just as `let a = @[]` is (lacks type - # of elements) - doAssert: not compiles(mapIt(@[], it)) - doAssert: not compiles(mapIt([], it)) - doAssert newSeq[int](0).mapIt(it) == @[] - - block: # mapIt redifinition check, see https://github.com/nim-lang/Nim/issues/8580 - let s2 = [1, 2].mapIt(it) - doAssert s2 == @[1, 2] - - block: - counter = 0 - doAssert [1, 2].identity().mapIt(it*2).mapIt(it*10) == @[20, 40] - # https://github.com/nim-lang/Nim/issues/7187 test case - doAssert counter == 1 - - block: # mapIt with invalid RHS for `let` (#8566) - type X = enum - A, B - doAssert mapIt(X, $it) == @["A", "B"] - - block: - # bug #9093 - let inp = "a:b,c:d" - - let outp = inp.split(",").mapIt(it.split(":")) - doAssert outp == @[@["a", "b"], @["c", "d"]] - - - block: - proc iter(len: int): auto = - result = iterator(): int = - for i in 0..<len: - yield i - - doAssert: iter(3).mapIt(2*it).foldl(a + b) == 6 - - when not defined(testing): - echo "Finished doc tests" diff --git a/lib/pure/collections/setimpl.nim b/lib/pure/collections/setimpl.nim index d2d1490ff..360a075d6 100644 --- a/lib/pure/collections/setimpl.nim +++ b/lib/pure/collections/setimpl.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -# An ``include`` file for the different hash set implementations. +# An `include` file for the different hash set implementations. template maxHash(t): untyped = high(t.data) @@ -16,12 +16,12 @@ template dataLen(t): untyped = len(t.data) include hashcommon template initImpl(s: typed, size: int) = - assert isPowerOfTwo(size) + let correctSize = slotsNeeded(size) when s is OrderedSet: s.first = -1 s.last = -1 s.counter = 0 - newSeq(s.data, size) + newSeq(s.data, correctSize) template rawInsertImpl() {.dirty.} = if data.len == 0: @@ -48,7 +48,7 @@ template inclImpl() {.dirty.} = var hc: Hash var index = rawGet(s, key, hc) if index < 0: - if mustRehash(len(s.data), s.counter): + if mustRehash(s): enlarge(s) index = rawGetKnownHC(s, key, hc) rawInsert(s, s.data, key, hc, -1 - index) @@ -62,7 +62,8 @@ template containsOrInclImpl() {.dirty.} = if index >= 0: result = true else: - if mustRehash(len(s.data), s.counter): + result = false + if mustRehash(s): enlarge(s) index = rawGetKnownHC(s, key, hc) rawInsert(s, s.data, key, hc, -1 - index) @@ -86,7 +87,9 @@ proc exclImpl[A](s: var HashSet[A], key: A): bool {.inline.} = var j = i # The correctness of this depends on (h+1) in nextTry, var r = j # though may be adaptable to other simple sequences. s.data[i].hcode = 0 # mark current EMPTY - s.data[i].key = default(type(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} doWhile((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): i = (i + 1) and msk # increment mod table size if isEmpty(s.data[i].hcode): # end of collision cluster; So all done diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 9dd72cb56..af13135aa 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## The ``sets`` module implements an efficient `hash set`:idx: and +## The `sets` module implements an efficient `hash set`:idx: and ## ordered hash set. ## ## Hash sets are different from the `built in set type @@ -16,7 +16,7 @@ ## ## Common usages of sets: ## * removing duplicates from a container by converting it with `toHashSet proc -## <#toHashSet,openArray[A]>`_ (see also `sequtils.deduplicate proc +## <#toHashSet,openArray[A]>`_ (see also `sequtils.deduplicate func ## <sequtils.html#deduplicate,openArray[T],bool>`_) ## * membership testing ## * mathematical operations on two sets, such as @@ -25,7 +25,9 @@ ## `difference <#difference,HashSet[A],HashSet[A]>`_, and ## `symmetric difference <#symmetricDifference,HashSet[A],HashSet[A]>`_ ## -## .. code-block:: +## **Examples:** +## +## ```Nim ## echo toHashSet([9, 5, 1]) # {9, 1, 5} ## echo toOrderedSet([9, 5, 1]) # {9, 5, 1} ## @@ -37,10 +39,10 @@ ## echo s1 - s2 # {1, 9} ## echo s1 * s2 # {5} ## echo s1 -+- s2 # {9, 1, 3, 7} -## +## ``` ## ## Note: The data types declared here have *value semantics*: This means -## that ``=`` performs a copy of the set. +## that `=` performs a copy of the set. ## ## **See also:** ## * `intsets module <intsets.html>`_ for efficient int sets @@ -48,12 +50,12 @@ import - hashes, math + std/[hashes, math] -{.pragma: myShallow.} -when not defined(nimhygiene): - {.pragma: dirty.} +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} +{.pragma: myShallow.} # For "integer-like A" that are too big for intsets/bit-vectors to be practical, # it would be best to shrink hcode to the same size as the integer. Larger # codes should never be needed, and this can pack more entries per cache-line. @@ -64,7 +66,7 @@ type HashSet*[A] {.myShallow.} = object ## \ ## A generic hash set. ## - ## Use `init proc <#init,HashSet[A],int>`_ or `initHashSet proc <#initHashSet,int>`_ + ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int @@ -76,19 +78,18 @@ type OrderedSet*[A] {.myShallow.} = object ## \ ## A generic hash set that remembers insertion order. ## - ## Use `init proc <#init,OrderedSet[A],int>`_ or `initOrderedSet proc - ## <#initOrderedSet,int>`_ before calling other procs on it. + ## Use `init proc <#init,OrderedSet[A]>`_ or `initOrderedSet proc + ## <#initOrderedSet>`_ before calling other procs on it. data: OrderedKeyValuePairSeq[A] counter, first, last: int + SomeSet*[A] = HashSet[A] | OrderedSet[A] + ## Type union representing `HashSet` or `OrderedSet`. const defaultInitialSize* = 64 include setimpl -proc rightSize*(count: Natural): int {.inline.} - - # --------------------------------------------------------------------- # ------------------------------ HashSet ------------------------------ # --------------------------------------------------------------------- @@ -97,11 +98,6 @@ proc rightSize*(count: Natural): int {.inline.} proc init*[A](s: var HashSet[A], initialSize = defaultInitialSize) = ## Initializes a hash set. ## - ## The `initialSize` parameter needs to be a power of two (default: 64). - ## If you need to accept runtime values for this, you can use - ## `math.nextPowerOfTwo proc <math.html#nextPowerOfTwo,int>`_ or - ## `rightSize proc <#rightSize,Natural>`_ from this module. - ## ## Starting from Nim v0.20, sets are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -110,21 +106,19 @@ proc init*[A](s: var HashSet[A], initialSize = defaultInitialSize) = ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. ## ## See also: - ## * `initHashSet proc <#initHashSet,int>`_ + ## * `initHashSet proc <#initHashSet>`_ ## * `toHashSet proc <#toHashSet,openArray[A]>`_ runnableExamples: var a: HashSet[int] - assert(not a.isValid) init(a) - assert a.isValid initImpl(s, initialSize) proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] = - ## Wrapper around `init proc <#init,HashSet[A],int>`_ for initialization of + ## Wrapper around `init proc <#init,HashSet[A]>`_ for initialization of ## hash sets. ## - ## Returns an empty hash set you can assign directly in ``var`` blocks in a + ## Returns an empty hash set you can assign directly in `var` blocks in a ## single line. ## ## Starting from Nim v0.20, sets are initialized by default and it is @@ -136,12 +130,12 @@ proc initHashSet*[A](initialSize = defaultInitialSize): HashSet[A] = var a = initHashSet[int]() a.incl(3) assert len(a) == 1 - + result = default(HashSet[A]) result.init(initialSize) proc `[]`*[A](s: var HashSet[A], key: A): var A = ## Returns the element that is actually stored in `s` which has the same - ## value as `key` or raises the ``KeyError`` exception. + ## value as `key` or raises the `KeyError` exception. ## ## This is useful when one overloaded `hash` and `==` but still needs ## reference semantics for sharing. @@ -175,6 +169,27 @@ proc contains*[A](s: HashSet[A], key: A): bool = var index = rawGet(s, key, hc) result = index >= 0 +proc len*[A](s: HashSet[A]): int = + ## Returns the number of elements in `s`. + ## + ## Due to an implementation detail you can call this proc on variables which + ## have not been initialized yet. The proc will return zero as the length + ## then. + runnableExamples: + var a: HashSet[string] + assert len(a) == 0 + let s = toHashSet([3, 5, 7]) + assert len(s) == 3 + + result = s.counter + +proc card*[A](s: HashSet[A]): int = + ## Alias for `len() <#len,HashSet[A]>`_. + ## + ## Card stands for the `cardinality + ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. + result = s.counter + proc incl*[A](s: var HashSet[A], key: A) = ## Includes an element `key` in `s`. ## @@ -217,7 +232,7 @@ proc toHashSet*[A](keys: openArray[A]): HashSet[A] = ## Duplicates are removed. ## ## See also: - ## * `initHashSet proc <#initHashSet,int>`_ + ## * `initHashSet proc <#initHashSet>`_ runnableExamples: let a = toHashSet([5, 3, 2]) @@ -227,7 +242,7 @@ proc toHashSet*[A](keys: openArray[A]): HashSet[A] = assert len(b) == 5 ## b == {'a', 'b', 'c', 'd', 'r'} - result = initHashSet[A](rightSize(keys.len)) + result = initHashSet[A](keys.len) for key in items(keys): result.incl(key) iterator items*[A](s: HashSet[A]): A = @@ -236,7 +251,7 @@ iterator items*[A](s: HashSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim ## type ## pair = tuple[a, b: int] ## var @@ -249,8 +264,12 @@ iterator items*[A](s: HashSet[A]): A = ## assert a.len == 2 ## echo b ## # --> {(a: 1, b: 3), (a: 0, b: 4)} + ## ``` + let length = s.len for h in 0 .. high(s.data): - if isFilled(s.data[h].hcode): yield s.data[h].key + if isFilled(s.data[h].hcode): + yield s.data[h].key + assert(len(s) == length, "the length of the HashSet changed while iterating over it") proc containsOrIncl*[A](s: var HashSet[A], key: A): bool = ## Includes `key` in the set `s` and tells if `key` was already in `s`. @@ -329,16 +348,15 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = exclImpl(s, key) proc pop*[A](s: var HashSet[A]): A = - ## Remove and return an arbitrary element from the set `s`. + ## Removes and returns an arbitrary element from the set `s`. ## - ## Raises KeyError if the set `s` is empty. + ## Raises `KeyError` if the set `s` is empty. ## ## See also: ## * `clear proc <#clear,HashSet[A]>`_ runnableExamples: var s = toHashSet([2, 1]) - assert s.pop == 1 - assert s.pop == 2 + assert [s.pop, s.pop] in [[1, 2], [2,1]] # order unspecified doAssertRaises(KeyError, echo s.pop) for h in 0 .. high(s.data): @@ -364,28 +382,9 @@ proc clear*[A](s: var HashSet[A]) = s.counter = 0 for i in 0 ..< s.data.len: s.data[i].hcode = 0 - s.data[i].key = default(type(s.data[i].key)) - -proc len*[A](s: HashSet[A]): int = - ## Returns the number of elements in `s`. - ## - ## Due to an implementation detail you can call this proc on variables which - ## have not been initialized yet. The proc will return zero as the length - ## then. - runnableExamples: - var a: HashSet[string] - assert len(a) == 0 - let s = toHashSet([3, 5, 7]) - assert len(s) == 3 - - result = s.counter - -proc card*[A](s: HashSet[A]): int = - ## Alias for `len() <#len,HashSet[A]>`_. - ## - ## Card stands for the `cardinality - ## <http://en.wikipedia.org/wiki/Cardinality>`_ of a set. - result = s.counter + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc union*[A](s1, s2: HashSet[A]): HashSet[A] = @@ -431,8 +430,15 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = assert c == toHashSet(["b"]) result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) - for item in s1: - if item in s2: incl(result, item) + + # iterate over the elements of the smaller set + if s1.data.len < s2.data.len: + for item in s1: + if item in s2: incl(result, item) + else: + for item in s2: + if item in s1: incl(result, item) + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. @@ -558,7 +564,7 @@ proc `==`*[A](s, t: HashSet[A]): bool = s.counter == t.counter and s <= t -proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] = +proc map*[A, B](data: HashSet[A], op: proc (x: A): B {.closure.}): HashSet[B] {.effectsOf: op.} = ## Returns a new set after applying `op` proc on each of the elements of ##`data` set. ## @@ -585,23 +591,14 @@ proc `$`*[A](s: HashSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toHashSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toHashSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() -proc rightSize*(count: Natural): int {.inline.} = - ## Return the value of `initialSize` to support `count` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - ## - ## Internally, we want `mustRehash(rightSize(x), x) == false`. - result = nextPowerOfTwo(count * 3 div 2 + 4) - proc initSet*[A](initialSize = defaultInitialSize): HashSet[A] {.deprecated: "Deprecated since v0.20, use 'initHashSet'".} = initHashSet[A](initialSize) @@ -612,14 +609,12 @@ proc toSet*[A](keys: openArray[A]): HashSet[A] {.deprecated: proc isValid*[A](s: HashSet[A]): bool {.deprecated: "Deprecated since v0.20; sets are initialized by default".} = ## Returns `true` if the set has been initialized (with `initHashSet proc - ## <#initHashSet,int>`_ or `init proc <#init,HashSet[A],int>`_). + ## <#initHashSet>`_ or `init proc <#init,HashSet[A]>`_). ## - ## **Examples:** - ## - ## .. code-block :: - ## proc savePreferences(options: HashSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! + runnableExamples: + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + # Do stuff here, may crash in release builds! result = s.data.len > 0 @@ -643,11 +638,6 @@ template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = proc init*[A](s: var OrderedSet[A], initialSize = defaultInitialSize) = ## Initializes an ordered hash set. ## - ## The `initialSize` parameter needs to be a power of two (default: 64). - ## If you need to accept runtime values for this, you can use - ## `math.nextPowerOfTwo proc <math.html#nextPowerOfTwo,int>`_ or - ## `rightSize proc <#rightSize,Natural>`_ from this module. - ## ## Starting from Nim v0.20, sets are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -656,21 +646,19 @@ proc init*[A](s: var OrderedSet[A], initialSize = defaultInitialSize) = ## existing values and calling `excl() <#excl,HashSet[A],A>`_ on them. ## ## See also: - ## * `initOrderedSet proc <#initOrderedSet,int>`_ + ## * `initOrderedSet proc <#initOrderedSet>`_ ## * `toOrderedSet proc <#toOrderedSet,openArray[A]>`_ runnableExamples: var a: OrderedSet[int] - assert(not a.isValid) init(a) - assert a.isValid initImpl(s, initialSize) proc initOrderedSet*[A](initialSize = defaultInitialSize): OrderedSet[A] = - ## Wrapper around `init proc <#init,OrderedSet[A],int>`_ for initialization of + ## Wrapper around `init proc <#init,OrderedSet[A]>`_ for initialization of ## ordered hash sets. ## - ## Returns an empty ordered hash set you can assign directly in ``var`` blocks + ## Returns an empty ordered hash set you can assign directly in `var` blocks ## in a single line. ## ## Starting from Nim v0.20, sets are initialized by default and it is @@ -692,7 +680,7 @@ proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = ## Duplicates are removed. ## ## See also: - ## * `initOrderedSet proc <#initOrderedSet,int>`_ + ## * `initOrderedSet proc <#initOrderedSet>`_ runnableExamples: let a = toOrderedSet([5, 3, 2]) @@ -702,7 +690,7 @@ proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = assert len(b) == 5 ## b == {'a', 'b', 'r', 'c', 'd'} # different than in HashSet - result = initOrderedSet[A](rightSize(keys.len)) + result = initOrderedSet[A](keys.len) for key in items(keys): result.incl(key) proc contains*[A](s: OrderedSet[A], key: A): bool = @@ -830,7 +818,9 @@ proc clear*[A](s: var OrderedSet[A]) = for i in 0 ..< s.data.len: s.data[i].hcode = 0 s.data[i].next = 0 - s.data[i].key = default(type(s.data[i].key)) + {.push warning[UnsafeDefault]:off.} + reset(s.data[i].key) + {.pop.} proc len*[A](s: OrderedSet[A]): int {.inline.} = ## Returns the number of elements in `s`. @@ -891,12 +881,12 @@ proc `$`*[A](s: OrderedSet[A]): string = ## any moment and values are not escaped. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## echo toOrderedSet([2, 4, 5]) ## # --> {2, 4, 5} ## echo toOrderedSet(["no", "esc'aping", "is \" provided"]) ## # --> {no, esc'aping, is " provided} + ## ``` dollarImpl() @@ -907,7 +897,7 @@ iterator items*[A](s: OrderedSet[A]): A = ## If you need a sequence with the elements you can use `sequtils.toSeq ## template <sequtils.html#toSeq.t,untyped>`_. ## - ## .. code-block:: + ## ```Nim ## var a = initOrderedSet[int]() ## for value in [9, 2, 1, 5, 1, 8, 4, 2]: ## a.incl(value) @@ -919,8 +909,11 @@ iterator items*[A](s: OrderedSet[A]): A = ## # --> Got 5 ## # --> Got 8 ## # --> Got 4 + ## ``` + let length = s.len forAllOrderedPairs: yield s.data[h].key + assert(len(s) == length, "the length of the OrderedSet changed while iterating over it") iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = ## Iterates through (position, value) tuples of OrderedSet `s`. @@ -931,311 +924,7 @@ iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] = p.add(x) assert p == @[(0, 'a'), (1, 'b'), (2, 'r'), (3, 'c'), (4, 'd')] + let length = s.len forAllOrderedPairs: yield (idx, s.data[h].key) - - - -proc isValid*[A](s: OrderedSet[A]): bool {.deprecated: - "Deprecated since v0.20; sets are initialized by default".} = - ## - ## Returns `true` if the set has been initialized (with `initHashSet proc - ## <#initOrderedSet,int>`_ or `init proc <#init,OrderedSet[A],int>`_). - ## - ## **Examples:** - ## - ## .. code-block :: - ## proc savePreferences(options: OrderedSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! - result = s.data.len > 0 - - -# ----------------------------------------------------------------------- - - - -when isMainModule and not defined(release): - proc testModule() = - ## Internal micro test to validate docstrings and such. - block isValidTest: # isValid is deprecated - var options: HashSet[string] - proc savePreferences(options: HashSet[string]) = - assert options.isValid, "Pass an initialized set!" - options = initHashSet[string]() - options.savePreferences - - block lenTest: - var values: HashSet[int] - assert(not values.isValid) - assert values.len == 0 - assert values.card == 0 - - block setIterator: - type pair = tuple[a, b: int] - var a, b = initHashSet[pair]() - a.incl((2, 3)) - a.incl((3, 2)) - a.incl((2, 3)) - for x, y in a.items: - b.incl((x - 2, y + 1)) - assert a.len == b.card - assert a.len == 2 - #echo b - - block setContains: - var values = initHashSet[int]() - assert(not values.contains(2)) - values.incl(2) - assert values.contains(2) - values.excl(2) - assert(not values.contains(2)) - - values.incl(4) - var others = toHashSet([6, 7]) - values.incl(others) - assert values.len == 3 - - values.init - assert values.containsOrIncl(2) == false - assert values.containsOrIncl(2) == true - var - a = toHashSet([1, 2]) - b = toHashSet([1]) - b.incl(2) - assert a == b - - block exclusions: - var s = toHashSet([2, 3, 6, 7]) - s.excl(2) - s.excl(2) - assert s.len == 3 - - var - numbers = toHashSet([1, 2, 3, 4, 5]) - even = toHashSet([2, 4, 6, 8]) - numbers.excl(even) - #echo numbers - # --> {1, 3, 5} - - block toSeqAndString: - var a = toHashSet([2, 7, 5]) - var b = initHashSet[int]() - for x in [2, 7, 5]: b.incl(x) - assert($a == $b) - #echo a - #echo toHashSet(["no", "esc'aping", "is \" provided"]) - - #block orderedToSeqAndString: - # echo toOrderedSet([2, 4, 5]) - # echo toOrderedSet(["no", "esc'aping", "is \" provided"]) - - block setOperations: - var - a = toHashSet(["a", "b"]) - b = toHashSet(["b", "c"]) - c = union(a, b) - assert c == toHashSet(["a", "b", "c"]) - var d = intersection(a, b) - assert d == toHashSet(["b"]) - var e = difference(a, b) - assert e == toHashSet(["a"]) - var f = symmetricDifference(a, b) - assert f == toHashSet(["a", "c"]) - assert d < a and d < b - assert((a < a) == false) - assert d <= a and d <= b - assert((a <= a)) - # Alias test. - assert a + b == toHashSet(["a", "b", "c"]) - assert a * b == toHashSet(["b"]) - assert a - b == toHashSet(["a"]) - assert a -+- b == toHashSet(["a", "c"]) - assert disjoint(a, b) == false - assert disjoint(a, b - a) == true - - block mapSet: - var a = toHashSet([1, 2, 3]) - var b = a.map(proc (x: int): string = $x) - assert b == toHashSet(["1", "2", "3"]) - - block isValidTest: # isValid is deprecated - var cards: OrderedSet[string] - proc saveTarotCards(cards: OrderedSet[string]) = - assert cards.isValid, "Pass an initialized set!" - cards = initOrderedSet[string]() - cards.saveTarotCards - - block lenTest: - var values: OrderedSet[int] - assert(not values.isValid) - assert values.len == 0 - assert values.card == 0 - - block setIterator: - type pair = tuple[a, b: int] - var a, b = initOrderedSet[pair]() - a.incl((2, 3)) - a.incl((3, 2)) - a.incl((2, 3)) - for x, y in a.items: - b.incl((x - 2, y + 1)) - assert a.len == b.card - assert a.len == 2 - - block setPairsIterator: - var s = toOrderedSet([1, 3, 5, 7]) - var items = newSeq[tuple[a: int, b: int]]() - for idx, item in s: items.add((idx, item)) - assert items == @[(0, 1), (1, 3), (2, 5), (3, 7)] - - block exclusions: - var s = toOrderedSet([1, 2, 3, 6, 7, 4]) - - s.excl(3) - s.excl(3) - s.excl(1) - s.excl(4) - - var items = newSeq[int]() - for item in s: items.add item - assert items == @[2, 6, 7] - - block: #9005 - var s = initOrderedSet[(int, int)]() - for i in 0 .. 30: incl(s, (i, 0)) - for i in 0 .. 30: excl(s, (i, 0)) - doAssert s.len == 0 - - #block orderedSetIterator: - # var a = initOrderedSet[int]() - # for value in [9, 2, 1, 5, 1, 8, 4, 2]: - # a.incl(value) - # for value in a.items: - # echo "Got ", value - - block setContains: - var values = initOrderedSet[int]() - assert(not values.contains(2)) - values.incl(2) - assert values.contains(2) - - block toSeqAndString: - var a = toOrderedSet([2, 4, 5]) - var b = initOrderedSet[int]() - for x in [2, 4, 5]: b.incl(x) - assert($a == $b) - assert(a == b) # https://github.com/Araq/Nim/issues/1413 - - block initBlocks: - var a: OrderedSet[int] - a.init(4) - a.incl(2) - a.init - assert a.len == 0 and a.isValid - a = initOrderedSet[int](4) - a.incl(2) - assert a.len == 1 - - var b: HashSet[int] - b.init(4) - b.incl(2) - b.init - assert b.len == 0 and b.isValid - b = initHashSet[int](4) - b.incl(2) - assert b.len == 1 - - for i in 0 .. 32: - var s = rightSize(i) - if s <= i or mustRehash(s, i): - echo "performance issue: rightSize() will not elide enlarge() at ", i - - block missingOrExcl: - var s = toOrderedSet([2, 3, 6, 7]) - assert s.missingOrExcl(4) == true - assert s.missingOrExcl(6) == false - - block orderedSetEquality: - type pair = tuple[a, b: int] - - var aa = initOrderedSet[pair]() - var bb = initOrderedSet[pair]() - - var x = (a: 1, b: 2) - var y = (a: 3, b: 4) - - aa.incl(x) - aa.incl(y) - - bb.incl(x) - bb.incl(y) - assert aa == bb - - block setsWithoutInit: - var - a: HashSet[int] - b: HashSet[int] - c: HashSet[int] - d: HashSet[int] - e: HashSet[int] - - doAssert a.containsOrIncl(3) == false - doAssert a.contains(3) - doAssert a.len == 1 - doAssert a.containsOrIncl(3) - a.incl(3) - doAssert a.len == 1 - a.incl(6) - doAssert a.len == 2 - - b.incl(5) - doAssert b.len == 1 - b.excl(5) - b.excl(c) - doAssert b.missingOrExcl(5) - doAssert b.disjoint(c) - - d = b + c - doAssert d.len == 0 - d = b * c - doAssert d.len == 0 - d = b - c - doAssert d.len == 0 - d = b -+- c - doAssert d.len == 0 - - doAssert (d < e) == false - doAssert d <= e - doAssert d == e - - block setsWithoutInit: - var - a: OrderedSet[int] - b: OrderedSet[int] - c: OrderedSet[int] - d: HashSet[int] - - - doAssert a.containsOrIncl(3) == false - doAssert a.contains(3) - doAssert a.len == 1 - doAssert a.containsOrIncl(3) - a.incl(3) - doAssert a.len == 1 - a.incl(6) - doAssert a.len == 2 - - b.incl(5) - doAssert b.len == 1 - doAssert b.missingOrExcl(5) == false - doAssert b.missingOrExcl(5) - - doAssert c.missingOrExcl(9) - d.incl(c) - doAssert d.len == 0 - - when not defined(testing): - echo "Micro tests run successfully." - - testModule() + assert(len(s) == length, "the length of the OrderedSet changed while iterating over it") diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index f9182acce..ec8f1cd86 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -11,10 +11,12 @@ ## ## Unstable API. +{.deprecated.} + {.push stackTrace: off.} import - locks + std/locks const ElemsPerNode = 100 @@ -35,8 +37,10 @@ template withLock(t, x: untyped) = release(t.lock) proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = - ## iterates over the list. If 'action' returns true, the + ## Iterates over the list. If `action` returns true, the ## current item is removed from the list. + ## + ## .. warning:: It may not preserve the element order after some modifications. withLock(x): var n = x.head while n != nil: @@ -47,8 +51,9 @@ proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = if action(n.d[i]): acquire(x.lock) let t = x.tail + dec t.dataLen # TODO considering t.dataLen == 0, + # probably the module should be refactored using doubly linked lists n.d[i] = t.d[t.dataLen] - dec t.dataLen else: acquire(x.lock) inc i @@ -65,11 +70,14 @@ iterator items*[A](x: var SharedList[A]): A = proc add*[A](x: var SharedList[A]; y: A) = withLock(x): var node: SharedListNode[A] - if x.tail == nil or x.tail.dataLen == ElemsPerNode: - node = cast[type node](allocShared0(sizeof(node[]))) - node.next = x.tail + if x.tail == nil: + node = cast[typeof node](allocShared0(sizeof(node[]))) + x.tail = node + x.head = node + elif x.tail.dataLen == ElemsPerNode: + node = cast[typeof node](allocShared0(sizeof(node[]))) + x.tail.next = node x.tail = node - if x.head == nil: x.head = node else: node = x.tail node.d[node.dataLen] = y @@ -94,10 +102,4 @@ proc deinitSharedList*[A](t: var SharedList[A]) = clear(t) deinitLock t.lock -proc initSharedList*[A](): SharedList[A] {.deprecated: "use 'init' instead".} = - ## This is not posix compliant, may introduce undefined behavior. - initLock result.lock - result.head = nil - result.tail = nil - {.pop.} diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 1a6148752..b474ecd31 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -14,10 +14,10 @@ ## ## Unstable API. -import - hashes, math, locks +{.deprecated.} -include "system/inclrtl" +import + std/[hashes, math, locks] type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] @@ -32,7 +32,7 @@ template maxHash(t): untyped = t.dataLen-1 include tableimpl template st_maybeRehashPutImpl(enlarge) {.dirty.} = - if mustRehash(t.dataLen, t.counter): + if mustRehash(t): enlarge(t) index = rawGetKnownHC(t, key, hc) index = -1 - index # important to transform for mgetOrPutImpl @@ -62,17 +62,27 @@ template withLock(t, x: untyped) = template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body: untyped) = - ## retrieves the value at ``t[key]``. - ## `value` can be modified in the scope of the ``withValue`` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if ``key`` in ``t`` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## + ## Retrieves the value at `t[key]`. + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + table.withValue("a", value): + assert value[] == "x" + + table.withValue("b", value): + value[] = "modified" + + table.withValue("b", value): + assert value[] == "modified" + + table.withValue("nonexistent", value): + assert false # not called acquire(t.lock) try: var hc: Hash @@ -86,20 +96,33 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at ``t[key]``. - ## `value` can be modified in the scope of the ``withValue`` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if ``key`` in ``t`` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## do: - ## # block is executed when ``key`` not in ``t`` - ## raise newException(KeyError, "Key not found") - ## + ## Retrieves the value at `t[key]`. + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + + table.withValue("a", value): + value[] = "m" + + var flag = false + table.withValue("d", value): + discard value + doAssert false + do: # if "d" notin table + flag = true + + if flag: + table["d"] = "n" + + assert table.mget("a") == "m" + assert table.mget("d") == "n" + acquire(t.lock) try: var hc: Hash @@ -114,8 +137,8 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, release(t.lock) proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If `key` is not in `t`, the ``KeyError`` exception is raised. + ## Retrieves the value at `t[key]`. The value can be modified. + ## If `key` is not in `t`, the `KeyError` exception is raised. withLock t: var hc: Hash var index = rawGet(t, key, hc) @@ -128,44 +151,47 @@ proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = raise newException(KeyError, "key not found") proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = - ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. **Note**: This is inherently ## unsafe in the context of multi-threading since it returns a pointer - ## to ``B``. + ## to `B`. withLock t: mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = - ## returns true iff `key` is in the table, otherwise inserts `value`. + ## Returns true if `key` is in the table, otherwise inserts `value`. withLock t: hasKeyOrPutImpl(enlarge) +template tabMakeEmpty(i) = t.data[i].hcode = 0 +template tabCellEmpty(i) = isEmpty(t.data[i].hcode) +template tabCellHash(i) = t.data[i].hcode + proc withKey*[A, B](t: var SharedTable[A, B], key: A, mapper: proc(key: A, val: var B, pairExists: var bool)) = - ## Computes a new mapping for the ``key`` with the specified ``mapper`` + ## Computes a new mapping for the `key` with the specified `mapper` ## procedure. ## - ## The ``mapper`` takes 3 arguments: + ## The `mapper` takes 3 arguments: ## - ## 1. ``key`` - the current key, if it exists, or the key passed to - ## ``withKey`` otherwise; - ## 2. ``val`` - the current value, if the key exists, or default value + ## 1. `key` - the current key, if it exists, or the key passed to + ## `withKey` otherwise; + ## 2. `val` - the current value, if the key exists, or default value ## of the type otherwise; - ## 3. ``pairExists`` - ``true`` if the key exists, ``false`` otherwise. + ## 3. `pairExists` - `true` if the key exists, `false` otherwise. ## - ## The ``mapper`` can can modify ``val`` and ``pairExists`` values to change + ## The `mapper` can can modify `val` and `pairExists` values to change ## the mapping of the key or delete it from the table. - ## When adding a value, make sure to set ``pairExists`` to ``true`` along - ## with modifying the ``val``. + ## When adding a value, make sure to set `pairExists` to `true` along + ## with modifying the `val`. ## ## The operation is performed atomically and other operations on the table - ## will be blocked while the ``mapper`` is invoked, so it should be short and + ## will be blocked while the `mapper` is invoked, so it should be short and ## simple. ## ## Example usage: ## - ## .. code-block:: nim - ## + ## ```nim ## # If value exists, decrement it. ## # If it becomes zero or less, delete the key ## t.withKey(1'i64) do (k: int64, v: var int, pairExists: var bool): @@ -173,6 +199,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, ## dec v ## if v <= 0: ## pairExists = false + ## ``` withLock t: var hc: Hash var index = rawGet(t, key, hc) @@ -181,7 +208,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, if pairExists: mapper(t.data[index].key, t.data[index].val, pairExists) if not pairExists: - delImplIdx(t, index) + delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash) else: var val: B mapper(key, val, pairExists) @@ -189,30 +216,31 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, st_maybeRehashPutImpl(enlarge) proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. + ## Puts a (key, value)-pair into `t`. withLock t: putImpl(enlarge) proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + ## Puts a new (key, value)-pair into `t` even if `t[key]` already exists. ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) proc del*[A, B](t: var SharedTable[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. + withLock t: + delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) + +proc len*[A, B](t: var SharedTable[A, B]): int = + ## Number of elements in `t`. withLock t: - delImpl() + result = t.counter -proc init*[A, B](t: var SharedTable[A, B], initialSize = 64) = - ## creates a new hash table that is empty. +proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = + ## Creates a new hash table that is empty. ## ## This proc must be called before any other usage of `t`. - ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc from this module. - assert isPowerOfTwo(initialSize) + let initialSize = slotsNeeded(initialSize) t.counter = 0 t.dataLen = initialSize t.data = cast[KeyValuePairSeq[A, B]](allocShared0( @@ -222,13 +250,3 @@ proc init*[A, B](t: var SharedTable[A, B], initialSize = 64) = proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) = deallocShared(t.data) deinitLock t.lock - -proc initSharedTable*[A, B](initialSize = 64): SharedTable[A, B] {.deprecated: - "use 'init' instead".} = - ## This is not posix compliant, may introduce undefined behavior. - assert isPowerOfTwo(initialSize) - result.counter = 0 - result.dataLen = initialSize - result.data = cast[KeyValuePairSeq[A, B]](allocShared0( - sizeof(KeyValuePair[A, B]) * initialSize)) - initLock result.lock diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index 85bffd60f..3542741fa 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -7,10 +7,13 @@ # distribution, for details about the copyright. # -# An ``include`` file for the different table implementations. +# An `include` file for the different table implementations. include hashcommon +const + defaultInitialSize* = 32 + template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add genHashImpl(key, hc) var h: Hash = hc and maxHash(t) @@ -23,29 +26,28 @@ template rawInsertImpl() {.dirty.} = data[h].val = val data[h].hcode = hc -proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline.} = +proc rawGetDeep[X, A](t: X, key: A, hc: var Hash): int {.inline, outParamsAt: [3].} = rawGetDeepImpl() proc rawInsert[X, A, B](t: var X, data: var KeyValuePairSeq[A, B], - key: A, val: B, hc: Hash, h: Hash) = + key: A, val: sink B, hc: Hash, h: Hash) = rawInsertImpl() template checkIfInitialized() = - when compiles(defaultInitialSize): - if t.dataLen == 0: - initImpl(t, defaultInitialSize) + if t.dataLen == 0: + initImpl(t, defaultInitialSize) template addImpl(enlarge) {.dirty.} = checkIfInitialized() - if mustRehash(t.dataLen, t.counter): enlarge(t) + if mustRehash(t): enlarge(t) var hc: Hash var j = rawGetDeep(t, key, hc) rawInsert(t, t.data, key, val, hc, j) inc(t.counter) -template maybeRehashPutImpl(enlarge) {.dirty.} = +template maybeRehashPutImpl(enlarge, val) {.dirty.} = checkIfInitialized() - if mustRehash(t.dataLen, t.counter): + if mustRehash(t): enlarge(t) index = rawGetKnownHC(t, key, hc) index = -1 - index # important to transform for mgetOrPutImpl @@ -54,46 +56,91 @@ template maybeRehashPutImpl(enlarge) {.dirty.} = template putImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index >= 0: t.data[index].val = val - else: maybeRehashPutImpl(enlarge) + else: maybeRehashPutImpl(enlarge, val) template mgetOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: # not present: insert (flipping index) - maybeRehashPutImpl(enlarge) + when declared(val): + maybeRehashPutImpl(enlarge, val) + else: + maybeRehashPutImpl(enlarge, default(B)) # either way return modifiable val result = t.data[index].val +# template mgetOrPutDefaultImpl(enlarge) {.dirty.} = +# checkIfInitialized() +# var hc: Hash = default(Hash) +# var index = rawGet(t, key, hc) +# if index < 0: +# # not present: insert (flipping index) +# maybeRehashPutImpl(enlarge, default(B)) +# # either way return modifiable val +# result = t.data[index].val + template hasKeyOrPutImpl(enlarge) {.dirty.} = checkIfInitialized() - var hc: Hash + var hc: Hash = default(Hash) var index = rawGet(t, key, hc) if index < 0: result = false - maybeRehashPutImpl(enlarge) + maybeRehashPutImpl(enlarge, val) else: result = true -template delImplIdx(t, i) = +# delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to +# be called "back shift delete". It shifts elements in the collision cluster of +# a victim backward to make things as-if the victim were never inserted in the +# first place. This is desirable to keep things "ageless" after many deletes. +# It is trickier than you might guess since initial probe (aka "home") locations +# of keys in a cluster may collide and since table addresses wrap around. +# +# A before-after diagram might look like ('.' means empty): +# slot: 0 1 2 3 4 5 6 7 +# before(1) +# hash1: 6 7 . 3 . 5 5 6 ; Really hash() and msk +# data1: E F . A . B C D ; About to delete C @index 6 +# after(2) +# hash2: 7 . . 3 . 5 6 6 ; Really hash() and msk +# data2: F . . A . B D E ; After deletion of C +# +# This lowers total search depth over the whole table from 1+1+2+2+2+2=10 to 7. +# Had the victim been B@5, C would need back shifting to slot 5. Total depth is +# always lowered by at least 1, e.g. victim A@3. This is all quite fast when +# empty slots are frequent (also needed to keep insert/miss searches fast) and +# hash() is either fast or avoided (via `.hcode`). It need not compare keys. +# +# delImplIdx realizes the above transformation, but only works for dense Linear +# Probing, nextTry(h)=h+1. This is not an important limitation since that's the +# fastest sequence on any CPU made since the 1980s. { Performance analysis often +# overweights "key cmp" neglecting cache behavior, giving bad ideas how big/slow +# tables behave (when perf matters most!). Comparing hcode first means usually +# only 1 key cmp is needed for *any* seq. Timing only predictable activity, +# small tables, and/or integer keys often perpetuates such bad ideas. } + +template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) = let msk = maxHash(t) if i >= 0: dec(t.counter) block outer: while true: # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1 - var j = i # The correctness of this depends on (h+1) in nextTry, + var j = i # The correctness of this depends on (h+1) in nextTry var r = j # though may be adaptable to other simple sequences. - t.data[i].hcode = 0 # mark current EMPTY - t.data[i].key = default(type(t.data[i].key)) - t.data[i].val = default(type(t.data[i].val)) + makeEmpty(i) # mark current EMPTY + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} while true: i = (i + 1) and msk # increment mod table size - if isEmpty(t.data[i].hcode): # end of collision cluster; So all done + if cellEmpty(i): # end of collision cluster; So all done break outer - r = t.data[i].hcode and msk # "home" location of key@i + r = cellHash(i) and msk # initial probe index for key@slot i if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)): break when defined(js): @@ -101,30 +148,50 @@ template delImplIdx(t, i) = else: t.data[j] = move(t.data[i]) # data[j] will be marked EMPTY next loop -template delImpl() {.dirty.} = +template delImpl(makeEmpty, cellEmpty, cellHash) {.dirty.} = var hc: Hash var i = rawGet(t, key, hc) - delImplIdx(t, i) + delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) + +template delImplNoHCode(makeEmpty, cellEmpty, cellHash) {.dirty.} = + if t.dataLen > 0: + var i: Hash = hash(key) and maxHash(t) + while not cellEmpty(i): + if t.data[i].key == key: + delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) + break + i = nextTry(i, maxHash(t)) template clearImpl() {.dirty.} = for i in 0 ..< t.dataLen: when compiles(t.data[i].hcode): # CountTable records don't contain a hcode t.data[i].hcode = 0 - t.data[i].key = default(type(t.data[i].key)) - t.data[i].val = default(type(t.data[i].val)) + {.push warning[UnsafeDefault]:off.} + reset(t.data[i].key) + reset(t.data[i].val) + {.pop.} t.counter = 0 +template ctAnd(a, b): bool = + when a: + when b: true + else: false + else: false + template initImpl(result: typed, size: int) = - assert isPowerOfTwo(size) - result.counter = 0 - newSeq(result.data, size) - when compiles(result.first): - result.first = -1 - result.last = -1 + let correctSize = slotsNeeded(size) + when ctAnd(declared(SharedTable), typeof(result) is SharedTable): + init(result, correctSize) + else: + result.counter = 0 + newSeq(result.data, correctSize) + when compiles(result.first): + result.first = -1 + result.last = -1 template insertImpl() = # for CountTable if t.dataLen == 0: initImpl(t, defaultInitialSize) - if mustRehash(len(t.data), t.counter): enlarge(t) + if mustRehash(t): enlarge(t) ctRawInsert(t, t.data, key, val) inc(t.counter) @@ -160,3 +227,5 @@ template equalsImpl(s, t: typed) = if not t.hasKey(key): return false if t.getOrDefault(key) != val: return false return true + else: + return false diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 20fe12fc8..d414caeed 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -7,219 +7,198 @@ # distribution, for details about the copyright. # -## The ``tables`` module implements variants of an efficient `hash table`:idx: +## The `tables` module implements variants of an efficient `hash table`:idx: ## (also often named `dictionary`:idx: in other programming languages) that is ## a mapping from keys to values. ## ## There are several different types of hash tables available: ## * `Table<#Table>`_ is the usual hash table, -## * `OrderedTable<#OrderedTable>`_ is like ``Table`` but remembers insertion order, +## * `OrderedTable<#OrderedTable>`_ is like `Table` but remembers insertion order, ## * `CountTable<#CountTable>`_ is a mapping from a key to its number of occurrences ## ## For consistency with every other data type in Nim these have **value** -## semantics, this means that ``=`` performs a copy of the hash table. +## semantics, this means that `=` performs a copy of the hash table. ## ## For `ref semantics<manual.html#types-reference-and-pointer-types>`_ -## use their ``Ref`` variants: `TableRef<#TableRef>`_, +## use their `Ref` variants: `TableRef<#TableRef>`_, ## `OrderedTableRef<#OrderedTableRef>`_, and `CountTableRef<#CountTableRef>`_. ## -## To give an example, when ``a`` is a ``Table``, then ``var b = a`` gives ``b`` -## as a new independent table. ``b`` is initialised with the contents of ``a``. -## Changing ``b`` does not affect ``a`` and vice versa: -## -## .. code-block:: -## import tables -## -## var -## a = {1: "one", 2: "two"}.toTable # creates a Table -## b = a -## -## echo a, b # output: {1: one, 2: two}{1: one, 2: two} -## -## b[3] = "three" -## echo a, b # output: {1: one, 2: two}{1: one, 2: two, 3: three} -## echo a == b # output: false -## -## On the other hand, when ``a`` is a ``TableRef`` instead, then changes to ``b`` -## also affect ``a``. Both ``a`` and ``b`` **ref** the same data structure: -## -## .. code-block:: -## import tables -## -## var -## a = {1: "one", 2: "two"}.newTable # creates a TableRef -## b = a -## -## echo a, b # output: {1: one, 2: two}{1: one, 2: two} -## -## b[3] = "three" -## echo a, b # output: {1: one, 2: two, 3: three}{1: one, 2: two, 3: three} -## echo a == b # output: true +## To give an example, when `a` is a `Table`, then `var b = a` gives `b` +## as a new independent table. `b` is initialised with the contents of `a`. +## Changing `b` does not affect `a` and vice versa: + +runnableExamples: + var + a = {1: "one", 2: "two"}.toTable # creates a Table + b = a + + assert a == b + + b[3] = "three" + assert 3 notin a + assert 3 in b + assert a != b + +## On the other hand, when `a` is a `TableRef` instead, then changes to `b` +## also affect `a`. Both `a` and `b` **ref** the same data structure: + +runnableExamples: + var + a = {1: "one", 2: "two"}.newTable # creates a TableRef + b = a + + assert a == b + + b[3] = "three" + + assert 3 in a + assert 3 in b + assert a == b + ## ## ---- ## -## Basic usage -## =========== -## -## Table -## ----- -## -## .. code-block:: -## import tables -## from sequtils import zip -## -## let -## names = ["John", "Paul", "George", "Ringo"] -## years = [1940, 1942, 1943, 1940] -## -## var beatles = initTable[string, int]() -## -## for pairs in zip(names, years): -## let (name, birthYear) = pairs -## beatles[name] = birthYear -## -## echo beatles -## # {"George": 1943, "Ringo": 1940, "Paul": 1942, "John": 1940} -## -## -## var beatlesByYear = initTable[int, seq[string]]() -## -## for pairs in zip(years, names): -## let (birthYear, name) = pairs -## if not beatlesByYear.hasKey(birthYear): -## # if a key doesn't exist, we create one with an empty sequence -## # before we can add elements to it -## beatlesByYear[birthYear] = @[] -## beatlesByYear[birthYear].add(name) -## -## echo beatlesByYear -## # {1940: @["John", "Ringo"], 1942: @["Paul"], 1943: @["George"]} -## -## -## -## OrderedTable -## ------------ -## + +## # Basic usage + + +## ## Table +runnableExamples: + from std/sequtils import zip + + let + names = ["John", "Paul", "George", "Ringo"] + years = [1940, 1942, 1943, 1940] + + var beatles = initTable[string, int]() + + for pairs in zip(names, years): + let (name, birthYear) = pairs + beatles[name] = birthYear + + assert beatles == {"George": 1943, "Ringo": 1940, "Paul": 1942, "John": 1940}.toTable + + + var beatlesByYear = initTable[int, seq[string]]() + + for pairs in zip(years, names): + let (birthYear, name) = pairs + if not beatlesByYear.hasKey(birthYear): + # if a key doesn't exist, we create one with an empty sequence + # before we can add elements to it + beatlesByYear[birthYear] = @[] + beatlesByYear[birthYear].add(name) + + assert beatlesByYear == {1940: @["John", "Ringo"], 1942: @["Paul"], 1943: @["George"]}.toTable + +## ## OrderedTable ## `OrderedTable<#OrderedTable>`_ is used when it is important to preserve ## the insertion order of keys. -## -## .. code-block:: -## import tables -## -## let -## a = [('z', 1), ('y', 2), ('x', 3)] -## t = a.toTable # regular table -## ot = a.toOrderedTable # ordered tables -## -## echo t # {'x': 3, 'y': 2, 'z': 1} -## echo ot # {'z': 1, 'y': 2, 'x': 3} -## -## -## -## CountTable -## ---------- -## + +runnableExamples: + let + a = [('z', 1), ('y', 2), ('x', 3)] + ot = a.toOrderedTable # ordered tables + + assert $ot == """{'z': 1, 'y': 2, 'x': 3}""" + +## ## CountTable ## `CountTable<#CountTable>`_ is useful for counting number of items of some ## container (e.g. string, sequence or array), as it is a mapping where the ## items are the keys, and their number of occurrences are the values. ## For that purpose `toCountTable proc<#toCountTable,openArray[A]>`_ ## comes handy: -## -## .. code-block:: -## import tables -## -## let myString = "abracadabra" -## let letterFrequencies = toCountTable(myString) -## echo letterFrequencies -## # 'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r': 2} -## + +runnableExamples: + let myString = "abracadabra" + let letterFrequencies = toCountTable(myString) + assert $letterFrequencies == "{'a': 5, 'd': 1, 'b': 2, 'r': 2, 'c': 1}" + ## The same could have been achieved by manually iterating over a container ## and increasing each key's value with `inc proc -## <#inc,CountTable[A],A,Positive>`_: -## -## .. code-block:: -## import tables -## -## let myString = "abracadabra" -## var letterFrequencies = initCountTable[char]() -## for c in myString: -## letterFrequencies.inc(c) -## echo letterFrequencies -## # output: {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r': 2} +## <#inc,CountTable[A],A,int>`_: + +runnableExamples: + let myString = "abracadabra" + var letterFrequencies = initCountTable[char]() + for c in myString: + letterFrequencies.inc(c) + assert $letterFrequencies == "{'d': 1, 'r': 2, 'c': 1, 'a': 5, 'b': 2}" + ## ## ---- ## + +## ## Hashing ## -## -## Hashing -## ------- -## -## If you are using simple standard types like ``int`` or ``string`` for the +## If you are using simple standard types like `int` or `string` for the ## keys of the table you won't have any problems, but as soon as you try to use ## a more complex object as a key you will be greeted by a strange compiler ## error: ## -## Error: type mismatch: got (Person) -## but expected one of: -## hashes.hash(x: openArray[A]): Hash -## hashes.hash(x: int): Hash -## hashes.hash(x: float): Hash -## … +## Error: type mismatch: got (Person) +## but expected one of: +## hashes.hash(x: openArray[A]): Hash +## hashes.hash(x: int): Hash +## hashes.hash(x: float): Hash ## ## What is happening here is that the types used for table keys require to have -## a ``hash()`` proc which will convert them to a `Hash <hashes.html#Hash>`_ +## a `hash()` proc which will convert them to a `Hash <hashes.html#Hash>`_ ## value, and the compiler is listing all the hash functions it knows. -## Additionally there has to be a ``==`` operator that provides the same -## semantics as its corresponding ``hash`` proc. +## Additionally there has to be a `==` operator that provides the same +## semantics as its corresponding `hash` proc. ## -## After you add ``hash`` and ``==`` for your custom type everything will work. -## Currently, however, ``hash`` for objects is not defined, whereas -## ``system.==`` for objects does exist and performs a "deep" comparison (every +## After you add `hash` and `==` for your custom type everything will work. +## Currently, however, `hash` for objects is not defined, whereas +## `system.==` for objects does exist and performs a "deep" comparison (every ## field is compared) which is usually what you want. So in the following -## example implementing only ``hash`` suffices: -## -## .. code-block:: -## import tables, hashes -## -## type -## Person = object -## firstName, lastName: string -## -## proc hash(x: Person): Hash = -## ## Piggyback on the already available string hash proc. -## ## -## ## Without this proc nothing works! -## result = x.firstName.hash !& x.lastName.hash -## result = !$result -## -## var -## salaries = initTable[Person, int]() -## p1, p2: Person -## -## p1.firstName = "Jon" -## p1.lastName = "Ross" -## salaries[p1] = 30_000 -## -## p2.firstName = "소진" -## p2.lastName = "박" -## salaries[p2] = 45_000 +## example implementing only `hash` suffices: + +runnableExamples: + import std/hashes + + type + Person = object + firstName, lastName: string + + proc hash(x: Person): Hash = + ## Piggyback on the already available string hash proc. + ## + ## Without this proc nothing works! + result = x.firstName.hash !& x.lastName.hash + result = !$result + + var + salaries = initTable[Person, int]() + p1, p2: Person + + p1.firstName = "Jon" + p1.lastName = "Ross" + salaries[p1] = 30_000 + + p2.firstName = "소진" + p2.lastName = "박" + salaries[p2] = 45_000 + ## ## ---- ## -## See also -## ======== + +## # See also ## ## * `json module<json.html>`_ for table-like structure which allows ## heterogeneous members -## * `sharedtables module<sharedtables.html>`_ for shared hash table support ## * `strtabs module<strtabs.html>`_ for efficient hash tables ## mapping from strings to strings ## * `hashes module<hashes.html>`_ for helper functions for hashing -import hashes, math, algorithm +import std/private/since +import std/[hashes, math, algorithm] -include "system/inclrtl" + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] @@ -230,16 +209,14 @@ type ## `data` and `counter` are internal implementation details which ## can't be accessed. ## - ## For creating an empty Table, use `initTable proc<#initTable,int>`_. + ## For creating an empty Table, use `initTable proc<#initTable>`_. data: KeyValuePairSeq[A, B] counter: int TableRef*[A, B] = ref Table[A, B] ## Ref version of `Table<#Table>`_. ## ## For creating a new empty TableRef, use `newTable proc - ## <#newTable,int>`_. + ## <#newTable>`_. -const - defaultInitialSize* = 64 # ------------------------------ helpers --------------------------------- @@ -250,20 +227,21 @@ template dataLen(t): untyped = len(t.data) include tableimpl -proc rightSize*(count: Natural): int {.inline.} +proc raiseKeyError[T](key: T) {.noinline, noreturn.} = + when compiles($key): + raise newException(KeyError, "key not found: " & $key) + else: + raise newException(KeyError, "key not found") template get(t, key): untyped = - ## retrieves the value at ``t[key]``. The value can be modified. - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## retrieves the value at `t[key]`. The value can be modified. + ## If `key` is not in `t`, the `KeyError` exception is raised. mixin rawGet var hc: Hash var index = rawGet(t, key, hc) if index >= 0: result = t.data[index].val else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raiseKeyError(key) proc enlarge[A, B](t: var Table[A, B]) = var n: KeyValuePairSeq[A, B] @@ -290,26 +268,21 @@ proc enlarge[A, B](t: var Table[A, B]) = proc initTable*[A, B](initialSize = defaultInitialSize): Table[A, B] = ## Creates a new hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## Starting from Nim v0.20, tables are initialized by default and it is ## not necessary to call this function explicitly. ## ## See also: ## * `toTable proc<#toTable,openArray[]>`_ - ## * `newTable proc<#newTable,int>`_ for creating a `TableRef` + ## * `newTable proc<#newTable>`_ for creating a `TableRef` runnableExamples: let a = initTable[int, string]() b = initTable[char, seq[int]]() + result = default(Table[A, B]) initImpl(result, initialSize) -proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = - ## Inserts a ``(key, value)`` pair into ``t``. +proc `[]=`*[A, B](t: var Table[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. ## ## See also: ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key @@ -325,25 +298,25 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) = putImpl(enlarge) proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = - ## Creates a new hash table that contains the given ``pairs``. + ## Creates a new hash table that contains the given `pairs`. ## - ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## `pairs` is a container consisting of `(key, value)` tuples. ## ## See also: - ## * `initTable proc<#initTable,int>`_ + ## * `initTable proc<#initTable>`_ ## * `newTable proc<#newTable,openArray[]>`_ for a `TableRef` version runnableExamples: let a = [('a', 5), ('b', 9)] let b = toTable(a) assert b == {'a': 5, 'b': 9}.toTable - result = initTable[A, B](rightSize(pairs.len)) + result = initTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: Table[A, B], key: A): B = - ## Retrieves the value at ``t[key]``. +proc `[]`*[A, B](t: Table[A, B], key: A): lent B = + ## Retrieves the value at `t[key]`. ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## If `key` is not in `t`, the `KeyError` exception is raised. ## One can check with `hasKey proc<#hasKey,Table[A,B],A>`_ whether ## the key exists. ## @@ -352,7 +325,7 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in ## the table @@ -364,23 +337,23 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B = get(t, key) proc `[]`*[A, B](t: var Table[A, B], key: A): var B = - ## Retrieves the value at ``t[key]``. The value can be modified. + ## Retrieves the value at `t[key]`. The value can be modified. ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## If `key` is not in `t`, the `KeyError` exception is raised. ## ## See also: ## * `getOrDefault proc<#getOrDefault,Table[A,B],A>`_ to return ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in ## the table get(t, key) proc hasKey*[A, B](t: Table[A, B], key: A): bool = - ## Returns true if ``key`` is in the table ``t``. + ## Returns true if `key` is in the table `t`. ## ## See also: ## * `contains proc<#contains,Table[A,B],A>`_ for use with the `in` operator @@ -399,7 +372,7 @@ proc hasKey*[A, B](t: Table[A, B], key: A): bool = proc contains*[A, B](t: Table[A, B], key: A): bool = ## Alias of `hasKey proc<#hasKey,Table[A,B],A>`_ for use with - ## the ``in`` operator. + ## the `in` operator. runnableExamples: let a = {'a': 5, 'b': 9}.toTable doAssert 'b' in a == true @@ -408,7 +381,7 @@ proc contains*[A, B](t: Table[A, B], key: A): bool = return hasKey[A, B](t, key) proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = - ## Returns true if ``key`` is in the table, otherwise inserts ``value``. + ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: ## * `hasKey proc<#hasKey,Table[A,B],A>`_ @@ -428,8 +401,8 @@ proc hasKeyOrPut*[A, B](t: var Table[A, B], key: A, val: B): bool = hasKeyOrPutImpl(enlarge) proc getOrDefault*[A, B](t: Table[A, B], key: A): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any ## integer type). ## ## See also: @@ -443,12 +416,12 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise, ``default`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. ## ## See also: ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key @@ -461,13 +434,20 @@ proc getOrDefault*[A, B](t: Table[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = - ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. ## + ## + ## Note that while the value returned is of type `var B`, + ## it is easy to accidentally create a copy of the value at `t[key]`. + ## Remember that seqs and strings are value types, and therefore + ## cannot be copied into a separate variable for modification. + ## See the example below. + ## ## See also: ## * `[] proc<#[],Table[A,B],A>`_ for retrieving a value of a key ## * `hasKey proc<#hasKey,Table[A,B],A>`_ @@ -482,27 +462,58 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = doAssert a.mgetOrPut('z', 99) == 99 doAssert a == {'a': 5, 'b': 9, 'z': 99}.toTable + # An example of accidentally creating a copy + var t = initTable[int, seq[int]]() + # In this example, we expect t[10] to be modified, + # but it is not. + var copiedSeq = t.mgetOrPut(10, @[10]) + copiedSeq.add(20) + doAssert t[10] == @[10] + # Correct + t.mgetOrPut(25, @[25]).add(35) + doAssert t[25] == @[25, 35] + + mgetOrPutImpl(enlarge) + +proc mgetOrPut*[A, B](t: var Table[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + mgetOrPutImpl(enlarge) proc len*[A, B](t: Table[A, B]): int = - ## Returns the number of keys in ``t``. + ## Returns the number of keys in `t`. runnableExamples: let a = {'a': 5, 'b': 9}.toTable doAssert len(a) == 2 result = t.counter -proc add*[A, B](t: var Table[A, B], key: A, val: B) = - ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. +proc add*[A, B](t: var Table[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. ## ## **This can introduce duplicate keys into the table!** ## - ## Use `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new + ## Use `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table without introducing duplicates. addImpl(enlarge) +template tabMakeEmpty(i) = t.data[i].hcode = 0 +template tabCellEmpty(i) = isEmpty(t.data[i].hcode) +template tabCellHash(i) = t.data[i].hcode + proc del*[A, B](t: var Table[A, B], key: A) = - ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. + ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `pop proc<#pop,Table[A,B],A,B>`_ @@ -514,14 +525,17 @@ proc del*[A, B](t: var Table[A, B], key: A) = a.del('z') doAssert a == {'b': 9, 'c': 13}.toTable - delImpl() + delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool = - ## Deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. + ## ## See also: ## * `del proc<#del,Table[A,B],A>`_ ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table @@ -542,7 +556,7 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool = result = index >= 0 if result: val = move(t.data[index].val) - delImplIdx(t, index) + delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash) proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool {.inline.} = ## Alias for: @@ -564,12 +578,12 @@ proc clear*[A, B](t: var Table[A, B]) = clearImpl() proc `$`*[A, B](t: Table[A, B]): string = - ## The ``$`` operator for hash tables. Used internally when calling `echo` + ## The `$` operator for hash tables. Used internally when calling `echo` ## on a table. dollarImpl() proc `==`*[A, B](s, t: Table[A, B]): bool = - ## The ``==`` operator for hash tables. Returns ``true`` if the content of both + ## The `==` operator for hash tables. Returns `true` if the content of both ## tables contains the same key-value pairs. Insert order does not matter. runnableExamples: let @@ -579,15 +593,6 @@ proc `==`*[A, B](s, t: Table[A, B]): bool = equalsImpl(s, t) -proc rightSize*(count: Natural): int {.inline.} = - ## Return the value of ``initialSize`` to support ``count`` items. - ## - ## If more items are expected to be added, simply add that - ## expected extra amount to the parameter before calling this. - ## - ## Internally, we want mustRehash(rightSize(x), x) == false. - result = nextPowerOfTwo(count * 3 div 2 + 4) - proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] @@ -598,17 +603,31 @@ proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = - ## Retrieves the value at ``t[key]``. - ## - ## ``value`` can be modified in the scope of the ``withValue`` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if ``key`` in ``t`` - ## value.name = "username" - ## value.uid = 1000 + ## Retrieves the value at `t[key]`. ## + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + type + User = object + name: string + uid: int + + var t = initTable[int, User]() + let u = User(name: "Hello", uid: 99) + t[1] = u + + t.withValue(1, value): + # block is executed only if `key` in `t` + value.name = "Nim" + value.uid = 1314 + + t.withValue(2, value): + value.name = "No" + value.uid = 521 + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + mixin rawGet var hc: Hash var index = rawGet(t, key, hc) @@ -619,20 +638,35 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = template withValue*[A, B](t: var Table[A, B], key: A, value, body1, body2: untyped) = - ## Retrieves the value at ``t[key]``. - ## - ## ``value`` can be modified in the scope of the ``withValue`` call. - ## - ## .. code-block:: nim - ## - ## table.withValue(key, value) do: - ## # block is executed only if ``key`` in ``t`` - ## value.name = "username" - ## value.uid = 1000 - ## do: - ## # block is executed when ``key`` not in ``t`` - ## raise newException(KeyError, "Key not found") + ## Retrieves the value at `t[key]`. ## + ## `value` can be modified in the scope of the `withValue` call. + runnableExamples: + type + User = object + name: string + uid: int + + var t = initTable[int, User]() + let u = User(name: "Hello", uid: 99) + t[1] = u + + t.withValue(1, value): + # block is executed only if `key` in `t` + value.name = "Nim" + value.uid = 1314 + + t.withValue(521, value): + doAssert false + do: + # block is executed when `key` not in `t` + t[1314] = User(name: "exist", uid: 521) + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + assert t[1314].name == "exist" + assert t[1314].uid == 521 + mixin rawGet var hc: Hash var index = rawGet(t, key, hc) @@ -645,7 +679,7 @@ template withValue*[A, B](t: var Table[A, B], key: A, iterator pairs*[A, B](t: Table[A, B]): (A, B) = - ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## Iterates over any `(key, value)` pair in the table `t`. ## ## See also: ## * `mpairs iterator<#mpairs.i,Table[A,B]>`_ @@ -654,7 +688,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -668,6 +702,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -675,7 +710,7 @@ iterator pairs*[A, B](t: Table[A, B]): (A, B) = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = - ## Iterates over any ``(key, value)`` pair in the table ``t`` (must be + ## Iterates over any `(key, value)` pair in the table `t` (must be ## declared as `var`). The values can be modified. ## ## See also: @@ -696,8 +731,8 @@ iterator mpairs*[A, B](t: var Table[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: Table[A, B]): A = - ## Iterates over any key in the table ``t``. +iterator keys*[A, B](t: Table[A, B]): lent A = + ## Iterates over any key in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,Table[A,B]>`_ @@ -717,8 +752,8 @@ iterator keys*[A, B](t: Table[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: Table[A, B]): B = - ## Iterates over any value in the table ``t``. +iterator values*[A, B](t: Table[A, B]): lent B = + ## Iterates over any value in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,Table[A,B]>`_ @@ -739,7 +774,7 @@ iterator values*[A, B](t: Table[A, B]): B = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mvalues*[A, B](t: var Table[A, B]): var B = - ## Iterates over any value in the table ``t`` (must be + ## Iterates over any value in the table `t` (must be ## declared as `var`). The values can be modified. ## ## See also: @@ -760,25 +795,20 @@ iterator mvalues*[A, B](t: var Table[A, B]): var B = yield t.data[h].val assert(len(t) == L, "the length of the table changed while iterating over it") -iterator allValues*[A, B](t: Table[A, B]; key: A): B = - ## Iterates over any value in the table ``t`` that belongs to the given ``key``. +iterator allValues*[A, B](t: Table[A, B]; key: A): B {.deprecated: + "Deprecated since v1.4; tables with duplicated keys are deprecated".} = + ## Iterates over any value in the table `t` that belongs to the given `key`. ## ## Used if you have a table with duplicate keys (as a result of using - ## `add proc<#add,Table[A,B],A,B>`_). - ## - ## **Examples:** + ## `add proc<#add,Table[A,B],A,sinkB>`_). ## - ## .. code-block:: - ## var a = {'a': 3, 'b': 5}.toTable - ## for i in 1..3: - ## a.add('z', 10*i) - ## echo a # {'a': 3, 'b': 5, 'z': 10, 'z': 20, 'z': 30} - ## - ## for v in a.allValues('z'): - ## echo v - ## # 10 - ## # 20 - ## # 30 + runnableExamples: + import std/[sequtils, algorithm] + + var a = {'a': 3, 'b': 5}.toTable + for i in 1..3: a.add('z', 10*i) + doAssert toSeq(a.pairs).sorted == @[('a', 3), ('b', 5), ('z', 10), ('z', 20), ('z', 30)] + doAssert sorted(toSeq(a.allValues('z'))) == @[10, 20, 30] var h: Hash = genHash(key) and high(t.data) let L = len(t) while isFilled(t.data[h].hcode): @@ -794,34 +824,29 @@ iterator allValues*[A, B](t: Table[A, B]; key: A): B = # ------------------------------------------------------------------- -proc newTable*[A, B](initialSize = defaultInitialSize): <//>TableRef[A, B] = +proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = ## Creates a new ref hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## See also: ## * `newTable proc<#newTable,openArray[]>`_ for creating a `TableRef` ## from a collection of `(key, value)` pairs - ## * `initTable proc<#initTable,int>`_ for creating a `Table` + ## * `initTable proc<#initTable>`_ for creating a `Table` runnableExamples: let a = newTable[int, string]() b = newTable[char, seq[int]]() new(result) - result[] = initTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) -proc newTable*[A, B](pairs: openArray[(A, B)]): <//>TableRef[A, B] = - ## Creates a new ref hash table that contains the given ``pairs``. +proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = + ## Creates a new ref hash table that contains the given `pairs`. ## - ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## `pairs` is a container consisting of `(key, value)` tuples. ## ## See also: - ## * `newTable proc<#newTable,int>`_ + ## * `newTable proc<#newTable>`_ ## * `toTable proc<#toTable,openArray[]>`_ for a `Table` version runnableExamples: let a = [('a', 5), ('b', 9)] @@ -829,19 +854,21 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): <//>TableRef[A, B] = assert b == {'a': 5, 'b': 9}.newTable new(result) - result[] = toTable[A, B](pairs) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) -proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): <//>TableRef[C, B] = +proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() - for item in collection: - result[index(item)] = item + {.noSideEffect.}: + for item in collection: + result[index(item)] = item proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = - ## Retrieves the value at ``t[key]``. + ## Retrieves the value at `t[key]`. ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## If `key` is not in `t`, the `KeyError` exception is raised. ## One can check with `hasKey proc<#hasKey,TableRef[A,B],A>`_ whether ## the key exists. ## @@ -850,7 +877,7 @@ proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,TableRef[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,TableRef[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,TableRef[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ for checking if a key is in ## the table @@ -862,8 +889,8 @@ proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = result = t[][key] -proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = - ## Inserts a ``(key, value)`` pair into ``t``. +proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. ## ## See also: ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key @@ -879,7 +906,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) = t[][key] = val proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = - ## Returns true if ``key`` is in the table ``t``. + ## Returns true if `key` is in the table `t`. ## ## See also: ## * `contains proc<#contains,TableRef[A,B],A>`_ for use with the `in` @@ -898,7 +925,7 @@ proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = proc contains*[A, B](t: TableRef[A, B], key: A): bool = ## Alias of `hasKey proc<#hasKey,TableRef[A,B],A>`_ for use with - ## the ``in`` operator. + ## the `in` operator. runnableExamples: let a = {'a': 5, 'b': 9}.newTable doAssert 'b' in a == true @@ -906,8 +933,8 @@ proc contains*[A, B](t: TableRef[A, B], key: A): bool = return hasKey[A, B](t, key) -proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = - ## Returns true if ``key`` is in the table, otherwise inserts ``value``. +proc hasKeyOrPut*[A, B](t: TableRef[A, B], key: A, val: B): bool = + ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ @@ -927,8 +954,8 @@ proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool = t[].hasKeyOrPut(key, val) proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any ## integer type). ## ## See also: @@ -946,8 +973,8 @@ proc getOrDefault*[A, B](t: TableRef[A, B], key: A): B = getOrDefault(t[], key) proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise, ``default`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. ## ## See also: ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key @@ -964,9 +991,15 @@ proc getOrDefault*[A, B](t: TableRef[A, B], key: A, default: B): B = getOrDefault(t[], key, default) proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = - ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. ## + ## Note that while the value returned is of type `var B`, + ## it is easy to accidentally create an copy of the value at `t[key]`. + ## Remember that seqs and strings are value types, and therefore + ## cannot be copied into a separate variable for modification. + ## See the example below. + ## ## See also: ## * `[] proc<#[],TableRef[A,B],A>`_ for retrieving a value of a key ## * `hasKey proc<#hasKey,TableRef[A,B],A>`_ @@ -981,29 +1014,53 @@ proc mgetOrPut*[A, B](t: TableRef[A, B], key: A, val: B): var B = doAssert a.mgetOrPut('z', 99) == 99 doAssert a == {'a': 5, 'b': 9, 'z': 99}.newTable + # An example of accidentally creating a copy + var t = newTable[int, seq[int]]() + # In this example, we expect t[10] to be modified, + # but it is not. + var copiedSeq = t.mgetOrPut(10, @[10]) + copiedSeq.add(20) + doAssert t[10] == @[10] + # Correct + t.mgetOrPut(25, @[25]).add(35) + doAssert t[25] == @[25, 35] t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: TableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.newTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.newTable + + t[].mgetOrPut(key) + proc len*[A, B](t: TableRef[A, B]): int = - ## Returns the number of keys in ``t``. + ## Returns the number of keys in `t`. runnableExamples: let a = {'a': 5, 'b': 9}.newTable doAssert len(a) == 2 result = t.counter -proc add*[A, B](t: TableRef[A, B], key: A, val: B) = - ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. +proc add*[A, B](t: TableRef[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. ## ## **This can introduce duplicate keys into the table!** ## - ## Use `[]= proc<#[]=,TableRef[A,B],A,B>`_ for inserting a new + ## Use `[]= proc<#[]=,TableRef[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table without introducing duplicates. t[].add(key, val) proc del*[A, B](t: TableRef[A, B], key: A) = - ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## - ## **If duplicate keys were added, this may need to be called multiple times.** + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `pop proc<#pop,TableRef[A,B],A,B>`_ @@ -1018,12 +1075,13 @@ proc del*[A, B](t: TableRef[A, B], key: A) = t[].del(key) proc pop*[A, B](t: TableRef[A, B], key: A, val: var B): bool = - ## Deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## - ## **If duplicate keys were added, this may need to be called multiple times.** + ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc), + ## this may need to be called multiple times. ## ## See also: ## * `del proc<#del,TableRef[A,B],A>`_ @@ -1062,13 +1120,13 @@ proc clear*[A, B](t: TableRef[A, B]) = clearImpl() proc `$`*[A, B](t: TableRef[A, B]): string = - ## The ``$`` operator for hash tables. Used internally when calling `echo` + ## The `$` operator for hash tables. Used internally when calling `echo` ## on a table. dollarImpl() proc `==`*[A, B](s, t: TableRef[A, B]): bool = - ## The ``==`` operator for hash tables. Returns ``true`` if either both tables - ## are ``nil``, or neither is ``nil`` and the content of both tables contains the + ## The `==` operator for hash tables. Returns `true` if either both tables + ## are `nil`, or neither is `nil` and the content of both tables contains the ## same key-value pairs. Insert order does not matter. runnableExamples: let @@ -1083,7 +1141,7 @@ proc `==`*[A, B](s, t: TableRef[A, B]): bool = iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = - ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## Iterates over any `(key, value)` pair in the table `t`. ## ## See also: ## * `mpairs iterator<#mpairs.i,TableRef[A,B]>`_ @@ -1092,7 +1150,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1106,6 +1164,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = ## # value: [2, 4, 6, 8] ## # key: o ## # value: [1, 5, 7, 9] + ## ``` let L = len(t) for h in 0 .. high(t.data): if isFilled(t.data[h].hcode): @@ -1113,7 +1172,7 @@ iterator pairs*[A, B](t: TableRef[A, B]): (A, B) = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = - ## Iterates over any ``(key, value)`` pair in the table ``t``. The values + ## Iterates over any `(key, value)` pair in the table `t`. The values ## can be modified. ## ## See also: @@ -1134,8 +1193,8 @@ iterator mpairs*[A, B](t: TableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: TableRef[A, B]): A = - ## Iterates over any key in the table ``t``. +iterator keys*[A, B](t: TableRef[A, B]): lent A = + ## Iterates over any key in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ @@ -1155,8 +1214,8 @@ iterator keys*[A, B](t: TableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: TableRef[A, B]): B = - ## Iterates over any value in the table ``t``. +iterator values*[A, B](t: TableRef[A, B]): lent B = + ## Iterates over any value in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,TableRef[A,B]>`_ @@ -1177,7 +1236,7 @@ iterator values*[A, B](t: TableRef[A, B]): B = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mvalues*[A, B](t: TableRef[A, B]): var B = - ## Iterates over any value in the table ``t``. The values can be modified. + ## Iterates over any value in the table `t`. The values can be modified. ## ## See also: ## * `mpairs iterator<#mpairs.i,TableRef[A,B]>`_ @@ -1216,14 +1275,14 @@ type ## Hash table that remembers insertion order. ## ## For creating an empty OrderedTable, use `initOrderedTable proc - ## <#initOrderedTable,int>`_. + ## <#initOrderedTable>`_. data: OrderedKeyValuePairSeq[A, B] counter, first, last: int OrderedTableRef*[A, B] = ref OrderedTable[A, B] ## Ref version of ## `OrderedTable<#OrderedTable>`_. ## ## For creating a new empty OrderedTableRef, use `newOrderedTable proc - ## <#newOrderedTable,int>`_. + ## <#newOrderedTable>`_. # ------------------------------ helpers --------------------------------- @@ -1239,7 +1298,7 @@ proc rawGet[A, B](t: OrderedTable[A, B], key: A, hc: var Hash): int = proc rawInsert[A, B](t: var OrderedTable[A, B], data: var OrderedKeyValuePairSeq[A, B], - key: A, val: B, hc: Hash, h: Hash) = + key: A, val: sink B, hc: Hash, h: Hash) = rawInsertImpl() data[h].next = -1 if t.first < 0: t.first = h @@ -1260,7 +1319,7 @@ proc enlarge[A, B](t: var OrderedTable[A, B]) = var j: Hash = eh and maxHash(t) while isFilled(t.data[j].hcode): j = nextTry(j, maxHash(t)) - rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) + rawInsert(t, t.data, move n[h].key, move n[h].val, n[h].hcode, j) h = nxt template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = @@ -1277,27 +1336,22 @@ template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = proc initOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTable[A, B] = ## Creates a new ordered hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## Starting from Nim v0.20, tables are initialized by default and it is ## not necessary to call this function explicitly. ## ## See also: ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_ - ## * `newOrderedTable proc<#newOrderedTable,int>`_ for creating an + ## * `newOrderedTable proc<#newOrderedTable>`_ for creating an ## `OrderedTableRef` runnableExamples: let a = initOrderedTable[int, string]() b = initOrderedTable[char, seq[int]]() + result = default(OrderedTable[A, B]) initImpl(result, initialSize) -proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = - ## Inserts a ``(key, value)`` pair into ``t``. +proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. ## ## See also: ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key @@ -1313,12 +1367,12 @@ proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) = putImpl(enlarge) proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = - ## Creates a new ordered hash table that contains the given ``pairs``. + ## Creates a new ordered hash table that contains the given `pairs`. ## - ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## `pairs` is a container consisting of `(key, value)` tuples. ## ## See also: - ## * `initOrderedTable proc<#initOrderedTable,int>`_ + ## * `initOrderedTable proc<#initOrderedTable>`_ ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for an ## `OrderedTableRef` version runnableExamples: @@ -1326,13 +1380,13 @@ proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = let b = toOrderedTable(a) assert b == {'a': 5, 'b': 9}.toOrderedTable - result = initOrderedTable[A, B](rightSize(pairs.len)) + result = initOrderedTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val -proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = - ## Retrieves the value at ``t[key]``. +proc `[]`*[A, B](t: OrderedTable[A, B], key: A): lent B = + ## Retrieves the value at `t[key]`. ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## If `key` is not in `t`, the `KeyError` exception is raised. ## One can check with `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ whether ## the key exists. ## @@ -1341,7 +1395,7 @@ proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,OrderedTable[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,OrderedTable[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for checking if a ## key is in the table @@ -1354,23 +1408,23 @@ proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = get(t, key) proc `[]`*[A, B](t: var OrderedTable[A, B], key: A): var B = - ## Retrieves the value at ``t[key]``. The value can be modified. + ## Retrieves the value at `t[key]`. The value can be modified. ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## If `key` is not in `t`, the `KeyError` exception is raised. ## ## See also: ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A>`_ to return ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,OrderedTable[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,OrderedTable[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,OrderedTable[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for checking if a ## key is in the table get(t, key) proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = - ## Returns true if ``key`` is in the table ``t``. + ## Returns true if `key` is in the table `t`. ## ## See also: ## * `contains proc<#contains,OrderedTable[A,B],A>`_ for use with the `in` @@ -1385,12 +1439,12 @@ proc hasKey*[A, B](t: OrderedTable[A, B], key: A): bool = doAssert a.hasKey('a') == true doAssert a.hasKey('z') == false - var hc: Hash + var hc: Hash = default(Hash) result = rawGet(t, key, hc) >= 0 proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = ## Alias of `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ for use with - ## the ``in`` operator. + ## the `in` operator. runnableExamples: let a = {'a': 5, 'b': 9}.toOrderedTable doAssert 'b' in a == true @@ -1399,7 +1453,7 @@ proc contains*[A, B](t: OrderedTable[A, B], key: A): bool = return hasKey[A, B](t, key) proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = - ## Returns true if ``key`` is in the table, otherwise inserts ``value``. + ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: ## * `hasKey proc<#hasKey,OrderedTable[A,B],A>`_ @@ -1419,8 +1473,8 @@ proc hasKeyOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): bool = hasKeyOrPutImpl(enlarge) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any ## integer type). ## ## See also: @@ -1434,12 +1488,12 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a') == 5 doAssert a.getOrDefault('z') == 0 - + result = default(B) getOrDefaultImpl(t, key) proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise, ``default`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. ## ## See also: ## * `[] proc<#[],OrderedTable[A,B],A>`_ for retrieving a value of a key @@ -1452,11 +1506,11 @@ proc getOrDefault*[A, B](t: OrderedTable[A, B], key: A, default: B): B = let a = {'a': 5, 'b': 9}.toOrderedTable doAssert a.getOrDefault('a', 99) == 5 doAssert a.getOrDefault('z', 99) == 99 - + result = default(B) getOrDefaultImpl(t, key, default) proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = - ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. ## ## See also: @@ -1475,25 +1529,38 @@ proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) +proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + mgetOrPutImpl(enlarge) + proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = - ## Returns the number of keys in ``t``. + ## Returns the number of keys in `t`. runnableExamples: let a = {'a': 5, 'b': 9}.toOrderedTable doAssert len(a) == 2 result = t.counter -proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) = - ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. +proc add*[A, B](t: var OrderedTable[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. ## ## **This can introduce duplicate keys into the table!** ## - ## Use `[]= proc<#[]=,OrderedTable[A,B],A,B>`_ for inserting a new + ## Use `[]= proc<#[]=,OrderedTable[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table without introducing duplicates. addImpl(enlarge) proc del*[A, B](t: var OrderedTable[A, B], key: A) = - ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## ## O(n) complexity. ## @@ -1522,13 +1589,13 @@ proc del*[A, B](t: var OrderedTable[A, B], key: A) = dec t.counter else: var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) - rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) + rawInsert(t, t.data, move n[h].key, move n[h].val, n[h].hcode, j) h = nxt proc pop*[A, B](t: var OrderedTable[A, B], key: A, val: var B): bool {.since: (1, 1).} = - ## Deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## ## O(n) complexity. @@ -1572,15 +1639,15 @@ proc clear*[A, B](t: var OrderedTable[A, B]) = t.last = -1 proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = - ## Sorts ``t`` according to the function ``cmp``. + order = SortOrder.Ascending) {.effectsOf: cmp.} = + ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list ## that kept the insertion order, so insertion order is lost after this - ## call but key lookup and insertions remain possible after ``sort`` (in + ## call but key lookup and insertions remain possible after `sort` (in ## contrast to the `sort proc<#sort,CountTable[A]>`_ for count tables). runnableExamples: - import algorithm + import std/[algorithm] var a = initOrderedTable[char, int]() for i, c in "cab": a[c] = 10*i @@ -1631,12 +1698,12 @@ proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x, y: (A, B)): int, t.last = tail proc `$`*[A, B](t: OrderedTable[A, B]): string = - ## The ``$`` operator for ordered hash tables. Used internally when calling + ## The `$` operator for ordered hash tables. Used internally when calling ## `echo` on a table. dollarImpl() proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = - ## The ``==`` operator for ordered hash tables. Returns ``true`` if both the + ## The `==` operator for ordered hash tables. Returns `true` if both the ## content and the order are equal. runnableExamples: let @@ -1646,6 +1713,8 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = if s.counter != t.counter: return false + if s.counter == 0 and t.counter == 0: + return true var ht = t.first var hs = s.first while ht >= 0 and hs >= 0: @@ -1661,7 +1730,7 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = - ## Iterates over any ``(key, value)`` pair in the table ``t`` in insertion + ## Iterates over any `(key, value)` pair in the table `t` in insertion ## order. ## ## See also: @@ -1671,7 +1740,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -1685,6 +1754,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -1692,7 +1762,7 @@ iterator pairs*[A, B](t: OrderedTable[A, B]): (A, B) = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = - ## Iterates over any ``(key, value)`` pair in the table ``t`` (must be + ## Iterates over any `(key, value)` pair in the table `t` (must be ## declared as `var`) in insertion order. The values can be modified. ## ## See also: @@ -1713,8 +1783,8 @@ iterator mpairs*[A, B](t: var OrderedTable[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTable[A, B]): A = - ## Iterates over any key in the table ``t`` in insertion order. +iterator keys*[A, B](t: OrderedTable[A, B]): lent A = + ## Iterates over any key in the table `t` in insertion order. ## ## See also: ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ @@ -1734,8 +1804,8 @@ iterator keys*[A, B](t: OrderedTable[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTable[A, B]): B = - ## Iterates over any value in the table ``t`` in insertion order. +iterator values*[A, B](t: OrderedTable[A, B]): lent B = + ## Iterates over any value in the table `t` in insertion order. ## ## See also: ## * `pairs iterator<#pairs.i,OrderedTable[A,B]>`_ @@ -1755,7 +1825,7 @@ iterator values*[A, B](t: OrderedTable[A, B]): B = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = - ## Iterates over any value in the table ``t`` (must be + ## Iterates over any value in the table `t` (must be ## declared as `var`) in insertion order. The values ## can be modified. ## @@ -1777,42 +1847,33 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = yield t.data[h].val assert(len(t) == L, "the length of the table changed while iterating over it") - - - - # --------------------------------------------------------------------------- # --------------------------- OrderedTableRef ------------------------------- # --------------------------------------------------------------------------- -proc newOrderedTable*[A, B](initialSize = defaultInitialSize): <//>OrderedTableRef[A, B] = +proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## See also: ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for creating ## an `OrderedTableRef` from a collection of `(key, value)` pairs - ## * `initOrderedTable proc<#initOrderedTable,int>`_ for creating an + ## * `initOrderedTable proc<#initOrderedTable>`_ for creating an ## `OrderedTable` runnableExamples: let a = newOrderedTable[int, string]() b = newOrderedTable[char, seq[int]]() new(result) - result[] = initOrderedTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) -proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B] = - ## Creates a new ordered ref hash table that contains the given ``pairs``. +proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = + ## Creates a new ordered ref hash table that contains the given `pairs`. ## - ## ``pairs`` is a container consisting of ``(key, value)`` tuples. + ## `pairs` is a container consisting of `(key, value)` tuples. ## ## See also: - ## * `newOrderedTable proc<#newOrderedTable,int>`_ + ## * `newOrderedTable proc<#newOrderedTable>`_ ## * `toOrderedTable proc<#toOrderedTable,openArray[]>`_ for an ## `OrderedTable` version runnableExamples: @@ -1820,14 +1881,15 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B] let b = newOrderedTable(a) assert b == {'a': 5, 'b': 9}.newOrderedTable - result = newOrderedTable[A, B](rightSize(pairs.len)) - for key, val in items(pairs): result.add(key, val) + result = newOrderedTable[A, B](pairs.len) + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = - ## Retrieves the value at ``t[key]``. + ## Retrieves the value at `t[key]`. ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. + ## If `key` is not in `t`, the `KeyError` exception is raised. ## One can check with `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ whether ## the key exists. ## @@ -1836,7 +1898,7 @@ proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = ## a default value (e.g. zero for int) if the key doesn't exist ## * `getOrDefault proc<#getOrDefault,OrderedTableRef[A,B],A,B>`_ to return ## a custom value if the key doesn't exist - ## * `[]= proc<#[]=,OrderedTableRef[A,B],A,B>`_ for inserting a new + ## * `[]= proc<#[]=,OrderedTableRef[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ for checking if ## a key is in the table @@ -1847,8 +1909,8 @@ proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = echo a['z'] result = t[][key] -proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = - ## Inserts a ``(key, value)`` pair into ``t``. +proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: sink B) = + ## Inserts a `(key, value)` pair into `t`. ## ## See also: ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key @@ -1864,7 +1926,7 @@ proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = t[][key] = val proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = - ## Returns true if ``key`` is in the table ``t``. + ## Returns true if `key` is in the table `t`. ## ## See also: ## * `contains proc<#contains,OrderedTableRef[A,B],A>`_ for use with the `in` @@ -1883,7 +1945,7 @@ proc hasKey*[A, B](t: OrderedTableRef[A, B], key: A): bool = proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = ## Alias of `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ for use with - ## the ``in`` operator. + ## the `in` operator. runnableExamples: let a = {'a': 5, 'b': 9}.newOrderedTable doAssert 'b' in a == true @@ -1891,8 +1953,8 @@ proc contains*[A, B](t: OrderedTableRef[A, B], key: A): bool = return hasKey[A, B](t, key) -proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = - ## Returns true if ``key`` is in the table, otherwise inserts ``value``. +proc hasKeyOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): bool = + ## Returns true if `key` is in the table, otherwise inserts `value`. ## ## See also: ## * `hasKey proc<#hasKey,OrderedTableRef[A,B],A>`_ @@ -1912,8 +1974,8 @@ proc hasKeyOrPut*[A, B](t: var OrderedTableRef[A, B], key: A, val: B): bool = result = t[].hasKeyOrPut(key, val) proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. Otherwise, the - ## default initialization value for type ``B`` is returned (e.g. 0 for any + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## default initialization value for type `B` is returned (e.g. 0 for any ## integer type). ## ## See also: @@ -1931,8 +1993,8 @@ proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A): B = getOrDefault(t[], key) proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise, ``default`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise, `default` is returned. ## ## See also: ## * `[] proc<#[],OrderedTableRef[A,B],A>`_ for retrieving a value of a key @@ -1949,7 +2011,7 @@ proc getOrDefault*[A, B](t: OrderedTableRef[A, B], key: A, default: B): B = getOrDefault(t[], key, default) proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = - ## Retrieves value at ``t[key]`` or puts ``val`` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. ## ## See also: @@ -1968,25 +2030,38 @@ proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A, val: B): var B = result = t[].mgetOrPut(key, val) +proc mgetOrPut*[A, B](t: OrderedTableRef[A, B], key: A): var B = + ## Retrieves the value at `t[key]` or puts the + ## default initialization value for type `B` (e.g. 0 for any + ## integer type). + runnableExamples: + var a = {'a': 5}.toOrderedTable + doAssert a.mgetOrPut('a') == 5 + a.mgetOrPut('z').inc + doAssert a == {'a': 5, 'z': 1}.toOrderedTable + + t[].mgetOrPut(key) + proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = - ## Returns the number of keys in ``t``. + ## Returns the number of keys in `t`. runnableExamples: let a = {'a': 5, 'b': 9}.newOrderedTable doAssert len(a) == 2 result = t.counter -proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) = - ## Puts a new ``(key, value)`` pair into ``t`` even if ``t[key]`` already exists. +proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: sink B) {.deprecated: + "Deprecated since v1.4; it was more confusing than useful, use `[]=`".} = + ## Puts a new `(key, value)` pair into `t` even if `t[key]` already exists. ## ## **This can introduce duplicate keys into the table!** ## - ## Use `[]= proc<#[]=,OrderedTableRef[A,B],A,B>`_ for inserting a new + ## Use `[]= proc<#[]=,OrderedTableRef[A,B],A,sinkB>`_ for inserting a new ## (key, value) pair in the table without introducing duplicates. t[].add(key, val) proc del*[A, B](t: OrderedTableRef[A, B], key: A) = - ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist. + ## Deletes `key` from hash table `t`. Does nothing if the key does not exist. ## ## See also: ## * `clear proc<#clear,OrderedTableRef[A,B]>`_ to empty the whole table @@ -2000,9 +2075,9 @@ proc del*[A, B](t: OrderedTableRef[A, B], key: A) = t[].del(key) proc pop*[A, B](t: OrderedTableRef[A, B], key: A, val: var B): bool {.since: (1, 1).} = - ## Deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## ## See also: @@ -2036,15 +2111,15 @@ proc clear*[A, B](t: OrderedTableRef[A, B]) = clear(t[]) proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x, y: (A, B)): int, - order = SortOrder.Ascending) = - ## Sorts ``t`` according to the function ``cmp``. + order = SortOrder.Ascending) {.effectsOf: cmp.} = + ## Sorts `t` according to the function `cmp`. ## ## This modifies the internal list ## that kept the insertion order, so insertion order is lost after this - ## call but key lookup and insertions remain possible after ``sort`` (in + ## call but key lookup and insertions remain possible after `sort` (in ## contrast to the `sort proc<#sort,CountTableRef[A]>`_ for count tables). runnableExamples: - import algorithm + import std/[algorithm] var a = newOrderedTable[char, int]() for i, c in "cab": a[c] = 10*i @@ -2057,13 +2132,13 @@ proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x, y: (A, B)): int, t[].sort(cmp, order = order) proc `$`*[A, B](t: OrderedTableRef[A, B]): string = - ## The ``$`` operator for hash tables. Used internally when calling `echo` + ## The `$` operator for hash tables. Used internally when calling `echo` ## on a table. dollarImpl() proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = - ## The ``==`` operator for ordered hash tables. Returns true if either both - ## tables are ``nil``, or neither is ``nil`` and the content and the order of + ## The `==` operator for ordered hash tables. Returns true if either both + ## tables are `nil`, or neither is `nil` and the content and the order of ## both are equal. runnableExamples: let @@ -2078,7 +2153,7 @@ proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = - ## Iterates over any ``(key, value)`` pair in the table ``t`` in insertion + ## Iterates over any `(key, value)` pair in the table `t` in insertion ## order. ## ## See also: @@ -2088,7 +2163,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = { ## 'o': [1, 5, 7, 9], ## 'e': [2, 4, 6, 8] @@ -2102,6 +2177,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## # value: [1, 5, 7, 9] ## # key: e ## # value: [2, 4, 6, 8] + ## ``` let L = len(t) forAllOrderedPairs: @@ -2109,7 +2185,7 @@ iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = - ## Iterates over any ``(key, value)`` pair in the table ``t`` in insertion + ## Iterates over any `(key, value)` pair in the table `t` in insertion ## order. The values can be modified. ## ## See also: @@ -2130,8 +2206,8 @@ iterator mpairs*[A, B](t: OrderedTableRef[A, B]): (A, var B) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A, B](t: OrderedTableRef[A, B]): A = - ## Iterates over any key in the table ``t`` in insertion order. +iterator keys*[A, B](t: OrderedTableRef[A, B]): lent A = + ## Iterates over any key in the table `t` in insertion order. ## ## See also: ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ @@ -2151,8 +2227,8 @@ iterator keys*[A, B](t: OrderedTableRef[A, B]): A = yield t.data[h].key assert(len(t) == L, "the length of the table changed while iterating over it") -iterator values*[A, B](t: OrderedTableRef[A, B]): B = - ## Iterates over any value in the table ``t`` in insertion order. +iterator values*[A, B](t: OrderedTableRef[A, B]): lent B = + ## Iterates over any value in the table `t` in insertion order. ## ## See also: ## * `pairs iterator<#pairs.i,OrderedTableRef[A,B]>`_ @@ -2172,7 +2248,7 @@ iterator values*[A, B](t: OrderedTableRef[A, B]): B = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mvalues*[A, B](t: OrderedTableRef[A, B]): var B = - ## Iterates over any value in the table ``t`` in insertion order. The values + ## Iterates over any value in the table `t` in insertion order. The values ## can be modified. ## ## See also: @@ -2208,7 +2284,7 @@ type ## Hash table that counts the number of each key. ## ## For creating an empty CountTable, use `initCountTable proc - ## <#initCountTable,int>`_. + ## <#initCountTable>`_. data: seq[tuple[key: A, val: int]] counter: int isSorted: bool @@ -2216,7 +2292,7 @@ type ## `CountTable<#CountTable>`_. ## ## For creating a new empty CountTableRef, use `newCountTable proc - ## <#newCountTable,int>`_. + ## <#newCountTable>`_. # ------------------------------ helpers --------------------------------- @@ -2232,22 +2308,9 @@ proc enlarge[A](t: var CountTable[A]) = var n: seq[tuple[key: A, val: int]] newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): - if t.data[i].val != 0: ctRawInsert(t, n, t.data[i].key, t.data[i].val) + if t.data[i].val != 0: ctRawInsert(t, n, move t.data[i].key, move t.data[i].val) swap(t.data, n) -proc remove[A](t: var CountTable[A], key: A) = - var n: seq[tuple[key: A, val: int]] - newSeq(n, len(t.data)) - var removed: bool - for i in countup(0, high(t.data)): - if t.data[i].val != 0: - if t.data[i].key != key: - ctRawInsert(t, n, t.data[i].key, t.data[i].val) - else: - removed = true - swap(t.data, n) - if removed: dec(t.counter) - proc rawGet[A](t: CountTable[A], key: A): int = if t.data.len == 0: return -1 @@ -2261,42 +2324,36 @@ template ctget(t, key, default: untyped): untyped = var index = rawGet(t, key) result = if index >= 0: t.data[index].val else: default -proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) +proc inc*[A](t: var CountTable[A], key: A, val = 1) # ---------------------------------------------------------------------- proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = ## Creates a new count table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## Starting from Nim v0.20, tables are initialized by default and it is ## not necessary to call this function explicitly. ## ## See also: ## * `toCountTable proc<#toCountTable,openArray[A]>`_ - ## * `newCountTable proc<#newCountTable,int>`_ for creating a + ## * `newCountTable proc<#newCountTable>`_ for creating a ## `CountTableRef` + result = default(CountTable[A]) initImpl(result, initialSize) proc toCountTable*[A](keys: openArray[A]): CountTable[A] = - ## Creates a new count table with every member of a container ``keys`` + ## Creates a new count table with every member of a container `keys` ## having a count of how many times it occurs in that container. - result = initCountTable[A](rightSize(keys.len)) + result = initCountTable[A](keys.len) for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTable[A], key: A): int = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise ``0`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise `0` is returned. ## ## See also: ## * `getOrDefault<#getOrDefault,CountTable[A],A,int>`_ to return ## a custom value if the key doesn't exist - ## * `mget proc<#mget,CountTable[A],A>`_ ## * `[]= proc<#[]%3D,CountTable[A],A,int>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,CountTable[A],A>`_ for checking if a key @@ -2304,24 +2361,21 @@ proc `[]`*[A](t: CountTable[A], key: A): int = assert(not t.isSorted, "CountTable must not be used after sorting") ctget(t, key, 0) -proc mget*[A](t: var CountTable[A], key: A): var int = - ## Retrieves the value at ``t[key]``. The value can be modified. - ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - assert(not t.isSorted, "CountTable must not be used after sorting") - get(t, key) +template cntMakeEmpty(i) = t.data[i].val = 0 +template cntCellEmpty(i) = t.data[i].val == 0 +template cntCellHash(i) = hash(t.data[i].key) proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = - ## Inserts a ``(key, value)`` pair into ``t``. + ## Inserts a `(key, value)` pair into `t`. ## ## See also: ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key - ## * `inc proc<#inc,CountTable[A],A,Positive>`_ for incrementing a + ## * `inc proc<#inc,CountTable[A],A,int>`_ for incrementing a ## value of a key assert(not t.isSorted, "CountTable must not be used after sorting") assert val >= 0 if val == 0: - t.remove(key) + delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash) else: let h = rawGet(t, key) if h >= 0: @@ -2329,11 +2383,8 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) = else: insertImpl() -proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) = - ## Increments ``t[key]`` by ``val`` (default: 1). - ## - ## ``val`` must be a positive number. If you need to decrement a value, - ## use a regular ``Table`` instead. +proc inc*[A](t: var CountTable[A], key: A, val = 1) = + ## Increments `t[key]` by `val` (default: 1). runnableExamples: var a = toCountTable("aab") a.inc('a') @@ -2344,16 +2395,22 @@ proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) = var index = rawGet(t, key) if index >= 0: inc(t.data[index].val, val) - if t.data[index].val == 0: dec(t.counter) + if t.data[index].val == 0: + delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash) else: - insertImpl() + if val != 0: + insertImpl() + +proc len*[A](t: CountTable[A]): int = + ## Returns the number of keys in `t`. + result = t.counter proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) + ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) ## ## See also: ## * `largest proc<#largest,CountTable[A]>`_ - assert t.len > 0 + assert t.len > 0, "counttable is empty" var minIdx = -1 for h in 0 .. high(t.data): if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val): @@ -2362,11 +2419,11 @@ proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = result.val = t.data[minIdx].val proc largest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## Returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) + ## Returns the `(key, value)` pair with the largest `val`. Efficiency: O(n) ## ## See also: ## * `smallest proc<#smallest,CountTable[A]>`_ - assert t.len > 0 + assert t.len > 0, "counttable is empty" var maxIdx = 0 for h in 1 .. high(t.data): if t.data[maxIdx].val < t.data[h].val: maxIdx = h @@ -2374,7 +2431,7 @@ proc largest*[A](t: CountTable[A]): tuple[key: A, val: int] = result.val = t.data[maxIdx].val proc hasKey*[A](t: CountTable[A], key: A): bool = - ## Returns true if ``key`` is in the table ``t``. + ## Returns true if `key` is in the table `t`. ## ## See also: ## * `contains proc<#contains,CountTable[A],A>`_ for use with the `in` @@ -2387,12 +2444,12 @@ proc hasKey*[A](t: CountTable[A], key: A): bool = proc contains*[A](t: CountTable[A], key: A): bool = ## Alias of `hasKey proc<#hasKey,CountTable[A],A>`_ for use with - ## the ``in`` operator. + ## the `in` operator. return hasKey[A](t, key) proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at ``t[key]`` if``key`` is in ``t``. Otherwise, the - ## integer value of ``default`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. ## ## See also: ## * `[] proc<#[],CountTable[A],A>`_ for retrieving a value of a key @@ -2400,14 +2457,8 @@ proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = ## is in the table ctget(t, key, default) -proc len*[A](t: CountTable[A]): int = - ## Returns the number of keys in ``t``. - result = t.counter - proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} = - ## Deletes ``key`` from table ``t``. Does nothing if the key does not exist. - ## - ## O(n) complexity. + ## Deletes `key` from table `t`. Does nothing if the key does not exist. ## ## See also: ## * `pop proc<#pop,CountTable[A],A,int>`_ @@ -2421,16 +2472,14 @@ proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} = a.del('c') assert a == toCountTable("aa") - remove(t, key) + delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash) proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).} = - ## Deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## - ## O(n) complexity. - ## ## See also: ## * `del proc<#del,CountTable[A],A>`_ ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table @@ -2447,7 +2496,7 @@ proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).} result = index >= 0 if result: val = move(t.data[index].val) - remove(t, key) + delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash) proc clear*[A](t: var CountTable[A]) = ## Resets the table so that it is empty. @@ -2465,13 +2514,13 @@ proc sort*[A](t: var CountTable[A], order = SortOrder.Descending) = ## Sorts the count table so that, by default, the entry with the ## highest counter comes first. ## - ## **WARNING:** This is destructive! Once sorted, you must not modify ``t`` afterwards! + ## .. warning:: This is destructive! Once sorted, you must not modify `t` afterwards! ## ## You can use the iterators `pairs<#pairs.i,CountTable[A]>`_, ## `keys<#keys.i,CountTable[A]>`_, and `values<#values.i,CountTable[A]>`_ - ## to iterate over ``t`` in the sorted order. + ## to iterate over `t` in the sorted order. runnableExamples: - import algorithm, sequtils + import std/[algorithm, sequtils] var a = toCountTable("abracadabra") doAssert a == "aaaaabbrrcd".toCountTable a.sort() @@ -2494,32 +2543,33 @@ proc merge*[A](s: var CountTable[A], t: CountTable[A]) = for key, value in t: s.inc(key, value) -proc merge*[A](s, t: CountTable[A]): CountTable[A] = - ## Merges the two tables into a new one. - runnableExamples: - let - a = toCountTable("aaabbc") - b = toCountTable("bcc") - doAssert merge(a, b) == toCountTable("aaabbbccc") +when (NimMajor, NimMinor) <= (1, 0): + proc merge*[A](s, t: CountTable[A]): CountTable[A] = + ## Merges the two tables into a new one. + runnableExamples: + let + a = toCountTable("aaabbc") + b = toCountTable("bcc") + doAssert merge(a, b) == toCountTable("aaabbbccc") - result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len))) - for table in @[s, t]: - for key, value in table: - result.inc(key, value) + result = initCountTable[A](nextPowerOfTwo(max(s.len, t.len))) + for table in @[s, t]: + for key, value in table: + result.inc(key, value) proc `$`*[A](t: CountTable[A]): string = - ## The ``$`` operator for count tables. Used internally when calling `echo` + ## The `$` operator for count tables. Used internally when calling `echo` ## on a table. dollarImpl() proc `==`*[A](s, t: CountTable[A]): bool = - ## The ``==`` operator for count tables. Returns ``true`` if both tables + ## The `==` operator for count tables. Returns `true` if both tables ## contain the same keys with the same count. Insert order does not matter. equalsImpl(s, t) iterator pairs*[A](t: CountTable[A]): (A, int) = - ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## Iterates over any `(key, value)` pair in the table `t`. ## ## See also: ## * `mpairs iterator<#mpairs.i,CountTable[A]>`_ @@ -2528,7 +2578,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = toCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2545,6 +2595,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2552,7 +2603,7 @@ iterator pairs*[A](t: CountTable[A]): (A, int) = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mpairs*[A](t: var CountTable[A]): (A, var int) = - ## Iterates over any ``(key, value)`` pair in the table ``t`` (must be + ## Iterates over any `(key, value)` pair in the table `t` (must be ## declared as `var`). The values can be modified. ## ## See also: @@ -2570,8 +2621,8 @@ iterator mpairs*[A](t: var CountTable[A]): (A, var int) = yield (t.data[h].key, t.data[h].val) assert(len(t) == L, "the length of the table changed while iterating over it") -iterator keys*[A](t: CountTable[A]): A = - ## Iterates over any key in the table ``t``. +iterator keys*[A](t: CountTable[A]): lent A = + ## Iterates over any key in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,CountTable[A]>`_ @@ -2589,7 +2640,7 @@ iterator keys*[A](t: CountTable[A]): A = assert(len(t) == L, "the length of the table changed while iterating over it") iterator values*[A](t: CountTable[A]): int = - ## Iterates over any value in the table ``t``. + ## Iterates over any value in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,CountTable[A]>`_ @@ -2607,7 +2658,7 @@ iterator values*[A](t: CountTable[A]): int = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mvalues*[A](t: var CountTable[A]): var int = - ## Iterates over any value in the table ``t`` (must be + ## Iterates over any value in the table `t` (must be ## declared as `var`). The values can be modified. ## ## See also: @@ -2637,84 +2688,76 @@ iterator mvalues*[A](t: var CountTable[A]): var int = proc inc*[A](t: CountTableRef[A], key: A, val = 1) -proc newCountTable*[A](initialSize = defaultInitialSize): <//>CountTableRef[A] = +proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## Creates a new ref count table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## See also: ## * `newCountTable proc<#newCountTable,openArray[A]>`_ for creating ## a `CountTableRef` from a collection - ## * `initCountTable proc<#initCountTable,int>`_ for creating a + ## * `initCountTable proc<#initCountTable>`_ for creating a ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) -proc newCountTable*[A](keys: openArray[A]): <//>CountTableRef[A] = - ## Creates a new ref count table with every member of a container ``keys`` +proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = + ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. - result = newCountTable[A](rightSize(keys.len)) - for key in items(keys): result.inc(key) + result = newCountTable[A](keys.len) + {.noSideEffect.}: + for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = - ## Retrieves the value at ``t[key]`` if ``key`` is in ``t``. - ## Otherwise ``0`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. + ## Otherwise `0` is returned. ## ## See also: ## * `getOrDefault<#getOrDefault,CountTableRef[A],A,int>`_ to return ## a custom value if the key doesn't exist - ## * `mget proc<#mget,CountTableRef[A],A>`_ + ## * `inc proc<#inc,CountTableRef[A],A,int>`_ to inc even if missing ## * `[]= proc<#[]%3D,CountTableRef[A],A,int>`_ for inserting a new ## (key, value) pair in the table ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key ## is in the table result = t[][key] -proc mget*[A](t: CountTableRef[A], key: A): var int = - ## Retrieves the value at ``t[key]``. The value can be modified. - ## - ## If ``key`` is not in ``t``, the ``KeyError`` exception is raised. - mget(t[], key) - proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = - ## Inserts a ``(key, value)`` pair into ``t``. + ## Inserts a `(key, value)` pair into `t`. ## ## See also: ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a ## value of a key assert val > 0 - t[][key] = val + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = - ## Increments ``t[key]`` by ``val`` (default: 1). + ## Increments `t[key]` by `val` (default: 1). runnableExamples: var a = newCountTable("aab") a.inc('a') a.inc('b', 10) doAssert a == newCountTable("aaabbbbbbbbbbb") - t[].inc(key, val) + {.noSideEffect.}: + t[].inc(key, val) -proc smallest*[A](t: CountTableRef[A]): (A, int) = - ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) +proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = + ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) ## ## See also: ## * `largest proc<#largest,CountTableRef[A]>`_ t[].smallest -proc largest*[A](t: CountTableRef[A]): (A, int) = - ## Returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) +proc largest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = + ## Returns the `(key, value)` pair with the largest `val`. Efficiency: O(n) ## ## See also: ## * `smallest proc<#smallest,CountTable[A]>`_ t[].largest proc hasKey*[A](t: CountTableRef[A], key: A): bool = - ## Returns true if ``key`` is in the table ``t``. + ## Returns true if `key` is in the table `t`. ## ## See also: ## * `contains proc<#contains,CountTableRef[A],A>`_ for use with the `in` @@ -2726,12 +2769,12 @@ proc hasKey*[A](t: CountTableRef[A], key: A): bool = proc contains*[A](t: CountTableRef[A], key: A): bool = ## Alias of `hasKey proc<#hasKey,CountTableRef[A],A>`_ for use with - ## the ``in`` operator. + ## the `in` operator. return hasKey[A](t, key) proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## Retrieves the value at ``t[key]`` if``key`` is in ``t``. Otherwise, the - ## integer value of ``default`` is returned. + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the + ## integer value of `default` is returned. ## ## See also: ## * `[] proc<#[],CountTableRef[A],A>`_ for retrieving a value of a key @@ -2740,13 +2783,11 @@ proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = result = t[].getOrDefault(key, default) proc len*[A](t: CountTableRef[A]): int = - ## Returns the number of keys in ``t``. + ## Returns the number of keys in `t`. result = t.counter proc del*[A](t: CountTableRef[A], key: A) {.since: (1, 1).} = - ## Deletes ``key`` from table ``t``. Does nothing if the key does not exist. - ## - ## O(n) complexity. + ## Deletes `key` from table `t`. Does nothing if the key does not exist. ## ## See also: ## * `pop proc<#pop,CountTableRef[A],A,int>`_ @@ -2754,13 +2795,11 @@ proc del*[A](t: CountTableRef[A], key: A) {.since: (1, 1).} = del(t[], key) proc pop*[A](t: CountTableRef[A], key: A, val: var int): bool {.since: (1, 1).} = - ## Deletes the ``key`` from the table. - ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the - ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## Deletes the `key` from the table. + ## Returns `true`, if the `key` existed, and sets `val` to the + ## mapping of the key. Otherwise, returns `false`, and the `val` is ## unchanged. ## - ## O(n) complexity. - ## ## See also: ## * `del proc<#del,CountTableRef[A],A>`_ ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table @@ -2782,7 +2821,7 @@ proc sort*[A](t: CountTableRef[A], order = SortOrder.Descending) = ## ## You can use the iterators `pairs<#pairs.i,CountTableRef[A]>`_, ## `keys<#keys.i,CountTableRef[A]>`_, and `values<#values.i,CountTableRef[A]>`_ - ## to iterate over ``t`` in the sorted order. + ## to iterate over `t` in the sorted order. t[].sort(order = order) proc merge*[A](s, t: CountTableRef[A]) = @@ -2797,13 +2836,13 @@ proc merge*[A](s, t: CountTableRef[A]) = s[].merge(t[]) proc `$`*[A](t: CountTableRef[A]): string = - ## The ``$`` operator for count tables. Used internally when calling `echo` + ## The `$` operator for count tables. Used internally when calling `echo` ## on a table. dollarImpl() proc `==`*[A](s, t: CountTableRef[A]): bool = - ## The ``==`` operator for count tables. Returns ``true`` if either both tables - ## are ``nil``, or neither is ``nil`` and both contain the same keys with the same + ## The `==` operator for count tables. Returns `true` if either both tables + ## are `nil`, or neither is `nil` and both contain the same keys with the same ## count. Insert order does not matter. if isNil(s): result = isNil(t) elif isNil(t): result = false @@ -2811,7 +2850,7 @@ proc `==`*[A](s, t: CountTableRef[A]): bool = iterator pairs*[A](t: CountTableRef[A]): (A, int) = - ## Iterates over any ``(key, value)`` pair in the table ``t``. + ## Iterates over any `(key, value)` pair in the table `t`. ## ## See also: ## * `mpairs iterator<#mpairs.i,CountTableRef[A]>`_ @@ -2820,7 +2859,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## let a = newCountTable("abracadabra") ## ## for k, v in pairs(a): @@ -2837,6 +2876,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = ## # value: 1 ## # key: r ## # value: 2 + ## ``` let L = len(t) for h in 0 .. high(t.data): if t.data[h].val != 0: @@ -2844,7 +2884,7 @@ iterator pairs*[A](t: CountTableRef[A]): (A, int) = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = - ## Iterates over any ``(key, value)`` pair in the table ``t``. The values can + ## Iterates over any `(key, value)` pair in the table `t`. The values can ## be modified. ## ## See also: @@ -2863,7 +2903,7 @@ iterator mpairs*[A](t: CountTableRef[A]): (A, var int) = assert(len(t) == L, "table modified while iterating over it") iterator keys*[A](t: CountTableRef[A]): A = - ## Iterates over any key in the table ``t``. + ## Iterates over any key in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,CountTable[A]>`_ @@ -2881,7 +2921,7 @@ iterator keys*[A](t: CountTableRef[A]): A = assert(len(t) == L, "the length of the table changed while iterating over it") iterator values*[A](t: CountTableRef[A]): int = - ## Iterates over any value in the table ``t``. + ## Iterates over any value in the table `t`. ## ## See also: ## * `pairs iterator<#pairs.i,CountTableRef[A]>`_ @@ -2899,7 +2939,7 @@ iterator values*[A](t: CountTableRef[A]): int = assert(len(t) == L, "the length of the table changed while iterating over it") iterator mvalues*[A](t: CountTableRef[A]): var int = - ## Iterates over any value in the table ``t``. The values can be modified. + ## Iterates over any value in the table `t`. The values can be modified. ## ## See also: ## * `mpairs iterator<#mpairs.i,CountTableRef[A]>`_ @@ -2916,323 +2956,17 @@ iterator mvalues*[A](t: CountTableRef[A]): var int = yield t.data[h].val assert(len(t) == L, "the length of the table changed while iterating over it") +proc hash*[K,V](s: Table[K,V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result +proc hash*[K,V](s: OrderedTable[K,V]): Hash = + for p in pairs(s): + result = result !& hash(p) + result = !$result - -when isMainModule: - type - Person = object - firstName, lastName: string - - proc hash(x: Person): Hash = - ## Piggyback on the already available string hash proc. - ## - ## Without this proc nothing works! - result = x.firstName.hash !& x.lastName.hash - result = !$result - - var - salaries = initTable[Person, int]() - p1, p2: Person - p1.firstName = "Jon" - p1.lastName = "Ross" - salaries[p1] = 30_000 - p2.firstName = "소진" - p2.lastName = "박" - salaries[p2] = 45_000 - var - s2 = initOrderedTable[Person, int]() - s3 = initCountTable[Person]() - s2[p1] = 30_000 - s2[p2] = 45_000 - s3[p1] = 30_000 - s3[p2] = 45_000 - - block: # Ordered table should preserve order after deletion - var - s4 = initOrderedTable[int, int]() - s4[1] = 1 - s4[2] = 2 - s4[3] = 3 - - var prev = 0 - for i in s4.values: - doAssert(prev < i) - prev = i - - s4.del(2) - doAssert(2 notin s4) - doAssert(s4.len == 2) - prev = 0 - for i in s4.values: - doAssert(prev < i) - prev = i - - block: # Deletion from OrderedTable should account for collision groups. See issue #5057. - # The bug is reproducible only with exact keys - const key1 = "boy_jackpot.inGamma" - const key2 = "boy_jackpot.outBlack" - - var t = { - key1: 0, - key2: 0 - }.toOrderedTable() - - t.del(key1) - assert(t.len == 1) - assert(key2 in t) - - var - t1 = initCountTable[string]() - t2 = initCountTable[string]() - t1.inc("foo") - t1.inc("bar", 2) - t1.inc("baz", 3) - t2.inc("foo", 4) - t2.inc("bar") - t2.inc("baz", 11) - merge(t1, t2) - assert(t1["foo"] == 5) - assert(t1["bar"] == 3) - assert(t1["baz"] == 14) - - let - t1r = newCountTable[string]() - t2r = newCountTable[string]() - t1r.inc("foo") - t1r.inc("bar", 2) - t1r.inc("baz", 3) - t2r.inc("foo", 4) - t2r.inc("bar") - t2r.inc("baz", 11) - merge(t1r, t2r) - assert(t1r["foo"] == 5) - assert(t1r["bar"] == 3) - assert(t1r["baz"] == 14) - - var - t1l = initCountTable[string]() - t2l = initCountTable[string]() - t1l.inc("foo") - t1l.inc("bar", 2) - t1l.inc("baz", 3) - t2l.inc("foo", 4) - t2l.inc("bar") - t2l.inc("baz", 11) - let - t1merging = t1l - t2merging = t2l - let merged = merge(t1merging, t2merging) - assert(merged["foo"] == 5) - assert(merged["bar"] == 3) - assert(merged["baz"] == 14) - - block: - const testKey = "TESTKEY" - let t: CountTableRef[string] = newCountTable[string]() - - # Before, does not compile with error message: - #test_counttable.nim(7, 43) template/generic instantiation from here - #lib/pure/collections/tables.nim(117, 21) template/generic instantiation from here - #lib/pure/collections/tableimpl.nim(32, 27) Error: undeclared field: 'hcode - doAssert 0 == t[testKey] - t.inc(testKey, 3) - doAssert 3 == t[testKey] - - block: - # Clear tests - var clearTable = newTable[int, string]() - clearTable[42] = "asd" - clearTable[123123] = "piuyqwb " - doAssert clearTable[42] == "asd" - clearTable.clear() - doAssert(not clearTable.hasKey(123123)) - doAssert clearTable.getOrDefault(42) == "" - - block: #5482 - var a = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable() - var b = newOrderedTable[string, string](initialSize = 2) - b.add("wrong?", "foo") - b.add("wrong?", "foo2") - assert a == b - - block: #5482 - var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() - var b = newOrderedTable[string, string](initialSize = 2) - b.add("wrong?", "foo") - b.add("wrong?", "foo2") - assert a == b - - block: #5487 - var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() - var b = newOrderedTable[string, string]() # notice, default size! - b.add("wrong?", "foo") - b.add("wrong?", "foo2") - assert a == b - - block: #5487 - var a = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable() - var b = newOrderedTable[string, string]() # notice, default size! - b.add("wrong?", "foo") - b.add("wrong?", "foo2") - assert a == b - - block: - var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() - var b = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable() - var c = newOrderedTable[string, string]() # notice, default size! - c.add("wrong?", "foo") - c.add("wrong?", "foo2") - assert a == b - assert a == c - - block: #6250 - let - a = {3: 1}.toOrderedTable - b = {3: 2}.toOrderedTable - assert((a == b) == false) - assert((b == a) == false) - - block: #6250 - let - a = {3: 2}.toOrderedTable - b = {3: 2}.toOrderedTable - assert((a == b) == true) - assert((b == a) == true) - - block: # CountTable.smallest - let t = toCountTable([0, 0, 5, 5, 5]) - doAssert t.smallest == (0, 2) - - block: #10065 - let t = toCountTable("abracadabra") - doAssert t['z'] == 0 - - var t_mut = toCountTable("abracadabra") - doAssert t_mut['z'] == 0 - # the previous read may not have modified the table. - doAssert t_mut.hasKey('z') == false - t_mut['z'] = 1 - doAssert t_mut['z'] == 1 - doAssert t_mut.hasKey('z') == true - - block: #12813 #13079 - var t = toCountTable("abracadabra") - doAssert len(t) == 5 - - t['a'] = 0 # remove a key - doAssert len(t) == 4 - - block: - var tp: Table[string, string] = initTable[string, string]() - doAssert "test1" == tp.getOrDefault("test1", "test1") - tp["test2"] = "test2" - doAssert "test2" == tp.getOrDefault("test2", "test1") - var tr: TableRef[string, string] = newTable[string, string]() - doAssert "test1" == tr.getOrDefault("test1", "test1") - tr["test2"] = "test2" - doAssert "test2" == tr.getOrDefault("test2", "test1") - var op: OrderedTable[string, string] = initOrderedTable[string, string]() - doAssert "test1" == op.getOrDefault("test1", "test1") - op["test2"] = "test2" - doAssert "test2" == op.getOrDefault("test2", "test1") - var orf: OrderedTableRef[string, string] = newOrderedTable[string, string]() - doAssert "test1" == orf.getOrDefault("test1", "test1") - orf["test2"] = "test2" - doAssert "test2" == orf.getOrDefault("test2", "test1") - - block tableWithoutInit: - var - a: Table[string, int] - b: Table[string, int] - c: Table[string, int] - d: Table[string, int] - e: Table[string, int] - - a["a"] = 7 - doAssert a.hasKey("a") - doAssert a.len == 1 - doAssert a["a"] == 7 - a["a"] = 9 - doAssert a.len == 1 - doAssert a["a"] == 9 - - doAssert b.hasKeyOrPut("b", 5) == false - doAssert b.hasKey("b") - doAssert b.hasKeyOrPut("b", 8) - doAssert b["b"] == 5 - - doAssert c.getOrDefault("a") == 0 - doAssert c.getOrDefault("a", 3) == 3 - c["a"] = 6 - doAssert c.getOrDefault("a", 3) == 6 - - doAssert d.mgetOrPut("a", 3) == 3 - doAssert d.mgetOrPut("a", 6) == 3 - - var x = 99 - doAssert e.pop("a", x) == false - doAssert x == 99 - e["a"] = 77 - doAssert e.pop("a", x) - doAssert x == 77 - - block orderedTableWithoutInit: - var - a: OrderedTable[string, int] - b: OrderedTable[string, int] - c: OrderedTable[string, int] - d: OrderedTable[string, int] - - a["a"] = 7 - doAssert a.hasKey("a") - doAssert a.len == 1 - doAssert a["a"] == 7 - a["a"] = 9 - doAssert a.len == 1 - doAssert a["a"] == 9 - - doAssert b.hasKeyOrPut("b", 5) == false - doAssert b.hasKey("b") - doAssert b.hasKeyOrPut("b", 8) - doAssert b["b"] == 5 - - doAssert c.getOrDefault("a") == 0 - doAssert c.getOrDefault("a", 3) == 3 - c["a"] = 6 - doAssert c.getOrDefault("a", 3) == 6 - - doAssert d.mgetOrPut("a", 3) == 3 - doAssert d.mgetOrPut("a", 6) == 3 - - block countTableWithoutInit: - var - a: CountTable[string] - b: CountTable[string] - c: CountTable[string] - d: CountTable[string] - e: CountTable[string] - - a["a"] = 7 - doAssert a.hasKey("a") - doAssert a.len == 1 - doAssert a["a"] == 7 - a["a"] = 9 - doAssert a.len == 1 - doAssert a["a"] == 9 - - doAssert b["b"] == 0 - b.inc("b") - doAssert b["b"] == 1 - - doAssert c.getOrDefault("a") == 0 - doAssert c.getOrDefault("a", 3) == 3 - c["a"] = 6 - doAssert c.getOrDefault("a", 3) == 6 - - e["f"] = 3 - merge(d, e) - doAssert d.hasKey("f") - d.inc("f") - merge(d, e) - doAssert d["f"] == 7 +proc hash*[V](s: CountTable[V]): Hash = + for p in pairs(s): + result = result xor hash(p) + result = !$result diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index d57e309e9..d3e6dc063 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -6,24 +6,26 @@ # distribution, for details about the copyright. # -## This module implements color handling for Nim. +## This module implements color handling for Nim, +## namely color mixing and parsing the CSS color names. -import strutils -from algorithm import binarySearch +import std/strutils +from std/algorithm import binarySearch type Color* = distinct int ## A color stored as RGB, e.g. `0xff00cc`. -proc `==` *(a, b: Color): bool {.borrow.} +proc `==`*(a, b: Color): bool {.borrow.} ## Compares two colors. ## - ## .. code-block:: + ## ```Nim ## var ## a = Color(0xff_00_ff) ## b = colFuchsia ## c = Color(0x00_ff_cc) ## assert a == b - ## assert not a == c + ## assert not (a == c) + ## ``` template extract(a: Color, r, g, b: untyped) = var r = a.int shr 16 and 0xff @@ -170,6 +172,7 @@ const colDarkGoldenRod* = Color(0xB8860B) colDarkGray* = Color(0xA9A9A9) colDarkGreen* = Color(0x006400) + colDarkGrey* = Color(0xA9A9A9) colDarkKhaki* = Color(0xBDB76B) colDarkMagenta* = Color(0x8B008B) colDarkOliveGreen* = Color(0x556B2F) @@ -180,11 +183,13 @@ const colDarkSeaGreen* = Color(0x8FBC8F) colDarkSlateBlue* = Color(0x483D8B) colDarkSlateGray* = Color(0x2F4F4F) + colDarkSlateGrey* = Color(0x2F4F4F) colDarkTurquoise* = Color(0x00CED1) colDarkViolet* = Color(0x9400D3) colDeepPink* = Color(0xFF1493) colDeepSkyBlue* = Color(0x00BFFF) colDimGray* = Color(0x696969) + colDimGrey* = Color(0x696969) colDodgerBlue* = Color(0x1E90FF) colFireBrick* = Color(0xB22222) colFloralWhite* = Color(0xFFFAF0) @@ -197,6 +202,7 @@ const colGray* = Color(0x808080) colGreen* = Color(0x008000) colGreenYellow* = Color(0xADFF2F) + colGrey* = Color(0x808080) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) colIndianRed* = Color(0xCD5C5C) @@ -211,13 +217,15 @@ const colLightCoral* = Color(0xF08080) colLightCyan* = Color(0xE0FFFF) colLightGoldenRodYellow* = Color(0xFAFAD2) - colLightGrey* = Color(0xD3D3D3) + colLightGray* = Color(0xD3D3D3) colLightGreen* = Color(0x90EE90) + colLightGrey* = Color(0xD3D3D3) colLightPink* = Color(0xFFB6C1) colLightSalmon* = Color(0xFFA07A) colLightSeaGreen* = Color(0x20B2AA) colLightSkyBlue* = Color(0x87CEFA) colLightSlateGray* = Color(0x778899) + colLightSlateGrey* = Color(0x778899) colLightSteelBlue* = Color(0xB0C4DE) colLightYellow* = Color(0xFFFFE0) colLime* = Color(0x00FF00) @@ -228,7 +236,7 @@ const colMediumAquaMarine* = Color(0x66CDAA) colMediumBlue* = Color(0x0000CD) colMediumOrchid* = Color(0xBA55D3) - colMediumPurple* = Color(0x9370D8) + colMediumPurple* = Color(0x9370DB) colMediumSeaGreen* = Color(0x3CB371) colMediumSlateBlue* = Color(0x7B68EE) colMediumSpringGreen* = Color(0x00FA9A) @@ -249,7 +257,7 @@ const colPaleGoldenRod* = Color(0xEEE8AA) colPaleGreen* = Color(0x98FB98) colPaleTurquoise* = Color(0xAFEEEE) - colPaleVioletRed* = Color(0xD87093) + colPaleVioletRed* = Color(0xDB7093) colPapayaWhip* = Color(0xFFEFD5) colPeachPuff* = Color(0xFFDAB9) colPeru* = Color(0xCD853F) @@ -257,6 +265,7 @@ const colPlum* = Color(0xDDA0DD) colPowderBlue* = Color(0xB0E0E6) colPurple* = Color(0x800080) + colRebeccaPurple* = Color(0x663399) colRed* = Color(0xFF0000) colRosyBrown* = Color(0xBC8F8F) colRoyalBlue* = Color(0x4169E1) @@ -270,6 +279,7 @@ const colSkyBlue* = Color(0x87CEEB) colSlateBlue* = Color(0x6A5ACD) colSlateGray* = Color(0x708090) + colSlateGrey* = Color(0x708090) colSnow* = Color(0xFFFAFA) colSpringGreen* = Color(0x00FF7F) colSteelBlue* = Color(0x4682B4) @@ -285,147 +295,155 @@ const colYellow* = Color(0xFFFF00) colYellowGreen* = Color(0x9ACD32) - colorNames = [ - ("aliceblue", colAliceBlue), - ("antiquewhite", colAntiqueWhite), - ("aqua", colAqua), - ("aquamarine", colAquamarine), - ("azure", colAzure), - ("beige", colBeige), - ("bisque", colBisque), - ("black", colBlack), - ("blanchedalmond", colBlanchedAlmond), - ("blue", colBlue), - ("blueviolet", colBlueViolet), - ("brown", colBrown), - ("burlywood", colBurlyWood), - ("cadetblue", colCadetBlue), - ("chartreuse", colChartreuse), - ("chocolate", colChocolate), - ("coral", colCoral), - ("cornflowerblue", colCornflowerBlue), - ("cornsilk", colCornsilk), - ("crimson", colCrimson), - ("cyan", colCyan), - ("darkblue", colDarkBlue), - ("darkcyan", colDarkCyan), - ("darkgoldenrod", colDarkGoldenRod), - ("darkgray", colDarkGray), - ("darkgreen", colDarkGreen), - ("darkkhaki", colDarkKhaki), - ("darkmagenta", colDarkMagenta), - ("darkolivegreen", colDarkOliveGreen), - ("darkorange", colDarkorange), - ("darkorchid", colDarkOrchid), - ("darkred", colDarkRed), - ("darksalmon", colDarkSalmon), - ("darkseagreen", colDarkSeaGreen), - ("darkslateblue", colDarkSlateBlue), - ("darkslategray", colDarkSlateGray), - ("darkturquoise", colDarkTurquoise), - ("darkviolet", colDarkViolet), - ("deeppink", colDeepPink), - ("deepskyblue", colDeepSkyBlue), - ("dimgray", colDimGray), - ("dodgerblue", colDodgerBlue), - ("firebrick", colFireBrick), - ("floralwhite", colFloralWhite), - ("forestgreen", colForestGreen), - ("fuchsia", colFuchsia), - ("gainsboro", colGainsboro), - ("ghostwhite", colGhostWhite), - ("gold", colGold), - ("goldenrod", colGoldenRod), - ("gray", colGray), - ("green", colGreen), - ("greenyellow", colGreenYellow), - ("honeydew", colHoneyDew), - ("hotpink", colHotPink), - ("indianred", colIndianRed), - ("indigo", colIndigo), - ("ivory", colIvory), - ("khaki", colKhaki), - ("lavender", colLavender), - ("lavenderblush", colLavenderBlush), - ("lawngreen", colLawnGreen), - ("lemonchiffon", colLemonChiffon), - ("lightblue", colLightBlue), - ("lightcoral", colLightCoral), - ("lightcyan", colLightCyan), - ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgrey", colLightGrey), - ("lightgreen", colLightGreen), - ("lightpink", colLightPink), - ("lightsalmon", colLightSalmon), - ("lightseagreen", colLightSeaGreen), - ("lightskyblue", colLightSkyBlue), - ("lightslategray", colLightSlateGray), - ("lightsteelblue", colLightSteelBlue), - ("lightyellow", colLightYellow), - ("lime", colLime), - ("limegreen", colLimeGreen), - ("linen", colLinen), - ("magenta", colMagenta), - ("maroon", colMaroon), - ("mediumaquamarine", colMediumAquaMarine), - ("mediumblue", colMediumBlue), - ("mediumorchid", colMediumOrchid), - ("mediumpurple", colMediumPurple), - ("mediumseagreen", colMediumSeaGreen), - ("mediumslateblue", colMediumSlateBlue), - ("mediumspringgreen", colMediumSpringGreen), - ("mediumturquoise", colMediumTurquoise), - ("mediumvioletred", colMediumVioletRed), - ("midnightblue", colMidnightBlue), - ("mintcream", colMintCream), - ("mistyrose", colMistyRose), - ("moccasin", colMoccasin), - ("navajowhite", colNavajoWhite), - ("navy", colNavy), - ("oldlace", colOldLace), - ("olive", colOlive), - ("olivedrab", colOliveDrab), - ("orange", colOrange), - ("orangered", colOrangeRed), - ("orchid", colOrchid), - ("palegoldenrod", colPaleGoldenRod), - ("palegreen", colPaleGreen), - ("paleturquoise", colPaleTurquoise), - ("palevioletred", colPaleVioletRed), - ("papayawhip", colPapayaWhip), - ("peachpuff", colPeachPuff), - ("peru", colPeru), - ("pink", colPink), - ("plum", colPlum), - ("powderblue", colPowderBlue), - ("purple", colPurple), - ("red", colRed), - ("rosybrown", colRosyBrown), - ("royalblue", colRoyalBlue), - ("saddlebrown", colSaddleBrown), - ("salmon", colSalmon), - ("sandybrown", colSandyBrown), - ("seagreen", colSeaGreen), - ("seashell", colSeaShell), - ("sienna", colSienna), - ("silver", colSilver), - ("skyblue", colSkyBlue), - ("slateblue", colSlateBlue), - ("slategray", colSlateGray), - ("snow", colSnow), - ("springgreen", colSpringGreen), - ("steelblue", colSteelBlue), - ("tan", colTan), - ("teal", colTeal), - ("thistle", colThistle), - ("tomato", colTomato), - ("turquoise", colTurquoise), - ("violet", colViolet), - ("wheat", colWheat), - ("white", colWhite), - ("whitesmoke", colWhiteSmoke), - ("yellow", colYellow), - ("yellowgreen", colYellowGreen)] + colorNames = { + "aliceblue": colAliceBlue, + "antiquewhite": colAntiqueWhite, + "aqua": colAqua, + "aquamarine": colAquamarine, + "azure": colAzure, + "beige": colBeige, + "bisque": colBisque, + "black": colBlack, + "blanchedalmond": colBlanchedAlmond, + "blue": colBlue, + "blueviolet": colBlueViolet, + "brown": colBrown, + "burlywood": colBurlyWood, + "cadetblue": colCadetBlue, + "chartreuse": colChartreuse, + "chocolate": colChocolate, + "coral": colCoral, + "cornflowerblue": colCornflowerBlue, + "cornsilk": colCornsilk, + "crimson": colCrimson, + "cyan": colCyan, + "darkblue": colDarkBlue, + "darkcyan": colDarkCyan, + "darkgoldenrod": colDarkGoldenRod, + "darkgray": colDarkGray, + "darkgreen": colDarkGreen, + "darkgrey": colDarkGrey, + "darkkhaki": colDarkKhaki, + "darkmagenta": colDarkMagenta, + "darkolivegreen": colDarkOliveGreen, + "darkorange": colDarkorange, + "darkorchid": colDarkOrchid, + "darkred": colDarkRed, + "darksalmon": colDarkSalmon, + "darkseagreen": colDarkSeaGreen, + "darkslateblue": colDarkSlateBlue, + "darkslategray": colDarkSlateGray, + "darkslategrey": colDarkSlateGrey, + "darkturquoise": colDarkTurquoise, + "darkviolet": colDarkViolet, + "deeppink": colDeepPink, + "deepskyblue": colDeepSkyBlue, + "dimgray": colDimGray, + "dimgrey": colDimGrey, + "dodgerblue": colDodgerBlue, + "firebrick": colFireBrick, + "floralwhite": colFloralWhite, + "forestgreen": colForestGreen, + "fuchsia": colFuchsia, + "gainsboro": colGainsboro, + "ghostwhite": colGhostWhite, + "gold": colGold, + "goldenrod": colGoldenRod, + "gray": colGray, + "green": colGreen, + "greenyellow": colGreenYellow, + "grey": colGrey, + "honeydew": colHoneyDew, + "hotpink": colHotPink, + "indianred": colIndianRed, + "indigo": colIndigo, + "ivory": colIvory, + "khaki": colKhaki, + "lavender": colLavender, + "lavenderblush": colLavenderBlush, + "lawngreen": colLawnGreen, + "lemonchiffon": colLemonChiffon, + "lightblue": colLightBlue, + "lightcoral": colLightCoral, + "lightcyan": colLightCyan, + "lightgoldenrodyellow": colLightGoldenRodYellow, + "lightgray": colLightGray, + "lightgreen": colLightGreen, + "lightgrey": colLightGrey, + "lightpink": colLightPink, + "lightsalmon": colLightSalmon, + "lightseagreen": colLightSeaGreen, + "lightskyblue": colLightSkyBlue, + "lightslategray": colLightSlateGray, + "lightslategrey": colLightSlateGrey, + "lightsteelblue": colLightSteelBlue, + "lightyellow": colLightYellow, + "lime": colLime, + "limegreen": colLimeGreen, + "linen": colLinen, + "magenta": colMagenta, + "maroon": colMaroon, + "mediumaquamarine": colMediumAquaMarine, + "mediumblue": colMediumBlue, + "mediumorchid": colMediumOrchid, + "mediumpurple": colMediumPurple, + "mediumseagreen": colMediumSeaGreen, + "mediumslateblue": colMediumSlateBlue, + "mediumspringgreen": colMediumSpringGreen, + "mediumturquoise": colMediumTurquoise, + "mediumvioletred": colMediumVioletRed, + "midnightblue": colMidnightBlue, + "mintcream": colMintCream, + "mistyrose": colMistyRose, + "moccasin": colMoccasin, + "navajowhite": colNavajoWhite, + "navy": colNavy, + "oldlace": colOldLace, + "olive": colOlive, + "olivedrab": colOliveDrab, + "orange": colOrange, + "orangered": colOrangeRed, + "orchid": colOrchid, + "palegoldenrod": colPaleGoldenRod, + "palegreen": colPaleGreen, + "paleturquoise": colPaleTurquoise, + "palevioletred": colPaleVioletRed, + "papayawhip": colPapayaWhip, + "peachpuff": colPeachPuff, + "peru": colPeru, + "pink": colPink, + "plum": colPlum, + "powderblue": colPowderBlue, + "purple": colPurple, + "rebeccapurple": colRebeccaPurple, + "red": colRed, + "rosybrown": colRosyBrown, + "royalblue": colRoyalBlue, + "saddlebrown": colSaddleBrown, + "salmon": colSalmon, + "sandybrown": colSandyBrown, + "seagreen": colSeaGreen, + "seashell": colSeaShell, + "sienna": colSienna, + "silver": colSilver, + "skyblue": colSkyBlue, + "slateblue": colSlateBlue, + "slategray": colSlateGray, + "slategrey": colSlateGrey, + "snow": colSnow, + "springgreen": colSpringGreen, + "steelblue": colSteelBlue, + "tan": colTan, + "teal": colTeal, + "thistle": colThistle, + "tomato": colTomato, + "turquoise": colTurquoise, + "violet": colViolet, + "wheat": colWheat, + "white": colWhite, + "whitesmoke": colWhiteSmoke, + "yellow": colYellow, + "yellowgreen": colYellowGreen} proc `$`*(c: Color): string = ## Converts a color into its textual representation. @@ -440,7 +458,7 @@ proc colorNameCmp(x: tuple[name: string, col: Color], y: string): int = proc parseColor*(name: string): Color = ## Parses `name` to a color value. ## - ## If no valid color could be parsed ``ValueError`` is raised. + ## If no valid color could be parsed `ValueError` is raised. ## Case insensitive. ## runnableExamples: @@ -452,7 +470,7 @@ proc parseColor*(name: string): Color = assert parseColor(b) == Color(0x01_79_fc) doAssertRaises(ValueError): discard parseColor(c) - if name[0] == '#': + if name.len > 0 and name[0] == '#': result = Color(parseHexInt(name)) else: var idx = binarySearch(colorNames, name, colorNameCmp) @@ -461,7 +479,7 @@ proc parseColor*(name: string): Color = proc isColor*(name: string): bool = ## Returns true if `name` is a known color name or a hexadecimal color - ## prefixed with ``#``. Case insensitive. + ## prefixed with `#`. Case insensitive. ## runnableExamples: var @@ -472,9 +490,10 @@ proc isColor*(name: string): bool = assert b.isColor assert not c.isColor + if name.len == 0: return false if name[0] == '#': for i in 1 .. name.len-1: - if name[i] notin {'0'..'9', 'a'..'f', 'A'..'F'}: return false + if name[i] notin HexDigits: return false result = true else: result = binarySearch(colorNames, name, colorNameCmp) >= 0 diff --git a/lib/pure/complex.nim b/lib/pure/complex.nim index d57adeb92..b48811eae 100644 --- a/lib/pure/complex.nim +++ b/lib/pure/complex.nim @@ -7,157 +7,186 @@ # distribution, for details about the copyright. # -## This module implements complex numbers. -## Complex numbers are currently implemented as generic on a 64-bit or 32-bit float. +## This module implements complex numbers +## and basic mathematical operations on them. +## +## Complex numbers are currently generic over 64-bit or 32-bit floats. + +runnableExamples: + from std/math import almostEqual, sqrt + + let + z1 = complex(1.0, 2.0) + z2 = complex(3.0, -4.0) + + assert almostEqual(z1 + z2, complex(4.0, -2.0)) + assert almostEqual(z1 - z2, complex(-2.0, 6.0)) + assert almostEqual(z1 * z2, complex(11.0, 2.0)) + assert almostEqual(z1 / z2, complex(-0.2, 0.4)) + + assert almostEqual(abs(z1), sqrt(5.0)) + assert almostEqual(conjugate(z1), complex(1.0, -2.0)) + + let (r, phi) = z1.polar + assert almostEqual(rect(r, phi), z1) {.push checks: off, line_dir: off, stack_trace: off, debugger: off.} # the user does not want to trace a part of the standard library! -import math +import std/[math, strformat] type Complex*[T: SomeFloat] = object - re*, im*: T ## A complex number, consisting of a real and an imaginary part. + re*, im*: T Complex64* = Complex[float64] - ## Alias for a pair of 64-bit floats. + ## Alias for a complex number using 64-bit floats. Complex32* = Complex[float32] - ## Alias for a pair of 32-bit floats. + ## Alias for a complex number using 32-bit floats. -proc complex*[T: SomeFloat](re: T; im: T = 0.0): Complex[T] = +func complex*[T: SomeFloat](re: T; im: T = 0.0): Complex[T] = + ## Returns a `Complex[T]` with real part `re` and imaginary part `im`. result.re = re result.im = im -proc complex32*(re: float32; im: float32 = 0.0): Complex[float32] = +func complex32*(re: float32; im: float32 = 0.0): Complex32 = + ## Returns a `Complex32` with real part `re` and imaginary part `im`. result.re = re result.im = im -proc complex64*(re: float64; im: float64 = 0.0): Complex[float64] = +func complex64*(re: float64; im: float64 = 0.0): Complex64 = + ## Returns a `Complex64` with real part `re` and imaginary part `im`. result.re = re result.im = im -template im*(arg: typedesc[float32]): Complex32 = complex[float32](0, 1) -template im*(arg: typedesc[float64]): Complex64 = complex[float64](0, 1) -template im*(arg: float32): Complex32 = complex[float32](0, arg) -template im*(arg: float64): Complex64 = complex[float64](0, arg) - -proc abs*[T](z: Complex[T]): T = - ## Return the distance from (0,0) to ``z``. +template im*(arg: typedesc[float32]): Complex32 = complex32(0, 1) + ## Returns the imaginary unit (`complex32(0, 1)`). +template im*(arg: typedesc[float64]): Complex64 = complex64(0, 1) + ## Returns the imaginary unit (`complex64(0, 1)`). +template im*(arg: float32): Complex32 = complex32(0, arg) + ## Returns `arg` as an imaginary number (`complex32(0, arg)`). +template im*(arg: float64): Complex64 = complex64(0, arg) + ## Returns `arg` as an imaginary number (`complex64(0, arg)`). + +func abs*[T](z: Complex[T]): T = + ## Returns the absolute value of `z`, + ## that is the distance from (0, 0) to `z`. result = hypot(z.re, z.im) -proc abs2*[T](z: Complex[T]): T = - ## Return the squared distance from (0,0) to ``z``. - result = z.re*z.re + z.im*z.im - -proc conjugate*[T](z: Complex[T]): Complex[T] = - ## Conjugate of complex number ``z``. +func abs2*[T](z: Complex[T]): T = + ## Returns the squared absolute value of `z`, + ## that is the squared distance from (0, 0) to `z`. + ## This is more efficient than `abs(z) ^ 2`. + result = z.re * z.re + z.im * z.im + +func sgn*[T](z: Complex[T]): Complex[T] = + ## Returns the phase of `z` as a unit complex number, + ## or 0 if `z` is 0. + let a = abs(z) + if a != 0: + result = z / a + +func conjugate*[T](z: Complex[T]): Complex[T] = + ## Returns the complex conjugate of `z` (`complex(z.re, -z.im)`). result.re = z.re result.im = -z.im -proc inv*[T](z: Complex[T]): Complex[T] = - ## Multiplicative inverse of complex number ``z``. +func inv*[T](z: Complex[T]): Complex[T] = + ## Returns the multiplicative inverse of `z` (`1/z`). conjugate(z) / abs2(z) -proc `==` *[T](x, y: Complex[T]): bool = - ## Compare two complex numbers ``x`` and ``y`` for equality. +func `==`*[T](x, y: Complex[T]): bool = + ## Compares two complex numbers for equality. result = x.re == y.re and x.im == y.im -proc `+` *[T](x: T; y: Complex[T]): Complex[T] = - ## Add a real number to a complex number. +func `+`*[T](x: T; y: Complex[T]): Complex[T] = + ## Adds a real number to a complex number. result.re = x + y.re result.im = y.im -proc `+` *[T](x: Complex[T]; y: T): Complex[T] = - ## Add a complex number to a real number. +func `+`*[T](x: Complex[T]; y: T): Complex[T] = + ## Adds a complex number to a real number. result.re = x.re + y result.im = x.im -proc `+` *[T](x, y: Complex[T]): Complex[T] = - ## Add two complex numbers. +func `+`*[T](x, y: Complex[T]): Complex[T] = + ## Adds two complex numbers. result.re = x.re + y.re result.im = x.im + y.im -proc `-` *[T](z: Complex[T]): Complex[T] = +func `-`*[T](z: Complex[T]): Complex[T] = ## Unary minus for complex numbers. result.re = -z.re result.im = -z.im -proc `-` *[T](x: T; y: Complex[T]): Complex[T] = - ## Subtract a complex number from a real number. - x + (-y) +func `-`*[T](x: T; y: Complex[T]): Complex[T] = + ## Subtracts a complex number from a real number. + result.re = x - y.re + result.im = -y.im -proc `-` *[T](x: Complex[T]; y: T): Complex[T] = - ## Subtract a real number from a complex number. +func `-`*[T](x: Complex[T]; y: T): Complex[T] = + ## Subtracts a real number from a complex number. result.re = x.re - y result.im = x.im -proc `-` *[T](x, y: Complex[T]): Complex[T] = - ## Subtract two complex numbers. +func `-`*[T](x, y: Complex[T]): Complex[T] = + ## Subtracts two complex numbers. result.re = x.re - y.re result.im = x.im - y.im -proc `/` *[T](x: Complex[T]; y: T): Complex[T] = - ## Divide complex number ``x`` by real number ``y``. - result.re = x.re / y - result.im = x.im / y - -proc `/` *[T](x: T; y: Complex[T]): Complex[T] = - ## Divide real number ``x`` by complex number ``y``. - result = x * inv(y) - -proc `/` *[T](x, y: Complex[T]): Complex[T] = - ## Divide ``x`` by ``y``. - var r, den: T - if abs(y.re) < abs(y.im): - r = y.re / y.im - den = y.im + r * y.re - result.re = (x.re * r + x.im) / den - result.im = (x.im * r - x.re) / den - else: - r = y.im / y.re - den = y.re + r * y.im - result.re = (x.re + r * x.im) / den - result.im = (x.im - r * x.re) / den - -proc `*` *[T](x: T; y: Complex[T]): Complex[T] = - ## Multiply a real number and a complex number. +func `*`*[T](x: T; y: Complex[T]): Complex[T] = + ## Multiplies a real number with a complex number. result.re = x * y.re result.im = x * y.im -proc `*` *[T](x: Complex[T]; y: T): Complex[T] = - ## Multiply a complex number with a real number. +func `*`*[T](x: Complex[T]; y: T): Complex[T] = + ## Multiplies a complex number with a real number. result.re = x.re * y result.im = x.im * y -proc `*` *[T](x, y: Complex[T]): Complex[T] = - ## Multiply ``x`` with ``y``. +func `*`*[T](x, y: Complex[T]): Complex[T] = + ## Multiplies two complex numbers. result.re = x.re * y.re - x.im * y.im result.im = x.im * y.re + x.re * y.im +func `/`*[T](x: Complex[T]; y: T): Complex[T] = + ## Divides a complex number by a real number. + result.re = x.re / y + result.im = x.im / y -proc `+=` *[T](x: var Complex[T]; y: Complex[T]) = - ## Add ``y`` to ``x``. +func `/`*[T](x: T; y: Complex[T]): Complex[T] = + ## Divides a real number by a complex number. + result = x * inv(y) + +func `/`*[T](x, y: Complex[T]): Complex[T] = + ## Divides two complex numbers. + x * conjugate(y) / abs2(y) + +func `+=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Adds `y` to `x`. x.re += y.re x.im += y.im -proc `-=` *[T](x: var Complex[T]; y: Complex[T]) = - ## Subtract ``y`` from ``x``. +func `-=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Subtracts `y` from `x`. x.re -= y.re x.im -= y.im -proc `*=` *[T](x: var Complex[T]; y: Complex[T]) = - ## Multiply ``y`` to ``x``. +func `*=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Multiplies `x` by `y`. let im = x.im * y.re + x.re * y.im x.re = x.re * y.re - x.im * y.im x.im = im -proc `/=` *[T](x: var Complex[T]; y: Complex[T]) = - ## Divide ``x`` by ``y`` in place. +func `/=`*[T](x: var Complex[T]; y: Complex[T]) = + ## Divides `x` by `y` in place. x = x / y -proc sqrt*[T](z: Complex[T]): Complex[T] = - ## Square root for a complex number ``z``. +func sqrt*[T](z: Complex[T]): Complex[T] = + ## Computes the + ## ([principal](https://en.wikipedia.org/wiki/Square_root#Principal_square_root_of_a_complex_number)) + ## square root of a complex number `z`. var x, y, w, r: T if z.re == 0.0 and z.im == 0.0: @@ -179,29 +208,37 @@ proc sqrt*[T](z: Complex[T]): Complex[T] = result.im = if z.im >= 0.0: w else: -w result.re = z.im / (result.im + result.im) -proc exp*[T](z: Complex[T]): Complex[T] = - ## ``e`` raised to the power ``z``. - var +func exp*[T](z: Complex[T]): Complex[T] = + ## Computes the exponential function (`e^z`). + let rho = exp(z.re) theta = z.im result.re = rho * cos(theta) result.im = rho * sin(theta) -proc ln*[T](z: Complex[T]): Complex[T] = - ## Returns the natural log of ``z``. +func ln*[T](z: Complex[T]): Complex[T] = + ## Returns the + ## ([principal value](https://en.wikipedia.org/wiki/Complex_logarithm#Principal_value) + ## of the) natural logarithm of `z`. result.re = ln(abs(z)) result.im = arctan2(z.im, z.re) -proc log10*[T](z: Complex[T]): Complex[T] = - ## Returns the log base 10 of ``z``. +func log10*[T](z: Complex[T]): Complex[T] = + ## Returns the logarithm base 10 of `z`. + ## + ## **See also:** + ## * `ln func<#ln,Complex[T]>`_ result = ln(z) / ln(10.0) -proc log2*[T](z: Complex[T]): Complex[T] = - ## Returns the log base 2 of ``z``. +func log2*[T](z: Complex[T]): Complex[T] = + ## Returns the logarithm base 2 of `z`. + ## + ## **See also:** + ## * `ln func<#ln,Complex[T]>`_ result = ln(z) / ln(2.0) -proc pow*[T](x, y: Complex[T]): Complex[T] = - ## ``x`` raised to the power ``y``. +func pow*[T](x, y: Complex[T]): Complex[T] = + ## `x` raised to the power of `y`. if x.re == 0.0 and x.im == 0.0: if y.re == 0.0 and y.im == 0.0: result.re = 1.0 @@ -209,12 +246,33 @@ proc pow*[T](x, y: Complex[T]): Complex[T] = else: result.re = 0.0 result.im = 0.0 - elif y.re == 1.0 and y.im == 0.0: - result = x - elif y.re == -1.0 and y.im == 0.0: - result = T(1.0) / x + elif y.im == 0.0: + if y.re == 1.0: + result = x + elif y.re == -1.0: + result = T(1.0) / x + elif y.re == 2.0: + result = x * x + elif y.re == 0.5: + result = sqrt(x) + elif x.im == 0.0: + # Revert to real pow when both base and exponent are real + result.re = pow(x.re, y.re) + result.im = 0.0 + else: + # Special case when the exponent is real + let + rho = abs(x) + theta = arctan2(x.im, x.re) + s = pow(rho, y.re) + r = y.re * theta + result.re = s * cos(r) + result.im = s * sin(r) + elif x.im == 0.0 and x.re == E: + # Special case Euler's formula + result = exp(y) else: - var + let rho = abs(x) theta = arctan2(x.im, x.re) s = pow(rho, y.re) * exp(-y.im * theta) @@ -222,235 +280,194 @@ proc pow*[T](x, y: Complex[T]): Complex[T] = result.re = s * cos(r) result.im = s * sin(r) -proc pow*[T](x: Complex[T]; y: T): Complex[T] = - ## Complex number ``x`` raised to the power ``y``. +func pow*[T](x: Complex[T]; y: T): Complex[T] = + ## The complex number `x` raised to the power of the real number `y`. pow(x, complex[T](y)) -proc sin*[T](z: Complex[T]): Complex[T] = - ## Returns the sine of ``z``. +func sin*[T](z: Complex[T]): Complex[T] = + ## Returns the sine of `z`. result.re = sin(z.re) * cosh(z.im) result.im = cos(z.re) * sinh(z.im) -proc arcsin*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse sine of ``z``. +func arcsin*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse sine of `z`. result = -im(T) * ln(im(T) * z + sqrt(T(1.0) - z*z)) -proc cos*[T](z: Complex[T]): Complex[T] = - ## Returns the cosine of ``z``. +func cos*[T](z: Complex[T]): Complex[T] = + ## Returns the cosine of `z`. result.re = cos(z.re) * cosh(z.im) result.im = -sin(z.re) * sinh(z.im) -proc arccos*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse cosine of ``z``. +func arccos*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse cosine of `z`. result = -im(T) * ln(z + sqrt(z*z - T(1.0))) -proc tan*[T](z: Complex[T]): Complex[T] = - ## Returns the tangent of ``z``. +func tan*[T](z: Complex[T]): Complex[T] = + ## Returns the tangent of `z`. result = sin(z) / cos(z) -proc arctan*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse tangent of ``z``. +func arctan*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse tangent of `z`. result = T(0.5)*im(T) * (ln(T(1.0) - im(T)*z) - ln(T(1.0) + im(T)*z)) -proc cot*[T](z: Complex[T]): Complex[T] = - ## Returns the cotangent of ``z``. +func cot*[T](z: Complex[T]): Complex[T] = + ## Returns the cotangent of `z`. result = cos(z)/sin(z) -proc arccot*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse cotangent of ``z``. +func arccot*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse cotangent of `z`. result = T(0.5)*im(T) * (ln(T(1.0) - im(T)/z) - ln(T(1.0) + im(T)/z)) -proc sec*[T](z: Complex[T]): Complex[T] = - ## Returns the secant of ``z``. +func sec*[T](z: Complex[T]): Complex[T] = + ## Returns the secant of `z`. result = T(1.0) / cos(z) -proc arcsec*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse secant of ``z``. +func arcsec*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse secant of `z`. result = -im(T) * ln(im(T) * sqrt(1.0 - 1.0/(z*z)) + T(1.0)/z) -proc csc*[T](z: Complex[T]): Complex[T] = - ## Returns the cosecant of ``z``. +func csc*[T](z: Complex[T]): Complex[T] = + ## Returns the cosecant of `z`. result = T(1.0) / sin(z) -proc arccsc*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse cosecant of ``z``. +func arccsc*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse cosecant of `z`. result = -im(T) * ln(sqrt(T(1.0) - T(1.0)/(z*z)) + im(T)/z) -proc sinh*[T](z: Complex[T]): Complex[T] = - ## Returns the hyperbolic sine of ``z``. +func sinh*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic sine of `z`. result = T(0.5) * (exp(z) - exp(-z)) -proc arcsinh*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse hyperbolic sine of ``z``. +func arcsinh*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic sine of `z`. result = ln(z + sqrt(z*z + 1.0)) -proc cosh*[T](z: Complex[T]): Complex[T] = - ## Returns the hyperbolic cosine of ``z``. +func cosh*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic cosine of `z`. result = T(0.5) * (exp(z) + exp(-z)) -proc arccosh*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse hyperbolic cosine of ``z``. +func arccosh*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic cosine of `z`. result = ln(z + sqrt(z*z - T(1.0))) -proc tanh*[T](z: Complex[T]): Complex[T] = - ## Returns the hyperbolic tangent of ``z``. +func tanh*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic tangent of `z`. result = sinh(z) / cosh(z) -proc arctanh*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse hyperbolic tangent of ``z``. +func arctanh*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic tangent of `z`. result = T(0.5) * (ln((T(1.0)+z) / (T(1.0)-z))) -proc sech*[T](z: Complex[T]): Complex[T] = - ## Returns the hyperbolic secant of ``z``. +func coth*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic cotangent of `z`. + result = cosh(z) / sinh(z) + +func arccoth*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic cotangent of `z`. + result = T(0.5) * (ln(T(1.0) + T(1.0)/z) - ln(T(1.0) - T(1.0)/z)) + +func sech*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic secant of `z`. result = T(2.0) / (exp(z) + exp(-z)) -proc arcsech*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse hyperbolic secant of ``z``. +func arcsech*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic secant of `z`. result = ln(1.0/z + sqrt(T(1.0)/z+T(1.0)) * sqrt(T(1.0)/z-T(1.0))) -proc csch*[T](z: Complex[T]): Complex[T] = - ## Returns the hyperbolic cosecant of ``z``. +func csch*[T](z: Complex[T]): Complex[T] = + ## Returns the hyperbolic cosecant of `z`. result = T(2.0) / (exp(z) - exp(-z)) -proc arccsch*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse hyperbolic cosecant of ``z``. +func arccsch*[T](z: Complex[T]): Complex[T] = + ## Returns the inverse hyperbolic cosecant of `z`. result = ln(T(1.0)/z + sqrt(T(1.0)/(z*z) + T(1.0))) -proc coth*[T](z: Complex[T]): Complex[T] = - ## Returns the hyperbolic cotangent of ``z``. - result = cosh(z) / sinh(z) - -proc arccoth*[T](z: Complex[T]): Complex[T] = - ## Returns the inverse hyperbolic cotangent of ``z``. - result = T(0.5) * (ln(T(1.0) + T(1.0)/z) - ln(T(1.0) - T(1.0)/z)) - -proc phase*[T](z: Complex[T]): T = - ## Returns the phase of ``z``. +func phase*[T](z: Complex[T]): T = + ## Returns the phase (or argument) of `z`, that is the angle in polar representation. + ## + ## | `result = arctan2(z.im, z.re)` arctan2(z.im, z.re) -proc polar*[T](z: Complex[T]): tuple[r, phi: T] = - ## Returns ``z`` in polar coordinates. +func polar*[T](z: Complex[T]): tuple[r, phi: T] = + ## Returns `z` in polar coordinates. + ## + ## | `result.r = abs(z)` + ## | `result.phi = phase(z)` + ## + ## **See also:** + ## * `rect func<#rect,T,T>`_ for the inverse operation (r: abs(z), phi: phase(z)) -proc rect*[T](r, phi: T): Complex[T] = - ## Returns the complex number with polar coordinates ``r`` and ``phi``. +func rect*[T](r, phi: T): Complex[T] = + ## Returns the complex number with polar coordinates `r` and `phi`. + ## + ## | `result.re = r * cos(phi)` + ## | `result.im = r * sin(phi)` ## - ## | ``result.re = r * cos(phi)`` - ## | ``result.im = r * sin(phi)`` + ## **See also:** + ## * `polar func<#polar,Complex[T]>`_ for the inverse operation complex(r * cos(phi), r * sin(phi)) +func almostEqual*[T: SomeFloat](x, y: Complex[T]; unitsInLastPlace: Natural = 4): bool = + ## Checks if two complex values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## Two complex values are considered almost equal if their real and imaginary + ## components are almost equal. + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + almostEqual(x.re, y.re, unitsInLastPlace = unitsInLastPlace) and + almostEqual(x.im, y.im, unitsInLastPlace = unitsInLastPlace) -proc `$`*(z: Complex): string = - ## Returns ``z``'s string representation as ``"(re, im)"``. - result = "(" & $z.re & ", " & $z.im & ")" +func `$`*(z: Complex): string = + ## Returns `z`'s string representation as `"(re, im)"`. + runnableExamples: + doAssert $complex(1.0, 2.0) == "(1.0, 2.0)" -{.pop.} + result = "(" & $z.re & ", " & $z.im & ")" +proc formatValueAsTuple(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (real, imaginary) tuple. + result.add "(" + formatValue(result, value.re, specifier) + result.add ", " + formatValue(result, value.im, specifier) + result.add ")" + +proc formatValueAsComplexNumber(result: var string; value: Complex; specifier: string) = + ## Format implementation for `Complex` representing the value as a (RE+IMj) number + ## By default, the real and imaginary parts are formatted using the general ('g') format + let specifier = if specifier.contains({'e', 'E', 'f', 'F', 'g', 'G'}): + specifier.replace("j") + else: + specifier.replace('j', 'g') + result.add "(" + formatValue(result, value.re, specifier) + if value.im >= 0 and not specifier.contains({'+', '-'}): + result.add "+" + formatValue(result, value.im, specifier) + result.add "j)" + +proc formatValue*(result: var string; value: Complex; specifier: string) = + ## Standard format implementation for `Complex`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + ## For complex numbers, we add a specific 'j' specifier, which formats + ## the value as (A+Bj) like in mathematics. + if specifier.len == 0: + result.add $value + elif 'j' in specifier: + formatValueAsComplexNumber(result, value, specifier) + else: + formatValueAsTuple(result, value, specifier) -when isMainModule: - proc `=~`[T](x, y: Complex[T]): bool = - result = abs(x.re-y.re) < 1e-6 and abs(x.im-y.im) < 1e-6 - - proc `=~`[T](x: Complex[T]; y: T): bool = - result = abs(x.re-y) < 1e-6 and abs(x.im) < 1e-6 - - var - z: Complex64 = complex(0.0, 0.0) - oo: Complex64 = complex(1.0, 1.0) - a: Complex64 = complex(1.0, 2.0) - b: Complex64 = complex(-1.0, -2.0) - m1: Complex64 = complex(-1.0, 0.0) - i: Complex64 = complex(0.0, 1.0) - one: Complex64 = complex(1.0, 0.0) - tt: Complex64 = complex(10.0, 20.0) - ipi: Complex64 = complex(0.0, -PI) - - doAssert(a/2.0 =~ complex(0.5, 1.0)) - doAssert(a == a) - doAssert((a-a) == z) - doAssert((a+b) == z) - doAssert((a+b) =~ 0.0) - doAssert((a/b) == m1) - doAssert((1.0/a) =~ complex(0.2, -0.4)) - doAssert((a*b) == complex(3.0, -4.0)) - doAssert(10.0*a == tt) - doAssert(a*10.0 == tt) - doAssert(tt/10.0 == a) - doAssert(oo+(-1.0) == i) - doAssert( (-1.0)+oo == i) - doAssert(abs(oo) == sqrt(2.0)) - doAssert(conjugate(a) == complex(1.0, -2.0)) - doAssert(sqrt(m1) == i) - doAssert(exp(ipi) =~ m1) - - doAssert(pow(a, b) =~ complex(-3.72999124927876, -1.68815826725068)) - doAssert(pow(z, a) =~ complex(0.0, 0.0)) - doAssert(pow(z, z) =~ complex(1.0, 0.0)) - doAssert(pow(a, one) =~ a) - doAssert(pow(a, m1) =~ complex(0.2, -0.4)) - doAssert(pow(a, 2.0) =~ complex(-3.0, 4.0)) - doAssert(pow(a, 2) =~ complex(-3.0, 4.0)) - doAssert(not(pow(a, 2.0) =~ a)) - - doAssert(ln(a) =~ complex(0.804718956217050, 1.107148717794090)) - doAssert(log10(a) =~ complex(0.349485002168009, 0.480828578784234)) - doAssert(log2(a) =~ complex(1.16096404744368, 1.59727796468811)) - - doAssert(sin(a) =~ complex(3.16577851321617, 1.95960104142161)) - doAssert(cos(a) =~ complex(2.03272300701967, -3.05189779915180)) - doAssert(tan(a) =~ complex(0.0338128260798967, 1.0147936161466335)) - doAssert(cot(a) =~ 1.0 / tan(a)) - doAssert(sec(a) =~ 1.0 / cos(a)) - doAssert(csc(a) =~ 1.0 / sin(a)) - doAssert(arcsin(a) =~ complex(0.427078586392476, 1.528570919480998)) - doAssert(arccos(a) =~ complex(1.14371774040242, -1.52857091948100)) - doAssert(arctan(a) =~ complex(1.338972522294494, 0.402359478108525)) - doAssert(arccot(a) =~ complex(0.2318238045004031, -0.402359478108525)) - doAssert(arcsec(a) =~ complex(1.384478272687081, 0.3965682301123288)) - doAssert(arccsc(a) =~ complex(0.1863180541078155, -0.3965682301123291)) - - doAssert(cosh(a) =~ complex(-0.642148124715520, 1.068607421382778)) - doAssert(sinh(a) =~ complex(-0.489056259041294, 1.403119250622040)) - doAssert(tanh(a) =~ complex(1.1667362572409199, -0.243458201185725)) - doAssert(sech(a) =~ 1.0 / cosh(a)) - doAssert(csch(a) =~ 1.0 / sinh(a)) - doAssert(coth(a) =~ 1.0 / tanh(a)) - doAssert(arccosh(a) =~ complex(1.528570919480998, 1.14371774040242)) - doAssert(arcsinh(a) =~ complex(1.469351744368185, 1.06344002357775)) - doAssert(arctanh(a) =~ complex(0.173286795139986, 1.17809724509617)) - doAssert(arcsech(a) =~ arccosh(1.0/a)) - doAssert(arccsch(a) =~ arcsinh(1.0/a)) - doAssert(arccoth(a) =~ arctanh(1.0/a)) - - doAssert(phase(a) == 1.1071487177940904) - var t = polar(a) - doAssert(rect(t.r, t.phi) =~ a) - doAssert(rect(1.0, 2.0) =~ complex(-0.4161468365471424, 0.9092974268256817)) - - - var - i64: Complex32 = complex(0.0f, 1.0f) - a64: Complex32 = 2.0f*i64 + 1.0.float32 - b64: Complex32 = complex(-1.0'f32, -2.0'f32) - - doAssert(a64 == a64) - doAssert(a64 == -b64) - doAssert(a64 + b64 =~ 0.0'f32) - doAssert(not(pow(a64, b64) =~ a64)) - doAssert(pow(a64, 0.5f) =~ sqrt(a64)) - doAssert(pow(a64, 2) =~ complex(-3.0'f32, 4.0'f32)) - doAssert(sin(arcsin(b64)) =~ b64) - doAssert(cosh(arccosh(a64)) =~ a64) - - doAssert(phase(a64) - 1.107149f < 1e-6) - var t64 = polar(a64) - doAssert(rect(t64.r, t64.phi) =~ a64) - doAssert(rect(1.0f, 2.0f) =~ complex(-0.4161468f, 0.90929742f)) - doAssert(sizeof(a64) == 8) - doAssert(sizeof(a) == 16) - - doAssert 123.0.im + 456.0 == complex64(456, 123) - - var localA = complex(0.1'f32) - doAssert localA.im is float32 +{.pop.} diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index 29491f68c..818f1b37a 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -10,10 +10,50 @@ ## Types and operations for atomic operations and lockless algorithms. ## ## Unstable API. - -import macros - -when defined(cpp) or defined(nimdoc): +## +## By default, C++ uses C11 atomic primitives. To use C++ `std::atomic`, +## `-d:nimUseCppAtomics` can be defined. + +runnableExamples: + # Atomic + var loc: Atomic[int] + loc.store(4) + assert loc.load == 4 + loc.store(2) + assert loc.load(moRelaxed) == 2 + loc.store(9) + assert loc.load(moAcquire) == 9 + loc.store(0, moRelease) + assert loc.load == 0 + + assert loc.exchange(7) == 0 + assert loc.load == 7 + + var expected = 7 + assert loc.compareExchange(expected, 5, moRelaxed, moRelaxed) + assert expected == 7 + assert loc.load == 5 + + assert not loc.compareExchange(expected, 12, moRelaxed, moRelaxed) + assert expected == 5 + assert loc.load == 5 + + assert loc.fetchAdd(1) == 5 + assert loc.fetchAdd(2) == 6 + assert loc.fetchSub(3) == 8 + + loc.atomicInc(1) + assert loc.load == 6 + + # AtomicFlag + var flag: AtomicFlag + + assert not flag.testAndSet + assert flag.testAndSet + flag.clear(moRelaxed) + assert not flag.testAndSet + +when (defined(cpp) and defined(nimUseCppAtomics)) or defined(nimdoc): # For the C++ backend, types and operations map directly to C++11 atomics. {.push, header: "<atomic>".} @@ -51,10 +91,11 @@ when defined(cpp) or defined(nimdoc): ## with other moSequentiallyConsistent operations. type - Atomic* {.importcpp: "std::atomic".} [T] = object + Atomic*[T] {.importcpp: "std::atomic", completeStruct.} = object ## An atomic object with underlying type `T`. + raw: T - AtomicFlag* {.importcpp: "std::atomic_flag".} = object + AtomicFlag* {.importcpp: "std::atomic_flag", size: 1.} = object ## An atomic boolean state. # Access operations @@ -172,8 +213,8 @@ else: # MSVC intrinsics proc interlockedExchange(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8".} - proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange".} - proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16".} + proc interlockedExchange(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange".} proc interlockedExchange(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64".} proc interlockedCompareExchange(location: pointer; desired, expected: int8): int8 {.importc: "_InterlockedCompareExchange8".} @@ -235,10 +276,17 @@ else: cast[T](interlockedXor(addr(location.value), cast[nonAtomicType(T)](value))) else: - {.push, header: "<stdatomic.h>".} + when defined(cpp): + {.push, header: "<atomic>".} + template maybeWrapStd(x: string): string = + "std::" & x + else: + {.push, header: "<stdatomic.h>".} + template maybeWrapStd(x: string): string = + x type - MemoryOrder* {.importc: "memory_order".} = enum + MemoryOrder* {.importc: "memory_order".maybeWrapStd.} = enum moRelaxed moConsume moAcquire @@ -246,58 +294,64 @@ else: moAcquireRelease moSequentiallyConsistent - type - # Atomic* {.importcpp: "_Atomic('0)".} [T] = object + when defined(cpp): + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object - AtomicInt8 {.importc: "_Atomic NI8".} = object - AtomicInt16 {.importc: "_Atomic NI16".} = object - AtomicInt32 {.importc: "_Atomic NI32".} = object - AtomicInt64 {.importc: "_Atomic NI64".} = object + AtomicInt8 {.importc: "std::atomic<NI8>".} = int8 + AtomicInt16 {.importc: "std::atomic<NI16>".} = int16 + AtomicInt32 {.importc: "std::atomic<NI32>".} = int32 + AtomicInt64 {.importc: "std::atomic<NI64>".} = int64 + else: + type + # Atomic*[T] {.importcpp: "_Atomic('0)".} = object - template atomicType*(T: typedesc[Trivial]): untyped = - # Maps the size of a trivial type to it's internal atomic type - when sizeof(T) == 1: AtomicInt8 - elif sizeof(T) == 2: AtomicInt16 - elif sizeof(T) == 4: AtomicInt32 - elif sizeof(T) == 8: AtomicInt64 + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 type - AtomicFlag* {.importc: "atomic_flag".} = object + AtomicFlag* {.importc: "atomic_flag".maybeWrapStd, size: 1.} = object Atomic*[T] = object when T is Trivial: - value: T.atomicType + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: value: AtomicInt8 + elif sizeof(T) == 2: value: AtomicInt16 + elif sizeof(T) == 4: value: AtomicInt32 + elif sizeof(T) == 8: value: AtomicInt64 else: nonAtomicValue: T guard: AtomicFlag #proc init*[T](location: var Atomic[T]; value: T): T {.importcpp: "atomic_init(@)".} - proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc.} - proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc.} - proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} - proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc.} + proc atomic_load_explicit[T, A](location: ptr A; order: MemoryOrder): T {.importc: "atomic_load_explicit".maybeWrapStd.} + proc atomic_store_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_store_explicit".maybeWrapStd.} + proc atomic_exchange_explicit[T, A](location: ptr A; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_exchange_explicit".maybeWrapStd.} + proc atomic_compare_exchange_strong_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_strong_explicit".maybeWrapStd.} + proc atomic_compare_exchange_weak_explicit[T, A](location: ptr A; expected: ptr T; desired: T; success, failure: MemoryOrder): bool {.importc: "atomic_compare_exchange_weak_explicit".maybeWrapStd.} # Numerical operations - proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} - proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc.} + proc atomic_fetch_add_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_add_explicit".maybeWrapStd.} + proc atomic_fetch_sub_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_sub_explicit".maybeWrapStd.} + proc atomic_fetch_and_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_and_explicit".maybeWrapStd.} + proc atomic_fetch_or_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_or_explicit".maybeWrapStd.} + proc atomic_fetch_xor_explicit[T, A](location: ptr A; value: T; order: MemoryOrder = moSequentiallyConsistent): T {.importc: "atomic_fetch_xor_explicit".maybeWrapStd.} # Flag operations # var ATOMIC_FLAG_INIT {.importc, nodecl.}: AtomicFlag # proc init*(location: var AtomicFlag) {.inline.} = location = ATOMIC_FLAG_INIT - proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".} - proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".} + proc testAndSet*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent): bool {.importc: "atomic_flag_test_and_set_explicit".maybeWrapStd.} + proc clear*(location: var AtomicFlag; order: MemoryOrder = moSequentiallyConsistent) {.importc: "atomic_flag_clear_explicit".maybeWrapStd.} - proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".} - proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".} + proc fence*(order: MemoryOrder) {.importc: "atomic_thread_fence".maybeWrapStd.} + proc signalFence*(order: MemoryOrder) {.importc: "atomic_signal_fence".maybeWrapStd.} {.pop.} proc load*[T: Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = - cast[T](atomic_load_explicit[nonAtomicType(T), type(location.value)](addr(location.value), order)) + cast[T](atomic_load_explicit[nonAtomicType(T), typeof(location.value)](addr(location.value), order)) proc store*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent) {.inline.} = atomic_store_explicit(addr(location.value), cast[nonAtomicType(T)](desired), order) proc exchange*[T: Trivial](location: var Atomic[T]; desired: T; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = @@ -325,9 +379,11 @@ else: cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = - while location.guard.testAndSet(moAcquire): discard - body - location.guard.clear(moRelease) + while testAndSet(location.guard, moAcquire): discard + try: + body + finally: + clear(location.guard, moRelease) proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = withLock(location, order): @@ -345,16 +401,14 @@ else: proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = withLock(location, success): if location.nonAtomicValue != expected: + expected = location.nonAtomicValue return false + expected = desired swap(location.nonAtomicValue, expected) return true proc compareExchangeWeak*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; success, failure: MemoryOrder): bool {.inline.} = - withLock(location, success): - if location.nonAtomicValue != expected: - return false - swap(location.nonAtomicValue, expected) - return true + compareExchange(location, expected, desired, success, failure) proc compareExchange*[T: not Trivial](location: var Atomic[T]; expected: var T; desired: T; order: MemoryOrder = moSequentiallyConsistent): bool {.inline.} = compareExchange(location, expected, desired, order, order) diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index 515d7e2da..9bc3fd579 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -7,91 +7,104 @@ # distribution, for details about the copyright. # -## This module implements procs to determine the number of CPUs / cores. +## This module implements a proc to determine the number of CPUs / cores. + +runnableExamples: + doAssert countProcessors() > 0 + include "system/inclrtl" -when not defined(windows): - import posix +when defined(js): + import std/jsffi + proc countProcessorsImpl(): int = + when defined(nodejs): + let jsOs = require("os") + let jsObj = jsOs.cpus().length + else: + # `navigator.hardwareConcurrency` + # works on browser as well as deno. + let navigator{.importcpp.}: JsObject + let jsObj = navigator.hardwareConcurrency + result = jsObj.to int +else: + when defined(posix) and not (defined(macosx) or defined(bsd)): + import std/posix + + when defined(windows): + import std/private/win_getsysteminfo + + when defined(freebsd) or defined(macosx): + {.emit: "#include <sys/types.h>".} -when defined(freebsd) or defined(macosx): - {.emit:"#include <sys/types.h>".} + when defined(openbsd) or defined(netbsd): + {.emit: "#include <sys/param.h>".} -when defined(openbsd) or defined(netbsd): - {.emit:"#include <sys/param.h>".} + when defined(macosx) or defined(bsd): + # we HAVE to emit param.h before sysctl.h so we cannot use .header here + # either. The amount of archaic bullshit in Poonix based OSes is just insane. + {.emit: "#include <sys/sysctl.h>".} + {.push nodecl.} + when defined(macosx): + proc sysctlbyname(name: cstring, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + let + CTL_HW{.importc.}: cint + HW_NCPU{.importc.}: cint + proc sysctl[I: static[int]](name: var array[I, cint], namelen: cuint, + oldp: pointer, oldlenp: var csize_t, + newp: pointer, newlen: csize_t): cint {.importc.} + {.pop.} -when defined(macosx) or defined(bsd): - # we HAVE to emit param.h before sysctl.h so we cannot use .header here - # either. The amount of archaic bullshit in Poonix based OSes is just insane. - {.emit:"#include <sys/sysctl.h>".} - const - CTL_HW = 6 - HW_AVAILCPU = 25 - HW_NCPU = 3 - proc sysctl(x: ptr array[0..3, cint], y: cint, z: pointer, - a: var csize_t, b: pointer, c: csize_t): cint {. - importc: "sysctl", nodecl.} + when defined(genode): + import genode/env -when defined(genode): - include genode/env + proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. + importcpp: "@->cpu().affinity_space().total()".} - proc affinitySpaceTotal(env: GenodeEnvPtr): cuint {. - importcpp: "@->cpu().affinity_space().total()".} + when defined(haiku): + type + SystemInfo {.importc: "system_info", header: "<OS.h>".} = object + cpuCount {.importc: "cpu_count".}: uint32 + + proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", + header: "<OS.h>".} + + proc countProcessorsImpl(): int {.inline.} = + when defined(windows): + var + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) + elif defined(macosx) or defined(bsd): + let dest = addr result + var len = sizeof(result).csize_t + when defined(macosx): + # alias of "hw.activecpu" + if sysctlbyname("hw.logicalcpu", dest, len, nil, 0) == 0: + return + var mib = [CTL_HW, HW_NCPU] + if sysctl(mib, 2, dest, len, nil, 0) == 0: + return + elif defined(hpux): + result = mpctl(MPC_GETNUMSPUS, nil, nil) + elif defined(irix): + var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint + result = sysconf(SC_NPROC_ONLN) + elif defined(genode): + result = runtimeEnv.affinitySpaceTotal().int + elif defined(haiku): + var sysinfo: SystemInfo + if getSystemInfo(addr sysinfo) == 0: + result = sysinfo.cpuCount.int + else: + result = sysconf(SC_NPROCESSORS_ONLN) + if result < 0: result = 0 -when defined(haiku): - type - SystemInfo {.importc: "system_info", header: "<OS.h>".} = object - cpuCount {.importc: "cpu_count".}: uint32 - proc getSystemInfo(info: ptr SystemInfo): int32 {.importc: "get_system_info", - header: "<OS.h>".} proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = - ## returns the number of the processors/cores the machine has. + ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. - when defined(windows): - type - SYSTEM_INFO {.final, pure.} = object - u1: int32 - dwPageSize: int32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr int32 - dwNumberOfProcessors: int32 - dwProcessorType: int32 - dwAllocationGranularity: int32 - wProcessorLevel: int16 - wProcessorRevision: int16 - - proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} - - var - si: SYSTEM_INFO - GetSystemInfo(si) - result = si.dwNumberOfProcessors - elif defined(macosx) or defined(bsd): - var - mib: array[0..3, cint] - numCPU: int - mib[0] = CTL_HW - mib[1] = HW_AVAILCPU - var len = sizeof(numCPU).csize_t - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - if numCPU < 1: - mib[1] = HW_NCPU - discard sysctl(addr(mib), 2, addr(numCPU), len, nil, 0) - result = numCPU - elif defined(hpux): - result = mpctl(MPC_GETNUMSPUS, nil, nil) - elif defined(irix): - var SC_NPROC_ONLN {.importc: "_SC_NPROC_ONLN", header: "<unistd.h>".}: cint - result = sysconf(SC_NPROC_ONLN) - elif defined(genode): - result = runtimeEnv.affinitySpaceTotal().int - elif defined(haiku): - var sysinfo: SystemInfo - if getSystemInfo(addr sysinfo) == 0: - result = sysinfo.cpuCount.int - else: - result = sysconf(SC_NPROCESSORS_ONLN) - if result <= 0: result = 0 + countProcessorsImpl() diff --git a/lib/pure/concurrency/cpuload.nim b/lib/pure/concurrency/cpuload.nim index 3ee7336f0..bfbf16721 100644 --- a/lib/pure/concurrency/cpuload.nim +++ b/lib/pure/concurrency/cpuload.nim @@ -13,11 +13,14 @@ ## Unstable API. when defined(windows): - import winlean, os, strutils, math + import std/[winlean, os, strutils, math] proc `-`(a, b: FILETIME): int64 = a.rdFileTime - b.rdFileTime elif defined(linux): - from cpuinfo import countProcessors + from std/cpuinfo import countProcessors + +when defined(nimPreviewSlimSystem): + import std/syncio type ThreadPoolAdvice* = enum @@ -84,11 +87,11 @@ proc advice*(s: var ThreadPoolState): ThreadPoolAdvice = inc s.calls when not defined(testing) and isMainModule and not defined(nimdoc): - import random + import std/random proc busyLoop() = while true: - discard random(80) + discard rand(80) os.sleep(100) spawn busyLoop() diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 1f7df7c00..06ed2fe54 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -7,20 +7,25 @@ # distribution, for details about the copyright. # -## Implements Nim's `spawn <manual_experimental.html#parallel-amp-spawn>`_. -## -## **See also:** -## * `threads module <threads.html>`_ -## * `channels module <channels.html>`_ -## * `locks module <locks.html>`_ -## * `asyncdispatch module <asyncdispatch.html>`_ +{.deprecated: "use the nimble packages `malebolgia`, `taskpools` or `weave` instead".} + +## Implements Nim's `parallel & spawn statements <manual_experimental.html#parallel-amp-spawn>`_. ## ## Unstable API. +## +## See also +## ======== +## * `threads module <typedthreads.html>`_ for basic thread support +## * `locks module <locks.html>`_ for locks and condition variables +## * `asyncdispatch module <asyncdispatch.html>`_ for asynchronous IO when not compileOption("threads"): {.error: "Threadpool requires --threads:on option.".} -import cpuinfo, cpuload, locks, os +import std/[cpuinfo, cpuload, locks, os] + +when defined(nimPreviewSlimSystem): + import std/[assertions, typedthreads, sysatomics] {.push stackTrace:off.} @@ -51,38 +56,35 @@ proc signal(cv: var Semaphore) = release(cv.L) signal(cv.c) -const CacheLineSize = 32 # true for most archs +const CacheLineSize = 64 # true for most archs type - Barrier {.compilerProc.} = object + Barrier {.compilerproc.} = object entered: int cv: Semaphore # Semaphore takes 3 words at least - when sizeof(int) < 8: - cacheAlign: array[CacheLineSize-4*sizeof(int), byte] - left: int - cacheAlign2: array[CacheLineSize-sizeof(int), byte] - interest: bool # whether the master is interested in the "all done" event + left {.align(CacheLineSize).}: int + interest {.align(CacheLineSize).} : bool # whether the master is interested in the "all done" event -proc barrierEnter(b: ptr Barrier) {.compilerProc, inline.} = +proc barrierEnter(b: ptr Barrier) {.compilerproc, inline.} = # due to the signaling between threads, it is ensured we are the only # one with access to 'entered' so we don't need 'atomicInc' here: inc b.entered # also we need no 'fence' instructions here as soon 'nimArgsPassingDone' # will be called which already will perform a fence for us. -proc barrierLeave(b: ptr Barrier) {.compilerProc, inline.} = +proc barrierLeave(b: ptr Barrier) {.compilerproc, inline.} = atomicInc b.left when not defined(x86): fence() # We may not have seen the final value of b.entered yet, # so we need to check for >= instead of ==. if b.interest and b.left >= b.entered: signal(b.cv) -proc openBarrier(b: ptr Barrier) {.compilerProc, inline.} = +proc openBarrier(b: ptr Barrier) {.compilerproc, inline.} = b.entered = 0 b.left = 0 b.interest = false -proc closeBarrier(b: ptr Barrier) {.compilerProc.} = +proc closeBarrier(b: ptr Barrier) {.compilerproc.} = fence() if b.left != b.entered: b.cv.initSemaphore() @@ -101,8 +103,8 @@ type cv: Semaphore idx: int - FlowVarBase* = ref FlowVarBaseObj ## Untyped base class for ``FlowVar[T]``. - FlowVarBaseObj = object of RootObj + FlowVarBase* = ref FlowVarBaseObj ## Untyped base class for `FlowVar[T] <#FlowVar>`_. + FlowVarBaseObj {.acyclic.} = object of RootObj ready, usesSemaphore, awaited: bool cv: Semaphore # for 'blockUntilAny' support ai: ptr AwaitInfo @@ -111,10 +113,10 @@ type # be RootRef here otherwise the wrong GC keeps track of it! owner: pointer # ptr Worker - FlowVarObj[T] = object of FlowVarBaseObj + FlowVarObj[T] {.acyclic.} = object of FlowVarBaseObj blob: T - FlowVar*{.compilerProc.}[T] = ref FlowVarObj[T] ## A data flow variable. + FlowVar*[T] {.compilerproc.} = ref FlowVarObj[T] ## A data flow variable. ToFreeQueue = object len: int @@ -137,8 +139,8 @@ type const threadpoolWaitMs {.intdefine.}: int = 100 -proc blockUntil*(fv: FlowVarBase) = - ## Waits until the value for the ``fv`` arrives. +proc blockUntil*(fv: var FlowVarBaseObj) = + ## Waits until the value for `fv` arrives. ## ## Usually it is not necessary to call this explicitly. if fv.usesSemaphore and not fv.awaited: @@ -185,7 +187,7 @@ proc attach(fv: FlowVarBase; i: int): bool = result = false release(fv.cv.L) -proc finished(fv: FlowVarBase) = +proc finished(fv: var FlowVarBaseObj) = doAssert fv.ai.isNil, "flowVar is still attached to an 'blockUntilAny'" # we have to protect against the rare cases where the owner of the flowVar # simply disregards the flowVar and yet the "flowVar" has not yet written @@ -208,16 +210,18 @@ proc finished(fv: FlowVarBase) = # the worker thread waits for "data" to be set to nil before shutting down owner.data = nil -proc fvFinalizer[T](fv: FlowVar[T]) = finished(fv) +proc `=destroy`[T](fv: var FlowVarObj[T]) = + finished(fv) + `=destroy`(fv.blob) -proc nimCreateFlowVar[T](): FlowVar[T] {.compilerProc.} = - new(result, fvFinalizer) +proc nimCreateFlowVar[T](): FlowVar[T] {.compilerproc.} = + new(result) -proc nimFlowVarCreateSemaphore(fv: FlowVarBase) {.compilerProc.} = +proc nimFlowVarCreateSemaphore(fv: FlowVarBase) {.compilerproc.} = fv.cv.initSemaphore() fv.usesSemaphore = true -proc nimFlowVarSignal(fv: FlowVarBase) {.compilerProc.} = +proc nimFlowVarSignal(fv: FlowVarBase) {.compilerproc.} = if fv.ai != nil: acquire(fv.ai.cv.L) fv.ai.idx = fv.idx @@ -228,60 +232,51 @@ proc nimFlowVarSignal(fv: FlowVarBase) {.compilerProc.} = signal(fv.cv) proc awaitAndThen*[T](fv: FlowVar[T]; action: proc (x: T) {.closure.}) = - ## Blocks until the ``fv`` is available and then passes its value - ## to ``action``. + ## Blocks until `fv` is available and then passes its value + ## to `action`. ## - ## Note that due to Nim's parameter passing semantics this - ## means that ``T`` doesn't need to be copied so ``awaitAndThen`` can - ## sometimes be more efficient than `^ proc <#^,FlowVar[T]>`_. - blockUntil(fv) - when T is string or T is seq: + ## Note that due to Nim's parameter passing semantics, this + ## means that `T` doesn't need to be copied, so `awaitAndThen` can + ## sometimes be more efficient than the `^ proc <#^,FlowVar[T]>`_. + blockUntil(fv[]) + when defined(nimV2): + action(fv.blob) + elif T is string or T is seq: action(cast[T](fv.data)) elif T is ref: {.error: "'awaitAndThen' not available for FlowVar[ref]".} else: action(fv.blob) - finished(fv) + finished(fv[]) proc unsafeRead*[T](fv: FlowVar[ref T]): ptr T = ## Blocks until the value is available and then returns this value. - blockUntil(fv) - result = cast[ptr T](fv.data) - finished(fv) - -proc `^`*[T](fv: FlowVar[ref T]): ref T = - ## Blocks until the value is available and then returns this value. - blockUntil(fv) - let src = cast[ref T](fv.data) + blockUntil(fv[]) when defined(nimV2): - result = src + result = cast[ptr T](fv.blob) else: - deepCopy result, src - finished(fv) + result = cast[ptr T](fv.data) + finished(fv[]) proc `^`*[T](fv: FlowVar[T]): T = ## Blocks until the value is available and then returns this value. - blockUntil(fv) - when T is string or T is seq: - let src = cast[T](fv.data) - when defined(nimV2): - result = src - else: - deepCopy result, src + blockUntil(fv[]) + when not defined(nimV2) and (T is string or T is seq or T is ref): + deepCopy result, cast[T](fv.data) else: result = fv.blob - finished(fv) + finished(fv[]) proc blockUntilAny*(flowVars: openArray[FlowVarBase]): int = - ## Awaits any of the given ``flowVars``. Returns the index of one ``flowVar`` + ## Awaits any of the given `flowVars`. Returns the index of one `flowVar` ## for which a value arrived. ## - ## A ``flowVar`` only supports one call to ``blockUntilAny`` at the same time. - ## That means if you ``blockUntilAny([a,b])`` and ``blockUntilAny([b,c])`` - ## the second call will only block until ``c``. If there is no ``flowVar`` left + ## A `flowVar` only supports one call to `blockUntilAny` at the same time. + ## That means if you `blockUntilAny([a,b])` and `blockUntilAny([b,c])` + ## the second call will only block until `c`. If there is no `flowVar` left ## to be able to wait on, -1 is returned. ## - ## **Note**: This results in non-deterministic behaviour and should be avoided. + ## **Note:** This results in non-deterministic behaviour and should be avoided. var ai: AwaitInfo ai.cv.initSemaphore() var conflicts = 0 @@ -302,9 +297,9 @@ proc blockUntilAny*(flowVars: openArray[FlowVarBase]): int = destroySemaphore(ai.cv) proc isReady*(fv: FlowVarBase): bool = - ## Determines whether the specified ``FlowVarBase``'s value is available. + ## Determines whether the specified `FlowVarBase`'s value is available. ## - ## If ``true``, awaiting ``fv`` will not block. + ## If `true`, awaiting `fv` will not block. if fv.usesSemaphore and not fv.awaited: acquire(fv.cv.L) result = fv.cv.counter > 0 @@ -312,31 +307,31 @@ proc isReady*(fv: FlowVarBase): bool = else: result = true -proc nimArgsPassingDone(p: pointer) {.compilerProc.} = +proc nimArgsPassingDone(p: pointer) {.compilerproc.} = let w = cast[ptr Worker](p) signal(w.taskStarted) const - MaxThreadPoolSize* = 256 ## Maximum size of the thread pool. 256 threads - ## should be good enough for anybody ;-) - MaxDistinguishedThread* = 32 ## Maximum number of "distinguished" threads. + MaxThreadPoolSize* {.intdefine.} = 256 ## Maximum size of the thread pool. 256 threads + ## should be good enough for anybody ;-) + MaxDistinguishedThread* {.intdefine.} = 32 ## Maximum number of "distinguished" threads. type - ThreadId* = range[0..MaxDistinguishedThread-1] + ThreadId* = range[0..MaxDistinguishedThread-1] ## A thread identifier. var currentPoolSize: int maxPoolSize = MaxThreadPoolSize minPoolSize = 4 - gSomeReady : Semaphore + gSomeReady: Semaphore readyWorker: ptr Worker # A workaround for recursion deadlock issue # https://github.com/nim-lang/Nim/issues/4597 var numSlavesLock: Lock - numSlavesRunning {.guard: numSlavesLock}: int - numSlavesWaiting {.guard: numSlavesLock}: int + numSlavesRunning {.guard: numSlavesLock.}: int + numSlavesWaiting {.guard: numSlavesLock.}: int isSlave {.threadvar.}: bool numSlavesLock.initLock @@ -409,7 +404,7 @@ proc setMinPoolSize*(size: range[1..MaxThreadPoolSize]) = proc setMaxPoolSize*(size: range[1..MaxThreadPoolSize]) = ## Sets the maximum thread pool size. The default value of this - ## is ``MaxThreadPoolSize`` (256). + ## is `MaxThreadPoolSize <#MaxThreadPoolSize>`_. maxPoolSize = size if currentPoolSize > maxPoolSize: for i in maxPoolSize..currentPoolSize-1: @@ -449,43 +444,45 @@ proc setup() = for i in 0..<currentPoolSize: activateWorkerThread(i) proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a ``spawn`` or a direct call is + ## Use this proc to determine quickly if a `spawn` or a direct call is ## preferable. ## - ## If it returns ``true``, a ``spawn`` may make sense. In general - ## it is not necessary to call this directly; use `spawnX template + ## If it returns `true`, a `spawn` may make sense. In general + ## it is not necessary to call this directly; use the `spawnX template ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: typed): void {.magic: "Spawn".} - ## Always spawns a new task, so that the ``call`` is never executed on +proc spawn*(call: sink typed) {.magic: "Spawn".} = + ## Always spawns a new task, so that the `call` is never executed on ## the calling thread. ## - ## ``call`` has to be proc call ``p(...)`` where ``p`` is gcsafe and has a - ## return type that is either ``void`` or compatible with ``FlowVar[T]``. + ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a + ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn3` internally" -proc pinnedSpawn*(id: ThreadId; call: typed): void {.magic: "Spawn".} - ## Always spawns a new task on the worker thread with ``id``, so that - ## the ``call`` is **always** executed on the thread. +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} = + ## Always spawns a new task on the worker thread with `id`, so that + ## the `call` is **always** executed on the thread. ## - ## ``call`` has to be proc call ``p(...)`` where ``p`` is gcsafe and has a - ## return type that is either ``void`` or compatible with ``FlowVar[T]``. + ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a + ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn4` internally" -template spawnX*(call): void = +template spawnX*(call) = ## Spawns a new task if a CPU core is ready, otherwise executes the ## call in the calling thread. ## - ## Usually it is advised to use `spawn proc <#spawn,typed>`_ in order to - ## not block the producer for an unknown amount of time. + ## Usually, it is advised to use the `spawn proc <#spawn,sinktyped>`_ + ## in order to not block the producer for an unknown amount of time. ## - ## ``call`` has to be proc call ``p(...)`` where ``p`` is gcsafe and has a - ## return type that is either 'void' or compatible with ``FlowVar[T]``. + ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a + ## return type that is either 'void' or compatible with `FlowVar[T]`. (if preferSpawn(): spawn call else: call) proc parallel*(body: untyped) {.magic: "Parallel".} ## A parallel section can be used to execute a block in parallel. ## - ## ``body`` has to be in a DSL that is a particular subset of the language. + ## `body` has to be in a DSL that is a particular subset of the language. ## ## Please refer to `the manual <manual_experimental.html#parallel-amp-spawn>`_ ## for further information. @@ -496,7 +493,7 @@ var initLock stateLock -proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = +proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerproc.} = # implementation of 'spawn' that is used by the code generator. while true: if selectWorker(readyWorker, fn, data): return @@ -581,7 +578,7 @@ var initLock distinguishedLock -proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} = +proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerproc.} = acquire(distinguishedLock) if not distinguishedData[id].initialized: activateDistinguishedThread(id) @@ -592,7 +589,7 @@ proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} = proc sync*() = - ## A simple barrier to wait for all ``spawn``'ed tasks. + ## A simple barrier to wait for all `spawn`ed tasks. ## ## If you need more elaborate waiting, you have to use an explicit barrier. while true: diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index eaff86ae6..f628aaf6b 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -9,18 +9,28 @@ ## This module implements helper procs for parsing Cookies. -import strtabs, times +import std/[strtabs, times, options] + +when defined(nimPreviewSlimSystem): + import std/assertions + + +type + SameSite* {.pure.} = enum ## The SameSite cookie attribute. + ## `Default` means that `setCookie` + ## proc will not set `SameSite` attribute. + Default, None, Lax, Strict proc parseCookies*(s: string): StringTableRef = - ## parses cookies into a string table. + ## Parses cookies into a string table. ## ## The proc is meant to parse the Cookie header set by a client, not the ## "Set-Cookie" header set by servers. - ## - ## Example: - ## - ## .. code-block::Nim - ## doAssert parseCookies("a=1; foo=bar") == {"a": 1, "foo": "bar"}.newStringTable + runnableExamples: + import std/strtabs + let cookieJar = parseCookies("a=1; foo=bar") + assert cookieJar["a"] == "1" + assert cookieJar["foo"] == "bar" result = newStringTable(modeCaseInsensitive) var i = 0 @@ -39,9 +49,12 @@ proc parseCookies*(s: string): StringTableRef = proc setCookie*(key, value: string, domain = "", path = "", expires = "", noName = false, - secure = false, httpOnly = false): string = + secure = false, httpOnly = false, + maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of - ## ``Set-Cookie: key=value; Domain=...; ...`` + ## `Set-Cookie: key=value; Domain=...; ...` + ## + ## .. tip:: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value @@ -50,27 +63,19 @@ proc setCookie*(key, value: string, domain = "", path = "", if expires != "": result.add("; Expires=" & expires) if secure: result.add("; Secure") if httpOnly: result.add("; HttpOnly") + if maxAge.isSome: result.add("; Max-Age=" & $maxAge.unsafeGet) + + if sameSite != SameSite.Default: + if sameSite == SameSite.None: + doAssert secure, "Cookies with SameSite=None must specify the Secure attribute!" + result.add("; SameSite=" & $sameSite) proc setCookie*(key, value: string, expires: DateTime|Time, domain = "", path = "", noName = false, - secure = false, httpOnly = false): string = + secure = false, httpOnly = false, + maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of - ## ``Set-Cookie: key=value; Domain=...; ...`` - return setCookie(key, value, domain, path, + ## `Set-Cookie: key=value; Domain=...; ...` + result = setCookie(key, value, domain, path, format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), - noname, secure, httpOnly) - -when isMainModule: - let expire = fromUnix(0) + 1.seconds - - let cookies = [ - setCookie("test", "value", expire), - setCookie("test", "value", expire.local), - setCookie("test", "value", expire.utc) - ] - let expected = "Set-Cookie: test=value; Expires=Thu, 01 Jan 1970 00:00:01 GMT" - doAssert cookies == [expected, expected, expected] - - let table = parseCookies("uid=1; kp=2") - doAssert table["uid"] == "1" - doAssert table["kp"] == "2" + noName, secure, httpOnly, maxAge, sameSite) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index 350b9f187..24836e316 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -8,11 +8,11 @@ # ## Nim coroutines implementation, supports several context switching methods: -## -------- ------------ +## ======== ============ ## ucontext available on unix and alike (default) ## setjmp available on unix and alike (x86/64 only) ## fibers available and required on windows. -## -------- ------------ +## ======== ============ ## ## -d:nimCoroutines Required to build this module. ## -d:nimCoroutinesUcontext Use ucontext backend. @@ -21,21 +21,30 @@ ## ## Unstable API. +import system/coro_detection + when not nimCoroutines and not defined(nimdoc): when defined(noNimCoroutines): {.error: "Coroutines can not be used with -d:noNimCoroutines".} else: {.error: "Coroutines require -d:nimCoroutines".} -import os -import lists +import std/[os, lists] include system/timers +when defined(nimPreviewSlimSystem): + import std/assertions + const defaultStackSize = 512 * 1024 +const useOrcArc = defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc) + +when useOrcArc: + proc nimGC_setStackBottom*(theStackBottom: pointer) = discard proc GC_addStack(bottom: pointer) {.cdecl, importc.} proc GC_removeStack(bottom: pointer) {.cdecl, importc.} proc GC_setActiveStack(bottom: pointer) {.cdecl, importc.} +proc GC_getActiveStack() : pointer {.cdecl, importc.} const CORO_BACKEND_UCONTEXT = 0 @@ -48,7 +57,7 @@ when defined(windows): {.warning: "ucontext coroutine backend is not available on windows, defaulting to fibers.".} when defined(nimCoroutinesSetjmp): {.warning: "setjmp coroutine backend is not available on windows, defaulting to fibers.".} -elif defined(haiku): +elif defined(haiku) or defined(openbsd): const coroBackend = CORO_BACKEND_SETJMP when defined(nimCoroutinesUcontext): {.warning: "ucontext coroutine backend is not available on haiku, defaulting to setjmp".} @@ -58,7 +67,7 @@ else: const coroBackend = CORO_BACKEND_UCONTEXT when coroBackend == CORO_BACKEND_FIBERS: - import windows.winlean + import std/winlean type Context = pointer @@ -158,7 +167,7 @@ type ## CoroutineRef holds a pointer to actual coroutine object. Public API always returns ## CoroutineRef instead of CoroutinePtr in order to allow holding a reference to coroutine ## object while it can be safely deallocated by coroutine scheduler loop. In this case - ## Coroutine.reference.coro is set to nil. Public API checks for for it being nil and + ## Coroutine.reference.coro is set to nil. Public API checks for it being nil and ## gracefully fails if it is nil. coro: CoroutinePtr @@ -166,6 +175,7 @@ type coroutines: DoublyLinkedList[CoroutinePtr] current: DoublyLinkedNode[CoroutinePtr] loop: Coroutine + ncbottom: pointer # non coroutine stack botttom var ctx {.threadvar.}: CoroutineLoopContext @@ -183,6 +193,8 @@ proc initialize() = ctx.coroutines = initDoublyLinkedList[CoroutinePtr]() ctx.loop = Coroutine() ctx.loop.state = CORO_EXECUTING + when not useOrcArc: + ctx.ncbottom = GC_getActiveStack() when coroBackend == CORO_BACKEND_FIBERS: ctx.loop.execContext = ConvertThreadToFiberEx(nil, FIBER_FLAG_FLOAT_SWITCH) @@ -192,7 +204,9 @@ proc switchTo(current, to: CoroutinePtr) = ## Switches execution from `current` into `to` context. to.lastRun = getTicks() # Update position of current stack so gc invoked from another stack knows how much to scan. - GC_setActiveStack(current.stack.bottom) + when not useOrcArc: + GC_setActiveStack(current.stack.bottom) + nimGC_setStackBottom(current.stack.bottom) var frame = getFrameState() block: # Execution will switch to another fiber now. We do not need to update current stack @@ -209,18 +223,21 @@ proc switchTo(current, to: CoroutinePtr) = elif to.state == CORO_CREATED: # Coroutine is started. coroExecWithStack(runCurrentTask, to.stack.bottom) - #doAssert false + #raiseAssert "unreachable" else: {.error: "Invalid coroutine backend set.".} # Execution was just resumed. Restore frame information and set active stack. setFrameState(frame) - GC_setActiveStack(current.stack.bottom) + when not useOrcArc: + GC_setActiveStack(current.stack.bottom) + nimGC_setStackBottom(ctx.ncbottom) proc suspend*(sleepTime: float = 0) = - ## Stops coroutine execution and resumes no sooner than after ``sleeptime`` seconds. + ## Stops coroutine execution and resumes no sooner than after `sleeptime` seconds. ## Until then other coroutines are executed. var current = getCurrent() current.sleepTime = sleepTime + nimGC_setStackBottom(ctx.ncbottom) switchTo(current, addr(ctx.loop)) proc runCurrentTask() = @@ -230,13 +247,15 @@ proc runCurrentTask() = block: var current = getCurrent() current.stack.bottom = sp + nimGC_setStackBottom(current.stack.bottom) # Execution of new fiber just started. Since it was entered not through `switchTo` we # have to set active stack here as well. GC_removeStack() has to be called in main loop # because we still need stack available in final suspend(0) call from which we will not # return. - GC_addStack(sp) - # Activate current stack because we are executing in a new coroutine. - GC_setActiveStack(sp) + when not useOrcArc: + GC_addStack(sp) + # Activate current stack because we are executing in a new coroutine. + GC_setActiveStack(sp) current.state = CORO_EXECUTING try: current.fn() # Start coroutine execution @@ -244,8 +263,9 @@ proc runCurrentTask() = echo "Unhandled exception in coroutine." writeStackTrace() current.state = CORO_FINISHED + nimGC_setStackBottom(ctx.ncbottom) suspend(0) # Exit coroutine without returning from coroExecWithStack() - doAssert false + raiseAssert "unreachable" proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discardable.} = ## Schedule coroutine for execution. It does not run immediately. @@ -257,13 +277,11 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine))) coro.execContext = CreateFiberEx(stacksize, stacksize, FIBER_FLAG_FLOAT_SWITCH, - (proc(p: pointer): void {.stdcall.} = runCurrentTask()), - nil) - coro.stack.size = stacksize + (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) - coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) - coro.stack.bottom = cast[pointer](cast[ByteAddress](coro.stack.top) + stacksize) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) when coroBackend == CORO_BACKEND_UCONTEXT: discard getcontext(coro.execContext) coro.execContext.uc_stack.ss_sp = coro.stack.top @@ -278,9 +296,9 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar return coro.reference proc run*() = - initialize() ## Starts main coroutine scheduler loop which exits when all coroutines exit. ## Calling this proc starts execution of first coroutine. + initialize() ctx.current = ctx.coroutines.head var minDelay: float = 0 while ctx.current != nil: @@ -304,7 +322,8 @@ proc run*() = next = ctx.current.next current.reference.coro = nil ctx.coroutines.remove(ctx.current) - GC_removeStack(current.stack.bottom) + when not useOrcArc: + GC_removeStack(current.stack.bottom) when coroBackend == CORO_BACKEND_FIBERS: DeleteFiber(current.execContext) else: @@ -318,9 +337,9 @@ proc run*() = ctx.current = ctx.current.next proc alive*(c: CoroutineRef): bool = c.coro != nil and c.coro.state != CORO_FINISHED - ## Returns ``true`` if coroutine has not returned, ``false`` otherwise. + ## Returns `true` if coroutine has not returned, `false` otherwise. proc wait*(c: CoroutineRef, interval = 0.01) = - ## Returns only after coroutine ``c`` has returned. ``interval`` is time in seconds how often. + ## Returns only after coroutine `c` has returned. `interval` is time in seconds how often. while alive(c): suspend(interval) diff --git a/lib/pure/cstrutils.nim b/lib/pure/cstrutils.nim index fe9ceb68b..c907e54d8 100644 --- a/lib/pure/cstrutils.nim +++ b/lib/pure/cstrutils.nim @@ -7,75 +7,116 @@ # distribution, for details about the copyright. # -## This module supports helper routines for working with ``cstring`` -## without having to convert ``cstring`` to ``string`` in order to +## This module supports helper routines for working with `cstring` +## without having to convert `cstring` to `string`, in order to ## save allocations. +## +## See also +## ======== +## * `strutils module <strutils.html>`_ for working with `string` -include "system/inclrtl" +include system/inclrtl +import std/private/strimpl -proc toLowerAscii(c: char): char {.inline.} = - if c in {'A'..'Z'}: - result = chr(ord(c) + (ord('a') - ord('A'))) - else: - result = c -proc startsWith*(s, prefix: cstring): bool {.noSideEffect, - rtl, extern: "csuStartsWith".} = - ## Returns true iff ``s`` starts with ``prefix``. - ## - ## If ``prefix == ""`` true is returned. - var i = 0 - while true: - if prefix[i] == '\0': return true - if s[i] != prefix[i]: return false - inc(i) +when defined(js): + func jsStartsWith(s, prefix: cstring): bool {.importjs: "#.startsWith(#)".} + func jsEndsWith(s, suffix: cstring): bool {.importjs: "#.endsWith(#)".} -proc endsWith*(s, suffix: cstring): bool {.noSideEffect, - rtl, extern: "csuEndsWith".} = - ## Returns true iff ``s`` ends with ``suffix``. + +func startsWith*(s, prefix: cstring): bool {.rtl, extern: "csuStartsWith".} = + ## Returns true if `s` starts with `prefix`. ## - ## If ``suffix == ""`` true is returned. - let slen = s.len - var i = 0 - var j = slen - len(suffix) - while i+j <% slen: - if s[i+j] != suffix[i]: return false - inc(i) - if suffix[i] == '\0': return true + ## The JS backend uses the native `String.prototype.startsWith` function. + runnableExamples: + assert startsWith(cstring"Hello, Nimion", cstring"Hello") + assert not startsWith(cstring"Hello, Nimion", cstring"Nimion") + assert startsWith(cstring"Hello", cstring"") -proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect, - rtl, extern: "csuCmpIgnoreStyle".} = - ## Semantically the same as ``cmp(normalize($a), normalize($b))``. It - ## is just optimized to not allocate temporary strings. This should - ## NOT be used to compare Nim identifier names. use `macros.eqIdent` - ## for that. Returns: + when nimvm: + startsWithImpl(s, prefix) + else: + when defined(js): + result = jsStartsWith(s, prefix) + else: + var i = 0 + while true: + if prefix[i] == '\0': return true + if s[i] != prefix[i]: return false + inc(i) + +func endsWith*(s, suffix: cstring): bool {.rtl, extern: "csuEndsWith".} = + ## Returns true if `s` ends with `suffix`. ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b - var i = 0 - var j = 0 - while true: - while a[i] == '_': inc(i) - while b[j] == '_': inc(j) # BUGFIX: typo - var aa = toLowerAscii(a[i]) - var bb = toLowerAscii(b[j]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) - inc(j) + ## The JS backend uses the native `String.prototype.endsWith` function. + runnableExamples: + assert endsWith(cstring"Hello, Nimion", cstring"Nimion") + assert not endsWith(cstring"Hello, Nimion", cstring"Hello") + assert endsWith(cstring"Hello", cstring"") + + when nimvm: + endsWithImpl(s, suffix) + else: + when defined(js): + result = jsEndsWith(s, suffix) + else: + let slen = s.len + var i = 0 + var j = slen - len(suffix) + while i + j <% slen: + if s[i + j] != suffix[i]: return false + inc(i) + if suffix[i] == '\0': return true -proc cmpIgnoreCase*(a, b: cstring): int {.noSideEffect, - rtl, extern: "csuCmpIgnoreCase".} = +func cmpIgnoreStyle*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreStyle".} = + ## Semantically the same as `cmp(normalize($a), normalize($b))`. It + ## is just optimized to not allocate temporary strings. This should + ## NOT be used to compare Nim identifier names, use `macros.eqIdent` + ## for that. Returns: + ## * 0 if `a == b` + ## * < 0 if `a < b` + ## * \> 0 if `a > b` + runnableExamples: + assert cmpIgnoreStyle(cstring"hello", cstring"H_e_L_Lo") == 0 + + when nimvm: + cmpIgnoreStyleImpl(a, b) + else: + when defined(js): + cmpIgnoreStyleImpl(a, b) + else: + var i = 0 + var j = 0 + while true: + while a[i] == '_': inc(i) + while b[j] == '_': inc(j) # BUGFIX: typo + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[j]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) + inc(j) + +func cmpIgnoreCase*(a, b: cstring): int {.rtl, extern: "csuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: - ## - ## | 0 iff a == b - ## | < 0 iff a < b - ## | > 0 iff a > b - var i = 0 - while true: - var aa = toLowerAscii(a[i]) - var bb = toLowerAscii(b[i]) - result = ord(aa) - ord(bb) - if result != 0 or aa == '\0': break - inc(i) + ## * 0 if `a == b` + ## * < 0 if `a < b` + ## * \> 0 if `a > b` + runnableExamples: + assert cmpIgnoreCase(cstring"hello", cstring"HeLLo") == 0 + assert cmpIgnoreCase(cstring"echo", cstring"hello") < 0 + assert cmpIgnoreCase(cstring"yellow", cstring"hello") > 0 + + when nimvm: + cmpIgnoreCaseImpl(a, b) + else: + when defined(js): + cmpIgnoreCaseImpl(a, b) + else: + var i = 0 + while true: + var aa = toLowerAscii(a[i]) + var bb = toLowerAscii(b[i]) + result = ord(aa) - ord(bb) + if result != 0 or aa == '\0': break + inc(i) diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim deleted file mode 100644 index 96c13c003..000000000 --- a/lib/pure/db_common.nim +++ /dev/null @@ -1,100 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Common datatypes and definitions for all ``db_*.nim`` ( -## `db_mysql <db_mysql.html>`_, `db_postgres <db_postgres.html>`_, -## and `db_sqlite <db_sqlite.html>`_) modules. - -type - DbError* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string - - - DbEffect* = object of IOEffect ## effect that denotes a database operation - ReadDbEffect* = object of DbEffect ## effect that denotes a read operation - WriteDbEffect* = object of DbEffect ## effect that denotes a write operation - - DbTypeKind* = enum ## a superset of datatypes that might be supported. - dbUnknown, ## unknown datatype - dbSerial, ## datatype used for primary auto-increment keys - dbNull, ## datatype used for the NULL value - dbBit, ## bit datatype - dbBool, ## boolean datatype - dbBlob, ## blob datatype - dbFixedChar, ## string of fixed length - dbVarchar, ## string datatype - dbJson, ## JSON datatype - dbXml, ## XML datatype - dbInt, ## some integer type - dbUInt, ## some unsigned integer type - dbDecimal, ## decimal numbers (fixed-point number) - dbFloat, ## some floating point type - dbDate, ## a year-month-day description - dbTime, ## HH:MM:SS information - dbDatetime, ## year-month-day and HH:MM:SS information, - ## plus optional time or timezone information - dbTimestamp, ## Timestamp values are stored as the number of seconds - ## since the epoch ('1970-01-01 00:00:00' UTC). - dbTimeInterval, ## an interval [a,b] of times - dbEnum, ## some enum - dbSet, ## set of enum values - dbArray, ## an array of values - dbComposite, ## composite type (record, struct, etc) - dbUrl, ## a URL - dbUuid, ## a UUID - dbInet, ## an IP address - dbMacAddress, ## a MAC address - dbGeometry, ## some geometric type - dbPoint, ## Point on a plane (x,y) - dbLine, ## Infinite line ((x1,y1),(x2,y2)) - dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) - dbBox, ## Rectangular box ((x1,y1),(x2,y2)) - dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) - dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) - dbCircle, ## Circle <(x,y),r> (center point and radius) - dbUser1, ## user definable datatype 1 (for unknown extensions) - dbUser2, ## user definable datatype 2 (for unknown extensions) - dbUser3, ## user definable datatype 3 (for unknown extensions) - dbUser4, ## user definable datatype 4 (for unknown extensions) - dbUser5 ## user definable datatype 5 (for unknown extensions) - - DbType* = object ## describes a database type - kind*: DbTypeKind ## the kind of the described type - notNull*: bool ## does the type contain NULL? - name*: string ## the name of the type - size*: Natural ## the size of the datatype; 0 if of variable size - maxReprLen*: Natural ## maximal length required for the representation - precision*, scale*: Natural ## precision and scale of the number - min*, max*: BiggestInt ## the minimum and maximum of allowed values - validValues*: seq[string] ## valid values of an enum or a set - - DbColumn* = object ## information about a database column - name*: string ## name of the column - tableName*: string ## name of the table the column belongs to (optional) - typ*: DbType ## type of the column - primaryKey*: bool ## is this a primary key? - foreignKey*: bool ## is this a foreign key? - DbColumns* = seq[DbColumn] - -template sql*(query: string): SqlQuery = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - SqlQuery(query) - -proc dbError*(msg: string) {.noreturn, noinline.} = - ## raises an DbError exception with message `msg`. - var e: ref DbError - new(e) - e.msg = msg - raise e diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 63cfe6719..9e71d4ce0 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -9,34 +9,33 @@ ## This module implements the basics for Linux distribution ("distro") ## detection and the OS's native package manager. Its primary purpose is to -## produce output for Nimble packages like:: +## produce output for Nimble packages, like: ## -## To complete the installation, run: +## To complete the installation, run: ## -## sudo apt-get libblas-dev -## sudo apt-get libvoodoo +## sudo apt-get install libblas-dev +## sudo apt-get install libvoodoo ## ## The above output could be the result of a code snippet like: ## -## .. code-block:: nim -## +## ```nim ## if detectOs(Ubuntu): ## foreignDep "lbiblas-dev" ## foreignDep "libvoodoo" -## +## ``` ## ## See `packaging <packaging.html>`_ for hints on distributing Nim using OS packages. -from strutils import contains, toLowerAscii +from std/strutils import contains, toLowerAscii when not defined(nimscript): - from osproc import execProcess - from os import existsEnv + from std/osproc import execProcess + from std/envvars import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions Windows ## some version of Windows - Posix ## some Posix system + Posix ## some POSIX system MacOSX ## some version of OSX Linux ## some version of Linux Ubuntu @@ -52,6 +51,7 @@ type CentOS Deepin ArchLinux + Artix Antergos PCLinuxOS Mageia @@ -108,7 +108,7 @@ type Clonezilla SteamOS Absolute - NixOS ## NixOS or a Nix build environment + NixOS ## NixOS or a Nix build environment AUSTRUMI Arya Porteus @@ -123,9 +123,11 @@ type ExTiX Rockstor GoboLinux + Void BSD FreeBSD + NetBSD OpenBSD DragonFlyBSD @@ -134,66 +136,85 @@ type const LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, - Distribution.ArchLinux} + Distribution.ArchLinux, Distribution.Artix, Distribution.Antergos, + Distribution.BlackArch, Distribution.ArchBang} -# we cache the result of the 'uname -a' +# we cache the result of the 'cmdRelease' # execution for faster platform detections. -var unameRes, releaseRes, hostnamectlRes: string +var unameRes, osReleaseIDRes, releaseRes, hostnamectlRes: string -template unameRelease(cmd, cache): untyped = +template cmdRelease(cmd, cache): untyped = if cache.len == 0: cache = (when defined(nimscript): gorge(cmd) else: execProcess(cmd)) cache -template uname(): untyped = unameRelease("uname -a", unameRes) -template release(): untyped = unameRelease("lsb_release -d", releaseRes) -template hostnamectl(): untyped = unameRelease("hostnamectl", hostnamectlRes) +template uname(): untyped = cmdRelease("uname -a", unameRes) +template osReleaseID(): untyped = cmdRelease("cat /etc/os-release | grep ^ID=", osReleaseIDRes) +template release(): untyped = cmdRelease("lsb_release -d", releaseRes) +template hostnamectl(): untyped = cmdRelease("hostnamectl", hostnamectlRes) + +proc detectOsWithAllCmd(d: Distribution): bool = + let dd = toLowerAscii($d) + result = dd in toLowerAscii(osReleaseID()) or dd in toLowerAscii(release()) or + dd in toLowerAscii(uname()) or ("operating system: " & dd) in + toLowerAscii(hostnamectl()) proc detectOsImpl(d: Distribution): bool = case d - of Distribution.Windows: ## some version of Windows - result = defined(windows) + of Distribution.Windows: result = defined(windows) of Distribution.Posix: result = defined(posix) of Distribution.MacOSX: result = defined(macosx) of Distribution.Linux: result = defined(linux) - of Distribution.Ubuntu, Distribution.Gentoo, Distribution.FreeBSD, - Distribution.OpenBSD: - result = ("-" & $d & " ") in uname() - of Distribution.RedHat: - result = "Red Hat" in uname() of Distribution.BSD: result = defined(bsd) - of Distribution.ArchLinux: - result = "arch" in toLowerAscii(uname()) - of Distribution.NixOS: - result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") - # Check if this is a Nix build or NixOS environment - of Distribution.OpenSUSE: - result = "suse" in toLowerAscii(uname()) or "suse" in toLowerAscii(release()) - of Distribution.GoboLinux: - result = "-Gobo " in uname() - of Distribution.OpenMandriva: - result = "mandriva" in toLowerAscii(uname()) - of Distribution.Solaris: - let uname = toLowerAscii(uname()) - result = ("sun" in uname) or ("solaris" in uname) - of Distribution.Haiku: - result = defined(haiku) else: - let dd = toLowerAscii($d) - result = dd in toLowerAscii(uname()) or dd in toLowerAscii(release()) or - ("operating system: " & dd) in toLowerAscii(hostnamectl()) + when defined(bsd): + case d + of Distribution.FreeBSD, Distribution.NetBSD, Distribution.OpenBSD: + result = $d in uname() + else: + result = false + elif defined(linux): + case d + of Distribution.Gentoo: + result = ("-" & $d & " ") in uname() + of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, + Distribution.Fedora, Distribution.OpenMandriva, Distribution.CentOS, + Distribution.Alpine, Distribution.Mageia, Distribution.Zorin, Distribution.Void: + result = toLowerAscii($d) in osReleaseID() + of Distribution.RedHat: + result = "rhel" in osReleaseID() + of Distribution.ArchLinux: + result = "arch" in osReleaseID() + of Distribution.Artix: + result = "artix" in osReleaseID() + of Distribution.NixOS: + # Check if this is a Nix build or NixOS environment + result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") + of Distribution.OpenSUSE: + result = "suse" in toLowerAscii(uname()) or "suse" in toLowerAscii(release()) + of Distribution.GoboLinux: + result = "-Gobo " in uname() + of Distribution.Solaris: + let uname = toLowerAscii(uname()) + result = ("sun" in uname) or ("solaris" in uname) + of Distribution.Haiku: + result = defined(haiku) + else: + result = detectOsWithAllCmd(d) + else: + result = false template detectOs*(d: untyped): bool = - ## Distro/OS detection. For convenience the - ## required ``Distribution.`` qualifier is added to the + ## Distro/OS detection. For convenience, the + ## required `Distribution.` qualifier is added to the ## enum value. detectOsImpl(Distribution.d) when not defined(nimble): - var foreignDeps: seq[string] = @[] + var foreignDeps*: seq[string] = @[] ## Registered foreign deps. proc foreignCmd*(cmd: string; requiresSudo = false) = - ## Registers a foreign command to the intern list of commands + ## Registers a foreign command to the internal list of commands ## that can be queried later. let c = (if requiresSudo: "sudo " else: "") & cmd when defined(nimble): @@ -202,11 +223,11 @@ proc foreignCmd*(cmd: string; requiresSudo = false) = foreignDeps.add(c) proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = - ## Returns the distro's native command line to install 'foreignPackageName' + ## Returns the distro's native command to install `foreignPackageName` ## and whether it requires root/admin rights. let p = foreignPackageName when defined(windows): - result = ("Chocolatey install " & p, false) + result = ("choco install " & p, false) elif defined(bsd): result = ("ports install " & p, true) elif defined(linux): @@ -229,12 +250,16 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("netpkg install " & p, true) elif detectOs(NixOS): result = ("nix-env -i " & p, false) - elif detectOs(Solaris): + elif detectOs(Solaris) or detectOs(FreeBSD): result = ("pkg install " & p, true) + elif detectOs(NetBSD) or detectOs(OpenBSD): + result = ("pkg_add " & p, true) elif detectOs(PCLinuxOS): result = ("rpm -ivh " & p, true) - elif detectOs(ArchLinux) or detectOs(Manjaro): + elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) + elif detectOs(Void): + result = ("xbps-install " & p, true) else: result = ("<your package manager here> install " & p, true) elif defined(haiku): @@ -243,17 +268,12 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("brew install " & p, false) proc foreignDep*(foreignPackageName: string) = - ## Registers 'foreignPackageName' to the internal list of foreign deps. - ## It is your job to ensure the package name + ## Registers `foreignPackageName` to the internal list of foreign deps. + ## It is your job to ensure that the package name is correct. let (installCmd, sudo) = foreignDepInstallCmd(foreignPackageName) - foreignCmd installCmd, sudo + foreignCmd(installCmd, sudo) proc echoForeignDeps*() = ## Writes the list of registered foreign deps to stdout. for d in foreignDeps: echo d - -when false: - foreignDep("libblas-dev") - foreignDep "libfoo" - echoForeignDeps() diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index f31ae94dd..a162fe37f 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -8,8 +8,8 @@ # ## This module implements the ability to access symbols from shared -## libraries. On POSIX this uses the ``dlsym`` mechanism, on -## Windows ``LoadLibrary``. +## libraries. On POSIX this uses the `dlsym` mechanism, on +## Windows `LoadLibrary`. ## ## Examples ## ======== @@ -17,75 +17,64 @@ ## Loading a simple C function ## --------------------------- ## -## The following example demonstrates loading a function called 'greet' +## The following example demonstrates loading a function called `greet` ## from a library that is determined at runtime based upon a language choice. -## If the library fails to load or the function 'greet' is not found, +## If the library fails to load or the function `greet` is not found, ## it quits with a failure error code. ## -## .. code-block::nim -## -## import dynlib -## -## type -## greetFunction = proc(): cstring {.gcsafe, stdcall.} -## -## let lang = stdin.readLine() -## -## let lib = case lang -## of "french": -## loadLib("french.dll") -## else: -## loadLib("english.dll") -## -## if lib == nil: -## echo "Error loading library" -## quit(QuitFailure) -## -## let greet = cast[greetFunction](lib.symAddr("greet")) -## -## if greet == nil: -## echo "Error loading 'greet' function from library" -## quit(QuitFailure) -## -## let greeting = greet() -## -## echo greeting -## -## unloadLib(lib) -## +runnableExamples: + type + GreetFunction = proc (): cstring {.gcsafe, stdcall.} + + proc loadGreet(lang: string) = + let lib = + case lang + of "french": + loadLib("french.dll") + else: + loadLib("english.dll") + assert lib != nil, "Error loading library" + + let greet = cast[GreetFunction](lib.symAddr("greet")) + assert greet != nil, "Error loading 'greet' function from library" + + echo greet() + + unloadLib(lib) -import strutils + +import std/strutils type - LibHandle* = pointer ## a handle to a dynamically loaded library + LibHandle* = pointer ## A handle to a dynamically loaded library. proc loadLib*(path: string, globalSymbols = false): LibHandle {.gcsafe.} - ## loads a library from `path`. Returns nil if the library could not + ## Loads a library from `path`. Returns nil if the library could not ## be loaded. proc loadLib*(): LibHandle {.gcsafe.} - ## gets the handle from the current executable. Returns nil if the + ## Gets the handle from the current executable. Returns nil if the ## library could not be loaded. proc unloadLib*(lib: LibHandle) {.gcsafe.} - ## unloads the library `lib` + ## Unloads the library `lib`. proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} = - ## raises an `EInvalidLibrary` exception. + ## Raises a `LibraryError` exception. raise newException(LibraryError, "could not find symbol: " & $name) proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.} - ## retrieves the address of a procedure/variable from `lib`. Returns nil + ## Retrieves the address of a procedure/variable from `lib`. Returns nil ## if the symbol could not be found. proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer = - ## retrieves the address of a procedure/variable from `lib`. Raises - ## `EInvalidLibrary` if the symbol could not be found. + ## Retrieves the address of a procedure/variable from `lib`. Raises + ## `LibraryError` if the symbol could not be found. result = symAddr(lib, name) if result == nil: raiseInvalidLibrary(name) proc libCandidates*(s: string, dest: var seq[string]) = - ## given a library name pattern `s` write possible library names to `dest`. + ## Given a library name pattern `s`, write possible library names to `dest`. var le = strutils.find(s, '(') var ri = strutils.find(s, ')', le+1) if le >= 0 and ri > le: @@ -97,9 +86,10 @@ proc libCandidates*(s: string, dest: var seq[string]) = add(dest, s) proc loadLibPattern*(pattern: string, globalSymbols = false): LibHandle = - ## loads a library with name matching `pattern`, similar to what `dlimport` + ## Loads a library with name matching `pattern`, similar to what the `dynlib` ## pragma does. Returns nil if the library could not be loaded. - ## Warning: this proc uses the GC and so cannot be used to load the GC. + ## + ## .. warning:: this proc uses the GC and so cannot be used to load the GC. var candidates = newSeq[string]() libCandidates(pattern, candidates) for c in candidates: @@ -115,7 +105,7 @@ when defined(posix) and not defined(nintendoswitch): # as an emulation layer on top of native functions. # ========================================================================= # - import posix + import std/posix proc loadLib(path: string, globalSymbols = false): LibHandle = let flags = @@ -150,6 +140,32 @@ elif defined(nintendoswitch): proc symAddr(lib: LibHandle, name: cstring): pointer = raise newException(OSError, "symAddr not implemented on Nintendo Switch!") +elif defined(genode): + # + # ========================================================================= + # Not implemented for Genode without POSIX. Raise an error if called. + # ========================================================================= + # + + template raiseErr(prc: string) = + raise newException(OSError, prc & " not implemented, compile with POSIX support") + + proc dlclose(lib: LibHandle) = + raiseErr(OSError, "dlclose") + proc dlopen(path: cstring, mode: int): LibHandle = + raiseErr(OSError, "dlopen") + proc dlsym(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "dlsym") + proc loadLib(path: string, global_symbols = false): LibHandle = + raiseErr(OSError, "loadLib") + proc loadLib(): LibHandle = + raiseErr(OSError, "loadLib") + proc unloadLib(lib: LibHandle) = + raiseErr(OSError, "unloadLib") + proc symAddr(lib: LibHandle, name: cstring): pointer = + raiseErr(OSError, "symAddr") + + elif defined(windows) or defined(dos): # # ======================================================================= diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index e3be1764b..bbadca655 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -7,15 +7,46 @@ # distribution, for details about the copyright. # -## Converts between different character encodings. On UNIX, this uses +## Routines for converting between different character encodings. On UNIX, this uses ## the `iconv`:idx: library, on Windows the Windows API. +## +## The following example shows how to change character encodings. +runnableExamples: + when defined(windows): + let + orig = "öäüß" + # convert `orig` from "UTF-8" to "CP1252" + cp1252 = convert(orig, "CP1252", "UTF-8") + # convert `cp1252` from "CP1252" to "ibm850" + ibm850 = convert(cp1252, "ibm850", "CP1252") + current = getCurrentEncoding() + assert orig == "\195\182\195\164\195\188\195\159" + assert ibm850 == "\148\132\129\225" + assert convert(ibm850, current, "ibm850") == orig + +## The example below uses a reuseable `EncodingConverter` object which is +## created by `open` with `destEncoding` and `srcEncoding` specified. You can use +## `convert` on this object multiple times. +runnableExamples: + when defined(windows): + var fromGB2312 = open("utf-8", "gb2312") + let first = "\203\173\197\194\163\191\210\187" & + "\203\242\209\204\211\234\200\206\198\189\201\250" + assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生" + + let second = "\211\208\176\215\205\183\200\231" & + "\208\194\163\172\199\227\184\199\200\231\185\202" + assert fromGB2312.convert(second) == "有白头如新，倾盖如故" -import os, parseutils, strutils + +import std/os +when defined(nimPreviewSlimSystem): + import std/assertions when not defined(windows): type ConverterObj = object - EncodingConverter* = ptr ConverterObj ## can convert between two character sets + EncodingConverter* = ptr ConverterObj ## Can convert between two character sets. else: type @@ -24,10 +55,11 @@ else: dest, src: CodePage type - EncodingError* = object of ValueError ## exception that is raised - ## for encoding errors + EncodingError* = object of ValueError ## Exception that is raised + ## for encoding errors. when defined(windows): + import std/[parseutils, strutils] proc eqEncodingNames(a, b: string): bool = var i = 0 var j = 0 @@ -71,6 +103,7 @@ when defined(windows): (875, "cp875"), # IBM EBCDIC Greek Modern (932, "shift_jis"), # ANSI/OEM Japanese; Japanese (Shift-JIS) (936, "gb2312"), # ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) + (936, "gbk"), # Alias for GB2312 encoding (949, "ks_c_5601-1987"), # ANSI/OEM Korean (Unified Hangul Code) (950, "big5"), # ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) (1026, "IBM1026"), # IBM EBCDIC Turkish (Latin 5) @@ -280,8 +313,10 @@ else: var errno {.importc, header: "<errno.h>".}: cint - when defined(freebsd) or defined(netbsd): + when defined(bsd): {.pragma: importIconv, cdecl, header: "<iconv.h>".} + when defined(openbsd): + {.passL: "-liconv".} else: {.pragma: importIconv, cdecl, dynlib: iconvDll.} @@ -289,15 +324,12 @@ else: importc: "iconv_open", importIconv.} proc iconvClose(c: EncodingConverter) {. importc: "iconv_close", importIconv.} - proc iconv(c: EncodingConverter, inbuf: var cstring, inbytesLeft: var int, - outbuf: var cstring, outbytesLeft: var int): int {. - importc: "iconv", importIconv.} - proc iconv(c: EncodingConverter, inbuf: pointer, inbytesLeft: pointer, - outbuf: var cstring, outbytesLeft: var int): int {. + proc iconv(c: EncodingConverter, inbuf: ptr cstring, inbytesLeft: ptr csize_t, + outbuf: ptr cstring, outbytesLeft: ptr csize_t): csize_t {. importc: "iconv", importIconv.} proc getCurrentEncoding*(uiApp = false): string = - ## retrieves the current encoding. On Unix, always "UTF-8" is returned. + ## Retrieves the current encoding. On Unix, "UTF-8" is always returned. ## The `uiApp` parameter is Windows specific. If true, the UI's code-page ## is returned, if false, the Console's code-page is returned. when defined(windows): @@ -306,11 +338,11 @@ proc getCurrentEncoding*(uiApp = false): string = result = "UTF-8" proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = - ## opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `IOError` if it cannot fulfill the request. + ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) - if result == nil: + if result == cast[EncodingConverter](-1): raise newException(EncodingError, "cannot create encoding converter from " & srcEncoding & " to " & destEncoding) @@ -325,7 +357,7 @@ proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = "cannot find encoding " & srcEncoding) proc close*(c: EncodingConverter) = - ## frees the resources the converter `c` holds. + ## Frees the resources the converter `c` holds. when not defined(windows): iconvClose(c) @@ -420,21 +452,22 @@ when defined(windows): else: convertFromWideString(codePageTo, wideString) proc convert*(c: EncodingConverter, s: string): string = - ## converts `s` to `destEncoding` that was given to the converter `c`. It - ## assumed that `s` is in `srcEncoding`. - ## utf-16BE, utf-32 conversions not supported on windows result = convertWin(c.src, c.dest, s) else: proc convert*(c: EncodingConverter, s: string): string = + ## Converts `s` to `destEncoding` that was given to the converter `c`. It + ## assumes that `s` is in `srcEncoding`. + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. result = newString(s.len) - var inLen = len(s) - var outLen = len(result) + var inLen = csize_t len(s) + var outLen = csize_t len(result) var src = cstring(s) var dst = cstring(result) - var iconvres: int + var iconvres: csize_t while inLen > 0: - iconvres = iconv(c, src, inLen, dst, outLen) - if iconvres == -1: + iconvres = iconv(c, addr src, addr inLen, addr dst, addr outLen) + if iconvres == high(csize_t): var lerr = errno if lerr == EILSEQ or lerr == EINVAL: # unknown char, skip @@ -445,105 +478,34 @@ else: dec(outLen) elif lerr == E2BIG: var offset = cast[int](dst) - cast[int](cstring(result)) - setLen(result, len(result)+inLen*2+5) + setLen(result, len(result) + inLen.int * 2 + 5) # 5 is minimally one utf-8 char dst = cast[cstring](cast[int](cstring(result)) + offset) - outLen = len(result) - offset + outLen = csize_t(len(result) - offset) else: raiseOSError(lerr.OSErrorCode) # iconv has a buffer that needs flushing, specially if the last char is # not '\0' - discard iconv(c, nil, nil, dst, outLen) - if iconvres == cint(-1) and errno == E2BIG: + discard iconv(c, nil, nil, addr dst, addr outLen) + if iconvres == high(csize_t) and errno == E2BIG: var offset = cast[int](dst) - cast[int](cstring(result)) - setLen(result, len(result)+inLen*2+5) + setLen(result, len(result) + inLen.int * 2 + 5) # 5 is minimally one utf-8 char dst = cast[cstring](cast[int](cstring(result)) + offset) - outLen = len(result) - offset - discard iconv(c, nil, nil, dst, outLen) + outLen = csize_t(len(result) - offset) + discard iconv(c, nil, nil, addr dst, addr outLen) # trim output buffer - setLen(result, len(result) - outLen) + setLen(result, len(result) - outLen.int) proc convert*(s: string, destEncoding = "UTF-8", srcEncoding = "CP1252"): string = - ## converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. + ## Converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`. ## This opens a converter, uses it and closes it again and is thus more ## convenient but also likely less efficient than re-using a converter. - ## utf-16BE, utf-32 conversions not supported on windows + ## + ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows. var c = open(destEncoding, srcEncoding) try: result = convert(c, s) finally: close(c) - -when not defined(testing) and isMainModule: - let - orig = "öäüß" - cp1252 = convert(orig, "CP1252", "UTF-8") - ibm850 = convert(cp1252, "ibm850", "CP1252") - current = getCurrentEncoding() - echo "Original string from source code: ", orig - echo "Forced ibm850 encoding: ", ibm850 - echo "Current encoding: ", current - echo "From ibm850 to current: ", convert(ibm850, current, "ibm850") - -when not defined(testing) and isMainModule and defined(windows): - block should_throw_on_unsupported_conversions: - let original = "some string" - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "unicodeFFFE") - - doAssertRaises(EncodingError): - discard convert(original, "utf-8", "utf-32BE") - - doAssertRaises(EncodingError): - discard convert(original, "unicodeFFFE", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32", "utf-8") - - doAssertRaises(EncodingError): - discard convert(original, "utf-32BE", "utf-8") - - block should_convert_from_utf16_to_utf8: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "utf-8", "utf-16") - doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82") - - block should_convert_from_utf16_to_win1251: - let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест" - let result = convert(original, "windows-1251", "utf-16") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_win1251_to_koi8r: - let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест" - let result = convert(original, "koi8-r", "windows-1251") - doAssert(result == "\xd4\xc5\xd3\xd4") - - block should_convert_from_koi8r_to_win1251: - let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест" - let result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_win1251: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "windows-1251", "utf-8") - doAssert(result == "\xf2\xe5\xf1\xf2") - - block should_convert_from_utf8_to_utf16: - let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест" - let result = convert(original, "utf-16", "utf-8") - doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04") - - block should_handle_empty_string_for_any_conversion: - let original = "" - var result = convert(original, "utf-16", "utf-8") - doAssert(result == "") - result = convert(original, "utf-8", "utf-16") - doAssert(result == "") - result = convert(original, "windows-1251", "koi8-r") - doAssert(result == "") diff --git a/lib/pure/endians.nim b/lib/pure/endians.nim index 29fde1c80..4c1d45ae5 100644 --- a/lib/pure/endians.nim +++ b/lib/pure/endians.nim @@ -10,6 +10,11 @@ ## This module contains helpers that deal with different byte orders ## (`endian`:idx:). ## +## Endianness is the order of bytes of a value in memory. Big-endian means that +## the most significant byte is stored at the smallest memory address, +## while little endian means that the least-significant byte is stored +## at the smallest address. See also https://en.wikipedia.org/wiki/Endianness. +## ## Unstable API. when defined(gcc) or defined(llvm_gcc) or defined(clang): @@ -47,7 +52,7 @@ else: when useBuiltinSwap: template swapOpImpl(T: typedesc, op: untyped) = - ## We have to use `copyMem` here instead of a simple deference because they + ## We have to use `copyMem` here instead of a simple dereference because they ## may point to a unaligned address. A sufficiently smart compiler _should_ ## be able to elide them when they're not necessary. var tmp: T @@ -56,18 +61,40 @@ when useBuiltinSwap: copyMem(outp, addr tmp, sizeof(T)) proc swapEndian64*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 8 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4, 5, 6, 7, 8] + var b: array[8, uint8] + swapEndian64(addr b, addr a) + assert b == [8'u8, 7, 6, 5, 4, 3, 2, 1] + swapOpImpl(uint64, builtin_bswap64) proc swapEndian32*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 4 bytes. + runnableExamples: + var a = [1'u8, 2, 3, 4] + var b: array[4, uint8] + swapEndian32(addr b, addr a) + assert b == [4'u8, 3, 2, 1] + swapOpImpl(uint32, builtin_bswap32) proc swapEndian16*(outp, inp: pointer) {.inline, noSideEffect.} = + ## Copies `inp` to `outp`, reversing the byte order. + ## Both buffers are supposed to contain at least 2 bytes. + runnableExamples: + var a = [1'u8, 2] + var b: array[2, uint8] + swapEndian16(addr b, addr a) + assert b == [2'u8, 1] + swapOpImpl(uint16, builtin_bswap16) else: proc swapEndian64*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 8 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[7] @@ -80,8 +107,6 @@ else: o[7] = i[0] proc swapEndian32*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 4 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[3] @@ -90,8 +115,6 @@ else: o[3] = i[0] proc swapEndian16*(outp, inp: pointer) = - ## copies `inp` to `outp` swapping bytes. Both buffers are supposed to - ## contain at least 2 bytes. var i = cast[cstring](inp) var o = cast[cstring](outp) o[0] = i[1] @@ -106,8 +129,20 @@ when system.cpuEndian == bigEndian: proc bigEndian16*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 2) else: proc littleEndian64*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 8) + ## Copies `inp` to `outp`, storing it in 64-bit little-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc littleEndian32*(outp, inp: pointer) {.inline.} = copyMem(outp, inp, 4) + ## Copies `inp` to `outp`, storing it in 32-bit little-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc littleEndian16*(outp, inp: pointer){.inline.} = copyMem(outp, inp, 2) + ## Copies `inp` to `outp`, storing it in 16-bit little-endian order. + ## Both buffers are supposed to contain at least 2 bytes. proc bigEndian64*(outp, inp: pointer) {.inline.} = swapEndian64(outp, inp) + ## Copies `inp` to `outp`, storing it in 64-bit big-endian order. + ## Both buffers are supposed to contain at least 8 bytes. proc bigEndian32*(outp, inp: pointer) {.inline.} = swapEndian32(outp, inp) + ## Copies `inp` to `outp`, storing it in 32-bit big-endian order. + ## Both buffers are supposed to contain at least 4 bytes. proc bigEndian16*(outp, inp: pointer) {.inline.} = swapEndian16(outp, inp) + ## Copies `inp` to `outp`, storing it in 16-bit big-endian order. + ## Both buffers are supposed to contain at least 2 bytes. diff --git a/lib/pure/fenv.nim b/lib/pure/fenv.nim index 6da6aedad..1d96fd6be 100644 --- a/lib/pure/fenv.nim +++ b/lib/pure/fenv.nim @@ -9,10 +9,10 @@ ## Floating-point environment. Handling of floating-point rounding and ## exceptions (overflow, division by zero, etc.). +## The types, vars and procs are bindings for the C standard library +## [<fenv.h>](https://en.cppreference.com/w/c/numeric/fenv) header. -{.deadCodeElim: on.} # dce option deprecated - -when defined(Posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} var @@ -37,8 +37,8 @@ var FE_UPWARD* {.importc, header: "<fenv.h>".}: cint ## round toward +Inf FE_DFL_ENV* {.importc, header: "<fenv.h>".}: cint - ## macro of type pointer to fenv_t to be used as the argument - ## to functions taking an argument of type fenv_t; in this + ## macro of type pointer to `fenv_t` to be used as the argument + ## to functions taking an argument of type `fenv_t`; in this ## case the default environment will be used type @@ -130,7 +130,7 @@ template fpRadix*: int = FLT_RADIX ## point type on the architecture used to build the program. template mantissaDigits*(T: typedesc[float32]): int = FLT_MANT_DIG - ## Number of digits (in base ``floatingPointRadix``) in the mantissa + ## Number of digits (in base `floatingPointRadix`) in the mantissa ## of 32-bit floating-point numbers. template digits*(T: typedesc[float32]): int = FLT_DIG ## Number of decimal digits that can be represented in a @@ -156,7 +156,7 @@ template epsilon*(T: typedesc[float32]): float32 = FLT_EPSILON ## 1.0 that can be represented in a 32-bit floating-point type. template mantissaDigits*(T: typedesc[float64]): int = DBL_MANT_DIG - ## Number of digits (in base ``floatingPointRadix``) in the mantissa + ## Number of digits (in base `floatingPointRadix`) in the mantissa ## of 64-bit floating-point numbers. template digits*(T: typedesc[float64]): int = DBL_DIG ## Number of decimal digits that can be represented in a @@ -180,11 +180,3 @@ template maximumPositiveValue*(T: typedesc[float64]): float64 = DBL_MAX template epsilon*(T: typedesc[float64]): float64 = DBL_EPSILON ## The difference between 1.0 and the smallest number greater than ## 1.0 that can be represented in a 64-bit floating-point type. - - -when isMainModule: - func is_significant(x: float): bool = - if x > minimumPositiveValue(float) and x < maximumPositiveValue(float): true - else: false - - doAssert is_significant(10.0) diff --git a/lib/pure/future.nim b/lib/pure/future.nim deleted file mode 100644 index 40dba7846..000000000 --- a/lib/pure/future.nim +++ /dev/null @@ -1,4 +0,0 @@ - -{.deprecated: "Use the new 'sugar' module instead".} - -include sugar diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index ac8498517..1038d55a1 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -10,49 +10,76 @@ ## This module implements efficient computations of hash values for diverse ## Nim types. All the procs are based on these two building blocks: ## - `!& proc <#!&,Hash,int>`_ used to start or mix a hash value, and -## - `!$ proc <#!$,Hash>`_ used to *finish* the hash value. +## - `!$ proc <#!$,Hash>`_ used to finish the hash value. ## ## If you want to implement hash procs for your custom types, ## you will end up writing the following kind of skeleton of code: + +runnableExamples: + type + Something = object + foo: int + bar: string + + iterator items(x: Something): Hash = + yield hash(x.foo) + yield hash(x.bar) + + proc hash(x: Something): Hash = + ## Computes a Hash from `x`. + var h: Hash = 0 + # Iterate over parts of `x`. + for xAtom in x: + # Mix the atom with the partial hash. + h = h !& xAtom + # Finish the hash. + result = !$h + +## If your custom types contain fields for which there already is a `hash` proc, +## you can simply hash together the hash values of the individual fields: + +runnableExamples: + type + Something = object + foo: int + bar: string + + proc hash(x: Something): Hash = + ## Computes a Hash from `x`. + var h: Hash = 0 + h = h !& hash(x.foo) + h = h !& hash(x.bar) + result = !$h + +## .. important:: Use `-d:nimPreviewHashRef` to +## enable hashing `ref`s. It is expected that this behavior +## becomes the new default in upcoming versions. ## -## .. code-block:: Nim -## proc hash(x: Something): Hash = -## ## Computes a Hash from `x`. -## var h: Hash = 0 -## # Iterate over parts of `x`. -## for xAtom in x: -## # Mix the atom with the partial hash. -## h = h !& xAtom -## # Finish the hash. -## result = !$h -## -## If your custom types contain fields for which there already is a hash proc, -## like for example objects made up of ``strings``, you can simply hash -## together the hash value of the individual fields: -## -## .. code-block:: Nim -## proc hash(x: Something): Hash = -## ## Computes a Hash from `x`. -## var h: Hash = 0 -## h = h !& hash(x.foo) -## h = h !& hash(x.bar) -## result = !$h +## .. note:: If the type has a `==` operator, the following must hold: +## If two values compare equal, their hashes must also be equal. ## -## **See also:** -## * `md5 module <md5.html>`_ for MD5 checksum algorithm -## * `base64 module <base64.html>`_ for a base64 encoder and decoder -## * `std/sha1 module <sha1.html>`_ for a sha1 encoder and decoder +## See also +## ======== +## * `md5 module <md5.html>`_ for the MD5 checksum algorithm +## * `base64 module <base64.html>`_ for a Base64 encoder and decoder +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm ## * `tables module <tables.html>`_ for hash tables +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + + type Hash* = int ## A hash value. Hash tables using these values should - ## always have a size of a power of two and can use the ``and`` - ## operator instead of ``mod`` for truncation of the hash value. + ## always have a size of a power of two so they can use the `and` + ## operator instead of `mod` for truncation of the hash value. proc `!&`*(h: Hash, val: int): Hash {.inline.} = ## Mixes a hash value `h` with `val` to produce a new hash value. ## - ## This is only needed if you need to implement a hash proc for a new datatype. + ## This is only needed if you need to implement a `hash` proc for a new datatype. let h = cast[uint](h) let val = cast[uint](val) var res = h + val @@ -63,19 +90,107 @@ proc `!&`*(h: Hash, val: int): Hash {.inline.} = proc `!$`*(h: Hash): Hash {.inline.} = ## Finishes the computation of the hash value. ## - ## This is only needed if you need to implement a hash proc for a new datatype. + ## This is only needed if you need to implement a `hash` proc for a new datatype. let h = cast[uint](h) # Hash is practically unsigned. var res = h + h shl 3 res = res xor (res shr 11) res = res + res shl 15 result = cast[Hash](res) +proc hiXorLoFallback64(a, b: uint64): uint64 {.inline.} = + let # Fall back in 64-bit arithmetic + aH = a shr 32 + aL = a and 0xFFFFFFFF'u64 + bH = b shr 32 + bL = b and 0xFFFFFFFF'u64 + rHH = aH * bH + rHL = aH * bL + rLH = aL * bH + rLL = aL * bL + t = rLL + (rHL shl 32) + var c = if t < rLL: 1'u64 else: 0'u64 + let lo = t + (rLH shl 32) + c += (if lo < t: 1'u64 else: 0'u64) + let hi = rHH + (rHL shr 32) + (rLH shr 32) + c + return hi xor lo + +proc hiXorLo(a, b: uint64): uint64 {.inline.} = + # XOR of the high & low 8 bytes of the full 16 byte product. + when nimvm: + result = hiXorLoFallback64(a, b) # `result =` is necessary here. + else: + when Hash.sizeof < 8: + result = hiXorLoFallback64(a, b) + elif defined(gcc) or defined(llvm_gcc) or defined(clang): + {.emit: """__uint128_t r = `a`; r *= `b`; `result` = (r >> 64) ^ r;""".} + elif defined(windows) and not defined(tcc): + proc umul128(a, b: uint64, c: ptr uint64): uint64 {.importc: "_umul128", header: "intrin.h".} + var b = b + let c = umul128(a, b, addr b) + result = c xor b + else: + result = hiXorLoFallback64(a, b) + +when defined(js): + import std/jsbigints + import std/private/jsutils + + proc hiXorLoJs(a, b: JsBigInt): JsBigInt = + let + prod = a * b + mask = big"0xffffffffffffffff" # (big"1" shl big"64") - big"1" + result = (prod shr big"64") xor (prod and mask) + + template hashWangYiJS(x: JsBigInt): Hash = + let + P0 = big"0xa0761d6478bd642f" + P1 = big"0xe7037ed1a0b428db" + P58 = big"0xeb44accab455d16d" # big"0xeb44accab455d165" xor big"8" + res = hiXorLoJs(hiXorLoJs(P0, x xor P1), P58) + cast[Hash](toNumber(wrapToInt(res, 32))) + + template toBits(num: float): JsBigInt = + let + x = newArrayBuffer(8) + y = newFloat64Array(x) + if hasBigUint64Array(): + let z = newBigUint64Array(x) + y[0] = num + z[0] + else: + let z = newUint32Array(x) + y[0] = num + big(z[0]) + big(z[1]) shl big(32) + +proc hashWangYi1*(x: int64|uint64|Hash): Hash {.inline.} = + ## Wang Yi's hash_v1 for 64-bit ints (see https://github.com/rurban/smhasher for + ## more details). This passed all scrambling tests in Spring 2019 and is simple. + ## + ## **Note:** It's ok to define `proc(x: int16): Hash = hashWangYi1(Hash(x))`. + const P0 = 0xa0761d6478bd642f'u64 + const P1 = 0xe7037ed1a0b428db'u64 + const P58 = 0xeb44accab455d165'u64 xor 8'u64 + template h(x): untyped = hiXorLo(hiXorLo(P0, uint64(x) xor P1), P58) + when nimvm: + when defined(js): # Nim int64<->JS Number & VM match => JS gets 32-bit hash + result = cast[Hash](h(x)) and cast[Hash](0xFFFFFFFF) + else: + result = cast[Hash](h(x)) + else: + when defined(js): + if hasJsBigInt(): + result = hashWangYiJS(big(x)) + else: + result = cast[Hash](x) and cast[Hash](0xFFFFFFFF) + else: + result = cast[Hash](h(x)) + proc hashData*(data: pointer, size: int): Hash = ## Hashes an array of bytes of size `size`. var h: Hash = 0 when defined(js): var p: cstring - asm """`p` = `Data`;""" + {.emit: """`p` = `Data`;""".} else: var p = cast[cstring](data) var i = 0 @@ -86,13 +201,23 @@ proc hashData*(data: pointer, size: int): Hash = dec(s) result = !$h +proc hashIdentity*[T: Ordinal|enum](x: T): Hash {.inline, since: (1, 3).} = + ## The identity hash, i.e. `hashIdentity(x) = x`. + cast[Hash](ord(x)) + +when defined(nimIntHash1): + proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = + ## Efficient hashing of integers. + cast[Hash](ord(x)) +else: + proc hash*[T: Ordinal|enum](x: T): Hash {.inline.} = + ## Efficient hashing of integers. + hashWangYi1(uint64(ord(x))) + when defined(js): var objectID = 0 - -proc hash*(x: pointer): Hash {.inline.} = - ## Efficient hashing of pointers. - when defined(js): - asm """ + proc getObjectId(x: pointer): int = + {.emit: """ if (typeof `x` == "object") { if ("_NimID" in `x`) `result` = `x`["_NimID"]; @@ -101,49 +226,60 @@ proc hash*(x: pointer): Hash {.inline.} = `x`["_NimID"] = `result`; } } - """ - else: - result = cast[Hash](cast[uint](x) shr 3) # skip the alignment + """.} -proc hash*[T: proc](x: T): Hash {.inline.} = - ## Efficient hashing of proc vars. Closures are supported too. - when T is "closure": - result = hash(rawProc(x)) !& hash(rawEnv(x)) +proc hash*(x: pointer): Hash {.inline.} = + ## Efficient `hash` overload. + when defined(js): + let y = getObjectId(x) else: - result = hash(pointer(x)) + let y = cast[int](x) + hash(y) # consistent with code expecting scrambled hashes depending on `nimIntHash1`. -const - prime = uint(11) - -proc hash*(x: int): Hash {.inline.} = - ## Efficient hashing of integers. - result = cast[Hash](cast[uint](x) * prime) - -proc hash*(x: int64): Hash {.inline.} = - ## Efficient hashing of `int64` integers. - result = cast[Hash](cast[uint](x) * prime) - -proc hash*(x: uint): Hash {.inline.} = - ## Efficient hashing of unsigned integers. - result = cast[Hash](x * prime) - -proc hash*(x: uint64): Hash {.inline.} = - ## Efficient hashing of `uint64` integers. - result = cast[Hash](cast[uint](x) * prime) - -proc hash*(x: char): Hash {.inline.} = - ## Efficient hashing of characters. - result = cast[Hash](cast[uint](ord(x)) * prime) - -proc hash*[T: Ordinal](x: T): Hash {.inline.} = - ## Efficient hashing of other ordinal types (e.g. enums). - result = cast[Hash](cast[uint](ord(x)) * prime) +proc hash*[T](x: ptr[T]): Hash {.inline.} = + ## Efficient `hash` overload. + runnableExamples: + var a: array[10, uint8] + assert a[0].addr.hash != a[1].addr.hash + assert cast[pointer](a[0].addr).hash == a[0].addr.hash + hash(cast[pointer](x)) + +when defined(nimPreviewHashRef) or defined(nimdoc): + proc hash*[T](x: ref[T]): Hash {.inline.} = + ## Efficient `hash` overload. + ## + ## .. important:: Use `-d:nimPreviewHashRef` to + ## enable hashing `ref`s. It is expected that this behavior + ## becomes the new default in upcoming versions. + runnableExamples("-d:nimPreviewHashRef"): + type A = ref object + x: int + let a = A(x: 3) + let ha = a.hash + assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. + a.x = 4 + assert ha == a.hash # the hash only depends on the address + runnableExamples("-d:nimPreviewHashRef"): + # you can overload `hash` if you want to customize semantics + type A[T] = ref object + x, y: T + proc hash(a: A): Hash = hash(a.x) + assert A[int](x: 3, y: 4).hash == A[int](x: 3, y: 5).hash + # xxx pending bug #17733, merge as `proc hash*(pointer | ref | ptr): Hash` + # or `proc hash*[T: ref | ptr](x: T): Hash` + hash(cast[pointer](x)) proc hash*(x: float): Hash {.inline.} = ## Efficient hashing of floats. - var y = x + 1.0 - result = cast[ptr Hash](addr(y))[] - + let y = x + 0.0 # for denormalization + when nimvm: + # workaround a JS VM bug: bug #16547 + result = hashWangYi1(cast[int64](float64(y))) + else: + when not defined(js): + result = hashWangYi1(cast[Hash](y)) + else: + result = hashWangYiJS(toBits(y)) # Forward declarations before methods that hash containers. This allows # containers to contain other containers @@ -183,16 +319,24 @@ proc murmurHash(x: openArray[byte]): Hash = h1: uint32 i = 0 + + template impl = + var j = stepSize + while j > 0: + dec j + k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + # body while i < n * stepSize: var k1: uint32 - when defined(js): - var j = stepSize - while j > 0: - dec j - k1 = (k1 shl 8) or (ord(x[i+j])).uint32 + + when nimvm: + impl() else: - k1 = cast[ptr uint32](unsafeAddr x[i])[] + when declared(copyMem): + copyMem(addr k1, addr x[i], 4) + else: + impl() inc i, stepSize k1 = imul(k1, c1) @@ -223,33 +367,185 @@ proc murmurHash(x: openArray[byte]): Hash = h1 = h1 xor (h1 shr 16) return cast[Hash](h1) +proc hashVmImpl(x: cstring, sPos, ePos: int): Hash = + raiseAssert "implementation override in compiler/vmops.nim" + proc hashVmImpl(x: string, sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplChar(x: openArray[char], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" proc hashVmImplByte(x: openArray[byte], sPos, ePos: int): Hash = - doAssert false, "implementation override in compiler/vmops.nim" + raiseAssert "implementation override in compiler/vmops.nim" + +const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses +const k1 = 0xb492b66fbe98f273u64 +const k2 = 0x9ae16a3b2f90404fu64 + +proc load4e(s: openArray[byte], o=0): uint32 {.inline.} = + uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or + uint32(s[o + 1]) shl 8 or uint32(s[o + 0]) + +proc load8e(s: openArray[byte], o=0): uint64 {.inline.} = + uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or + uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or + uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or + uint64(s[o + 1]) shl 8 or uint64(s[o + 0]) + +proc load4(s: openArray[byte], o=0): uint32 {.inline.} = + when nimvm: result = load4e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load4e(s, o) + +proc load8(s: openArray[byte], o=0): uint64 {.inline.} = + when nimvm: result = load8e(s, o) + else: + when declared copyMem: copyMem result.addr, s[o].addr, result.sizeof + else: result = load8e(s, o) + +proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64 + +proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47) + +proc rotR(v: uint64; bits: cint): uint64 {.inline.} = + (v shr bits) or (v shl (64 - bits)) + +proc len16(u: uint64; v: uint64; mul: uint64): uint64 {.inline.} = + var a = (u xor v)*mul + a = a xor (a shr 47) + var b = (v xor a)*mul + b = b xor (b shr 47) + b*mul + +proc len0_16(s: openArray[byte]): uint64 {.inline.} = + if s.len >= 8: + let mul = k2 + 2*s.lenU + let a = load8(s) + k2 + let b = load8(s, s.len - 8) + let c = rotR(b, 37)*mul + a + let d = (rotR(a, 25) + b)*mul + len16 c, d, mul + elif s.len >= 4: + let mul = k2 + 2*s.lenU + let a = load4(s).uint64 + len16 s.lenU + (a shl 3), load4(s, s.len - 4), mul + elif s.len > 0: + let a = uint32(s[0]) + let b = uint32(s[s.len shr 1]) + let c = uint32(s[s.len - 1]) + let y = a + (b shl 8) + let z = s.lenU + (c shl 2) + shiftMix(y*k2 xor z*k0)*k2 + else: k2 # s.len == 0 + +proc len17_32(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k1 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + len16 rotR(a + b, 43) + rotR(c, 30) + d, a + rotR(b + k2, 18) + c, mul + +proc len33_64(s: openArray[byte]): uint64 {.inline.} = + let mul = k2 + 2*s.lenU + let a = load8(s)*k2 + let b = load8(s, 8) + let c = load8(s, s.len - 8)*mul + let d = load8(s, s.len - 16)*k2 + let y = rotR(a + b, 43) + rotR(c, 30) + d + let z = len16(y, a + rotR(b + k2, 18) + c, mul) + let e = load8(s, 16)*mul + let f = load8(s, 24) + let g = (y + load8(s, s.len - 32))*mul + let h = (z + load8(s, s.len - 24))*mul + len16 rotR(e + f, 43) + rotR(g, 30) + h, e + rotR(f + a, 18) + g, mul + +type Pair = tuple[first, second: uint64] + +proc weakLen32withSeeds2(w, x, y, z, a, b: uint64): Pair {.inline.} = + var a = a + w + var b = rotR(b + a + z, 21) + let c = a + a += x + a += y + b += rotR(a, 44) + result[0] = a + z + result[1] = b + c + +proc weakLen32withSeeds(s: openArray[byte]; o: int; a,b: uint64): Pair {.inline.} = + weakLen32withSeeds2 load8(s, o ), load8(s, o + 8), + load8(s, o + 16), load8(s, o + 24), a, b + +proc hashFarm(s: openArray[byte]): uint64 {.inline.} = + if s.len <= 16: return len0_16(s) + if s.len <= 32: return len17_32(s) + if s.len <= 64: return len33_64(s) + const seed = 81u64 # not const to use input `h` + var + o = 0 # s[] ptr arith -> variable origin variable `o` + x = seed + y = seed*k1 + 113 + z = shiftMix(y*k2 + 113)*k2 + v, w: Pair + x = x*k2 + load8(s) + let eos = ((s.len - 1) div 64)*64 + let last64 = eos + ((s.len - 1) and 63) - 63 + while true: + x = rotR(x + y + v[0] + load8(s, o+8), 37)*k1 + y = rotR(y + v[1] + load8(s, o+48), 42)*k1 + x = x xor w[1] + y += v[0] + load8(s, o+40) + z = rotR(z + w[0], 33)*k1 + v = weakLen32withSeeds(s, o+0 , v[1]*k1, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + inc o, 64 + if o == eos: break + let mul = k1 + ((z and 0xff) shl 1) + o = last64 + w[0] += (s.lenU - 1) and 63 + v[0] += w[0] + w[0] += v[0] + x = rotR(x + y + v[0] + load8(s, o+8), 37)*mul + y = rotR(y + v[1] + load8(s, o+48), 42)*mul + x = x xor w[1]*9 + y += v[0]*9 + load8(s, o+40) + z = rotR(z + w[0], 33)*mul + v = weakLen32withSeeds(s, o+0 , v[1]*mul, x + w[0]) + w = weakLen32withSeeds(s, o+32, z + w[1], y + load8(s, o+16)) + swap z, x + len16 len16(v[0],w[0],mul) + shiftMix(y)*k0 + z, len16(v[1],w[1],mul) + x, mul + +template jsNoInt64: untyped = + when defined js: + when compiles(compileOption("jsbigint64")): + when not compileOption("jsbigint64"): true + else: false + else: false + else: false +const sHash2 = (when defined(nimStringHash2) or jsNoInt64(): true else: false) + +template maybeFailJS_Number = + when jsNoInt64() and not defined(nimStringHash2): + {.error: "Must use `-d:nimStringHash2` when using `--jsbigint64:off`".} proc hash*(x: string): Hash = ## Efficient hashing of strings. ## - ## See also: + ## **See also:** ## * `hashIgnoreStyle <#hashIgnoreStyle,string>`_ ## * `hashIgnoreCase <#hashIgnoreCase,string>`_ runnableExamples: doAssert hash("abracadabra") != hash("AbracadabrA") - - when not defined(nimToOpenArrayCString): - result = 0 - for c in x: - result = result !& ord(c) - result = !$result + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) else: - when nimvm: - result = hashVmImpl(x, 0, high(x)) - else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: result = murmurHash(toOpenArrayByte(x, 0, high(x))) proc hash*(x: cstring): Hash = @@ -259,43 +555,44 @@ proc hash*(x: cstring): Hash = doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - var i = 0 - while x[i] != '\0': - result = result !& ord(x[i]) - inc i - result = !$result - else: - when not defined(js) and defined(nimToOpenArrayCString): - murmurHash(toOpenArrayByte(x, 0, x.high)) - else: + maybeFailJS_Number() + when not sHash2: + when defined js: let xx = $x - murmurHash(toOpenArrayByte(xx, 0, high(xx))) + result = cast[Hash](hashFarm(toOpenArrayByte(xx, 0, xx.high))) + else: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) + else: + #when nimvm: + # result = hashVmImpl(x, 0, high(x)) + when true: + when not defined(js): + result = murmurHash(toOpenArrayByte(x, 0, x.high)) + else: + let xx = $x + result = murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting ## position `sPos` to ending position `ePos` (included). ## - ## ``hash(myStr, 0, myStr.high)`` is equivalent to ``hash(myStr)``. + ## `hash(myStr, 0, myStr.high)` is equivalent to `hash(myStr)`. runnableExamples: var a = "abracadabra" doAssert hash(a, 0, 3) == hash(a, 7, 10) - when not defined(nimToOpenArrayCString): - result = 0 - for i in sPos..ePos: - result = result !& ord(sBuf[i]) - result = !$result + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(sBuf, sPos, ePos))) else: murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) proc hashIgnoreStyle*(x: string): Hash = ## Efficient hashing of strings; style is ignored. ## - ## **Note:** This uses different hashing algorithm than `hash(string)`. + ## **Note:** This uses a different hashing algorithm than `hash(string)`. ## - ## See also: + ## **See also:** ## * `hashIgnoreCase <#hashIgnoreCase,string>`_ runnableExamples: doAssert hashIgnoreStyle("aBr_aCa_dAB_ra") == hashIgnoreStyle("abracadabra") @@ -319,10 +616,10 @@ proc hashIgnoreStyle*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting ## position `sPos` to ending position `ePos` (included); style is ignored. ## - ## **Note:** This uses different hashing algorithm than `hash(string)`. + ## **Note:** This uses a different hashing algorithm than `hash(string)`. ## - ## ``hashIgnoreStyle(myBuf, 0, myBuf.high)`` is equivalent - ## to ``hashIgnoreStyle(myBuf)``. + ## `hashIgnoreStyle(myBuf, 0, myBuf.high)` is equivalent + ## to `hashIgnoreStyle(myBuf)`. runnableExamples: var a = "ABracada_b_r_a" doAssert hashIgnoreStyle(a, 0, 3) == hashIgnoreStyle(a, 7, a.high) @@ -343,9 +640,9 @@ proc hashIgnoreStyle*(sBuf: string, sPos, ePos: int): Hash = proc hashIgnoreCase*(x: string): Hash = ## Efficient hashing of strings; case is ignored. ## - ## **Note:** This uses different hashing algorithm than `hash(string)`. + ## **Note:** This uses a different hashing algorithm than `hash(string)`. ## - ## See also: + ## **See also:** ## * `hashIgnoreStyle <#hashIgnoreStyle,string>`_ runnableExamples: doAssert hashIgnoreCase("ABRAcaDABRA") == hashIgnoreCase("abRACAdabra") @@ -363,10 +660,10 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting ## position `sPos` to ending position `ePos` (included); case is ignored. ## - ## **Note:** This uses different hashing algorithm than `hash(string)`. + ## **Note:** This uses a different hashing algorithm than `hash(string)`. ## - ## ``hashIgnoreCase(myBuf, 0, myBuf.high)`` is equivalent - ## to ``hashIgnoreCase(myBuf)``. + ## `hashIgnoreCase(myBuf, 0, myBuf.high)` is equivalent + ## to `hashIgnoreCase(myBuf)`. runnableExamples: var a = "ABracadabRA" doAssert hashIgnoreCase(a, 0, 3) == hashIgnoreCase(a, 7, 10) @@ -379,24 +676,64 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h +proc hash*[T: tuple | object | proc | iterator {.closure.}](x: T): Hash = + ## Efficient `hash` overload. + runnableExamples: + # for `tuple|object`, `hash` must be defined for each component of `x`. + type Obj = object + x: int + y: string + type Obj2[T] = object + x: int + y: string + assert hash(Obj(x: 520, y: "Nim")) != hash(Obj(x: 520, y: "Nim2")) + # you can define custom hashes for objects (even if they're generic): + proc hash(a: Obj2): Hash = hash((a.x)) + assert hash(Obj2[float](x: 520, y: "Nim")) == hash(Obj2[float](x: 520, y: "Nim2")) + runnableExamples: + # proc + proc fn1() = discard + const fn1b = fn1 + assert hash(fn1b) == hash(fn1) + + # closure + proc outer = + var a = 0 + proc fn2() = a.inc + assert fn2 is "closure" + let fn2b = fn2 + assert hash(fn2b) == hash(fn2) + assert hash(fn2) != hash(fn1) + outer() -proc hash*[T: tuple](x: T): Hash = - ## Efficient hashing of tuples. - for f in fields(x): - result = result !& hash(f) - result = !$result - + when T is "closure": + result = hash((rawProc(x), rawEnv(x))) + elif T is (proc): + result = hash(cast[pointer](x)) + else: + result = 0 + for f in fields(x): + result = result !& hash(f) + result = !$result proc hash*[A](x: openArray[A]): Hash = ## Efficient hashing of arrays and sequences. + ## There must be a `hash` proc defined for the element type `A`. when A is byte: - result = murmurHash(x) + when not sHash2: + result = cast[Hash](hashFarm(x)) + else: + result = murmurHash(x) elif A is char: - when nimvm: - result = hashVmImplChar(x, 0, x.high) + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(x, 0, x.high))) else: - result = murmurHash(toOpenArrayByte(x, 0, x.high)) + #when nimvm: + # result = hashVmImplChar(x, 0, x.high) + when true: + result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: + result = 0 for a in x: result = result !& hash(a) result = !$result @@ -404,22 +741,30 @@ proc hash*[A](x: openArray[A]): Hash = proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = ## Efficient hashing of portions of arrays and sequences, from starting ## position `sPos` to ending position `ePos` (included). + ## There must be a `hash` proc defined for the element type `A`. ## - ## ``hash(myBuf, 0, myBuf.high)`` is equivalent to ``hash(myBuf)``. + ## `hash(myBuf, 0, myBuf.high)` is equivalent to `hash(myBuf)`. runnableExamples: let a = [1, 2, 5, 1, 2, 6] doAssert hash(a, 0, 1) == hash(a, 3, 4) - when A is byte: - when nimvm: - result = hashVmImplByte(aBuf, sPos, ePos) + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArray(aBuf, sPos, ePos))) else: - result = murmurHash(toOpenArray(aBuf, sPos, ePos)) + #when nimvm: + # result = hashVmImplByte(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArray(aBuf, sPos, ePos)) elif A is char: - when nimvm: - result = hashVmImplChar(aBuf, sPos, ePos) + maybeFailJS_Number() + when not sHash2: + result = cast[Hash](hashFarm(toOpenArrayByte(aBuf, sPos, ePos))) else: - result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) + #when nimvm: + # result = hashVmImplChar(aBuf, sPos, ePos) + when true: + result = murmurHash(toOpenArrayByte(aBuf, sPos, ePos)) else: for i in sPos .. ePos: result = result !& hash(aBuf[i]) @@ -427,49 +772,8 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = proc hash*[A](x: set[A]): Hash = ## Efficient hashing of sets. + ## There must be a `hash` proc defined for the element type `A`. + result = 0 for it in items(x): result = result !& hash(it) result = !$result - - -when isMainModule: - block empty: - var - a = "" - b = newSeq[char]() - c = newSeq[int]() - d = cstring"" - e = "abcd" - doAssert hash(a) == 0 - doAssert hash(b) == 0 - doAssert hash(c) == 0 - doAssert hash(d) == 0 - doAssert hashIgnoreCase(a) == 0 - doAssert hashIgnoreStyle(a) == 0 - doAssert hash(e, 3, 2) == 0 - block sameButDifferent: - doAssert hash("aa bb aaaa1234") == hash("aa bb aaaa1234", 0, 13) - doAssert hash("aa bb aaaa1234") == hash(cstring"aa bb aaaa1234") - doAssert hashIgnoreCase("aA bb aAAa1234") == hashIgnoreCase("aa bb aaaa1234") - doAssert hashIgnoreStyle("aa_bb_AAaa1234") == hashIgnoreCase("aaBBAAAa1234") - block smallSize: # no multibyte hashing - let - xx = @['H', 'i'] - ii = @[72'u8, 105] - ss = "Hi" - doAssert hash(xx) == hash(ii) - doAssert hash(xx) == hash(ss) - doAssert hash(xx) == hash(xx, 0, xx.high) - doAssert hash(ss) == hash(ss, 0, ss.high) - block largeSize: # longer than 4 characters - let - xx = @['H', 'e', 'l', 'l', 'o'] - xxl = @['H', 'e', 'l', 'l', 'o', 'w', 'e', 'e', 'n', 's'] - ssl = "Helloweens" - doAssert hash(xxl) == hash(ssl) - doAssert hash(xxl) == hash(xxl, 0, xxl.high) - doAssert hash(ssl) == hash(ssl, 0, ssl.high) - doAssert hash(xx) == hash(xxl, 0, 4) - doAssert hash(xx) == hash(ssl, 0, 4) - doAssert hash(xx, 0, 3) == hash(xxl, 0, 3) - doAssert hash(xx, 0, 3) == hash(ssl, 0, 3) diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index eb9dd9a73..fafa72463 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -8,9 +8,9 @@ # ## Do yourself a favor and import the module -## as ``from htmlgen import nil`` and then fully qualify the macros. +## as `from std/htmlgen import nil` and then fully qualify the macros. ## -## *Note*: The Karax project (``nimble install karax``) has a better +## *Note*: The Karax project (`nimble install karax`) has a better ## way to achieve the same, see https://github.com/pragmagic/karax/blob/master/tests/nativehtmlgen.nim ## for an example. ## @@ -30,17 +30,18 @@ ## Examples ## ======== ## -## .. code-block:: Nim +## ```Nim ## var nim = "Nim" ## echo h1(a(href="https://nim-lang.org", nim)) +## ``` ## -## Writes the string:: +## Writes the string: ## -## <h1><a href="https://nim-lang.org">Nim</a></h1> +## <h1><a href="https://nim-lang.org">Nim</a></h1> ## import - macros, strutils + std/[macros, strutils] const coreAttr* = " accesskey class contenteditable dir hidden id lang " & @@ -56,7 +57,7 @@ const ariaAttr* = " role " ## HTML DOM Aria Attributes commonAttr* = coreAttr & eventAttr & ariaAttr ## HTML DOM Common Attributes -proc getIdent(e: NimNode): string {.compileTime.} = +proc getIdent(e: NimNode): string = case e.kind of nnkIdent: result = e.strVal.normalize @@ -64,7 +65,7 @@ proc getIdent(e: NimNode): string {.compileTime.} = result = getIdent(e[0]) for i in 1 .. e.len-1: result.add getIdent(e[i]) - else: error("cannot extract identifier from node: " & toStrLit(e).strVal) + else: error("cannot extract identifier from node: " & toStrLit(e).strVal, e) proc delete[T](s: var seq[T], attr: T): bool = var idx = find(s, attr) @@ -75,7 +76,7 @@ proc delete[T](s: var seq[T], attr: T): bool = result = true proc xmlCheckedTag*(argsList: NimNode, tag: string, optAttr = "", reqAttr = "", - isLeaf = false): NimNode {.compileTime.} = + isLeaf = false): NimNode = ## use this procedure to define a new XML tag # copy the attributes; when iterating over them these lists @@ -89,21 +90,21 @@ proc xmlCheckedTag*(argsList: NimNode, tag: string, optAttr = "", reqAttr = "", for i in 0 ..< argsList.len: if argsList[i].kind == nnkExprEqExpr: var name = getIdent(argsList[i][0]) - if delete(req, name) or delete(opt, name): + if name.startsWith("data-") or delete(req, name) or delete(opt, name): result.add(newStrLitNode(" ")) result.add(newStrLitNode(name)) result.add(newStrLitNode("=\"")) result.add(argsList[i][1]) result.add(newStrLitNode("\"")) else: - error("invalid attribute for '" & tag & "' element: " & name) + error("invalid attribute for '" & tag & "' element: " & name, argsList[i]) # check each required attribute exists: if req.len > 0: - error(req[0] & " attribute for '" & tag & "' element expected") + error(req[0] & " attribute for '" & tag & "' element expected", argsList) if isLeaf: for i in 0 ..< argsList.len: if argsList[i].kind != nnkExprEqExpr: - error("element " & tag & " cannot be nested") + error("element " & tag & " cannot be nested", argsList[i]) result.add(newStrLitNode(" />")) else: result.add(newStrLitNode(">")) @@ -113,236 +114,233 @@ proc xmlCheckedTag*(argsList: NimNode, tag: string, optAttr = "", reqAttr = "", result.add(newStrLitNode("</")) result.add(newStrLitNode(tag)) result.add(newStrLitNode(">")) - when compiles(nestList(ident"&", result)): - result = nestList(ident"&", result) - else: - result = nestList(!"&", result) + result = nestList(ident"&", result) macro a*(e: varargs[untyped]): untyped = - ## generates the HTML ``a`` element. + ## Generates the HTML `a` element. result = xmlCheckedTag(e, "a", "href target download rel hreflang type " & commonAttr) macro abbr*(e: varargs[untyped]): untyped = - ## generates the HTML ``abbr`` element. + ## Generates the HTML `abbr` element. result = xmlCheckedTag(e, "abbr", commonAttr) macro address*(e: varargs[untyped]): untyped = - ## generates the HTML ``address`` element. + ## Generates the HTML `address` element. result = xmlCheckedTag(e, "address", commonAttr) macro area*(e: varargs[untyped]): untyped = - ## generates the HTML ``area`` element. + ## Generates the HTML `area` element. result = xmlCheckedTag(e, "area", "coords download href hreflang rel " & "shape target type" & commonAttr, "alt", true) macro article*(e: varargs[untyped]): untyped = - ## generates the HTML ``article`` element. + ## Generates the HTML `article` element. result = xmlCheckedTag(e, "article", commonAttr) macro aside*(e: varargs[untyped]): untyped = - ## generates the HTML ``aside`` element. + ## Generates the HTML `aside` element. result = xmlCheckedTag(e, "aside", commonAttr) macro audio*(e: varargs[untyped]): untyped = - ## generates the HTML ``audio`` element. + ## Generates the HTML `audio` element. result = xmlCheckedTag(e, "audio", "src crossorigin preload " & "autoplay mediagroup loop muted controls" & commonAttr) macro b*(e: varargs[untyped]): untyped = - ## generates the HTML ``b`` element. + ## Generates the HTML `b` element. result = xmlCheckedTag(e, "b", commonAttr) macro base*(e: varargs[untyped]): untyped = - ## generates the HTML ``base`` element. + ## Generates the HTML `base` element. result = xmlCheckedTag(e, "base", "href target" & commonAttr, "", true) macro bdi*(e: varargs[untyped]): untyped = - ## generates the HTML ``bdi`` element. + ## Generates the HTML `bdi` element. result = xmlCheckedTag(e, "bdi", commonAttr) macro bdo*(e: varargs[untyped]): untyped = - ## generates the HTML ``bdo`` element. + ## Generates the HTML `bdo` element. result = xmlCheckedTag(e, "bdo", commonAttr) macro big*(e: varargs[untyped]): untyped = - ## generates the HTML ``big`` element. + ## Generates the HTML `big` element. result = xmlCheckedTag(e, "big", commonAttr) macro blockquote*(e: varargs[untyped]): untyped = - ## generates the HTML ``blockquote`` element. + ## Generates the HTML `blockquote` element. result = xmlCheckedTag(e, "blockquote", " cite" & commonAttr) macro body*(e: varargs[untyped]): untyped = - ## generates the HTML ``body`` element. + ## Generates the HTML `body` element. result = xmlCheckedTag(e, "body", "onafterprint onbeforeprint " & "onbeforeunload onhashchange onmessage onoffline ononline onpagehide " & "onpageshow onpopstate onstorage onunload" & commonAttr) macro br*(e: varargs[untyped]): untyped = - ## generates the HTML ``br`` element. + ## Generates the HTML `br` element. result = xmlCheckedTag(e, "br", commonAttr, "", true) macro button*(e: varargs[untyped]): untyped = - ## generates the HTML ``button`` element. + ## Generates the HTML `button` element. result = xmlCheckedTag(e, "button", "autofocus disabled form formaction " & "formenctype formmethod formnovalidate formtarget menu name type value" & commonAttr) macro canvas*(e: varargs[untyped]): untyped = - ## generates the HTML ``canvas`` element. + ## Generates the HTML `canvas` element. result = xmlCheckedTag(e, "canvas", "width height" & commonAttr) macro caption*(e: varargs[untyped]): untyped = - ## generates the HTML ``caption`` element. + ## Generates the HTML `caption` element. result = xmlCheckedTag(e, "caption", commonAttr) macro center*(e: varargs[untyped]): untyped = - ## Generates the HTML ``center`` element. + ## Generates the HTML `center` element. result = xmlCheckedTag(e, "center", commonAttr) macro cite*(e: varargs[untyped]): untyped = - ## generates the HTML ``cite`` element. + ## Generates the HTML `cite` element. result = xmlCheckedTag(e, "cite", commonAttr) macro code*(e: varargs[untyped]): untyped = - ## generates the HTML ``code`` element. + ## Generates the HTML `code` element. result = xmlCheckedTag(e, "code", commonAttr) macro col*(e: varargs[untyped]): untyped = - ## generates the HTML ``col`` element. + ## Generates the HTML `col` element. result = xmlCheckedTag(e, "col", "span" & commonAttr, "", true) macro colgroup*(e: varargs[untyped]): untyped = - ## generates the HTML ``colgroup`` element. + ## Generates the HTML `colgroup` element. result = xmlCheckedTag(e, "colgroup", "span" & commonAttr) macro data*(e: varargs[untyped]): untyped = - ## generates the HTML ``data`` element. + ## Generates the HTML `data` element. result = xmlCheckedTag(e, "data", "value" & commonAttr) macro datalist*(e: varargs[untyped]): untyped = - ## generates the HTML ``datalist`` element. + ## Generates the HTML `datalist` element. result = xmlCheckedTag(e, "datalist", commonAttr) macro dd*(e: varargs[untyped]): untyped = - ## generates the HTML ``dd`` element. + ## Generates the HTML `dd` element. result = xmlCheckedTag(e, "dd", commonAttr) macro del*(e: varargs[untyped]): untyped = - ## generates the HTML ``del`` element. + ## Generates the HTML `del` element. result = xmlCheckedTag(e, "del", "cite datetime" & commonAttr) macro details*(e: varargs[untyped]): untyped = - ## Generates the HTML ``details`` element. + ## Generates the HTML `details` element. result = xmlCheckedTag(e, "details", commonAttr & "open") macro dfn*(e: varargs[untyped]): untyped = - ## generates the HTML ``dfn`` element. + ## Generates the HTML `dfn` element. result = xmlCheckedTag(e, "dfn", commonAttr) macro dialog*(e: varargs[untyped]): untyped = - ## Generates the HTML ``dialog`` element. + ## Generates the HTML `dialog` element. result = xmlCheckedTag(e, "dialog", commonAttr & "open") macro `div`*(e: varargs[untyped]): untyped = - ## generates the HTML ``div`` element. + ## Generates the HTML `div` element. result = xmlCheckedTag(e, "div", commonAttr) macro dl*(e: varargs[untyped]): untyped = - ## generates the HTML ``dl`` element. + ## Generates the HTML `dl` element. result = xmlCheckedTag(e, "dl", commonAttr) macro dt*(e: varargs[untyped]): untyped = - ## generates the HTML ``dt`` element. + ## Generates the HTML `dt` element. result = xmlCheckedTag(e, "dt", commonAttr) macro em*(e: varargs[untyped]): untyped = - ## generates the HTML ``em`` element. + ## Generates the HTML `em` element. result = xmlCheckedTag(e, "em", commonAttr) macro embed*(e: varargs[untyped]): untyped = - ## generates the HTML ``embed`` element. + ## Generates the HTML `embed` element. result = xmlCheckedTag(e, "embed", "src type height width" & commonAttr, "", true) macro fieldset*(e: varargs[untyped]): untyped = - ## generates the HTML ``fieldset`` element. + ## Generates the HTML `fieldset` element. result = xmlCheckedTag(e, "fieldset", "disabled form name" & commonAttr) macro figure*(e: varargs[untyped]): untyped = - ## generates the HTML ``figure`` element. + ## Generates the HTML `figure` element. result = xmlCheckedTag(e, "figure", commonAttr) macro figcaption*(e: varargs[untyped]): untyped = - ## generates the HTML ``figcaption`` element. + ## Generates the HTML `figcaption` element. result = xmlCheckedTag(e, "figcaption", commonAttr) macro footer*(e: varargs[untyped]): untyped = - ## generates the HTML ``footer`` element. + ## Generates the HTML `footer` element. result = xmlCheckedTag(e, "footer", commonAttr) macro form*(e: varargs[untyped]): untyped = - ## generates the HTML ``form`` element. + ## Generates the HTML `form` element. result = xmlCheckedTag(e, "form", "accept-charset action autocomplete " & "enctype method name novalidate target" & commonAttr) macro h1*(e: varargs[untyped]): untyped = - ## generates the HTML ``h1`` element. + ## Generates the HTML `h1` element. result = xmlCheckedTag(e, "h1", commonAttr) macro h2*(e: varargs[untyped]): untyped = - ## generates the HTML ``h2`` element. + ## Generates the HTML `h2` element. result = xmlCheckedTag(e, "h2", commonAttr) macro h3*(e: varargs[untyped]): untyped = - ## generates the HTML ``h3`` element. + ## Generates the HTML `h3` element. result = xmlCheckedTag(e, "h3", commonAttr) macro h4*(e: varargs[untyped]): untyped = - ## generates the HTML ``h4`` element. + ## Generates the HTML `h4` element. result = xmlCheckedTag(e, "h4", commonAttr) macro h5*(e: varargs[untyped]): untyped = - ## generates the HTML ``h5`` element. + ## Generates the HTML `h5` element. result = xmlCheckedTag(e, "h5", commonAttr) macro h6*(e: varargs[untyped]): untyped = - ## generates the HTML ``h6`` element. + ## Generates the HTML `h6` element. result = xmlCheckedTag(e, "h6", commonAttr) macro head*(e: varargs[untyped]): untyped = - ## generates the HTML ``head`` element. + ## Generates the HTML `head` element. result = xmlCheckedTag(e, "head", commonAttr) macro header*(e: varargs[untyped]): untyped = - ## generates the HTML ``header`` element. + ## Generates the HTML `header` element. result = xmlCheckedTag(e, "header", commonAttr) macro html*(e: varargs[untyped]): untyped = - ## generates the HTML ``html`` element. + ## Generates the HTML `html` element. result = xmlCheckedTag(e, "html", "xmlns" & commonAttr, "") macro hr*(): untyped = - ## generates the HTML ``hr`` element. - result = xmlCheckedTag(newNimNode(nnkArglist), "hr", commonAttr, "", true) + ## Generates the HTML `hr` element. + result = xmlCheckedTag(newNimNode(nnkArgList), "hr", commonAttr, "", true) macro i*(e: varargs[untyped]): untyped = - ## generates the HTML ``i`` element. + ## Generates the HTML `i` element. result = xmlCheckedTag(e, "i", commonAttr) macro iframe*(e: varargs[untyped]): untyped = - ## generates the HTML ``iframe`` element. - result = xmlCheckedTag(e, "iframe", "src srcdoc name sandbox width height" & + ## Generates the HTML `iframe` element. + result = xmlCheckedTag(e, "iframe", "src srcdoc name sandbox width height loading" & commonAttr) macro img*(e: varargs[untyped]): untyped = - ## generates the HTML ``img`` element. - result = xmlCheckedTag(e, "img", "crossorigin usemap ismap height width" & + ## Generates the HTML `img` element. + result = xmlCheckedTag(e, "img", "crossorigin usemap ismap height width loading" & commonAttr, "src alt", true) macro input*(e: varargs[untyped]): untyped = - ## generates the HTML ``input`` element. + ## Generates the HTML `input` element. result = xmlCheckedTag(e, "input", "accept alt autocomplete autofocus " & "checked dirname disabled form formaction formenctype formmethod " & "formnovalidate formtarget height inputmode list max maxlength min " & @@ -350,397 +348,401 @@ macro input*(e: varargs[untyped]): untyped = "src step type value width" & commonAttr, "", true) macro ins*(e: varargs[untyped]): untyped = - ## generates the HTML ``ins`` element. + ## Generates the HTML `ins` element. result = xmlCheckedTag(e, "ins", "cite datetime" & commonAttr) macro kbd*(e: varargs[untyped]): untyped = - ## generates the HTML ``kbd`` element. + ## Generates the HTML `kbd` element. result = xmlCheckedTag(e, "kbd", commonAttr) macro keygen*(e: varargs[untyped]): untyped = - ## generates the HTML ``keygen`` element. + ## Generates the HTML `keygen` element. result = xmlCheckedTag(e, "keygen", "autofocus challenge disabled " & "form keytype name" & commonAttr) macro label*(e: varargs[untyped]): untyped = - ## generates the HTML ``label`` element. + ## Generates the HTML `label` element. result = xmlCheckedTag(e, "label", "form for" & commonAttr) macro legend*(e: varargs[untyped]): untyped = - ## generates the HTML ``legend`` element. + ## Generates the HTML `legend` element. result = xmlCheckedTag(e, "legend", commonAttr) macro li*(e: varargs[untyped]): untyped = - ## generates the HTML ``li`` element. + ## Generates the HTML `li` element. result = xmlCheckedTag(e, "li", "value" & commonAttr) macro link*(e: varargs[untyped]): untyped = - ## generates the HTML ``link`` element. + ## Generates the HTML `link` element. result = xmlCheckedTag(e, "link", "href crossorigin rel media hreflang " & "type sizes" & commonAttr, "", true) macro main*(e: varargs[untyped]): untyped = - ## generates the HTML ``main`` element. + ## Generates the HTML `main` element. result = xmlCheckedTag(e, "main", commonAttr) macro map*(e: varargs[untyped]): untyped = - ## generates the HTML ``map`` element. + ## Generates the HTML `map` element. result = xmlCheckedTag(e, "map", "name" & commonAttr) macro mark*(e: varargs[untyped]): untyped = - ## generates the HTML ``mark`` element. + ## Generates the HTML `mark` element. result = xmlCheckedTag(e, "mark", commonAttr) macro marquee*(e: varargs[untyped]): untyped = - ## Generates the HTML ``marquee`` element. + ## Generates the HTML `marquee` element. result = xmlCheckedTag(e, "marquee", coreAttr & "behavior bgcolor direction height hspace loop scrollamount " & "scrolldelay truespeed vspace width onbounce onfinish onstart") macro meta*(e: varargs[untyped]): untyped = - ## generates the HTML ``meta`` element. + ## Generates the HTML `meta` element. result = xmlCheckedTag(e, "meta", "name http-equiv content charset" & commonAttr, "", true) macro meter*(e: varargs[untyped]): untyped = - ## generates the HTML ``meter`` element. + ## Generates the HTML `meter` element. result = xmlCheckedTag(e, "meter", "value min max low high optimum" & commonAttr) macro nav*(e: varargs[untyped]): untyped = - ## generates the HTML ``nav`` element. + ## Generates the HTML `nav` element. result = xmlCheckedTag(e, "nav", commonAttr) macro noscript*(e: varargs[untyped]): untyped = - ## generates the HTML ``noscript`` element. + ## Generates the HTML `noscript` element. result = xmlCheckedTag(e, "noscript", commonAttr) macro `object`*(e: varargs[untyped]): untyped = - ## generates the HTML ``object`` element. + ## Generates the HTML `object` element. result = xmlCheckedTag(e, "object", "data type typemustmatch name usemap " & "form width height" & commonAttr) macro ol*(e: varargs[untyped]): untyped = - ## generates the HTML ``ol`` element. + ## Generates the HTML `ol` element. result = xmlCheckedTag(e, "ol", "reversed start type" & commonAttr) macro optgroup*(e: varargs[untyped]): untyped = - ## generates the HTML ``optgroup`` element. + ## Generates the HTML `optgroup` element. result = xmlCheckedTag(e, "optgroup", "disabled" & commonAttr, "label", false) macro option*(e: varargs[untyped]): untyped = - ## generates the HTML ``option`` element. + ## Generates the HTML `option` element. result = xmlCheckedTag(e, "option", "disabled label selected value" & commonAttr) macro output*(e: varargs[untyped]): untyped = - ## generates the HTML ``output`` element. + ## Generates the HTML `output` element. result = xmlCheckedTag(e, "output", "for form name" & commonAttr) macro p*(e: varargs[untyped]): untyped = - ## generates the HTML ``p`` element. + ## Generates the HTML `p` element. result = xmlCheckedTag(e, "p", commonAttr) macro param*(e: varargs[untyped]): untyped = - ## generates the HTML ``param`` element. + ## Generates the HTML `param` element. result = xmlCheckedTag(e, "param", commonAttr, "name value", true) macro picture*(e: varargs[untyped]): untyped = - ## Generates the HTML ``picture`` element. + ## Generates the HTML `picture` element. result = xmlCheckedTag(e, "picture", commonAttr) macro pre*(e: varargs[untyped]): untyped = - ## generates the HTML ``pre`` element. + ## Generates the HTML `pre` element. result = xmlCheckedTag(e, "pre", commonAttr) macro progress*(e: varargs[untyped]): untyped = - ## generates the HTML ``progress`` element. + ## Generates the HTML `progress` element. result = xmlCheckedTag(e, "progress", "value max" & commonAttr) macro q*(e: varargs[untyped]): untyped = - ## generates the HTML ``q`` element. + ## Generates the HTML `q` element. result = xmlCheckedTag(e, "q", "cite" & commonAttr) macro rb*(e: varargs[untyped]): untyped = - ## generates the HTML ``rb`` element. + ## Generates the HTML `rb` element. result = xmlCheckedTag(e, "rb", commonAttr) macro rp*(e: varargs[untyped]): untyped = - ## generates the HTML ``rp`` element. + ## Generates the HTML `rp` element. result = xmlCheckedTag(e, "rp", commonAttr) macro rt*(e: varargs[untyped]): untyped = - ## generates the HTML ``rt`` element. + ## Generates the HTML `rt` element. result = xmlCheckedTag(e, "rt", commonAttr) macro rtc*(e: varargs[untyped]): untyped = - ## generates the HTML ``rtc`` element. + ## Generates the HTML `rtc` element. result = xmlCheckedTag(e, "rtc", commonAttr) macro ruby*(e: varargs[untyped]): untyped = - ## generates the HTML ``ruby`` element. + ## Generates the HTML `ruby` element. result = xmlCheckedTag(e, "ruby", commonAttr) macro s*(e: varargs[untyped]): untyped = - ## generates the HTML ``s`` element. + ## Generates the HTML `s` element. result = xmlCheckedTag(e, "s", commonAttr) macro samp*(e: varargs[untyped]): untyped = - ## generates the HTML ``samp`` element. + ## Generates the HTML `samp` element. result = xmlCheckedTag(e, "samp", commonAttr) macro script*(e: varargs[untyped]): untyped = - ## generates the HTML ``script`` element. + ## Generates the HTML `script` element. result = xmlCheckedTag(e, "script", "src type charset async defer " & "crossorigin" & commonAttr) macro section*(e: varargs[untyped]): untyped = - ## generates the HTML ``section`` element. + ## Generates the HTML `section` element. result = xmlCheckedTag(e, "section", commonAttr) macro select*(e: varargs[untyped]): untyped = - ## generates the HTML ``select`` element. + ## Generates the HTML `select` element. result = xmlCheckedTag(e, "select", "autofocus disabled form multiple " & "name required size" & commonAttr) macro slot*(e: varargs[untyped]): untyped = - ## Generates the HTML ``slot`` element. + ## Generates the HTML `slot` element. result = xmlCheckedTag(e, "slot", commonAttr) macro small*(e: varargs[untyped]): untyped = - ## generates the HTML ``small`` element. + ## Generates the HTML `small` element. result = xmlCheckedTag(e, "small", commonAttr) macro source*(e: varargs[untyped]): untyped = - ## generates the HTML ``source`` element. + ## Generates the HTML `source` element. result = xmlCheckedTag(e, "source", "type" & commonAttr, "src", true) macro span*(e: varargs[untyped]): untyped = - ## generates the HTML ``span`` element. + ## Generates the HTML `span` element. result = xmlCheckedTag(e, "span", commonAttr) macro strong*(e: varargs[untyped]): untyped = - ## generates the HTML ``strong`` element. + ## Generates the HTML `strong` element. result = xmlCheckedTag(e, "strong", commonAttr) macro style*(e: varargs[untyped]): untyped = - ## generates the HTML ``style`` element. + ## Generates the HTML `style` element. result = xmlCheckedTag(e, "style", "media type" & commonAttr) macro sub*(e: varargs[untyped]): untyped = - ## generates the HTML ``sub`` element. + ## Generates the HTML `sub` element. result = xmlCheckedTag(e, "sub", commonAttr) macro summary*(e: varargs[untyped]): untyped = - ## Generates the HTML ``summary`` element. + ## Generates the HTML `summary` element. result = xmlCheckedTag(e, "summary", commonAttr) macro sup*(e: varargs[untyped]): untyped = - ## generates the HTML ``sup`` element. + ## Generates the HTML `sup` element. result = xmlCheckedTag(e, "sup", commonAttr) macro table*(e: varargs[untyped]): untyped = - ## generates the HTML ``table`` element. + ## Generates the HTML `table` element. result = xmlCheckedTag(e, "table", "border sortable" & commonAttr) macro tbody*(e: varargs[untyped]): untyped = - ## generates the HTML ``tbody`` element. + ## Generates the HTML `tbody` element. result = xmlCheckedTag(e, "tbody", commonAttr) macro td*(e: varargs[untyped]): untyped = - ## generates the HTML ``td`` element. + ## Generates the HTML `td` element. result = xmlCheckedTag(e, "td", "colspan rowspan headers" & commonAttr) macro `template`*(e: varargs[untyped]): untyped = - ## generates the HTML ``template`` element. + ## Generates the HTML `template` element. result = xmlCheckedTag(e, "template", commonAttr) macro textarea*(e: varargs[untyped]): untyped = - ## generates the HTML ``textarea`` element. + ## Generates the HTML `textarea` element. result = xmlCheckedTag(e, "textarea", "autocomplete autofocus cols " & "dirname disabled form inputmode maxlength minlength name placeholder " & "readonly required rows wrap" & commonAttr) macro tfoot*(e: varargs[untyped]): untyped = - ## generates the HTML ``tfoot`` element. + ## Generates the HTML `tfoot` element. result = xmlCheckedTag(e, "tfoot", commonAttr) macro th*(e: varargs[untyped]): untyped = - ## generates the HTML ``th`` element. + ## Generates the HTML `th` element. result = xmlCheckedTag(e, "th", "colspan rowspan headers abbr scope axis" & " sorted" & commonAttr) macro thead*(e: varargs[untyped]): untyped = - ## generates the HTML ``thead`` element. + ## Generates the HTML `thead` element. result = xmlCheckedTag(e, "thead", commonAttr) macro time*(e: varargs[untyped]): untyped = - ## generates the HTML ``time`` element. + ## Generates the HTML `time` element. result = xmlCheckedTag(e, "time", "datetime" & commonAttr) macro title*(e: varargs[untyped]): untyped = - ## generates the HTML ``title`` element. + ## Generates the HTML `title` element. result = xmlCheckedTag(e, "title", commonAttr) macro tr*(e: varargs[untyped]): untyped = - ## generates the HTML ``tr`` element. + ## Generates the HTML `tr` element. result = xmlCheckedTag(e, "tr", commonAttr) macro track*(e: varargs[untyped]): untyped = - ## generates the HTML ``track`` element. + ## Generates the HTML `track` element. result = xmlCheckedTag(e, "track", "kind srclang label default" & commonAttr, "src", true) macro tt*(e: varargs[untyped]): untyped = - ## generates the HTML ``tt`` element. + ## Generates the HTML `tt` element. result = xmlCheckedTag(e, "tt", commonAttr) macro u*(e: varargs[untyped]): untyped = - ## generates the HTML ``u`` element. + ## Generates the HTML `u` element. result = xmlCheckedTag(e, "u", commonAttr) macro ul*(e: varargs[untyped]): untyped = - ## generates the HTML ``ul`` element. + ## Generates the HTML `ul` element. result = xmlCheckedTag(e, "ul", commonAttr) macro `var`*(e: varargs[untyped]): untyped = - ## generates the HTML ``var`` element. + ## Generates the HTML `var` element. result = xmlCheckedTag(e, "var", commonAttr) macro video*(e: varargs[untyped]): untyped = - ## generates the HTML ``video`` element. + ## Generates the HTML `video` element. result = xmlCheckedTag(e, "video", "src crossorigin poster preload " & "autoplay mediagroup loop muted controls width height" & commonAttr) macro wbr*(e: varargs[untyped]): untyped = - ## generates the HTML ``wbr`` element. + ## Generates the HTML `wbr` element. result = xmlCheckedTag(e, "wbr", commonAttr, "", true) +macro portal*(e: varargs[untyped]): untyped = + ## Generates the HTML `portal` element. + result = xmlCheckedTag(e, "portal", "width height type src disabled" & commonAttr, "", false) + macro math*(e: varargs[untyped]): untyped = - ## Generates the HTML ``math`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `math` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/math#Examples result = xmlCheckedTag(e, "math", "mathbackground mathcolor href overflow" & commonAttr) macro maction*(e: varargs[untyped]): untyped = - ## Generates the HTML ``maction`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `maction` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/maction result = xmlCheckedTag(e, "maction", "mathbackground mathcolor href" & commonAttr) macro menclose*(e: varargs[untyped]): untyped = - ## Generates the HTML ``menclose`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `menclose` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/menclose result = xmlCheckedTag(e, "menclose", "mathbackground mathcolor href notation" & commonAttr) macro merror*(e: varargs[untyped]): untyped = - ## Generates the HTML ``merror`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `merror` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/merror result = xmlCheckedTag(e, "merror", "mathbackground mathcolor href" & commonAttr) macro mfenced*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mfenced`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mfenced` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mfenced result = xmlCheckedTag(e, "mfenced", "mathbackground mathcolor href open separators" & commonAttr) macro mfrac*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mfrac`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mfrac` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mfrac result = xmlCheckedTag(e, "mfrac", "mathbackground mathcolor href linethickness numalign" & commonAttr) macro mglyph*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mglyph`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mglyph` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mglyph result = xmlCheckedTag(e, "mglyph", "mathbackground mathcolor href src valign" & commonAttr) macro mi*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mi`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mi` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mi result = xmlCheckedTag(e, "mi", "mathbackground mathcolor href mathsize mathvariant" & commonAttr) macro mlabeledtr*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mlabeledtr`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mlabeledtr` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mlabeledtr result = xmlCheckedTag(e, "mlabeledtr", "mathbackground mathcolor href columnalign groupalign rowalign" & commonAttr) macro mmultiscripts*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mmultiscripts`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mmultiscripts` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mmultiscripts result = xmlCheckedTag(e, "mmultiscripts", "mathbackground mathcolor href subscriptshift superscriptshift" & commonAttr) macro mn*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mn`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mn` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mn result = xmlCheckedTag(e, "mn", "mathbackground mathcolor href mathsize mathvariant" & commonAttr) macro mo*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mo`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mo` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mo result = xmlCheckedTag(e, "mo", "mathbackground mathcolor fence form largeop lspace mathsize mathvariant movablelimits rspace separator stretchy symmetric" & commonAttr) macro mover*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mover`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mover` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mover result = xmlCheckedTag(e, "mover", "mathbackground mathcolor accent href" & commonAttr) macro mpadded*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mpadded`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mpadded` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mpadded result = xmlCheckedTag(e, "mpadded", "mathbackground mathcolor depth href lspace voffset" & commonAttr) macro mphantom*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mphantom`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mphantom` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mphantom result = xmlCheckedTag(e, "mphantom", "mathbackground" & commonAttr) macro mroot*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mroot`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mroot` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mroot result = xmlCheckedTag(e, "mroot", "mathbackground mathcolor href" & commonAttr) macro mrow*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mrow`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mrow` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mrow result = xmlCheckedTag(e, "mrow", "mathbackground mathcolor href" & commonAttr) macro ms*(e: varargs[untyped]): untyped = - ## Generates the HTML ``ms`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `ms` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/ms result = xmlCheckedTag(e, "ms", "mathbackground mathcolor href lquote mathsize mathvariant rquote" & commonAttr) macro mspace*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mspace`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mspace` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mspace result = xmlCheckedTag(e, "mspace", "mathbackground mathcolor href linebreak" & commonAttr) macro msqrt*(e: varargs[untyped]): untyped = - ## Generates the HTML ``msqrt`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `msqrt` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msqrt result = xmlCheckedTag(e, "msqrt", "mathbackground mathcolor href" & commonAttr) macro mstyle*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mstyle`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mstyle` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mstyle result = xmlCheckedTag(e, "mstyle", ("mathbackground mathcolor href decimalpoint displaystyle " & "infixlinebreakstyle scriptlevel scriptminsize scriptsizemultiplier" & commonAttr)) macro msub*(e: varargs[untyped]): untyped = - ## Generates the HTML ``msub`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `msub` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msub result = xmlCheckedTag(e, "msub", "mathbackground mathcolor href subscriptshift" & commonAttr) macro msubsup*(e: varargs[untyped]): untyped = - ## Generates the HTML ``msubsup`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `msubsup` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msubsup result = xmlCheckedTag(e, "msubsup", "mathbackground mathcolor href subscriptshift superscriptshift" & commonAttr) macro msup*(e: varargs[untyped]): untyped = - ## Generates the HTML ``msup`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `msup` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/msup result = xmlCheckedTag(e, "msup", "mathbackground mathcolor href superscriptshift" & commonAttr) macro mtable*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mtable`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mtable` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mtable result = xmlCheckedTag(e, "mtable", ("mathbackground mathcolor href align " & "alignmentscope columnalign columnlines columnspacing columnwidth " & @@ -748,38 +750,38 @@ macro mtable*(e: varargs[untyped]): untyped = "rowalign rowlines rowspacing side width" & commonAttr)) macro mtd*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mtd`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mtd` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mtd result = xmlCheckedTag(e, "mtd", "mathbackground mathcolor href columnalign columnspan groupalign rowalign rowspan" & commonAttr) macro mtext*(e: varargs[untyped]): untyped = - ## Generates the HTML ``mtext`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `mtext` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/mtext result = xmlCheckedTag(e, "mtext", "mathbackground mathcolor href mathsize mathvariant" & commonAttr) macro munder*(e: varargs[untyped]): untyped = - ## Generates the HTML ``munder`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `munder` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/munder result = xmlCheckedTag(e, "munder", "mathbackground mathcolor href accentunder align" & commonAttr) macro munderover*(e: varargs[untyped]): untyped = - ## Generates the HTML ``munderover`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `munderover` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/munderover result = xmlCheckedTag(e, "munderover", "mathbackground mathcolor href accentunder accent align" & commonAttr) macro semantics*(e: varargs[untyped]): untyped = - ## Generates the HTML ``semantics`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `semantics` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics result = xmlCheckedTag(e, "semantics", "mathbackground mathcolor href definitionURL encoding cd src" & commonAttr) macro annotation*(e: varargs[untyped]): untyped = - ## Generates the HTML ``annotation`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `annotation` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics result = xmlCheckedTag(e, "annotation", "mathbackground mathcolor href definitionURL encoding cd src" & commonAttr) macro `annotation-xml`*(e: varargs[untyped]): untyped = - ## Generates the HTML ``annotation-xml`` element. MathML https://wikipedia.org/wiki/MathML + ## Generates the HTML `annotation-xml` element. MathML https://wikipedia.org/wiki/MathML ## https://developer.mozilla.org/en-US/docs/Web/MathML/Element/semantics result = xmlCheckedTag(e, "annotation", "mathbackground mathcolor href definitionURL encoding cd src" & commonAttr) diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 4b305cfd6..62919546f 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -12,14 +12,13 @@ ## ## It can be used to parse a wild HTML document and output it as valid XHTML ## document (well, if you are lucky): -## -## .. code-block:: Nim -## +## ```Nim ## echo loadHtml("mydirty.html") +## ``` ## ## Every tag in the resulting tree is in lower case. ## -## **Note:** The resulting ``XmlNode`` already uses the ``clientData`` field, +## **Note:** The resulting `XmlNode` already uses the `clientData` field, ## so it cannot be used by clients of this library. ## ## Example: Transforming hyperlinks @@ -27,16 +26,14 @@ ## ## This code demonstrates how you can iterate over all the tags in an HTML file ## and write back the modified version. In this case we look for hyperlinks -## ending with the extension ``.rst`` and convert them to ``.html``. -## -## .. code-block:: Nim -## :test: +## ending with the extension `.rst` and convert them to `.html`. ## -## import htmlparser -## import xmltree # To use '$' for XmlNode -## import strtabs # To access XmlAttributes -## import os # To use splitFile -## import strutils # To use cmpIgnoreCase +## ```Nim test +## import std/htmlparser +## import std/xmltree # To use '$' for XmlNode +## import std/strtabs # To access XmlAttributes +## import std/os # To use splitFile +## import std/strutils # To use cmpIgnoreCase ## ## proc transformHyperlinks() = ## let html = loadHtml("input.html") @@ -48,136 +45,142 @@ ## a.attrs["href"] = dir / filename & ".html" ## ## writeFile("output.html", $html) +## ``` + +{.deprecated: "use `nimble install htmlparser` and import `pkg/htmlparser` instead".} + +import std/[strutils, streams, parsexml, xmltree, unicode, strtabs] -import strutils, streams, parsexml, xmltree, unicode, strtabs +when defined(nimPreviewSlimSystem): + import std/syncio type HtmlTag* = enum ## list of all supported HTML tags; order will always be ## alphabetically tagUnknown, ## unknown HTML element - tagA, ## the HTML ``a`` element - tagAbbr, ## the deprecated HTML ``abbr`` element - tagAcronym, ## the HTML ``acronym`` element - tagAddress, ## the HTML ``address`` element - tagApplet, ## the deprecated HTML ``applet`` element - tagArea, ## the HTML ``area`` element - tagArticle, ## the HTML ``article`` element - tagAside, ## the HTML ``aside`` element - tagAudio, ## the HTML ``audio`` element - tagB, ## the HTML ``b`` element - tagBase, ## the HTML ``base`` element - tagBdi, ## the HTML ``bdi`` element - tagBdo, ## the deprecated HTML ``dbo`` element - tagBasefont, ## the deprecated HTML ``basefont`` element - tagBig, ## the HTML ``big`` element - tagBlockquote, ## the HTML ``blockquote`` element - tagBody, ## the HTML ``body`` element - tagBr, ## the HTML ``br`` element - tagButton, ## the HTML ``button`` element - tagCanvas, ## the HTML ``canvas`` element - tagCaption, ## the HTML ``caption`` element - tagCenter, ## the deprecated HTML ``center`` element - tagCite, ## the HTML ``cite`` element - tagCode, ## the HTML ``code`` element - tagCol, ## the HTML ``col`` element - tagColgroup, ## the HTML ``colgroup`` element - tagCommand, ## the HTML ``command`` element - tagDatalist, ## the HTML ``datalist`` element - tagDd, ## the HTML ``dd`` element - tagDel, ## the HTML ``del`` element - tagDetails, ## the HTML ``details`` element - tagDfn, ## the HTML ``dfn`` element - tagDialog, ## the HTML ``dialog`` element - tagDiv, ## the HTML ``div`` element - tagDir, ## the deprecated HTLM ``dir`` element - tagDl, ## the HTML ``dl`` element - tagDt, ## the HTML ``dt`` element - tagEm, ## the HTML ``em`` element - tagEmbed, ## the HTML ``embed`` element - tagFieldset, ## the HTML ``fieldset`` element - tagFigcaption, ## the HTML ``figcaption`` element - tagFigure, ## the HTML ``figure`` element - tagFont, ## the deprecated HTML ``font`` element - tagFooter, ## the HTML ``footer`` element - tagForm, ## the HTML ``form`` element - tagFrame, ## the HTML ``frame`` element - tagFrameset, ## the deprecated HTML ``frameset`` element - tagH1, ## the HTML ``h1`` element - tagH2, ## the HTML ``h2`` element - tagH3, ## the HTML ``h3`` element - tagH4, ## the HTML ``h4`` element - tagH5, ## the HTML ``h5`` element - tagH6, ## the HTML ``h6`` element - tagHead, ## the HTML ``head`` element - tagHeader, ## the HTML ``header`` element - tagHgroup, ## the HTML ``hgroup`` element - tagHtml, ## the HTML ``html`` element - tagHr, ## the HTML ``hr`` element - tagI, ## the HTML ``i`` element - tagIframe, ## the deprecated HTML ``iframe`` element - tagImg, ## the HTML ``img`` element - tagInput, ## the HTML ``input`` element - tagIns, ## the HTML ``ins`` element - tagIsindex, ## the deprecated HTML ``isindex`` element - tagKbd, ## the HTML ``kbd`` element - tagKeygen, ## the HTML ``keygen`` element - tagLabel, ## the HTML ``label`` element - tagLegend, ## the HTML ``legend`` element - tagLi, ## the HTML ``li`` element - tagLink, ## the HTML ``link`` element - tagMap, ## the HTML ``map`` element - tagMark, ## the HTML ``mark`` element - tagMenu, ## the deprecated HTML ``menu`` element - tagMeta, ## the HTML ``meta`` element - tagMeter, ## the HTML ``meter`` element - tagNav, ## the HTML ``nav`` element - tagNobr, ## the deprecated HTML ``nobr`` element - tagNoframes, ## the deprecated HTML ``noframes`` element - tagNoscript, ## the HTML ``noscript`` element - tagObject, ## the HTML ``object`` element - tagOl, ## the HTML ``ol`` element - tagOptgroup, ## the HTML ``optgroup`` element - tagOption, ## the HTML ``option`` element - tagOutput, ## the HTML ``output`` element - tagP, ## the HTML ``p`` element - tagParam, ## the HTML ``param`` element - tagPre, ## the HTML ``pre`` element - tagProgress, ## the HTML ``progress`` element - tagQ, ## the HTML ``q`` element - tagRp, ## the HTML ``rp`` element - tagRt, ## the HTML ``rt`` element - tagRuby, ## the HTML ``ruby`` element - tagS, ## the deprecated HTML ``s`` element - tagSamp, ## the HTML ``samp`` element - tagScript, ## the HTML ``script`` element - tagSection, ## the HTML ``section`` element - tagSelect, ## the HTML ``select`` element - tagSmall, ## the HTML ``small`` element - tagSource, ## the HTML ``source`` element - tagSpan, ## the HTML ``span`` element - tagStrike, ## the deprecated HTML ``strike`` element - tagStrong, ## the HTML ``strong`` element - tagStyle, ## the HTML ``style`` element - tagSub, ## the HTML ``sub`` element - tagSummary, ## the HTML ``summary`` element - tagSup, ## the HTML ``sup`` element - tagTable, ## the HTML ``table`` element - tagTbody, ## the HTML ``tbody`` element - tagTd, ## the HTML ``td`` element - tagTextarea, ## the HTML ``textarea`` element - tagTfoot, ## the HTML ``tfoot`` element - tagTh, ## the HTML ``th`` element - tagThead, ## the HTML ``thead`` element - tagTime, ## the HTML ``time`` element - tagTitle, ## the HTML ``title`` element - tagTr, ## the HTML ``tr`` element - tagTrack, ## the HTML ``track`` element - tagTt, ## the HTML ``tt`` element - tagU, ## the deprecated HTML ``u`` element - tagUl, ## the HTML ``ul`` element - tagVar, ## the HTML ``var`` element - tagVideo, ## the HTML ``video`` element - tagWbr ## the HTML ``wbr`` element + tagA, ## the HTML `a` element + tagAbbr, ## the deprecated HTML `abbr` element + tagAcronym, ## the HTML `acronym` element + tagAddress, ## the HTML `address` element + tagApplet, ## the deprecated HTML `applet` element + tagArea, ## the HTML `area` element + tagArticle, ## the HTML `article` element + tagAside, ## the HTML `aside` element + tagAudio, ## the HTML `audio` element + tagB, ## the HTML `b` element + tagBase, ## the HTML `base` element + tagBdi, ## the HTML `bdi` element + tagBdo, ## the deprecated HTML `dbo` element + tagBasefont, ## the deprecated HTML `basefont` element + tagBig, ## the HTML `big` element + tagBlockquote, ## the HTML `blockquote` element + tagBody, ## the HTML `body` element + tagBr, ## the HTML `br` element + tagButton, ## the HTML `button` element + tagCanvas, ## the HTML `canvas` element + tagCaption, ## the HTML `caption` element + tagCenter, ## the deprecated HTML `center` element + tagCite, ## the HTML `cite` element + tagCode, ## the HTML `code` element + tagCol, ## the HTML `col` element + tagColgroup, ## the HTML `colgroup` element + tagCommand, ## the HTML `command` element + tagDatalist, ## the HTML `datalist` element + tagDd, ## the HTML `dd` element + tagDel, ## the HTML `del` element + tagDetails, ## the HTML `details` element + tagDfn, ## the HTML `dfn` element + tagDialog, ## the HTML `dialog` element + tagDiv, ## the HTML `div` element + tagDir, ## the deprecated HTLM `dir` element + tagDl, ## the HTML `dl` element + tagDt, ## the HTML `dt` element + tagEm, ## the HTML `em` element + tagEmbed, ## the HTML `embed` element + tagFieldset, ## the HTML `fieldset` element + tagFigcaption, ## the HTML `figcaption` element + tagFigure, ## the HTML `figure` element + tagFont, ## the deprecated HTML `font` element + tagFooter, ## the HTML `footer` element + tagForm, ## the HTML `form` element + tagFrame, ## the HTML `frame` element + tagFrameset, ## the deprecated HTML `frameset` element + tagH1, ## the HTML `h1` element + tagH2, ## the HTML `h2` element + tagH3, ## the HTML `h3` element + tagH4, ## the HTML `h4` element + tagH5, ## the HTML `h5` element + tagH6, ## the HTML `h6` element + tagHead, ## the HTML `head` element + tagHeader, ## the HTML `header` element + tagHgroup, ## the HTML `hgroup` element + tagHtml, ## the HTML `html` element + tagHr, ## the HTML `hr` element + tagI, ## the HTML `i` element + tagIframe, ## the deprecated HTML `iframe` element + tagImg, ## the HTML `img` element + tagInput, ## the HTML `input` element + tagIns, ## the HTML `ins` element + tagIsindex, ## the deprecated HTML `isindex` element + tagKbd, ## the HTML `kbd` element + tagKeygen, ## the HTML `keygen` element + tagLabel, ## the HTML `label` element + tagLegend, ## the HTML `legend` element + tagLi, ## the HTML `li` element + tagLink, ## the HTML `link` element + tagMap, ## the HTML `map` element + tagMark, ## the HTML `mark` element + tagMenu, ## the deprecated HTML `menu` element + tagMeta, ## the HTML `meta` element + tagMeter, ## the HTML `meter` element + tagNav, ## the HTML `nav` element + tagNobr, ## the deprecated HTML `nobr` element + tagNoframes, ## the deprecated HTML `noframes` element + tagNoscript, ## the HTML `noscript` element + tagObject, ## the HTML `object` element + tagOl, ## the HTML `ol` element + tagOptgroup, ## the HTML `optgroup` element + tagOption, ## the HTML `option` element + tagOutput, ## the HTML `output` element + tagP, ## the HTML `p` element + tagParam, ## the HTML `param` element + tagPre, ## the HTML `pre` element + tagProgress, ## the HTML `progress` element + tagQ, ## the HTML `q` element + tagRp, ## the HTML `rp` element + tagRt, ## the HTML `rt` element + tagRuby, ## the HTML `ruby` element + tagS, ## the deprecated HTML `s` element + tagSamp, ## the HTML `samp` element + tagScript, ## the HTML `script` element + tagSection, ## the HTML `section` element + tagSelect, ## the HTML `select` element + tagSmall, ## the HTML `small` element + tagSource, ## the HTML `source` element + tagSpan, ## the HTML `span` element + tagStrike, ## the deprecated HTML `strike` element + tagStrong, ## the HTML `strong` element + tagStyle, ## the HTML `style` element + tagSub, ## the HTML `sub` element + tagSummary, ## the HTML `summary` element + tagSup, ## the HTML `sup` element + tagTable, ## the HTML `table` element + tagTbody, ## the HTML `tbody` element + tagTd, ## the HTML `td` element + tagTextarea, ## the HTML `textarea` element + tagTfoot, ## the HTML `tfoot` element + tagTh, ## the HTML `th` element + tagThead, ## the HTML `thead` element + tagTime, ## the HTML `time` element + tagTitle, ## the HTML `title` element + tagTr, ## the HTML `tr` element + tagTrack, ## the HTML `track` element + tagTt, ## the HTML `tt` element + tagU, ## the deprecated HTML `u` element + tagUl, ## the HTML `ul` element + tagVar, ## the HTML `var` element + tagVideo, ## the HTML `video` element + tagWbr ## the HTML `wbr` element const tagToStr* = [ @@ -215,7 +218,7 @@ const tagMenu, tagNoframes} SingleTags* = {tagArea, tagBase, tagBasefont, tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, - tagLink, tagMeta, tagParam, tagWbr} + tagLink, tagMeta, tagParam, tagWbr, tagSource} proc allLower(s: string): bool = for c in s: @@ -351,13 +354,13 @@ proc toHtmlTag(s: string): HtmlTag = proc htmlTag*(n: XmlNode): HtmlTag = - ## Gets `n`'s tag as a ``HtmlTag``. + ## Gets `n`'s tag as a `HtmlTag`. if n.clientData == 0: n.clientData = toHtmlTag(n.tag).ord result = HtmlTag(n.clientData) proc htmlTag*(s: string): HtmlTag = - ## Converts `s` to a ``HtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is + ## Converts `s` to a `HtmlTag`. If `s` is no HTML tag, `tagUnknown` is ## returned. let s = if allLower(s): s else: toLowerAscii(s) result = toHtmlTag(s) @@ -365,7 +368,7 @@ proc htmlTag*(s: string): HtmlTag = proc runeToEntity*(rune: Rune): string = ## converts a Rune to its numeric HTML entity equivalent. runnableExamples: - import unicode + import std/unicode doAssert runeToEntity(Rune(0)) == "" doAssert runeToEntity(Rune(-1)) == "" doAssert runeToEntity("Ü".runeAt(0)) == "#220" @@ -374,11 +377,11 @@ proc runeToEntity*(rune: Rune): string = else: result = '#' & $rune.ord proc entityToRune*(entity: string): Rune = - ## Converts an HTML entity name like ``Ü`` or values like ``Ü`` - ## or ``Ü`` to its UTF-8 equivalent. + ## Converts an HTML entity name like `Ü` or values like `Ü` + ## or `Ü` to its UTF-8 equivalent. ## Rune(0) is returned if the entity name is unknown. runnableExamples: - import unicode + import std/unicode doAssert entityToRune("") == Rune(0) doAssert entityToRune("a") == Rune(0) doAssert entityToRune("gt") == ">".runeAt(0) @@ -391,11 +394,11 @@ proc entityToRune*(entity: string): Rune = case entity[1] of '0'..'9': try: runeValue = parseInt(entity[1..^1]) - except: discard + except ValueError: discard of 'x', 'X': # not case sensitive here try: runeValue = parseHexInt(entity[2..^1]) - except: discard - else: discard # other entities are not defined with prefix ``#`` + except ValueError: discard + else: discard # other entities are not defined with prefix `#` if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values return Rune(runeValue) case entity # entity names are case sensitive @@ -1867,8 +1870,8 @@ proc entityToRune*(entity: string): Rune = else: Rune(0) proc entityToUtf8*(entity: string): string = - ## Converts an HTML entity name like ``Ü`` or values like ``Ü`` - ## or ``Ü`` to its UTF-8 equivalent. + ## Converts an HTML entity name like `Ü` or values like `Ü` + ## or `Ü` to its UTF-8 equivalent. ## "" is returned if the entity name is unknown. The HTML parser ## already converts entities to UTF-8. runnableExamples: @@ -1905,7 +1908,7 @@ template adderr(x: untyped) = proc untilElementEnd(x: var XmlParser, result: XmlNode, errors: var seq[string]) = - # we parsed e.g. ``<br>`` and don't really expect a ``</br>``: + # we parsed e.g. `<br>` and don't really expect a `</br>`: if result.htmlTag in SingleTags: if x.kind != xmlElementEnd or cmpIgnoreCase(x.elemName, result.tag) != 0: return @@ -1914,8 +1917,8 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, of xmlElementStart, xmlElementOpen: case result.htmlTag of tagP, tagInput, tagOption: - # some tags are common to have no ``</end>``, like ``<li>`` but - # allow ``<p>`` in `<dd>`, `<dt>` and ``<li>`` in next case + # some tags are common to have no `</end>`, like `<li>` but + # allow `<p>` in `<dd>`, `<dt>` and `<li>` in next case if htmlTag(x.elemName) in {tagLi, tagP, tagDt, tagDd, tagInput, tagOption}: adderr(expected(x, result)) @@ -2012,7 +2015,7 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = proc parseHtml*(s: Stream, filename: string, errors: var seq[string]): XmlNode = - ## Parses the XML from stream `s` and returns a ``XmlNode``. Every + ## Parses the XML from stream `s` and returns a `XmlNode`. Every ## occurred parsing error is added to the `errors` sequence. var x: XmlParser open(x, s, filename, {reportComments, reportWhitespace, allowUnquotedAttribs, @@ -2036,32 +2039,32 @@ proc parseHtml*(s: Stream, filename: string, result = result[0] proc parseHtml*(s: Stream): XmlNode = - ## Parses the HTML from stream `s` and returns a ``XmlNode``. All parsing + ## Parses the HTML from stream `s` and returns a `XmlNode`. All parsing ## errors are ignored. var errors: seq[string] = @[] result = parseHtml(s, "unknown_html_doc", errors) proc parseHtml*(html: string): XmlNode = - ## Parses the HTML from string ``html`` and returns a ``XmlNode``. All parsing + ## Parses the HTML from string `html` and returns a `XmlNode`. All parsing ## errors are ignored. parseHtml(newStringStream(html)) proc loadHtml*(path: string, errors: var seq[string]): XmlNode = - ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``XmlNode``. Every occurred parsing error is added to + ## Loads and parses HTML from file specified by `path`, and returns + ## a `XmlNode`. Every occurred parsing error is added to ## the `errors` sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) result = parseHtml(s, path, errors) proc loadHtml*(path: string): XmlNode = - ## Loads and parses HTML from file specified by ``path``, and returns - ## a ``XmlNode``. All parsing errors are ignored. + ## Loads and parses HTML from file specified by `path`, and returns + ## a `XmlNode`. All parsing errors are ignored. var errors: seq[string] = @[] result = loadHtml(path, errors) when not defined(testing) and isMainModule: - import os + import std/os var errors: seq[string] = @[] var x = loadHtml(paramStr(1), errors) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 7c062c2e1..08ea99627 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -10,62 +10,103 @@ ## This module implements a simple HTTP client that can be used to retrieve ## webpages and other data. ## +## .. warning:: Validate untrusted inputs: URI parsers and getters are not detecting malicious URIs. +## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve -## ``http://google.com``: +## `http://google.com`: ## -## .. code-block:: Nim -## import httpClient +## ```Nim +## import std/httpclient ## var client = newHttpClient() -## echo client.getContent("http://google.com") +## try: +## echo client.getContent("http://google.com") +## finally: +## client.close() +## ``` ## ## The same action can also be performed asynchronously, simply use the -## ``AsyncHttpClient``: +## `AsyncHttpClient`: +## +## ```Nim +## import std/[asyncdispatch, httpclient] +## +## proc asyncProc(): Future[string] {.async.} = +## var client = newAsyncHttpClient() +## try: +## return await client.getContent("http://google.com") +## finally: +## client.close() ## -## .. code-block:: Nim -## import httpClient -## var client = newAsyncHttpClient() -## echo await client.getContent("http://google.com") +## echo waitFor asyncProc() +## ``` ## -## The functionality implemented by ``HttpClient`` and ``AsyncHttpClient`` +## The functionality implemented by `HttpClient` and `AsyncHttpClient` ## is the same, so you can use whichever one suits you best in the examples ## shown here. ## -## **Note:** You will need to run asynchronous examples in an async proc -## otherwise you will get an ``Undeclared identifier: 'await'`` error. +## **Note:** You need to run asynchronous examples in an async proc +## otherwise you will get an `Undeclared identifier: 'await'` error. +## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. ## ## Using HTTP POST ## =============== ## ## This example demonstrates the usage of the W3 HTML Validator, it -## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to be +## uses `multipart/form-data` as the `Content-Type` to send the HTML to be ## validated to the server. ## -## .. code-block:: Nim +## ```Nim ## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` +## +## To stream files from disk when performing the request, use `addFiles`. ## -## echo client.postContent("http://validator.w3.org/check", multipart=data) +## **Note:** This will allocate a new `Mimetypes` database every time you call +## it, you can pass your own via the `mimeDb` parameter to avoid this. +## +## ```Nim +## let mimes = newMimetypes() +## var client = newHttpClient() +## var data = newMultipartData() +## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` ## ## You can also make post requests with custom headers. -## This example sets ``Content-Type`` to ``application/json`` +## This example sets `Content-Type` to `application/json` ## and uses a json object for the body ## -## .. code-block:: Nim -## import httpclient, json +## ```Nim +## import std/[httpclient, json] ## ## let client = newHttpClient() ## client.headers = newHttpHeaders({ "Content-Type": "application/json" }) ## let body = %*{ ## "data": "some text" ## } -## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) -## echo response.status +## try: +## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) +## echo response.status +## finally: +## client.close() +## ``` ## ## Progress reporting ## ================== @@ -74,37 +115,64 @@ ## This callback will be executed every second with information about the ## progress of the HTTP request. ## -## .. code-block:: Nim -## import asyncdispatch, httpclient +## ```Nim +## import std/[asyncdispatch, httpclient] ## -## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = -## echo("Downloaded ", progress, " of ", total) -## echo("Current rate: ", speed div 1000, "kb/s") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") ## -## proc asyncProc() {.async.} = -## var client = newAsyncHttpClient() -## client.onProgressChanged = onProgressChanged -## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## try: +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## finally: +## client.close() ## -## waitFor asyncProc() +## waitFor asyncProc() +## ``` ## -## If you would like to remove the callback simply set it to ``nil``. +## If you would like to remove the callback simply set it to `nil`. ## -## .. code-block:: Nim +## ```Nim ## client.onProgressChanged = nil +## ``` ## -## **Warning:** The ``total`` reported by httpclient may be 0 in some cases. +## .. warning:: The `total` reported by httpclient may be 0 in some cases. ## ## ## SSL/TLS support ## =============== -## This requires the OpenSSL library, fortunately it's widely used and installed +## This requires the OpenSSL library. Fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give -## any of the functions a url with the ``https`` schema, for example: -## ``https://github.com/``. +## any of the functions a url with the `https` schema, for example: +## `https://github.com/`. +## +## You will also have to compile with `ssl` defined like so: +## `nim c -d:ssl ...`. +## +## Certificate validation is performed by default. +## +## A set of directories and files from the `ssl_certs <ssl_certs.html>`_ +## module are scanned to locate CA certificates. ## -## You will also have to compile with ``ssl`` defined like so: -## ``nim c -d:ssl ...``. +## Example of setting SSL verification parameters in a new client: +## +## ```Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ``` +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## +## See `newContext <net.html#newContext.string,string,string,string>`_ to tweak or disable certificate validation. ## ## Timeouts ## ======== @@ -119,70 +187,87 @@ ## individual internal calls on the socket are affected. In practice this means ## that as long as the server is sending data an exception will not be raised, ## if however data does not reach the client within the specified timeout a -## ``TimeoutError`` exception will be raised. +## `TimeoutError` exception will be raised. ## -## Here is how to set a timeout when creating an ``HttpClient`` instance: +## Here is how to set a timeout when creating an `HttpClient` instance: ## -## .. code-block:: Nim -## import httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(timeout = 42) +## let client = newHttpClient(timeout = 42) +## ``` ## ## Proxy ## ===== ## ## A proxy can be specified as a param to any of the procedures defined in -## this module. To do this, use the ``newProxy`` constructor. Unfortunately, +## this module. To do this, use the `newProxy` constructor. Unfortunately, ## only basic authentication is supported at the moment. ## -## Some examples on how to configure a Proxy for ``HttpClient``: +## Some examples on how to configure a Proxy for `HttpClient`: +## +## ```Nim +## import std/httpclient ## -## .. code-block:: Nim -## import httpclient +## let myProxy = newProxy("http://myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` ## -## let myProxy = newProxy("http://myproxy.network") -## let client = newHttpClient(proxy = myProxy) +## Use proxies with basic authentication: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Get Proxy URL from environment variables: ## -## .. code-block:: Nim -## import httpclient +## ```Nim +## import std/httpclient ## -## var url = "" -## try: -## if existsEnv("http_proxy"): -## url = getEnv("http_proxy") -## elif existsEnv("https_proxy"): -## url = getEnv("https_proxy") -## except ValueError: -## echo "Unable to parse proxy from environment variables." +## var url = "" +## try: +## if existsEnv("http_proxy"): +## url = getEnv("http_proxy") +## elif existsEnv("https_proxy"): +## url = getEnv("https_proxy") +## except ValueError: +## echo "Unable to parse proxy from environment variables." ## -## let myProxy = newProxy(url = url) -## let client = newHttpClient(proxy = myProxy) +## let myProxy = newProxy(url = url) +## let client = newHttpClient(proxy = myProxy) +## ``` ## ## Redirects ## ========= ## -## The maximum redirects can be set with the ``maxRedirects`` of ``int`` type, +## The maximum redirects can be set with the `maxRedirects` of `int` type, ## it specifies the maximum amount of redirects to follow, -## it defaults to ``5``, you can set it to ``0`` to disable redirects. +## it defaults to `5`, you can set it to `0` to disable redirects. ## -## Here you can see an example about how to set the ``maxRedirects`` of ``HttpClient``: +## Here you can see an example about how to set the `maxRedirects` of `HttpClient`: ## -## .. code-block:: Nim -## import httpclient +## ```Nim +## import std/httpclient ## -## let client = newHttpClient(maxRedirects = 0) +## let client = newHttpClient(maxRedirects = 0) +## ``` ## -include "system/inclrtl" +import std/private/since + +import std/[ + net, strutils, uri, parseutils, base64, os, mimetypes, + math, random, httpcore, times, tables, streams, monotimes, + asyncnet, asyncdispatch, asyncfile, nativesockets, +] -import net, strutils, uri, parseutils, base64, os, mimetypes, - math, random, httpcore, times, tables, streams, std/monotimes -import asyncnet, asyncdispatch, asyncfile -import nativesockets +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] -export httpcore except parseHeader # TODO: The ``except`` doesn't work +export httpcore except parseHeader # TODO: The `except` doesn't work type Response* = ref object @@ -200,14 +285,14 @@ type bodyStream*: FutureStream[string] proc code*(response: Response | AsyncResponse): HttpCode - {.raises: [ValueError, OverflowError].} = - ## Retrieves the specified response's ``HttpCode``. + {.raises: [ValueError, OverflowDefect].} = + ## Retrieves the specified response's `HttpCode`. ## - ## Raises a ``ValueError`` if the response's ``status`` does not have a - ## corresponding ``HttpCode``. + ## Raises a `ValueError` if the response's `status` does not have a + ## corresponding `HttpCode`. return response.status[0 .. 2].parseInt.HttpCode -proc contentType*(response: Response | AsyncResponse): string = +proc contentType*(response: Response | AsyncResponse): string {.inline.} = ## Retrieves the specified response's content type. ## ## This is effectively the value of the "Content-Type" header. @@ -218,16 +303,17 @@ proc contentLength*(response: Response | AsyncResponse): int = ## ## This is effectively the value of the "Content-Length" header. ## - ## A ``ValueError`` exception will be raised if the value is not an integer. - var contentLengthHeader = response.headers.getOrDefault("Content-Length") - return contentLengthHeader.parseInt() + ## A `ValueError` exception will be raised if the value is not an integer. + ## If the Content-Length header is not set in the response, ContentLength is set to the value -1. + var contentLengthHeader = response.headers.getOrDefault("Content-Length", HttpHeaderValues(@["-1"])) + result = contentLengthHeader.parseInt() proc lastModified*(response: Response | AsyncResponse): DateTime = ## Retrieves the specified response's last modified time. ## ## This is effectively the value of the "Last-Modified" header. ## - ## Raises a ``ValueError`` if the parsing fails or the value is not a correctly + ## Raises a `ValueError` if the parsing fails or the value is not a correctly ## formatted time. var lastModifiedHeader = response.headers.getOrDefault("last-modified") result = parse(lastModifiedHeader, "ddd, dd MMM yyyy HH:mm:ss 'GMT'", utc()) @@ -252,19 +338,28 @@ type url*: Uri auth*: string + MultipartEntry = object + name, content: string + case isFile: bool + of true: + filename, contentType: string + fileSize: int64 + isStream: bool + else: discard + MultipartEntries* = openArray[tuple[name, content: string]] MultipartData* = ref object - content: seq[string] + content: seq[MultipartEntry] ProtocolError* = object of IOError ## exception that is raised when server ## does not conform to the implemented ## protocol - HttpRequestError* = object of IOError ## Thrown in the ``getContent`` proc - ## and ``postContent`` proc, + HttpRequestError* = object of IOError ## Thrown in the `getContent` proc + ## and `postContent` proc, ## when the server returns an error -const defUserAgent* = "Nim httpclient/" & NimVersion +const defUserAgent* = "Nim-httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -278,7 +373,6 @@ proc fileError(msg: string) = e.msg = msg raise e - when not defined(ssl): type SslContext = ref object var defaultSslContext {.threadvar.}: SslContext @@ -287,34 +381,42 @@ proc getDefaultSSL(): SslContext = result = defaultSslContext when defined(ssl): if result == nil: - defaultSslContext = newContext(verifyMode = CVerifyNone) + defaultSslContext = newContext(verifyMode = CVerifyPeer) result = defaultSslContext doAssert result != nil, "failure to initialize the SSL context" -proc newProxy*(url: string, auth = ""): Proxy = - ## Constructs a new ``TProxy`` object. +proc newProxy*(url: string; auth = ""): Proxy = + ## Constructs a new `TProxy` object. result = Proxy(url: parseUri(url), auth: auth) -proc newMultipartData*: MultipartData = - ## Constructs a new ``MultipartData`` object. - MultipartData(content: @[]) +proc newProxy*(url: Uri; auth = ""): Proxy = + ## Constructs a new `TProxy` object. + result = Proxy(url: url, auth: auth) +proc newMultipartData*: MultipartData {.inline.} = + ## Constructs a new `MultipartData` object. + MultipartData() proc `$`*(data: MultipartData): string {.since: (1, 1).} = ## convert MultipartData to string so it's human readable when echo ## see https://github.com/nim-lang/Nim/issues/11863 - const prefixLen = "Content-Disposition: form-data; ".len - for pos, item in data.content: - result &= "------------------------------ " - result.addInt pos - result &= " ------------------------------\n" - result &= item[prefixLen .. item.high] - -proc add*(p: var MultipartData, name, content: string, filename: string = "", - contentType: string = "") = - ## Add a value to the multipart data. Raises a `ValueError` exception if + const sep = "-".repeat(30) + for pos, entry in data.content: + result.add(sep & center($pos, 3) & sep) + result.add("\nname=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"\n") + result.add("Content-Type: " & entry.contentType) + result.add("\n\n" & entry.content & "\n") + +proc add*(p: MultipartData, name, content: string, filename: string = "", + contentType: string = "", useStream = true) = + ## Add a value to the multipart data. + ## + ## When `useStream` is `false`, the file will be read into memory. + ## + ## Raises a `ValueError` exception if ## `name`, `filename` or `contentType` contain newline characters. - if {'\c', '\L'} in name: raise newException(ValueError, "name contains a newline character") if {'\c', '\L'} in filename: @@ -322,23 +424,27 @@ proc add*(p: var MultipartData, name, content: string, filename: string = "", if {'\c', '\L'} in contentType: raise newException(ValueError, "contentType contains a newline character") - var str = "Content-Disposition: form-data; name=\"" & name & "\"" - if filename.len > 0: - str.add("; filename=\"" & filename & "\"") - str.add("\c\L") - if contentType.len > 0: - str.add("Content-Type: " & contentType & "\c\L") - str.add("\c\L" & content & "\c\L") + var entry = MultipartEntry( + name: name, + content: content, + isFile: filename.len > 0 + ) - p.content.add(str) + if entry.isFile: + entry.isStream = useStream + entry.filename = filename + entry.contentType = contentType -proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData + p.content.add(entry) + +proc add*(p: MultipartData, xs: MultipartEntries): MultipartData {.discardable.} = ## Add a list of multipart entries to the multipart data `p`. All values are ## added without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data.add({"action": "login", "format": "json"}) + ## ``` for name, content in xs.items: p.add(name, content) result = p @@ -347,91 +453,96 @@ proc newMultipartData*(xs: MultipartEntries): MultipartData = ## Create a new multipart data object and fill it with the entries `xs` ## directly. ## - ## .. code-block:: Nim + ## ```Nim ## var data = newMultipartData({"action": "login", "format": "json"}) - result = MultipartData(content: @[]) - result.add(xs) - -proc addFiles*(p: var MultipartData, xs: openArray[tuple[name, file: string]]): - MultipartData {.discardable.} = - ## Add files to a multipart data object. The file will be opened from your - ## disk, read and sent with the automatically determined MIME type. Raises an - ## `IOError` if the file cannot be opened or reading fails. To manually - ## specify file content, filename and MIME type, use `[]=` instead. + ## ``` + result = MultipartData() + for entry in xs: + result.add(entry.name, entry.content) + +proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], + mimeDb = newMimetypes(), useStream = true): + MultipartData {.discardable.} = + ## Add files to a multipart data object. The files will be streamed from disk + ## when the request is being made. When `stream` is `false`, the files are + ## instead read into memory, but beware this is very memory ineffecient even + ## for small files. The MIME types will automatically be determined. + ## Raises an `IOError` if the file cannot be opened or reading fails. To + ## manually specify file content, filename and MIME type, use `[]=` instead. ## - ## .. code-block:: Nim + ## ```Nim ## data.addFiles({"uploaded_file": "public/test.html"}) - var m = newMimetypes() + ## ``` for name, file in xs.items: var contentType: string let (_, fName, ext) = splitFile(file) if ext.len > 0: - contentType = m.getMimetype(ext[1..ext.high], "") - p.add(name, readFile(file).string, fName & ext, contentType) + contentType = mimeDb.getMimetype(ext[1..ext.high], "") + let content = if useStream: file else: readFile(file) + p.add(name, content, fName & ext, contentType, useStream = useStream) result = p -proc `[]=`*(p: var MultipartData, name, content: string) = +proc `[]=`*(p: MultipartData, name, content: string) {.inline.} = ## Add a multipart entry to the multipart data `p`. The value is added ## without a filename and without a content type. ## - ## .. code-block:: Nim + ## ```Nim ## data["username"] = "NimUser" + ## ``` p.add(name, content) -proc `[]=`*(p: var MultipartData, name: string, - file: tuple[name, contentType, content: string]) = - ## Add a file to the multipart data `p`, specifying filename, contentType and - ## content manually. +proc `[]=`*(p: MultipartData, name: string, + file: tuple[name, contentType, content: string]) {.inline.} = + ## Add a file to the multipart data `p`, specifying filename, contentType + ## and content manually. ## - ## .. code-block:: Nim + ## ```Nim ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") - p.add(name, file.content, file.name, file.contentType) - -proc format(p: MultipartData): tuple[contentType, body: string] = - if p == nil or p.content.len == 0: - return ("", "") + ## ``` + p.add(name, file.content, file.name, file.contentType, useStream = false) - # Create boundary that is not in the data to be formatted - var bound: string +proc getBoundary(p: MultipartData): string = + if p == nil or p.content.len == 0: return while true: - bound = $random(int.high) - var found = false - for s in p.content: - if bound in s: - found = true - if not found: - break - - result.contentType = "multipart/form-data; boundary=" & bound - result.body = "" - for s in p.content: - result.body.add("--" & bound & "\c\L" & s) - result.body.add("--" & bound & "--\c\L") - -proc redirection(status: string): bool = - const redirectionNRs = ["301", "302", "303", "307"] - for i in items(redirectionNRs): - if status.startsWith(i): - return true - -proc getNewLocation(lastURL: string, headers: HttpHeaders): string = - result = headers.getOrDefault"Location" - if result == "": httpError("location header expected") + result = $rand(int.high) + for i, entry in p.content: + if result in entry.content: break + elif i == p.content.high: return + +proc sendFile(socket: Socket | AsyncSocket, + entry: MultipartEntry) {.multisync.} = + const chunkSize = 2^18 + let file = + when socket is AsyncSocket: openAsync(entry.content) + else: newFileStream(entry.content, fmRead) + + var buffer: string + while true: + buffer = + when socket is AsyncSocket: (await read(file, chunkSize)) + else: readStr(file, chunkSize) + if buffer.len == 0: break + await socket.send(buffer) + file.close() + +proc getNewLocation(lastURL: Uri, headers: HttpHeaders): Uri = + let newLocation = headers.getOrDefault"Location" + if newLocation == "": httpError("location header expected") # Relative URLs. (Not part of the spec, but soon will be.) - let r = parseUri(result) - if r.hostname == "" and r.path != "": - var parsed = parseUri(lastURL) - parsed.path = r.path - parsed.query = r.query - parsed.anchor = r.anchor - result = $parsed - -proc generateHeaders(requestUrl: Uri, httpMethod: string, - headers: HttpHeaders, body: string, proxy: Proxy): string = + let parsedLocation = parseUri(newLocation) + if parsedLocation.hostname == "" and parsedLocation.path != "": + result = lastURL + result.path = parsedLocation.path + result.query = parsedLocation.query + result.anchor = parsedLocation.anchor + else: + result = parsedLocation + +proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeaders, + proxy: Proxy): string = # GET - let upperMethod = httpMethod.toUpperAscii() - result = upperMethod + result = $httpMethod result.add ' ' if proxy.isNil or requestUrl.scheme == "https": @@ -447,34 +558,28 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string, result.add($modifiedUrl) # HTTP/1.1\c\l - result.add(" HTTP/1.1\c\L") + result.add(" HTTP/1.1" & httpNewLine) # Host header. if not headers.hasKey("Host"): if requestUrl.port == "": - add(result, "Host: " & requestUrl.hostname & "\c\L") + add(result, "Host: " & requestUrl.hostname & httpNewLine) else: - add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & "\c\L") + add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine) # Connection header. if not headers.hasKey("Connection"): - add(result, "Connection: Keep-Alive\c\L") - - # Content length header. - const requiresBody = ["POST", "PUT", "PATCH"] - let needsContentLength = body.len > 0 or upperMethod in requiresBody - if needsContentLength and not headers.hasKey("Content-Length"): - add(result, "Content-Length: " & $body.len & "\c\L") + add(result, "Connection: Keep-Alive" & httpNewLine) # Proxy auth header. if not proxy.isNil and proxy.auth != "": let auth = base64.encode(proxy.auth) - add(result, "Proxy-Authorization: basic " & auth & "\c\L") + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: - add(result, key & ": " & val & "\c\L") + add(result, key & ": " & val & httpNewLine) - add(result, "\c\L") + add(result, httpNewLine) type ProgressChangedProc*[ReturnType] = @@ -486,11 +591,11 @@ type connected: bool currentURL: Uri ## Where we are currently connected. headers*: HttpHeaders ## Headers to send in requests. - maxRedirects: Natural ## Maximum redirects, set to ``0`` to disable. + maxRedirects: Natural ## Maximum redirects, set to `0` to disable. userAgent: string timeout*: int ## Only used for blocking HttpClient for now. proxy: Proxy - ## ``nil`` or the callback to call when request progress changes. + ## `nil` or the callback to call when request progress changes. when SocketType is Socket: onProgressChanged*: ProgressChangedProc[void] else: @@ -511,26 +616,34 @@ type type HttpClient* = HttpClientBase[Socket] -proc newHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil, - timeout = -1, headers = newHttpHeaders()): HttpClient = +proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + timeout = -1, headers = newHttpHeaders()): HttpClient = ## Creates a new HttpClient instance. ## - ## ``userAgent`` specifies the user agent that will be used when making + ## `userAgent` specifies the user agent that will be used when making ## requests. ## - ## ``maxRedirects`` specifies the maximum amount of redirects to follow, + ## `maxRedirects` specifies the maximum amount of redirects to follow, ## default is 5. ## - ## ``sslContext`` specifies the SSL context to use for HTTPS requests. + ## `sslContext` specifies the SSL context to use for HTTPS requests. + ## See `SSL/TLS support <#sslslashtls-support>`_ ## - ## ``proxy`` specifies an HTTP proxy to use for this HTTP client's + ## `proxy` specifies an HTTP proxy to use for this HTTP client's ## connections. ## - ## ``timeout`` specifies the number of milliseconds to allow before a - ## ``TimeoutError`` is raised. + ## `timeout` specifies the number of milliseconds to allow before a + ## `TimeoutError` is raised. ## - ## ``headers`` specifies the HTTP Headers. + ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/strutils + + let exampleHtml = newHttpClient().getContent("http://example.com") + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + new result result.headers = headers result.userAgent = userAgent @@ -546,23 +659,34 @@ proc newHttpClient*(userAgent = defUserAgent, type AsyncHttpClient* = HttpClientBase[AsyncSocket] -proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = getDefaultSSL(), - proxy: Proxy = nil, headers = newHttpHeaders()): AsyncHttpClient = +proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + headers = newHttpHeaders()): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## - ## ``userAgent`` specifies the user agent that will be used when making + ## `userAgent` specifies the user agent that will be used when making ## requests. ## - ## ``maxRedirects`` specifies the maximum amount of redirects to follow, + ## `maxRedirects` specifies the maximum amount of redirects to follow, ## default is 5. ## - ## ``sslContext`` specifies the SSL context to use for HTTPS requests. + ## `sslContext` specifies the SSL context to use for HTTPS requests. ## - ## ``proxy`` specifies an HTTP proxy to use for this HTTP client's + ## `proxy` specifies an HTTP proxy to use for this HTTP client's ## connections. ## - ## ``headers`` specifies the HTTP Headers. + ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/[asyncdispatch, strutils] + + proc asyncProc(): Future[string] {.async.} = + let client = newAsyncHttpClient() + result = await client.getContent("http://example.com") + + let exampleHtml = waitFor asyncProc() + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + new result result.headers = headers result.userAgent = userAgent @@ -581,26 +705,26 @@ proc close*(client: HttpClient | AsyncHttpClient) = client.socket.close() client.connected = false -proc getSocket*(client: HttpClient): Socket = - ## Get network socket, useful if you want to find out more details about the connection +proc getSocket*(client: HttpClient): Socket {.inline.} = + ## Get network socket, useful if you want to find out more details about the connection. ## - ## this example shows info about local and remote endpoints + ## This example shows info about local and remote endpoints: ## - ## .. code-block:: Nim + ## ```Nim ## if client.connected: ## echo client.getSocket.getLocalAddr ## echo client.getSocket.getPeerAddr - ## + ## ``` return client.socket -proc getSocket*(client: AsyncHttpClient): AsyncSocket = +proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} = return client.socket proc reportProgress(client: HttpClient | AsyncHttpClient, progress: BiggestInt) {.multisync.} = client.contentProgress += progress client.oneSecondProgress += progress - if (getMonoTime() - client.lastProgressReport).inSeconds > 1: + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: if not client.onProgressChanged.isNil: await client.onProgressChanged(client.contentTotal, client.contentProgress, @@ -638,7 +762,7 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] {.multisync.} = while true: var chunkSize = 0 - var chunkSizeStr = (await client.socket.recvLine()).string + var chunkSizeStr = await client.socket.recvLine() var i = 0 if chunkSizeStr == "": httpError("Server terminated connection prematurely") @@ -671,8 +795,7 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(client: HttpClient | AsyncHttpClient, - headers: HttpHeaders, +proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpVersion: string): Future[void] {.multisync.} = # Reset progress from previous requests. client.contentTotal = 0 @@ -699,7 +822,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, httpError("Got disconnected while trying to read body.") if recvLen != length: httpError("Received length doesn't match expected length. Wanted " & - $length & " got " & $recvLen) + $length & " got: " & $recvLen) else: # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO @@ -733,19 +856,20 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, var parsedStatus = false var linei = 0 var fullyRead = false + var lastHeaderName = "" var line = "" result.headers = newHttpHeaders() while true: linei = 0 when client is HttpClient: - line = (await client.socket.recvLine(client.timeout)).string + line = await client.socket.recvLine(client.timeout) else: - line = (await client.socket.recvLine()).string + line = await client.socket.recvLine() if line == "": # We've been disconnected. client.close() break - if line == "\c\L": + if line == httpNewLine: fullyRead = true break if not parsedStatus: @@ -767,31 +891,52 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, parsedStatus = true else: # Parse headers - var name = "" - var le = parseUntil(line, name, ':', linei) - if le <= 0: httpError("invalid headers") - inc(linei, le) - if line[linei] != ':': httpError("invalid headers") - inc(linei) # Skip : - - result.headers.add(name, line[linei .. ^1].strip()) - if result.headers.len > headerLimit: - httpError("too many headers") + # There's at least one char because empty lines are handled above (with client.close) + if line[0] in {' ', '\t'}: + # Check if it's a multiline header value, if so, append to the header we're currently parsing + # This works because a line with a header must start with the header name without any leading space + # See https://datatracker.ietf.org/doc/html/rfc7230, section 3.2 and 3.2.4 + # Multiline headers are deprecated in the spec, but it's better to parse them than crash + if lastHeaderName == "": + # Some extra unparsable lines in the HTTP output - we ignore them + discard + else: + result.headers.table[result.headers.toCaseInsensitive(lastHeaderName)][^1].add "\n" & line + else: + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("Invalid headers - received empty header name") + if line.len == le: httpError("Invalid headers - no colon after header name") + inc(linei, le) # Skip the parsed header name + inc(linei) # Skip : + # If we want to be HTTP spec compliant later, error on linei == line.len (for empty header value) + lastHeaderName = name # Remember the header name for the possible multi-line header + result.headers.add(name, line[linei .. ^1].strip()) + if result.headers.len > headerLimit: + httpError("too many headers") if not fullyRead: httpError("Connection was closed before full request has been made") - if getBody: + when client is HttpClient: + result.bodyStream = newStringStream() + else: + result.bodyStream = newFutureStream[string]("parseResponse") + + if getBody and result.code != Http204: + client.bodyStream = result.bodyStream when client is HttpClient: - client.bodyStream = newStringStream() - result.bodyStream = client.bodyStream parseBody(client, result.headers, result.version) else: - client.bodyStream = newFutureStream[string]("parseResponse") - result.bodyStream = client.bodyStream assert(client.parseBodyFut.isNil or client.parseBodyFut.finished) + # do not wait here for the body request to complete client.parseBodyFut = parseBody(client, result.headers, result.version) - # do not wait here for the body request to complete + client.parseBodyFut.addCallback do(): + if client.parseBodyFut.failed: + client.bodyStream.fail(client.parseBodyFut.error) + else: + when client is AsyncHttpClient: + result.bodyStream.complete() proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = @@ -845,8 +990,8 @@ proc newConnection(client: HttpClient | AsyncHttpClient, connectUrl.hostname = url.hostname connectUrl.port = if url.port != "": url.port else: "443" - let proxyHeaderString = generateHeaders(connectUrl, $HttpConnect, - newHttpHeaders(), "", client.proxy) + let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, + newHttpHeaders(), client.proxy) await client.socket.send(proxyHeaderString) let proxyResp = await parseResponse(client, false) @@ -864,25 +1009,65 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.currentURL = url client.connected = true +proc readFileSizes(client: HttpClient | AsyncHttpClient, + multipart: MultipartData) {.multisync.} = + for entry in multipart.content.mitems(): + if not entry.isFile: continue + if not entry.isStream: + entry.fileSize = entry.content.len + continue + + # TODO: look into making getFileSize work with async + let fileSize = getFileSize(entry.content) + entry.fileSize = fileSize + +proc format(entry: MultipartEntry, boundary: string): string = + result = "--" & boundary & httpNewLine + result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"" & httpNewLine) + result.add("Content-Type: " & entry.contentType & httpNewLine) + else: + result.add(httpNewLine & httpNewLine & entry.content) + +proc format(client: HttpClient | AsyncHttpClient, + multipart: MultipartData): Future[seq[string]] {.multisync.} = + let bound = getBoundary(multipart) + client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound + + await client.readFileSizes(multipart) + + var length: int64 + for entry in multipart.content: + result.add(format(entry, bound) & httpNewLine) + if entry.isFile: + length += entry.fileSize + httpNewLine.len + + result.add "--" & bound & "--" & httpNewLine + + for s in result: length += s.len + client.headers["Content-Length"] = $length + proc override(fallback, override: HttpHeaders): HttpHeaders = # Right-biased map union for `HttpHeaders` - if override.isNil: - return fallback result = newHttpHeaders() # Copy by value result.table[] = fallback.table[] + + if override.isNil: + # Return the copy of fallback so it does not get modified + return result + for k, vs in override.table: result[k] = vs -proc requestAux(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = "", - headers: HttpHeaders = nil): Future[Response | AsyncResponse] +proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, + httpMethod: HttpMethod, body = "", headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = # Helper that actually makes the request. Does not handle redirects. - let requestUrl = parseUri(url) - - if requestUrl.scheme == "": + if url.scheme == "": raise newException(ValueError, "No uri scheme supplied.") when client is AsyncHttpClient: @@ -891,175 +1076,248 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, await client.parseBodyFut client.parseBodyFut = nil - await newConnection(client, requestUrl) - - let effectiveHeaders = client.headers.override(headers) - - if not effectiveHeaders.hasKey("user-agent") and client.userAgent != "": - effectiveHeaders["User-Agent"] = client.userAgent + await newConnection(client, url) - var headersString = generateHeaders(requestUrl, httpMethod, - effectiveHeaders, body, client.proxy) + var newHeaders: HttpHeaders - await client.socket.send(headersString) - if body != "": + var data: seq[string] + if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 + data = await client.format(multipart) + newHeaders = client.headers.override(headers) + else: + newHeaders = client.headers.override(headers) + # Only change headers if they have not been specified already + if not newHeaders.hasKey("Content-Length"): + if body.len != 0: + newHeaders["Content-Length"] = $body.len + elif httpMethod notin {HttpGet, HttpHead}: + newHeaders["Content-Length"] = "0" + + if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0: + newHeaders["User-Agent"] = client.userAgent + + let headerString = generateHeaders(url, httpMethod, newHeaders, + client.proxy) + await client.socket.send(headerString) + + if data.len > 0: + var buffer: string + for i, entry in multipart.content: + buffer.add data[i] + if not entry.isFile: continue + if buffer.len > 0: + await client.socket.send(buffer) + buffer.setLen(0) + if entry.isStream: + await client.socket.sendFile(entry) + else: + await client.socket.send(entry.content) + buffer.add httpNewLine + # send the rest and the last boundary + await client.socket.send(buffer & data[^1]) + elif body.len > 0: await client.socket.send(body) - let getBody = httpMethod.toLowerAscii() notin ["head", "connect"] and + let getBody = httpMethod notin {HttpHead, HttpConnect} and client.getBody result = await parseResponse(client, getBody) -proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = "", - headers: HttpHeaders = nil): Future[Response | AsyncResponse] +proc request*(client: HttpClient | AsyncHttpClient, url: Uri | string, + httpMethod: HttpMethod | string = HttpGet, body = "", + headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request - ## using the custom method string specified by ``httpMethod``. + ## using the custom method string specified by `httpMethod`. ## - ## Connection will be kept alive. Further requests on the same ``client`` to + ## Connection will be kept alive. Further requests on the same `client` to ## the same hostname will not require a new connection to be made. The - ## connection can be closed by using the ``close`` procedure. + ## connection can be closed by using the `close` procedure. ## ## This procedure will follow redirects up to a maximum number of redirects - ## specified in ``client.maxRedirects``. - result = await client.requestAux(url, httpMethod, body, headers) + ## specified in `client.maxRedirects`. + ## + ## You need to make sure that the `url` doesn't contain any newline + ## characters. Failing to do so will raise `AssertionDefect`. + ## + ## `headers` are HTTP headers that override the `client.headers` for + ## this specific request only and will not be persisted. + ## + ## **Deprecated since v1.5**: use HttpMethod enum instead; string parameter httpMethod is deprecated + when url is string: + doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters") + let url = parseUri(url) + + when httpMethod is string: + {.warning: + "Deprecated since v1.5; use HttpMethod enum instead; string parameter httpMethod is deprecated".} + let httpMethod = case httpMethod + of "HEAD": + HttpHead + of "GET": + HttpGet + of "POST": + HttpPost + of "PUT": + HttpPut + of "DELETE": + HttpDelete + of "TRACE": + HttpTrace + of "OPTIONS": + HttpOptions + of "CONNECT": + HttpConnect + of "PATCH": + HttpPatch + else: + raise newException(ValueError, "Invalid HTTP method name: " & httpMethod) + + result = await client.requestAux(url, httpMethod, body, headers, multipart) var lastURL = url for i in 1..client.maxRedirects: - if result.status.redirection(): - let redirectTo = getNewLocation(lastURL, result.headers) - # Guarantee method for HTTP 307: see https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307 - var meth = if result.status == "307": httpMethod else: "GET" - result = await client.requestAux(redirectTo, meth, body, headers) - lastURL = redirectTo + let statusCode = result.code + if statusCode notin {Http301, Http302, Http303, Http307, Http308}: + break -proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod = HttpGet, body = "", - headers: HttpHeaders = nil): Future[Response | AsyncResponse] - {.multisync.} = - ## Connects to the hostname specified by the URL and performs a request - ## using the method specified. - ## - ## Connection will be kept alive. Further requests on the same ``client`` to - ## the same hostname will not require a new connection to be made. The - ## connection can be closed by using the ``close`` procedure. - ## - ## When a request is made to a different hostname, the current connection will - ## be closed. - result = await request(client, url, $httpMethod, body, headers) + let redirectTo = getNewLocation(lastURL, result.headers) + var redirectMethod: HttpMethod + var redirectBody: string + # For more informations about the redirect methods see: + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections + case statusCode + of Http301, Http302, Http303: + # The method is changed to GET unless it is GET or HEAD (RFC2616) + if httpMethod notin {HttpGet, HttpHead}: + redirectMethod = HttpGet + else: + redirectMethod = httpMethod + # The body is stripped away + redirectBody = "" + # Delete any header value associated with the body + if not headers.isNil(): + headers.del("Content-Length") + headers.del("Content-Type") + headers.del("Transfer-Encoding") + of Http307, Http308: + # The method and the body are unchanged + redirectMethod = httpMethod + redirectBody = body + else: + # Unreachable + doAssert(false) + + # Check if the redirection is to the same domain or a sub-domain (foo.com + # -> sub.foo.com) + if redirectTo.hostname != lastURL.hostname and + not redirectTo.hostname.endsWith("." & lastURL.hostname): + # Perform some cleanup of the header values + if headers != nil: + # Delete the Host header + headers.del("Host") + # Do not send any sensitive info to a unknown host + headers.del("Authorization") + + result = await client.requestAux(redirectTo, redirectMethod, redirectBody, + headers, multipart) + lastURL = redirectTo proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} = ## Returns the content of a response as a string. ## - ## A ``HttpRequestError`` will be raised if the server responds with a + ## A `HttpRequestError` will be raised if the server responds with a ## client error (status code 4xx) or a server error (status code 5xx). if resp.code.is4xx or resp.code.is5xx: - raise newException(HttpRequestError, resp.status) + raise newException(HttpRequestError, resp.status.move) else: return await resp.bodyStream.readAll() proc head*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a HEAD request. ## - ## This procedure uses httpClient values such as ``client.maxRedirects``. + ## This procedure uses httpClient values such as `client.maxRedirects`. result = await client.request(url, HttpHead) proc get*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a GET request. ## - ## This procedure uses httpClient values such as ``client.maxRedirects``. + ## This procedure uses httpClient values such as `client.maxRedirects`. result = await client.request(url, HttpGet) proc getContent*(client: HttpClient | AsyncHttpClient, - url: string): Future[string] {.multisync.} = + url: Uri | string): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a GET request. let resp = await get(client, url) return await responseContent(resp) proc delete*(client: HttpClient | AsyncHttpClient, - url: string): Future[Response | AsyncResponse] {.multisync.} = + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a DELETE request. - ## This procedure uses httpClient values such as ``client.maxRedirects``. + ## This procedure uses httpClient values such as `client.maxRedirects`. result = await client.request(url, HttpDelete) proc deleteContent*(client: HttpClient | AsyncHttpClient, - url: string): Future[string] {.multisync.} = + url: Uri | string): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a DELETE request. let resp = await delete(client, url) return await responseContent(resp) -proc makeRequestContent(body = "", multipart: MultipartData = nil): (string, HttpHeaders) = - let (mpContentType, mpBody) = format(multipart) - # TODO: Support FutureStream for `body` parameter. - template withNewLine(x): untyped = - if x.len > 0 and not x.endsWith("\c\L"): - x & "\c\L" - else: - x - var xb = mpBody.withNewLine() & body - var headers = newHttpHeaders() - if multipart != nil: - headers["Content-Type"] = mpContentType - headers["Content-Length"] = $len(xb) - return (xb, headers) - -proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", +proc post*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[Response | AsyncResponse] {.multisync.} = ## Connects to the hostname specified by the URL and performs a POST request. - ## This procedure uses httpClient values such as ``client.maxRedirects``. - var (xb, headers) = makeRequestContent(body, multipart) - result = await client.request(url, $HttpPost, xb, headers) + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPost, body, multipart=multipart) -proc postContent*(client: HttpClient | AsyncHttpClient, url: string, - body = "", +proc postContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a POST request. let resp = await post(client, url, body, multipart) return await responseContent(resp) -proc put*(client: HttpClient | AsyncHttpClient, url: string, body = "", - multipart: MultipartData = nil): Future[Response | AsyncResponse] - {.multisync.} = +proc put*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = ## Connects to the hostname specified by the URL and performs a PUT request. - ## This procedure uses httpClient values such as ``client.maxRedirects``. - var (xb, headers) = makeRequestContent(body, multipart) - result = await client.request(url, $HttpPut, xb, headers) + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPut, body, multipart=multipart) -proc putContent*(client: HttpClient | AsyncHttpClient, url: string, - body = "", - multipart: MultipartData = nil): Future[string] - {.multisync.} = +proc putContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL andreturns the content of a PUT request. let resp = await put(client, url, body, multipart) return await responseContent(resp) -proc patch*(client: HttpClient | AsyncHttpClient, url: string, body = "", - multipart: MultipartData = nil): Future[Response | AsyncResponse] - {.multisync.} = +proc patch*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = ## Connects to the hostname specified by the URL and performs a PATCH request. - ## This procedure uses httpClient values such as ``client.maxRedirects``. - var (xb, headers) = makeRequestContent(body, multipart) - result = await client.request(url, $HttpPatch, xb, headers) + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPatch, body, multipart=multipart) -proc patchContent*(client: HttpClient | AsyncHttpClient, url: string, - body = "", - multipart: MultipartData = nil): Future[string] +proc patchContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] {.multisync.} = ## Connects to the hostname specified by the URL and returns the content of a PATCH request. let resp = await patch(client, url, body, multipart) return await responseContent(resp) -proc downloadFile*(client: HttpClient, url: string, filename: string) = - ## Downloads ``url`` and saves it to ``filename``. +proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) = + ## Downloads `url` and saves it to `filename`. client.getBody = false defer: client.getBody = true let resp = client.get(url) + + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) client.bodyStream = newFileStream(filename, fmWrite) if client.bodyStream.isNil: @@ -1067,30 +1325,30 @@ proc downloadFile*(client: HttpClient, url: string, filename: string) = parseBody(client, resp.headers, resp.version) client.bodyStream.close() +proc downloadFileEx(client: AsyncHttpClient, + url: Uri | string, filename: string): Future[void] {.async.} = + ## Downloads `url` and saves it to `filename`. + client.getBody = false + let resp = await client.get(url) + if resp.code.is4xx or resp.code.is5xx: raise newException(HttpRequestError, resp.status) -proc downloadFile*(client: AsyncHttpClient, url: string, + client.bodyStream = newFutureStream[string]("downloadFile") + var file = openAsync(filename, fmWrite) + defer: file.close() + # Let `parseBody` write response data into client.bodyStream in the + # background. + let parseBodyFut = parseBody(client, resp.headers, resp.version) + parseBodyFut.addCallback do(): + if parseBodyFut.failed: + client.bodyStream.fail(parseBodyFut.error) + # The `writeFromStream` proc will complete once all the data in the + # `bodyStream` has been written to the file. + await file.writeFromStream(client.bodyStream) + +proc downloadFile*(client: AsyncHttpClient, url: Uri | string, filename: string): Future[void] = - proc downloadFileEx(client: AsyncHttpClient, - url, filename: string): Future[void] {.async.} = - ## Downloads ``url`` and saves it to ``filename``. - client.getBody = false - let resp = await client.get(url) - - client.bodyStream = newFutureStream[string]("downloadFile") - var file = openAsync(filename, fmWrite) - # Let `parseBody` write response data into client.bodyStream in the - # background. - asyncCheck parseBody(client, resp.headers, resp.version) - # The `writeFromStream` proc will complete once all the data in the - # `bodyStream` has been written to the file. - await file.writeFromStream(client.bodyStream) - file.close() - - if resp.code.is4xx or resp.code.is5xx: - raise newException(HttpRequestError, resp.status) - result = newFuture[void]("downloadFile") try: result = downloadFileEx(client, url, filename) diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 1e9a85a57..5ccab379c 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -7,16 +7,17 @@ # distribution, for details about the copyright. # -## Contains functionality shared between the ``httpclient`` and -## ``asynchttpserver`` modules. +## Contains functionality shared between the `httpclient` and +## `asynchttpserver` modules. ## ## Unstable API. - -import tables, strutils, parseutils +import std/private/since +import std/[tables, strutils, parseutils] type HttpHeaders* = ref object table*: TableRef[string, seq[string]] + isTitleCase: bool HttpHeaderValues* = distinct seq[string] @@ -28,29 +29,33 @@ type HttpVer11, HttpVer10 - HttpMethod* = enum ## the requested HttpMethod - HttpHead, ## Asks for the response identical to the one that would - ## correspond to a GET request, but without the response - ## body. - HttpGet, ## Retrieves the specified resource. - HttpPost, ## Submits data to be processed to the identified - ## resource. The data is included in the body of the - ## request. - HttpPut, ## Uploads a representation of the specified resource. - HttpDelete, ## Deletes the specified resource. - HttpTrace, ## Echoes back the received request, so that a client - ## can see what intermediate servers are adding or - ## changing in the request. - HttpOptions, ## Returns the HTTP methods that the server supports - ## for specified address. - HttpConnect, ## Converts the request connection to a transparent - ## TCP/IP tunnel, usually used for proxies. - HttpPatch ## Applies partial modifications to a resource. + HttpMethod* = enum ## the requested HttpMethod + HttpHead = "HEAD" ## Asks for the response identical to the one that + ## would correspond to a GET request, but without + ## the response body. + HttpGet = "GET" ## Retrieves the specified resource. + HttpPost = "POST" ## Submits data to be processed to the identified + ## resource. The data is included in the body of + ## the request. + HttpPut = "PUT" ## Uploads a representation of the specified + ## resource. + HttpDelete = "DELETE" ## Deletes the specified resource. + HttpTrace = "TRACE" ## Echoes back the received request, so that a + ## client + ## can see what intermediate servers are adding or + ## changing in the request. + HttpOptions = "OPTIONS" ## Returns the HTTP methods that the server + ## supports for specified address. + HttpConnect = "CONNECT" ## Converts the request connection to a transparent + ## TCP/IP tunnel, usually used for proxies. + HttpPatch = "PATCH" ## Applies partial modifications to a resource. const Http100* = HttpCode(100) Http101* = HttpCode(101) + Http102* = HttpCode(102) ## https://tools.ietf.org/html/rfc2518.html WebDAV + Http103* = HttpCode(103) ## https://tools.ietf.org/html/rfc8297.html Early hints Http200* = HttpCode(200) Http201* = HttpCode(201) Http202* = HttpCode(202) @@ -58,6 +63,9 @@ const Http204* = HttpCode(204) Http205* = HttpCode(205) Http206* = HttpCode(206) + Http207* = HttpCode(207) ## https://tools.ietf.org/html/rfc4918.html WebDAV + Http208* = HttpCode(208) ## https://tools.ietf.org/html/rfc5842.html WebDAV, Section 7.1 + Http226* = HttpCode(226) ## https://tools.ietf.org/html/rfc3229.html Delta encoding, Section 10.4.1 Http300* = HttpCode(300) Http301* = HttpCode(301) Http302* = HttpCode(302) @@ -65,8 +73,10 @@ const Http304* = HttpCode(304) Http305* = HttpCode(305) Http307* = HttpCode(307) + Http308* = HttpCode(308) Http400* = HttpCode(400) Http401* = HttpCode(401) + Http402* = HttpCode(402) ## https://tools.ietf.org/html/rfc7231.html Payment required, Section 6.5.2 Http403* = HttpCode(403) Http404* = HttpCode(404) Http405* = HttpCode(405) @@ -85,6 +95,9 @@ const Http418* = HttpCode(418) Http421* = HttpCode(421) Http422* = HttpCode(422) + Http423* = HttpCode(423) ## https://tools.ietf.org/html/rfc4918.html WebDAV, Section 11.3 + Http424* = HttpCode(424) ## https://tools.ietf.org/html/rfc4918.html WebDAV, Section 11.3 + Http425* = HttpCode(425) ## https://tools.ietf.org/html/rfc8470.html Early data Http426* = HttpCode(426) Http428* = HttpCode(428) Http429* = HttpCode(429) @@ -96,68 +109,99 @@ const Http503* = HttpCode(503) Http504* = HttpCode(504) Http505* = HttpCode(505) + Http506* = HttpCode(506) ## https://tools.ietf.org/html/rfc2295.html Content negotiation, Section 8.1 + Http507* = HttpCode(507) ## https://tools.ietf.org/html/rfc4918.html WebDAV, Section 11.5 + Http508* = HttpCode(508) ## https://tools.ietf.org/html/rfc5842.html WebDAV, Section 7.2 + Http510* = HttpCode(510) ## https://tools.ietf.org/html/rfc2774.html Extension framework, Section 7 + Http511* = HttpCode(511) ## https://tools.ietf.org/html/rfc6585.html Additional status code, Section 6 + +const httpNewLine* = "\c\L" const headerLimit* = 10_000 -proc newHttpHeaders*(): HttpHeaders = - new result - result.table = newTable[string, seq[string]]() +func toTitleCase(s: string): string = + result = newString(len(s)) + var upper = true + for i in 0..len(s) - 1: + result[i] = if upper: toUpperAscii(s[i]) else: toLowerAscii(s[i]) + upper = s[i] == '-' + +func toCaseInsensitive*(headers: HttpHeaders, s: string): string {.inline.} = + ## For internal usage only. Do not use. + return if headers.isTitleCase: toTitleCase(s) else: toLowerAscii(s) + +func newHttpHeaders*(titleCase=false): HttpHeaders = + ## Returns a new `HttpHeaders` object. if `titleCase` is set to true, + ## headers are passed to the server in title case (e.g. "Content-Length") + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) + +func newHttpHeaders*(keyValuePairs: + openArray[tuple[key: string, val: string]], titleCase=false): HttpHeaders = + ## Returns a new `HttpHeaders` object from an array. if `titleCase` is set to true, + ## headers are passed to the server in title case (e.g. "Content-Length") + result = HttpHeaders(table: newTable[string, seq[string]](), isTitleCase: titleCase) -proc newHttpHeaders*(keyValuePairs: - openArray[tuple[key: string, val: string]]): HttpHeaders = - var pairs: seq[tuple[key: string, val: seq[string]]] = @[] for pair in keyValuePairs: - pairs.add((pair.key.toLowerAscii(), @[pair.val])) - new result - result.table = newTable[string, seq[string]](pairs) + let key = result.toCaseInsensitive(pair.key) + {.cast(noSideEffect).}: + if key in result.table: + result.table[key].add(pair.val) + else: + result.table[key] = @[pair.val] -proc `$`*(headers: HttpHeaders): string = - return $headers.table +func `$`*(headers: HttpHeaders): string {.inline.} = + $headers.table -proc clear*(headers: HttpHeaders) = +proc clear*(headers: HttpHeaders) {.inline.} = headers.table.clear() -proc `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = - ## Returns the values associated with the given ``key``. If the returned - ## values are passed to a procedure expecting a ``string``, the first +func `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = + ## Returns the values associated with the given `key`. If the returned + ## values are passed to a procedure expecting a `string`, the first ## value is automatically picked. If there are ## no values associated with the key, an exception is raised. ## - ## To access multiple values of a key, use the overloaded ``[]`` below or - ## to get all of them access the ``table`` field directly. - return headers.table[key.toLowerAscii].HttpHeaderValues + ## To access multiple values of a key, use the overloaded `[]` below or + ## to get all of them access the `table` field directly. + {.cast(noSideEffect).}: + let tmp = headers.table[headers.toCaseInsensitive(key)] + return HttpHeaderValues(tmp) converter toString*(values: HttpHeaderValues): string = return seq[string](values)[0] -proc `[]`*(headers: HttpHeaders, key: string, i: int): string = - ## Returns the ``i``'th value associated with the given key. If there are - ## no values associated with the key or the ``i``'th value doesn't exist, +func `[]`*(headers: HttpHeaders, key: string, i: int): string = + ## Returns the `i`'th value associated with the given key. If there are + ## no values associated with the key or the `i`'th value doesn't exist, ## an exception is raised. - return headers.table[key.toLowerAscii][i] + {.cast(noSideEffect).}: + return headers.table[headers.toCaseInsensitive(key)][i] proc `[]=`*(headers: HttpHeaders, key, value: string) = - ## Sets the header entries associated with ``key`` to the specified value. + ## Sets the header entries associated with `key` to the specified value. ## Replaces any existing values. - headers.table[key.toLowerAscii] = @[value] + headers.table[headers.toCaseInsensitive(key)] = @[value] proc `[]=`*(headers: HttpHeaders, key: string, value: seq[string]) = - ## Sets the header entries associated with ``key`` to the specified list of - ## values. - ## Replaces any existing values. - headers.table[key.toLowerAscii] = value + ## Sets the header entries associated with `key` to the specified list of + ## values. Replaces any existing values. If `value` is empty, + ## deletes the header entries associated with `key`. + if value.len > 0: + headers.table[headers.toCaseInsensitive(key)] = value + else: + headers.table.del(headers.toCaseInsensitive(key)) proc add*(headers: HttpHeaders, key, value: string) = ## Adds the specified value to the specified key. Appends to any existing ## values associated with the key. - if not headers.table.hasKey(key.toLowerAscii): - headers.table[key.toLowerAscii] = @[value] + if not headers.table.hasKey(headers.toCaseInsensitive(key)): + headers.table[headers.toCaseInsensitive(key)] = @[value] else: - headers.table[key.toLowerAscii].add(value) + headers.table[headers.toCaseInsensitive(key)].add(value) proc del*(headers: HttpHeaders, key: string) = - ## Delete the header entries associated with ``key`` - headers.table.del(key.toLowerAscii) + ## Deletes the header entries associated with `key` + headers.table.del(headers.toCaseInsensitive(key)) iterator pairs*(headers: HttpHeaders): tuple[key, value: string] = ## Yields each key, value pair. @@ -165,54 +209,57 @@ iterator pairs*(headers: HttpHeaders): tuple[key, value: string] = for value in v: yield (k, value) -proc contains*(values: HttpHeaderValues, value: string): bool = - ## Determines if ``value`` is one of the values inside ``values``. Comparison +func contains*(values: HttpHeaderValues, value: string): bool = + ## Determines if `value` is one of the values inside `values`. Comparison ## is performed without case sensitivity. for val in seq[string](values): if val.toLowerAscii == value.toLowerAscii: return true -proc hasKey*(headers: HttpHeaders, key: string): bool = - return headers.table.hasKey(key.toLowerAscii()) +func hasKey*(headers: HttpHeaders, key: string): bool = + return headers.table.hasKey(headers.toCaseInsensitive(key)) -proc getOrDefault*(headers: HttpHeaders, key: string, +func getOrDefault*(headers: HttpHeaders, key: string, default = @[""].HttpHeaderValues): HttpHeaderValues = - ## Returns the values associated with the given ``key``. If there are no - ## values associated with the key, then ``default`` is returned. + ## Returns the values associated with the given `key`. If there are no + ## values associated with the key, then `default` is returned. if headers.hasKey(key): return headers[key] else: return default -proc len*(headers: HttpHeaders): int = return headers.table.len +func len*(headers: HttpHeaders): int {.inline.} = headers.table.len -proc parseList(line: string, list: var seq[string], start: int): int = +func parseList(line: string, list: var seq[string], start: int): int = var i = 0 var current = "" while start+i < line.len and line[start + i] notin {'\c', '\l'}: i += line.skipWhitespace(start + i) i += line.parseUntil(current, {'\c', '\l', ','}, start + i) - list.add(current) + list.add(move current) # implicit current.setLen(0) if start+i < line.len and line[start + i] == ',': i.inc # Skip , - current.setLen(0) -proc parseHeader*(line: string): tuple[key: string, value: seq[string]] = +func parseHeader*(line: string): tuple[key: string, value: seq[string]] = ## Parses a single raw header HTTP line into key value pairs. ## - ## Used by ``asynchttpserver`` and ``httpclient`` internally and should not + ## Used by `asynchttpserver` and `httpclient` internally and should not ## be used by you. result.value = @[] var i = 0 i = line.parseUntil(result.key, ':') inc(i) # skip : if i < len(line): - i += parseList(line, result.value, i) + if cmpIgnoreCase(result.key, "cookie") == 0: + i += line.skipWhitespace(i) + result.value.add line.substr(i) + else: + i += parseList(line, result.value, i) elif result.key.len > 0: result.value = @[""] else: result.value = @[] -proc `==`*(protocol: tuple[orig: string, major, minor: int], +func `==`*(protocol: tuple[orig: string, major, minor: int], ver: HttpVersion): bool = let major = case ver @@ -223,19 +270,18 @@ proc `==`*(protocol: tuple[orig: string, major, minor: int], of HttpVer10: 0 result = protocol.major == major and protocol.minor == minor -proc contains*(methods: set[HttpMethod], x: string): bool = +func contains*(methods: set[HttpMethod], x: string): bool = return parseEnum[HttpMethod](x) in methods -proc `$`*(code: HttpCode): string = - ## Converts the specified ``HttpCode`` into a HTTP status. - ## - ## For example: - ## - ## .. code-block:: nim - ## doAssert($Http404 == "404 Not Found") +func `$`*(code: HttpCode): string = + ## Converts the specified `HttpCode` into a HTTP status. + runnableExamples: + doAssert($Http404 == "404 Not Found") case code.int of 100: "100 Continue" of 101: "101 Switching Protocols" + of 102: "102 Processing" + of 103: "103 Early Hints" of 200: "200 OK" of 201: "201 Created" of 202: "202 Accepted" @@ -243,6 +289,9 @@ proc `$`*(code: HttpCode): string = of 204: "204 No Content" of 205: "205 Reset Content" of 206: "206 Partial Content" + of 207: "207 Multi-Status" + of 208: "208 Already Reported" + of 226: "226 IM Used" of 300: "300 Multiple Choices" of 301: "301 Moved Permanently" of 302: "302 Found" @@ -250,8 +299,10 @@ proc `$`*(code: HttpCode): string = of 304: "304 Not Modified" of 305: "305 Use Proxy" of 307: "307 Temporary Redirect" + of 308: "308 Permanent Redirect" of 400: "400 Bad Request" of 401: "401 Unauthorized" + of 402: "402 Payment Required" of 403: "403 Forbidden" of 404: "404 Not Found" of 405: "405 Method Not Allowed" @@ -270,6 +321,9 @@ proc `$`*(code: HttpCode): string = of 418: "418 I'm a teapot" of 421: "421 Misdirected Request" of 422: "422 Unprocessable Entity" + of 423: "423 Locked" + of 424: "424 Failed Dependency" + of 425: "425 Too Early" of 426: "426 Upgrade Required" of 428: "428 Precondition Required" of 429: "429 Too Many Requests" @@ -281,46 +335,34 @@ proc `$`*(code: HttpCode): string = of 503: "503 Service Unavailable" of 504: "504 Gateway Timeout" of 505: "505 HTTP Version Not Supported" + of 506: "506 Variant Also Negotiates" + of 507: "507 Insufficient Storage" + of 508: "508 Loop Detected" + of 510: "510 Not Extended" + of 511: "511 Network Authentication Required" else: $(int(code)) -proc `==`*(a, b: HttpCode): bool {.borrow.} - -proc `==`*(rawCode: string, code: HttpCode): bool = - return cmpIgnoreCase(rawCode, $code) == 0 - -proc is2xx*(code: HttpCode): bool = - ## Determines whether ``code`` is a 2xx HTTP status code. - return code.int in {200 .. 299} - -proc is3xx*(code: HttpCode): bool = - ## Determines whether ``code`` is a 3xx HTTP status code. - return code.int in {300 .. 399} +func `==`*(a, b: HttpCode): bool {.borrow.} -proc is4xx*(code: HttpCode): bool = - ## Determines whether ``code`` is a 4xx HTTP status code. - return code.int in {400 .. 499} +func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = + ## Determines whether `code` is a 1xx HTTP status code. + runnableExamples: + doAssert is1xx(HttpCode(103)) -proc is5xx*(code: HttpCode): bool = - ## Determines whether ``code`` is a 5xx HTTP status code. - return code.int in {500 .. 599} + code.int in 100 .. 199 -proc `$`*(httpMethod: HttpMethod): string = - return (system.`$`(httpMethod))[4 .. ^1].toUpperAscii() +func is2xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 2xx HTTP status code. + code.int in 200 .. 299 -when isMainModule: - var test = newHttpHeaders() - test["Connection"] = @["Upgrade", "Close"] - doAssert test["Connection", 0] == "Upgrade" - doAssert test["Connection", 1] == "Close" - test.add("Connection", "Test") - doAssert test["Connection", 2] == "Test" - doAssert "upgrade" in test["Connection"] +func is3xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 3xx HTTP status code. + code.int in 300 .. 399 - # Bug #5344. - doAssert parseHeader("foobar: ") == ("foobar", @[""]) - let (key, value) = parseHeader("foobar: ") - test = newHttpHeaders() - test[key] = value - doAssert test["foobar"] == "" +func is4xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 4xx HTTP status code. + code.int in 400 .. 499 - doAssert parseHeader("foobar:") == ("foobar", @[""]) +func is5xx*(code: HttpCode): bool {.inline.} = + ## Determines whether `code` is a 5xx HTTP status code. + code.int in 500 .. 599 diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim deleted file mode 100644 index 30904c688..000000000 --- a/lib/pure/includes/osenv.nim +++ /dev/null @@ -1,229 +0,0 @@ -# Include file that implements 'getEnv' and friends. Do not import it! - -when not declared(os) and not declared(ospaths): - {.error: "This is an include file for os.nim!".} - -when defined(windows): - from parseutils import skipIgnoreCase - -proc c_getenv(env: cstring): cstring {. - importc: "getenv", header: "<stdlib.h>".} -proc c_putenv(env: cstring): cint {. - importc: "putenv", header: "<stdlib.h>".} -proc c_unsetenv(env: cstring): cint {. - importc: "unsetenv", header: "<stdlib.h>".} - -# Environment handling cannot be put into RTL, because the ``envPairs`` -# iterator depends on ``environment``. - -var - envComputed {.threadvar.}: bool - environment {.threadvar.}: seq[string] - -when defined(nimV2): - proc unpairedEnvAllocs*(): int = - result = environment.len - if result > 0: inc result - -when defined(windows) and not defined(nimscript): - # because we support Windows GUI applications, things get really - # messy here... - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "<string.h>".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {. - importc: "strchr", header: "<string.h>".} - - proc getEnvVarsC() = - if not envComputed: - environment = @[] - when useWinUnicode: - var - env = getEnvironmentStringsW() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[WideCString](cast[ByteAddress](eend)+2) - if eend[1].int == 0: break - discard freeEnvironmentStringsW(env) - else: - var - env = getEnvironmentStringsA() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[cstring](cast[ByteAddress](eend)+1) - if eend[1] == '\0': break - discard freeEnvironmentStringsA(env) - envComputed = true - -else: - const - useNSGetEnviron = (defined(macosx) and not defined(ios)) or defined(nimscript) - - when useNSGetEnviron: - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # <crt_externs.h>, can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {. - importc: "_NSGetEnviron", header: "<crt_externs.h>".} - else: - var gEnv {.importc: "environ".}: cstringArray - - proc getEnvVarsC() = - # retrieves the variables of char** env of C's main proc - if not envComputed: - environment = @[] - when useNSGetEnviron: - var gEnv = NSGetEnviron()[] - var i = 0 - while true: - if gEnv[i] == nil: break - add environment, $gEnv[i] - inc(i) - envComputed = true - -proc findEnvVar(key: string): int = - getEnvVarsC() - var temp = key & '=' - for i in 0..high(environment): - when defined(windows): - if skipIgnoreCase(environment[i], temp) == len(temp): return i - else: - if startsWith(environment[i], temp): return i - return -1 - -proc getEnv*(key: string, default = ""): TaintedString {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. - ## - ## If the variable does not exist, `""` is returned. To distinguish - ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. - ## - ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert getEnv("unknownEnv") == "" - assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - - when nimvm: - discard "built into the compiler" - else: - var i = findEnvVar(key) - if i >= 0: - return TaintedString(substr(environment[i], find(environment[i], '=')+1)) - else: - var env = c_getenv(key) - if env == nil: return TaintedString(default) - result = TaintedString($env) - -proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert not existsEnv("unknownEnv") - - when nimvm: - discard "built into the compiler" - else: - if c_getenv(key) != nil: return true - else: return findEnvVar(key) >= 0 - -proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `OSError` is raised. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - - # Note: by storing the string in the environment sequence, - # we guarantee that we don't free the memory before the program - # ends (this is needed for POSIX compliance). It is also needed so that - # the process itself may access its modified environment variables! - when nimvm: - discard "built into the compiler" - else: - var indx = findEnvVar(key) - if indx >= 0: - environment[indx] = key & '=' & val - else: - add environment, (key & '=' & val) - indx = high(environment) - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - var v = newWideCString(val) - if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError()) - else: - if c_putenv(environment[indx]) != 0'i32: - raiseOSError(osLastError()) - -proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - ## Deletes the `environment variable`:idx: named `key`. - ## If an error occurs, `OSError` is raised. - ## - ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - when nimvm: - discard "built into the compiler" - else: - var indx = findEnvVar(key) - if indx < 0: return # Do nothing if the env var is not already set - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - if setEnvironmentVariableW(k, nil) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, nil) == 0'i32: raiseOSError(osLastError()) - else: - if c_unsetenv(key) != 0'i32: - raiseOSError(osLastError()) - environment.delete(indx) - -iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. - ## - ## In the first component of the tuple is the name of the current variable stored, - ## in the second its value. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - getEnvVarsC() - for i in 0..high(environment): - var p = find(environment[i], '=') - yield (TaintedString(substr(environment[i], 0, p-1)), - TaintedString(substr(environment[i], p+1))) diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim deleted file mode 100644 index 138aa127d..000000000 --- a/lib/pure/includes/oserr.nim +++ /dev/null @@ -1,120 +0,0 @@ -# Include file that implements 'osErrorMsg' and friends. Do not import it! - -when not declared(os) and not declared(ospaths): - {.error: "This is an include file for os.nim!".} - -when not defined(nimscript): - var errno {.importc, header: "<errno.h>".}: cint - - proc c_strerror(errnum: cint): cstring {. - importc: "strerror", header: "<string.h>".} - - when defined(windows): - import winlean - -proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} -proc `$`*(err: OSErrorCode): string {.borrow.} - -proc osErrorMsg*(errorCode: OSErrorCode): string = - ## Converts an OS error code into a human readable string. - ## - ## The error code can be retrieved using the `osLastError proc <#osLastError>`_. - ## - ## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be - ## returned. - ## - ## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to - ## make this procedure use the non-unicode Win API calls to retrieve the - ## message. - ## - ## See also: - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - ## * `osLastError proc <#osLastError>`_ - runnableExamples: - when defined(linux): - assert osErrorMsg(OSErrorCode(0)) == "" - assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" - assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" - - result = "" - when defined(nimscript): - discard - elif defined(Windows): - if errorCode != OSErrorCode(0'i32): - when useWinUnicode: - var msgbuf: WideCString - if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(cast[pointer](msgbuf)) - else: - var msgbuf: cstring - if formatMessageA(0x00000100 or 0x00001000 or 0x00000200, - nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: - result = $msgbuf - if msgbuf != nil: localFree(msgbuf) - else: - if errorCode != OSErrorCode(0'i32): - result = $c_strerror(errorCode.int32) - -proc newOSError*( - errorCode: OSErrorCode, additionalInfo = "" -): owned(ref OSError) {.noinline.} = - ## Creates a new `OSError exception <system.html#OSError>`_. - ## - ## The ``errorCode`` will determine the - ## message, `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ will be used - ## to get this message. - ## - ## The error code can be retrieved using the `osLastError proc - ## <#osLastError>`_. - ## - ## If the error code is ``0`` or an error message could not be retrieved, - ## the message ``unknown OS error`` will be used. - ## - ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `osLastError proc <#osLastError>`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) - if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.addQuoted additionalInfo - if e.msg == "": - e.msg = "unknown OS error" - return e - -proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = - ## Raises an `OSError exception <system.html#OSError>`_. - ## - ## Read the description of the `newOSError proc <#newOSError,OSErrorCode,string>`_ to learn - ## how the exception object is created. - raise newOSError(errorCode, additionalInfo) - -{.push stackTrace:off.} -proc osLastError*(): OSErrorCode {.sideEffect.} = - ## Retrieves the last operating system error code. - ## - ## This procedure is useful in the event when an OS call fails. In that case - ## this procedure will return the error code describing the reason why the - ## OS call failed. The ``OSErrorMsg`` procedure can then be used to convert - ## this code into a string. - ## - ## **Warning**: - ## The behaviour of this procedure varies between Windows and POSIX systems. - ## On Windows some OS calls can reset the error code to ``0`` causing this - ## procedure to return ``0``. It is therefore advised to call this procedure - ## immediately after an OS call fails. On POSIX systems this is not a problem. - ## - ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - when defined(nimscript): - discard - elif defined(windows): - result = OSErrorCode(getLastError()) - else: - result = OSErrorCode(errno) -{.pop.} diff --git a/lib/pure/includes/unicode_ranges.nim b/lib/pure/includes/unicode_ranges.nim index 74c8c6399..04ccfb747 100644 --- a/lib/pure/includes/unicode_ranges.nim +++ b/lib/pure/includes/unicode_ranges.nim @@ -2,1988 +2,1979 @@ const toLowerRanges = [ - 0x00041, 0x0005A, 532, - 0x000C0, 0x000D6, 532, - 0x000D8, 0x000DE, 532, - 0x00189, 0x0018A, 705, - 0x001B1, 0x001B2, 717, - 0x00388, 0x0038A, 537, - 0x0038E, 0x0038F, 563, - 0x00391, 0x003A1, 532, - 0x003A3, 0x003AB, 532, - 0x003FD, 0x003FF, 370, - 0x00400, 0x0040F, 580, - 0x00410, 0x0042F, 532, - 0x00531, 0x00556, 548, - 0x010A0, 0x010C5, 7764, - 0x013A0, 0x013EF, 39364, - 0x013F0, 0x013F5, 508, - 0x01C90, 0x01CBA, -2508, - 0x01CBD, 0x01CBF, -2508, - 0x01F08, 0x01F0F, 492, - 0x01F18, 0x01F1D, 492, - 0x01F28, 0x01F2F, 492, - 0x01F38, 0x01F3F, 492, - 0x01F48, 0x01F4D, 492, - 0x01F68, 0x01F6F, 492, - 0x01F88, 0x01F8F, 492, - 0x01F98, 0x01F9F, 492, - 0x01FA8, 0x01FAF, 492, - 0x01FB8, 0x01FB9, 492, - 0x01FBA, 0x01FBB, 426, - 0x01FC8, 0x01FCB, 414, - 0x01FD8, 0x01FD9, 492, - 0x01FDA, 0x01FDB, 400, - 0x01FE8, 0x01FE9, 492, - 0x01FEA, 0x01FEB, 388, - 0x01FF8, 0x01FF9, 372, - 0x01FFA, 0x01FFB, 374, - 0x02C00, 0x02C2E, 548, - 0x02C7E, 0x02C7F, -10315, - 0x0FF21, 0x0FF3A, 532, - 0x10400, 0x10427, 540, - 0x104B0, 0x104D3, 540, - 0x10C80, 0x10CB2, 564, - 0x118A0, 0x118BF, 532, - 0x16E40, 0x16E5F, 532, - 0x1E900, 0x1E921, 534, + 0x00041'i32, 0x0005A'i32, 532, + 0x000C0'i32, 0x000D6'i32, 532, + 0x000D8'i32, 0x000DE'i32, 532, + 0x00189'i32, 0x0018A'i32, 705, + 0x001B1'i32, 0x001B2'i32, 717, + 0x00388'i32, 0x0038A'i32, 537, + 0x0038E'i32, 0x0038F'i32, 563, + 0x00391'i32, 0x003A1'i32, 532, + 0x003A3'i32, 0x003AB'i32, 532, + 0x003FD'i32, 0x003FF'i32, 370, + 0x00400'i32, 0x0040F'i32, 580, + 0x00410'i32, 0x0042F'i32, 532, + 0x00531'i32, 0x00556'i32, 548, + 0x010A0'i32, 0x010C5'i32, 7764, + 0x013A0'i32, 0x013EF'i32, 39364, + 0x013F0'i32, 0x013F5'i32, 508, + 0x01C90'i32, 0x01CBA'i32, -2508, + 0x01CBD'i32, 0x01CBF'i32, -2508, + 0x01F08'i32, 0x01F0F'i32, 492, + 0x01F18'i32, 0x01F1D'i32, 492, + 0x01F28'i32, 0x01F2F'i32, 492, + 0x01F38'i32, 0x01F3F'i32, 492, + 0x01F48'i32, 0x01F4D'i32, 492, + 0x01F68'i32, 0x01F6F'i32, 492, + 0x01F88'i32, 0x01F8F'i32, 492, + 0x01F98'i32, 0x01F9F'i32, 492, + 0x01FA8'i32, 0x01FAF'i32, 492, + 0x01FB8'i32, 0x01FB9'i32, 492, + 0x01FBA'i32, 0x01FBB'i32, 426, + 0x01FC8'i32, 0x01FCB'i32, 414, + 0x01FD8'i32, 0x01FD9'i32, 492, + 0x01FDA'i32, 0x01FDB'i32, 400, + 0x01FE8'i32, 0x01FE9'i32, 492, + 0x01FEA'i32, 0x01FEB'i32, 388, + 0x01FF8'i32, 0x01FF9'i32, 372, + 0x01FFA'i32, 0x01FFB'i32, 374, + 0x02C00'i32, 0x02C2E'i32, 548, + 0x02C7E'i32, 0x02C7F'i32, -10315, + 0x0FF21'i32, 0x0FF3A'i32, 532, + 0x10400'i32, 0x10427'i32, 540, + 0x104B0'i32, 0x104D3'i32, 540, + 0x10C80'i32, 0x10CB2'i32, 564, + 0x118A0'i32, 0x118BF'i32, 532, + 0x16E40'i32, 0x16E5F'i32, 532, + 0x1E900'i32, 0x1E921'i32, 534, ] toLowerSinglets = [ - 0x00100, 501, - 0x00102, 501, - 0x00104, 501, - 0x00106, 501, - 0x00108, 501, - 0x0010A, 501, - 0x0010C, 501, - 0x0010E, 501, - 0x00110, 501, - 0x00112, 501, - 0x00114, 501, - 0x00116, 501, - 0x00118, 501, - 0x0011A, 501, - 0x0011C, 501, - 0x0011E, 501, - 0x00120, 501, - 0x00122, 501, - 0x00124, 501, - 0x00126, 501, - 0x00128, 501, - 0x0012A, 501, - 0x0012C, 501, - 0x0012E, 501, - 0x00130, 301, - 0x00132, 501, - 0x00134, 501, - 0x00136, 501, - 0x00139, 501, - 0x0013B, 501, - 0x0013D, 501, - 0x0013F, 501, - 0x00141, 501, - 0x00143, 501, - 0x00145, 501, - 0x00147, 501, - 0x0014A, 501, - 0x0014C, 501, - 0x0014E, 501, - 0x00150, 501, - 0x00152, 501, - 0x00154, 501, - 0x00156, 501, - 0x00158, 501, - 0x0015A, 501, - 0x0015C, 501, - 0x0015E, 501, - 0x00160, 501, - 0x00162, 501, - 0x00164, 501, - 0x00166, 501, - 0x00168, 501, - 0x0016A, 501, - 0x0016C, 501, - 0x0016E, 501, - 0x00170, 501, - 0x00172, 501, - 0x00174, 501, - 0x00176, 501, - 0x00178, 379, - 0x00179, 501, - 0x0017B, 501, - 0x0017D, 501, - 0x00181, 710, - 0x00182, 501, - 0x00184, 501, - 0x00186, 706, - 0x00187, 501, - 0x0018B, 501, - 0x0018E, 579, - 0x0018F, 702, - 0x00190, 703, - 0x00191, 501, - 0x00193, 705, - 0x00194, 707, - 0x00196, 711, - 0x00197, 709, - 0x00198, 501, - 0x0019C, 711, - 0x0019D, 713, - 0x0019F, 714, - 0x001A0, 501, - 0x001A2, 501, - 0x001A4, 501, - 0x001A6, 718, - 0x001A7, 501, - 0x001A9, 718, - 0x001AC, 501, - 0x001AE, 718, - 0x001AF, 501, - 0x001B3, 501, - 0x001B5, 501, - 0x001B7, 719, - 0x001B8, 501, - 0x001BC, 501, - 0x001C4, 502, - 0x001C5, 501, - 0x001C7, 502, - 0x001C8, 501, - 0x001CA, 502, - 0x001CB, 501, - 0x001CD, 501, - 0x001CF, 501, - 0x001D1, 501, - 0x001D3, 501, - 0x001D5, 501, - 0x001D7, 501, - 0x001D9, 501, - 0x001DB, 501, - 0x001DE, 501, - 0x001E0, 501, - 0x001E2, 501, - 0x001E4, 501, - 0x001E6, 501, - 0x001E8, 501, - 0x001EA, 501, - 0x001EC, 501, - 0x001EE, 501, - 0x001F1, 502, - 0x001F2, 501, - 0x001F4, 501, - 0x001F6, 403, - 0x001F7, 444, - 0x001F8, 501, - 0x001FA, 501, - 0x001FC, 501, - 0x001FE, 501, - 0x00200, 501, - 0x00202, 501, - 0x00204, 501, - 0x00206, 501, - 0x00208, 501, - 0x0020A, 501, - 0x0020C, 501, - 0x0020E, 501, - 0x00210, 501, - 0x00212, 501, - 0x00214, 501, - 0x00216, 501, - 0x00218, 501, - 0x0021A, 501, - 0x0021C, 501, - 0x0021E, 501, - 0x00220, 370, - 0x00222, 501, - 0x00224, 501, - 0x00226, 501, - 0x00228, 501, - 0x0022A, 501, - 0x0022C, 501, - 0x0022E, 501, - 0x00230, 501, - 0x00232, 501, - 0x0023A, 11295, - 0x0023B, 501, - 0x0023D, 337, - 0x0023E, 11292, - 0x00241, 501, - 0x00243, 305, - 0x00244, 569, - 0x00245, 571, - 0x00246, 501, - 0x00248, 501, - 0x0024A, 501, - 0x0024C, 501, - 0x0024E, 501, - 0x00370, 501, - 0x00372, 501, - 0x00376, 501, - 0x0037F, 616, - 0x00386, 538, - 0x0038C, 564, - 0x003CF, 508, - 0x003D8, 501, - 0x003DA, 501, - 0x003DC, 501, - 0x003DE, 501, - 0x003E0, 501, - 0x003E2, 501, - 0x003E4, 501, - 0x003E6, 501, - 0x003E8, 501, - 0x003EA, 501, - 0x003EC, 501, - 0x003EE, 501, - 0x003F4, 440, - 0x003F7, 501, - 0x003F9, 493, - 0x003FA, 501, - 0x00460, 501, - 0x00462, 501, - 0x00464, 501, - 0x00466, 501, - 0x00468, 501, - 0x0046A, 501, - 0x0046C, 501, - 0x0046E, 501, - 0x00470, 501, - 0x00472, 501, - 0x00474, 501, - 0x00476, 501, - 0x00478, 501, - 0x0047A, 501, - 0x0047C, 501, - 0x0047E, 501, - 0x00480, 501, - 0x0048A, 501, - 0x0048C, 501, - 0x0048E, 501, - 0x00490, 501, - 0x00492, 501, - 0x00494, 501, - 0x00496, 501, - 0x00498, 501, - 0x0049A, 501, - 0x0049C, 501, - 0x0049E, 501, - 0x004A0, 501, - 0x004A2, 501, - 0x004A4, 501, - 0x004A6, 501, - 0x004A8, 501, - 0x004AA, 501, - 0x004AC, 501, - 0x004AE, 501, - 0x004B0, 501, - 0x004B2, 501, - 0x004B4, 501, - 0x004B6, 501, - 0x004B8, 501, - 0x004BA, 501, - 0x004BC, 501, - 0x004BE, 501, - 0x004C0, 515, - 0x004C1, 501, - 0x004C3, 501, - 0x004C5, 501, - 0x004C7, 501, - 0x004C9, 501, - 0x004CB, 501, - 0x004CD, 501, - 0x004D0, 501, - 0x004D2, 501, - 0x004D4, 501, - 0x004D6, 501, - 0x004D8, 501, - 0x004DA, 501, - 0x004DC, 501, - 0x004DE, 501, - 0x004E0, 501, - 0x004E2, 501, - 0x004E4, 501, - 0x004E6, 501, - 0x004E8, 501, - 0x004EA, 501, - 0x004EC, 501, - 0x004EE, 501, - 0x004F0, 501, - 0x004F2, 501, - 0x004F4, 501, - 0x004F6, 501, - 0x004F8, 501, - 0x004FA, 501, - 0x004FC, 501, - 0x004FE, 501, - 0x00500, 501, - 0x00502, 501, - 0x00504, 501, - 0x00506, 501, - 0x00508, 501, - 0x0050A, 501, - 0x0050C, 501, - 0x0050E, 501, - 0x00510, 501, - 0x00512, 501, - 0x00514, 501, - 0x00516, 501, - 0x00518, 501, - 0x0051A, 501, - 0x0051C, 501, - 0x0051E, 501, - 0x00520, 501, - 0x00522, 501, - 0x00524, 501, - 0x00526, 501, - 0x00528, 501, - 0x0052A, 501, - 0x0052C, 501, - 0x0052E, 501, - 0x010C7, 7764, - 0x010CD, 7764, - 0x01E00, 501, - 0x01E02, 501, - 0x01E04, 501, - 0x01E06, 501, - 0x01E08, 501, - 0x01E0A, 501, - 0x01E0C, 501, - 0x01E0E, 501, - 0x01E10, 501, - 0x01E12, 501, - 0x01E14, 501, - 0x01E16, 501, - 0x01E18, 501, - 0x01E1A, 501, - 0x01E1C, 501, - 0x01E1E, 501, - 0x01E20, 501, - 0x01E22, 501, - 0x01E24, 501, - 0x01E26, 501, - 0x01E28, 501, - 0x01E2A, 501, - 0x01E2C, 501, - 0x01E2E, 501, - 0x01E30, 501, - 0x01E32, 501, - 0x01E34, 501, - 0x01E36, 501, - 0x01E38, 501, - 0x01E3A, 501, - 0x01E3C, 501, - 0x01E3E, 501, - 0x01E40, 501, - 0x01E42, 501, - 0x01E44, 501, - 0x01E46, 501, - 0x01E48, 501, - 0x01E4A, 501, - 0x01E4C, 501, - 0x01E4E, 501, - 0x01E50, 501, - 0x01E52, 501, - 0x01E54, 501, - 0x01E56, 501, - 0x01E58, 501, - 0x01E5A, 501, - 0x01E5C, 501, - 0x01E5E, 501, - 0x01E60, 501, - 0x01E62, 501, - 0x01E64, 501, - 0x01E66, 501, - 0x01E68, 501, - 0x01E6A, 501, - 0x01E6C, 501, - 0x01E6E, 501, - 0x01E70, 501, - 0x01E72, 501, - 0x01E74, 501, - 0x01E76, 501, - 0x01E78, 501, - 0x01E7A, 501, - 0x01E7C, 501, - 0x01E7E, 501, - 0x01E80, 501, - 0x01E82, 501, - 0x01E84, 501, - 0x01E86, 501, - 0x01E88, 501, - 0x01E8A, 501, - 0x01E8C, 501, - 0x01E8E, 501, - 0x01E90, 501, - 0x01E92, 501, - 0x01E94, 501, - 0x01E9E, -7115, - 0x01EA0, 501, - 0x01EA2, 501, - 0x01EA4, 501, - 0x01EA6, 501, - 0x01EA8, 501, - 0x01EAA, 501, - 0x01EAC, 501, - 0x01EAE, 501, - 0x01EB0, 501, - 0x01EB2, 501, - 0x01EB4, 501, - 0x01EB6, 501, - 0x01EB8, 501, - 0x01EBA, 501, - 0x01EBC, 501, - 0x01EBE, 501, - 0x01EC0, 501, - 0x01EC2, 501, - 0x01EC4, 501, - 0x01EC6, 501, - 0x01EC8, 501, - 0x01ECA, 501, - 0x01ECC, 501, - 0x01ECE, 501, - 0x01ED0, 501, - 0x01ED2, 501, - 0x01ED4, 501, - 0x01ED6, 501, - 0x01ED8, 501, - 0x01EDA, 501, - 0x01EDC, 501, - 0x01EDE, 501, - 0x01EE0, 501, - 0x01EE2, 501, - 0x01EE4, 501, - 0x01EE6, 501, - 0x01EE8, 501, - 0x01EEA, 501, - 0x01EEC, 501, - 0x01EEE, 501, - 0x01EF0, 501, - 0x01EF2, 501, - 0x01EF4, 501, - 0x01EF6, 501, - 0x01EF8, 501, - 0x01EFA, 501, - 0x01EFC, 501, - 0x01EFE, 501, - 0x01F59, 492, - 0x01F5B, 492, - 0x01F5D, 492, - 0x01F5F, 492, - 0x01FBC, 491, - 0x01FCC, 491, - 0x01FEC, 493, - 0x01FFC, 491, - 0x02126, -7017, - 0x0212A, -7883, - 0x0212B, -7762, - 0x02132, 528, - 0x02183, 501, - 0x02C60, 501, - 0x02C62, -10243, - 0x02C63, -3314, - 0x02C64, -10227, - 0x02C67, 501, - 0x02C69, 501, - 0x02C6B, 501, - 0x02C6D, -10280, - 0x02C6E, -10249, - 0x02C6F, -10283, - 0x02C70, -10282, - 0x02C72, 501, - 0x02C75, 501, - 0x02C80, 501, - 0x02C82, 501, - 0x02C84, 501, - 0x02C86, 501, - 0x02C88, 501, - 0x02C8A, 501, - 0x02C8C, 501, - 0x02C8E, 501, - 0x02C90, 501, - 0x02C92, 501, - 0x02C94, 501, - 0x02C96, 501, - 0x02C98, 501, - 0x02C9A, 501, - 0x02C9C, 501, - 0x02C9E, 501, - 0x02CA0, 501, - 0x02CA2, 501, - 0x02CA4, 501, - 0x02CA6, 501, - 0x02CA8, 501, - 0x02CAA, 501, - 0x02CAC, 501, - 0x02CAE, 501, - 0x02CB0, 501, - 0x02CB2, 501, - 0x02CB4, 501, - 0x02CB6, 501, - 0x02CB8, 501, - 0x02CBA, 501, - 0x02CBC, 501, - 0x02CBE, 501, - 0x02CC0, 501, - 0x02CC2, 501, - 0x02CC4, 501, - 0x02CC6, 501, - 0x02CC8, 501, - 0x02CCA, 501, - 0x02CCC, 501, - 0x02CCE, 501, - 0x02CD0, 501, - 0x02CD2, 501, - 0x02CD4, 501, - 0x02CD6, 501, - 0x02CD8, 501, - 0x02CDA, 501, - 0x02CDC, 501, - 0x02CDE, 501, - 0x02CE0, 501, - 0x02CE2, 501, - 0x02CEB, 501, - 0x02CED, 501, - 0x02CF2, 501, - 0x0A640, 501, - 0x0A642, 501, - 0x0A644, 501, - 0x0A646, 501, - 0x0A648, 501, - 0x0A64A, 501, - 0x0A64C, 501, - 0x0A64E, 501, - 0x0A650, 501, - 0x0A652, 501, - 0x0A654, 501, - 0x0A656, 501, - 0x0A658, 501, - 0x0A65A, 501, - 0x0A65C, 501, - 0x0A65E, 501, - 0x0A660, 501, - 0x0A662, 501, - 0x0A664, 501, - 0x0A666, 501, - 0x0A668, 501, - 0x0A66A, 501, - 0x0A66C, 501, - 0x0A680, 501, - 0x0A682, 501, - 0x0A684, 501, - 0x0A686, 501, - 0x0A688, 501, - 0x0A68A, 501, - 0x0A68C, 501, - 0x0A68E, 501, - 0x0A690, 501, - 0x0A692, 501, - 0x0A694, 501, - 0x0A696, 501, - 0x0A698, 501, - 0x0A69A, 501, - 0x0A722, 501, - 0x0A724, 501, - 0x0A726, 501, - 0x0A728, 501, - 0x0A72A, 501, - 0x0A72C, 501, - 0x0A72E, 501, - 0x0A732, 501, - 0x0A734, 501, - 0x0A736, 501, - 0x0A738, 501, - 0x0A73A, 501, - 0x0A73C, 501, - 0x0A73E, 501, - 0x0A740, 501, - 0x0A742, 501, - 0x0A744, 501, - 0x0A746, 501, - 0x0A748, 501, - 0x0A74A, 501, - 0x0A74C, 501, - 0x0A74E, 501, - 0x0A750, 501, - 0x0A752, 501, - 0x0A754, 501, - 0x0A756, 501, - 0x0A758, 501, - 0x0A75A, 501, - 0x0A75C, 501, - 0x0A75E, 501, - 0x0A760, 501, - 0x0A762, 501, - 0x0A764, 501, - 0x0A766, 501, - 0x0A768, 501, - 0x0A76A, 501, - 0x0A76C, 501, - 0x0A76E, 501, - 0x0A779, 501, - 0x0A77B, 501, - 0x0A77D, -34832, - 0x0A77E, 501, - 0x0A780, 501, - 0x0A782, 501, - 0x0A784, 501, - 0x0A786, 501, - 0x0A78B, 501, - 0x0A78D, -41780, - 0x0A790, 501, - 0x0A792, 501, - 0x0A796, 501, - 0x0A798, 501, - 0x0A79A, 501, - 0x0A79C, 501, - 0x0A79E, 501, - 0x0A7A0, 501, - 0x0A7A2, 501, - 0x0A7A4, 501, - 0x0A7A6, 501, - 0x0A7A8, 501, - 0x0A7AA, -41808, - 0x0A7AB, -41819, - 0x0A7AC, -41815, - 0x0A7AD, -41805, - 0x0A7AE, -41808, - 0x0A7B0, -41758, - 0x0A7B1, -41782, - 0x0A7B2, -41761, - 0x0A7B3, 1428, - 0x0A7B4, 501, - 0x0A7B6, 501, - 0x0A7B8, 501, - 0x0A7BA, 501, - 0x0A7BC, 501, - 0x0A7BE, 501, - 0x0A7C2, 501, - 0x0A7C4, 452, - 0x0A7C5, -41807, - 0x0A7C6, -34884, + 0x00100'i32, 501, + 0x00102'i32, 501, + 0x00104'i32, 501, + 0x00106'i32, 501, + 0x00108'i32, 501, + 0x0010A'i32, 501, + 0x0010C'i32, 501, + 0x0010E'i32, 501, + 0x00110'i32, 501, + 0x00112'i32, 501, + 0x00114'i32, 501, + 0x00116'i32, 501, + 0x00118'i32, 501, + 0x0011A'i32, 501, + 0x0011C'i32, 501, + 0x0011E'i32, 501, + 0x00120'i32, 501, + 0x00122'i32, 501, + 0x00124'i32, 501, + 0x00126'i32, 501, + 0x00128'i32, 501, + 0x0012A'i32, 501, + 0x0012C'i32, 501, + 0x0012E'i32, 501, + 0x00130'i32, 301, + 0x00132'i32, 501, + 0x00134'i32, 501, + 0x00136'i32, 501, + 0x00139'i32, 501, + 0x0013B'i32, 501, + 0x0013D'i32, 501, + 0x0013F'i32, 501, + 0x00141'i32, 501, + 0x00143'i32, 501, + 0x00145'i32, 501, + 0x00147'i32, 501, + 0x0014A'i32, 501, + 0x0014C'i32, 501, + 0x0014E'i32, 501, + 0x00150'i32, 501, + 0x00152'i32, 501, + 0x00154'i32, 501, + 0x00156'i32, 501, + 0x00158'i32, 501, + 0x0015A'i32, 501, + 0x0015C'i32, 501, + 0x0015E'i32, 501, + 0x00160'i32, 501, + 0x00162'i32, 501, + 0x00164'i32, 501, + 0x00166'i32, 501, + 0x00168'i32, 501, + 0x0016A'i32, 501, + 0x0016C'i32, 501, + 0x0016E'i32, 501, + 0x00170'i32, 501, + 0x00172'i32, 501, + 0x00174'i32, 501, + 0x00176'i32, 501, + 0x00178'i32, 379, + 0x00179'i32, 501, + 0x0017B'i32, 501, + 0x0017D'i32, 501, + 0x00181'i32, 710, + 0x00182'i32, 501, + 0x00184'i32, 501, + 0x00186'i32, 706, + 0x00187'i32, 501, + 0x0018B'i32, 501, + 0x0018E'i32, 579, + 0x0018F'i32, 702, + 0x00190'i32, 703, + 0x00191'i32, 501, + 0x00193'i32, 705, + 0x00194'i32, 707, + 0x00196'i32, 711, + 0x00197'i32, 709, + 0x00198'i32, 501, + 0x0019C'i32, 711, + 0x0019D'i32, 713, + 0x0019F'i32, 714, + 0x001A0'i32, 501, + 0x001A2'i32, 501, + 0x001A4'i32, 501, + 0x001A6'i32, 718, + 0x001A7'i32, 501, + 0x001A9'i32, 718, + 0x001AC'i32, 501, + 0x001AE'i32, 718, + 0x001AF'i32, 501, + 0x001B3'i32, 501, + 0x001B5'i32, 501, + 0x001B7'i32, 719, + 0x001B8'i32, 501, + 0x001BC'i32, 501, + 0x001C4'i32, 502, + 0x001C5'i32, 501, + 0x001C7'i32, 502, + 0x001C8'i32, 501, + 0x001CA'i32, 502, + 0x001CB'i32, 501, + 0x001CD'i32, 501, + 0x001CF'i32, 501, + 0x001D1'i32, 501, + 0x001D3'i32, 501, + 0x001D5'i32, 501, + 0x001D7'i32, 501, + 0x001D9'i32, 501, + 0x001DB'i32, 501, + 0x001DE'i32, 501, + 0x001E0'i32, 501, + 0x001E2'i32, 501, + 0x001E4'i32, 501, + 0x001E6'i32, 501, + 0x001E8'i32, 501, + 0x001EA'i32, 501, + 0x001EC'i32, 501, + 0x001EE'i32, 501, + 0x001F1'i32, 502, + 0x001F2'i32, 501, + 0x001F4'i32, 501, + 0x001F6'i32, 403, + 0x001F7'i32, 444, + 0x001F8'i32, 501, + 0x001FA'i32, 501, + 0x001FC'i32, 501, + 0x001FE'i32, 501, + 0x00200'i32, 501, + 0x00202'i32, 501, + 0x00204'i32, 501, + 0x00206'i32, 501, + 0x00208'i32, 501, + 0x0020A'i32, 501, + 0x0020C'i32, 501, + 0x0020E'i32, 501, + 0x00210'i32, 501, + 0x00212'i32, 501, + 0x00214'i32, 501, + 0x00216'i32, 501, + 0x00218'i32, 501, + 0x0021A'i32, 501, + 0x0021C'i32, 501, + 0x0021E'i32, 501, + 0x00220'i32, 370, + 0x00222'i32, 501, + 0x00224'i32, 501, + 0x00226'i32, 501, + 0x00228'i32, 501, + 0x0022A'i32, 501, + 0x0022C'i32, 501, + 0x0022E'i32, 501, + 0x00230'i32, 501, + 0x00232'i32, 501, + 0x0023A'i32, 11295, + 0x0023B'i32, 501, + 0x0023D'i32, 337, + 0x0023E'i32, 11292, + 0x00241'i32, 501, + 0x00243'i32, 305, + 0x00244'i32, 569, + 0x00245'i32, 571, + 0x00246'i32, 501, + 0x00248'i32, 501, + 0x0024A'i32, 501, + 0x0024C'i32, 501, + 0x0024E'i32, 501, + 0x00370'i32, 501, + 0x00372'i32, 501, + 0x00376'i32, 501, + 0x0037F'i32, 616, + 0x00386'i32, 538, + 0x0038C'i32, 564, + 0x003CF'i32, 508, + 0x003D8'i32, 501, + 0x003DA'i32, 501, + 0x003DC'i32, 501, + 0x003DE'i32, 501, + 0x003E0'i32, 501, + 0x003E2'i32, 501, + 0x003E4'i32, 501, + 0x003E6'i32, 501, + 0x003E8'i32, 501, + 0x003EA'i32, 501, + 0x003EC'i32, 501, + 0x003EE'i32, 501, + 0x003F4'i32, 440, + 0x003F7'i32, 501, + 0x003F9'i32, 493, + 0x003FA'i32, 501, + 0x00460'i32, 501, + 0x00462'i32, 501, + 0x00464'i32, 501, + 0x00466'i32, 501, + 0x00468'i32, 501, + 0x0046A'i32, 501, + 0x0046C'i32, 501, + 0x0046E'i32, 501, + 0x00470'i32, 501, + 0x00472'i32, 501, + 0x00474'i32, 501, + 0x00476'i32, 501, + 0x00478'i32, 501, + 0x0047A'i32, 501, + 0x0047C'i32, 501, + 0x0047E'i32, 501, + 0x00480'i32, 501, + 0x0048A'i32, 501, + 0x0048C'i32, 501, + 0x0048E'i32, 501, + 0x00490'i32, 501, + 0x00492'i32, 501, + 0x00494'i32, 501, + 0x00496'i32, 501, + 0x00498'i32, 501, + 0x0049A'i32, 501, + 0x0049C'i32, 501, + 0x0049E'i32, 501, + 0x004A0'i32, 501, + 0x004A2'i32, 501, + 0x004A4'i32, 501, + 0x004A6'i32, 501, + 0x004A8'i32, 501, + 0x004AA'i32, 501, + 0x004AC'i32, 501, + 0x004AE'i32, 501, + 0x004B0'i32, 501, + 0x004B2'i32, 501, + 0x004B4'i32, 501, + 0x004B6'i32, 501, + 0x004B8'i32, 501, + 0x004BA'i32, 501, + 0x004BC'i32, 501, + 0x004BE'i32, 501, + 0x004C0'i32, 515, + 0x004C1'i32, 501, + 0x004C3'i32, 501, + 0x004C5'i32, 501, + 0x004C7'i32, 501, + 0x004C9'i32, 501, + 0x004CB'i32, 501, + 0x004CD'i32, 501, + 0x004D0'i32, 501, + 0x004D2'i32, 501, + 0x004D4'i32, 501, + 0x004D6'i32, 501, + 0x004D8'i32, 501, + 0x004DA'i32, 501, + 0x004DC'i32, 501, + 0x004DE'i32, 501, + 0x004E0'i32, 501, + 0x004E2'i32, 501, + 0x004E4'i32, 501, + 0x004E6'i32, 501, + 0x004E8'i32, 501, + 0x004EA'i32, 501, + 0x004EC'i32, 501, + 0x004EE'i32, 501, + 0x004F0'i32, 501, + 0x004F2'i32, 501, + 0x004F4'i32, 501, + 0x004F6'i32, 501, + 0x004F8'i32, 501, + 0x004FA'i32, 501, + 0x004FC'i32, 501, + 0x004FE'i32, 501, + 0x00500'i32, 501, + 0x00502'i32, 501, + 0x00504'i32, 501, + 0x00506'i32, 501, + 0x00508'i32, 501, + 0x0050A'i32, 501, + 0x0050C'i32, 501, + 0x0050E'i32, 501, + 0x00510'i32, 501, + 0x00512'i32, 501, + 0x00514'i32, 501, + 0x00516'i32, 501, + 0x00518'i32, 501, + 0x0051A'i32, 501, + 0x0051C'i32, 501, + 0x0051E'i32, 501, + 0x00520'i32, 501, + 0x00522'i32, 501, + 0x00524'i32, 501, + 0x00526'i32, 501, + 0x00528'i32, 501, + 0x0052A'i32, 501, + 0x0052C'i32, 501, + 0x0052E'i32, 501, + 0x010C7'i32, 7764, + 0x010CD'i32, 7764, + 0x01E00'i32, 501, + 0x01E02'i32, 501, + 0x01E04'i32, 501, + 0x01E06'i32, 501, + 0x01E08'i32, 501, + 0x01E0A'i32, 501, + 0x01E0C'i32, 501, + 0x01E0E'i32, 501, + 0x01E10'i32, 501, + 0x01E12'i32, 501, + 0x01E14'i32, 501, + 0x01E16'i32, 501, + 0x01E18'i32, 501, + 0x01E1A'i32, 501, + 0x01E1C'i32, 501, + 0x01E1E'i32, 501, + 0x01E20'i32, 501, + 0x01E22'i32, 501, + 0x01E24'i32, 501, + 0x01E26'i32, 501, + 0x01E28'i32, 501, + 0x01E2A'i32, 501, + 0x01E2C'i32, 501, + 0x01E2E'i32, 501, + 0x01E30'i32, 501, + 0x01E32'i32, 501, + 0x01E34'i32, 501, + 0x01E36'i32, 501, + 0x01E38'i32, 501, + 0x01E3A'i32, 501, + 0x01E3C'i32, 501, + 0x01E3E'i32, 501, + 0x01E40'i32, 501, + 0x01E42'i32, 501, + 0x01E44'i32, 501, + 0x01E46'i32, 501, + 0x01E48'i32, 501, + 0x01E4A'i32, 501, + 0x01E4C'i32, 501, + 0x01E4E'i32, 501, + 0x01E50'i32, 501, + 0x01E52'i32, 501, + 0x01E54'i32, 501, + 0x01E56'i32, 501, + 0x01E58'i32, 501, + 0x01E5A'i32, 501, + 0x01E5C'i32, 501, + 0x01E5E'i32, 501, + 0x01E60'i32, 501, + 0x01E62'i32, 501, + 0x01E64'i32, 501, + 0x01E66'i32, 501, + 0x01E68'i32, 501, + 0x01E6A'i32, 501, + 0x01E6C'i32, 501, + 0x01E6E'i32, 501, + 0x01E70'i32, 501, + 0x01E72'i32, 501, + 0x01E74'i32, 501, + 0x01E76'i32, 501, + 0x01E78'i32, 501, + 0x01E7A'i32, 501, + 0x01E7C'i32, 501, + 0x01E7E'i32, 501, + 0x01E80'i32, 501, + 0x01E82'i32, 501, + 0x01E84'i32, 501, + 0x01E86'i32, 501, + 0x01E88'i32, 501, + 0x01E8A'i32, 501, + 0x01E8C'i32, 501, + 0x01E8E'i32, 501, + 0x01E90'i32, 501, + 0x01E92'i32, 501, + 0x01E94'i32, 501, + 0x01E9E'i32, -7115, + 0x01EA0'i32, 501, + 0x01EA2'i32, 501, + 0x01EA4'i32, 501, + 0x01EA6'i32, 501, + 0x01EA8'i32, 501, + 0x01EAA'i32, 501, + 0x01EAC'i32, 501, + 0x01EAE'i32, 501, + 0x01EB0'i32, 501, + 0x01EB2'i32, 501, + 0x01EB4'i32, 501, + 0x01EB6'i32, 501, + 0x01EB8'i32, 501, + 0x01EBA'i32, 501, + 0x01EBC'i32, 501, + 0x01EBE'i32, 501, + 0x01EC0'i32, 501, + 0x01EC2'i32, 501, + 0x01EC4'i32, 501, + 0x01EC6'i32, 501, + 0x01EC8'i32, 501, + 0x01ECA'i32, 501, + 0x01ECC'i32, 501, + 0x01ECE'i32, 501, + 0x01ED0'i32, 501, + 0x01ED2'i32, 501, + 0x01ED4'i32, 501, + 0x01ED6'i32, 501, + 0x01ED8'i32, 501, + 0x01EDA'i32, 501, + 0x01EDC'i32, 501, + 0x01EDE'i32, 501, + 0x01EE0'i32, 501, + 0x01EE2'i32, 501, + 0x01EE4'i32, 501, + 0x01EE6'i32, 501, + 0x01EE8'i32, 501, + 0x01EEA'i32, 501, + 0x01EEC'i32, 501, + 0x01EEE'i32, 501, + 0x01EF0'i32, 501, + 0x01EF2'i32, 501, + 0x01EF4'i32, 501, + 0x01EF6'i32, 501, + 0x01EF8'i32, 501, + 0x01EFA'i32, 501, + 0x01EFC'i32, 501, + 0x01EFE'i32, 501, + 0x01F59'i32, 492, + 0x01F5B'i32, 492, + 0x01F5D'i32, 492, + 0x01F5F'i32, 492, + 0x01FBC'i32, 491, + 0x01FCC'i32, 491, + 0x01FEC'i32, 493, + 0x01FFC'i32, 491, + 0x02126'i32, -7017, + 0x0212A'i32, -7883, + 0x0212B'i32, -7762, + 0x02132'i32, 528, + 0x02183'i32, 501, + 0x02C60'i32, 501, + 0x02C62'i32, -10243, + 0x02C63'i32, -3314, + 0x02C64'i32, -10227, + 0x02C67'i32, 501, + 0x02C69'i32, 501, + 0x02C6B'i32, 501, + 0x02C6D'i32, -10280, + 0x02C6E'i32, -10249, + 0x02C6F'i32, -10283, + 0x02C70'i32, -10282, + 0x02C72'i32, 501, + 0x02C75'i32, 501, + 0x02C80'i32, 501, + 0x02C82'i32, 501, + 0x02C84'i32, 501, + 0x02C86'i32, 501, + 0x02C88'i32, 501, + 0x02C8A'i32, 501, + 0x02C8C'i32, 501, + 0x02C8E'i32, 501, + 0x02C90'i32, 501, + 0x02C92'i32, 501, + 0x02C94'i32, 501, + 0x02C96'i32, 501, + 0x02C98'i32, 501, + 0x02C9A'i32, 501, + 0x02C9C'i32, 501, + 0x02C9E'i32, 501, + 0x02CA0'i32, 501, + 0x02CA2'i32, 501, + 0x02CA4'i32, 501, + 0x02CA6'i32, 501, + 0x02CA8'i32, 501, + 0x02CAA'i32, 501, + 0x02CAC'i32, 501, + 0x02CAE'i32, 501, + 0x02CB0'i32, 501, + 0x02CB2'i32, 501, + 0x02CB4'i32, 501, + 0x02CB6'i32, 501, + 0x02CB8'i32, 501, + 0x02CBA'i32, 501, + 0x02CBC'i32, 501, + 0x02CBE'i32, 501, + 0x02CC0'i32, 501, + 0x02CC2'i32, 501, + 0x02CC4'i32, 501, + 0x02CC6'i32, 501, + 0x02CC8'i32, 501, + 0x02CCA'i32, 501, + 0x02CCC'i32, 501, + 0x02CCE'i32, 501, + 0x02CD0'i32, 501, + 0x02CD2'i32, 501, + 0x02CD4'i32, 501, + 0x02CD6'i32, 501, + 0x02CD8'i32, 501, + 0x02CDA'i32, 501, + 0x02CDC'i32, 501, + 0x02CDE'i32, 501, + 0x02CE0'i32, 501, + 0x02CE2'i32, 501, + 0x02CEB'i32, 501, + 0x02CED'i32, 501, + 0x02CF2'i32, 501, + 0x0A640'i32, 501, + 0x0A642'i32, 501, + 0x0A644'i32, 501, + 0x0A646'i32, 501, + 0x0A648'i32, 501, + 0x0A64A'i32, 501, + 0x0A64C'i32, 501, + 0x0A64E'i32, 501, + 0x0A650'i32, 501, + 0x0A652'i32, 501, + 0x0A654'i32, 501, + 0x0A656'i32, 501, + 0x0A658'i32, 501, + 0x0A65A'i32, 501, + 0x0A65C'i32, 501, + 0x0A65E'i32, 501, + 0x0A660'i32, 501, + 0x0A662'i32, 501, + 0x0A664'i32, 501, + 0x0A666'i32, 501, + 0x0A668'i32, 501, + 0x0A66A'i32, 501, + 0x0A66C'i32, 501, + 0x0A680'i32, 501, + 0x0A682'i32, 501, + 0x0A684'i32, 501, + 0x0A686'i32, 501, + 0x0A688'i32, 501, + 0x0A68A'i32, 501, + 0x0A68C'i32, 501, + 0x0A68E'i32, 501, + 0x0A690'i32, 501, + 0x0A692'i32, 501, + 0x0A694'i32, 501, + 0x0A696'i32, 501, + 0x0A698'i32, 501, + 0x0A69A'i32, 501, + 0x0A722'i32, 501, + 0x0A724'i32, 501, + 0x0A726'i32, 501, + 0x0A728'i32, 501, + 0x0A72A'i32, 501, + 0x0A72C'i32, 501, + 0x0A72E'i32, 501, + 0x0A732'i32, 501, + 0x0A734'i32, 501, + 0x0A736'i32, 501, + 0x0A738'i32, 501, + 0x0A73A'i32, 501, + 0x0A73C'i32, 501, + 0x0A73E'i32, 501, + 0x0A740'i32, 501, + 0x0A742'i32, 501, + 0x0A744'i32, 501, + 0x0A746'i32, 501, + 0x0A748'i32, 501, + 0x0A74A'i32, 501, + 0x0A74C'i32, 501, + 0x0A74E'i32, 501, + 0x0A750'i32, 501, + 0x0A752'i32, 501, + 0x0A754'i32, 501, + 0x0A756'i32, 501, + 0x0A758'i32, 501, + 0x0A75A'i32, 501, + 0x0A75C'i32, 501, + 0x0A75E'i32, 501, + 0x0A760'i32, 501, + 0x0A762'i32, 501, + 0x0A764'i32, 501, + 0x0A766'i32, 501, + 0x0A768'i32, 501, + 0x0A76A'i32, 501, + 0x0A76C'i32, 501, + 0x0A76E'i32, 501, + 0x0A779'i32, 501, + 0x0A77B'i32, 501, + 0x0A77D'i32, -34832, + 0x0A77E'i32, 501, + 0x0A780'i32, 501, + 0x0A782'i32, 501, + 0x0A784'i32, 501, + 0x0A786'i32, 501, + 0x0A78B'i32, 501, + 0x0A78D'i32, -41780, + 0x0A790'i32, 501, + 0x0A792'i32, 501, + 0x0A796'i32, 501, + 0x0A798'i32, 501, + 0x0A79A'i32, 501, + 0x0A79C'i32, 501, + 0x0A79E'i32, 501, + 0x0A7A0'i32, 501, + 0x0A7A2'i32, 501, + 0x0A7A4'i32, 501, + 0x0A7A6'i32, 501, + 0x0A7A8'i32, 501, + 0x0A7AA'i32, -41808, + 0x0A7AB'i32, -41819, + 0x0A7AC'i32, -41815, + 0x0A7AD'i32, -41805, + 0x0A7AE'i32, -41808, + 0x0A7B0'i32, -41758, + 0x0A7B1'i32, -41782, + 0x0A7B2'i32, -41761, + 0x0A7B3'i32, 1428, + 0x0A7B4'i32, 501, + 0x0A7B6'i32, 501, + 0x0A7B8'i32, 501, + 0x0A7BA'i32, 501, + 0x0A7BC'i32, 501, + 0x0A7BE'i32, 501, + 0x0A7C2'i32, 501, + 0x0A7C4'i32, 452, + 0x0A7C5'i32, -41807, + 0x0A7C6'i32, -34884, ] toUpperRanges = [ - 0x00061, 0x0007A, 468, - 0x000E0, 0x000F6, 468, - 0x000F8, 0x000FE, 468, - 0x0023F, 0x00240, 11315, - 0x00256, 0x00257, 295, - 0x0028A, 0x0028B, 283, - 0x0037B, 0x0037D, 630, - 0x003AD, 0x003AF, 463, - 0x003B1, 0x003C1, 468, - 0x003C3, 0x003CB, 468, - 0x003CD, 0x003CE, 437, - 0x00430, 0x0044F, 468, - 0x00450, 0x0045F, 420, - 0x00561, 0x00586, 452, - 0x010D0, 0x010FA, 3508, - 0x010FD, 0x010FF, 3508, - 0x013F8, 0x013FD, 492, - 0x01C83, 0x01C84, -5742, - 0x01F00, 0x01F07, 508, - 0x01F10, 0x01F15, 508, - 0x01F20, 0x01F27, 508, - 0x01F30, 0x01F37, 508, - 0x01F40, 0x01F45, 508, - 0x01F60, 0x01F67, 508, - 0x01F70, 0x01F71, 574, - 0x01F72, 0x01F75, 586, - 0x01F76, 0x01F77, 600, - 0x01F78, 0x01F79, 628, - 0x01F7A, 0x01F7B, 612, - 0x01F7C, 0x01F7D, 626, - 0x01F80, 0x01F87, 508, - 0x01F90, 0x01F97, 508, - 0x01FA0, 0x01FA7, 508, - 0x01FB0, 0x01FB1, 508, - 0x01FD0, 0x01FD1, 508, - 0x01FE0, 0x01FE1, 508, - 0x02C30, 0x02C5E, 452, - 0x02D00, 0x02D25, -6764, - 0x0AB70, 0x0ABBF, -38364, - 0x0FF41, 0x0FF5A, 468, - 0x10428, 0x1044F, 460, - 0x104D8, 0x104FB, 460, - 0x10CC0, 0x10CF2, 436, - 0x118C0, 0x118DF, 468, - 0x16E60, 0x16E7F, 468, - 0x1E922, 0x1E943, 466, + 0x00061'i32, 0x0007A'i32, 468, + 0x000E0'i32, 0x000F6'i32, 468, + 0x000F8'i32, 0x000FE'i32, 468, + 0x0023F'i32, 0x00240'i32, 11315, + 0x00256'i32, 0x00257'i32, 295, + 0x0028A'i32, 0x0028B'i32, 283, + 0x0037B'i32, 0x0037D'i32, 630, + 0x003AD'i32, 0x003AF'i32, 463, + 0x003B1'i32, 0x003C1'i32, 468, + 0x003C3'i32, 0x003CB'i32, 468, + 0x003CD'i32, 0x003CE'i32, 437, + 0x00430'i32, 0x0044F'i32, 468, + 0x00450'i32, 0x0045F'i32, 420, + 0x00561'i32, 0x00586'i32, 452, + 0x010D0'i32, 0x010FA'i32, 3508, + 0x010FD'i32, 0x010FF'i32, 3508, + 0x013F8'i32, 0x013FD'i32, 492, + 0x01C83'i32, 0x01C84'i32, -5742, + 0x01F00'i32, 0x01F07'i32, 508, + 0x01F10'i32, 0x01F15'i32, 508, + 0x01F20'i32, 0x01F27'i32, 508, + 0x01F30'i32, 0x01F37'i32, 508, + 0x01F40'i32, 0x01F45'i32, 508, + 0x01F60'i32, 0x01F67'i32, 508, + 0x01F70'i32, 0x01F71'i32, 574, + 0x01F72'i32, 0x01F75'i32, 586, + 0x01F76'i32, 0x01F77'i32, 600, + 0x01F78'i32, 0x01F79'i32, 628, + 0x01F7A'i32, 0x01F7B'i32, 612, + 0x01F7C'i32, 0x01F7D'i32, 626, + 0x01F80'i32, 0x01F87'i32, 508, + 0x01F90'i32, 0x01F97'i32, 508, + 0x01FA0'i32, 0x01FA7'i32, 508, + 0x01FB0'i32, 0x01FB1'i32, 508, + 0x01FD0'i32, 0x01FD1'i32, 508, + 0x01FE0'i32, 0x01FE1'i32, 508, + 0x02C30'i32, 0x02C5E'i32, 452, + 0x02D00'i32, 0x02D25'i32, -6764, + 0x0AB70'i32, 0x0ABBF'i32, -38364, + 0x0FF41'i32, 0x0FF5A'i32, 468, + 0x10428'i32, 0x1044F'i32, 460, + 0x104D8'i32, 0x104FB'i32, 460, + 0x10CC0'i32, 0x10CF2'i32, 436, + 0x118C0'i32, 0x118DF'i32, 468, + 0x16E60'i32, 0x16E7F'i32, 468, + 0x1E922'i32, 0x1E943'i32, 466, ] toUpperSinglets = [ - 0x000B5, 1243, - 0x000FF, 621, - 0x00101, 499, - 0x00103, 499, - 0x00105, 499, - 0x00107, 499, - 0x00109, 499, - 0x0010B, 499, - 0x0010D, 499, - 0x0010F, 499, - 0x00111, 499, - 0x00113, 499, - 0x00115, 499, - 0x00117, 499, - 0x00119, 499, - 0x0011B, 499, - 0x0011D, 499, - 0x0011F, 499, - 0x00121, 499, - 0x00123, 499, - 0x00125, 499, - 0x00127, 499, - 0x00129, 499, - 0x0012B, 499, - 0x0012D, 499, - 0x0012F, 499, - 0x00131, 268, - 0x00133, 499, - 0x00135, 499, - 0x00137, 499, - 0x0013A, 499, - 0x0013C, 499, - 0x0013E, 499, - 0x00140, 499, - 0x00142, 499, - 0x00144, 499, - 0x00146, 499, - 0x00148, 499, - 0x0014B, 499, - 0x0014D, 499, - 0x0014F, 499, - 0x00151, 499, - 0x00153, 499, - 0x00155, 499, - 0x00157, 499, - 0x00159, 499, - 0x0015B, 499, - 0x0015D, 499, - 0x0015F, 499, - 0x00161, 499, - 0x00163, 499, - 0x00165, 499, - 0x00167, 499, - 0x00169, 499, - 0x0016B, 499, - 0x0016D, 499, - 0x0016F, 499, - 0x00171, 499, - 0x00173, 499, - 0x00175, 499, - 0x00177, 499, - 0x0017A, 499, - 0x0017C, 499, - 0x0017E, 499, - 0x0017F, 200, - 0x00180, 695, - 0x00183, 499, - 0x00185, 499, - 0x00188, 499, - 0x0018C, 499, - 0x00192, 499, - 0x00195, 597, - 0x00199, 499, - 0x0019A, 663, - 0x0019E, 630, - 0x001A1, 499, - 0x001A3, 499, - 0x001A5, 499, - 0x001A8, 499, - 0x001AD, 499, - 0x001B0, 499, - 0x001B4, 499, - 0x001B6, 499, - 0x001B9, 499, - 0x001BD, 499, - 0x001BF, 556, - 0x001C5, 499, - 0x001C6, 498, - 0x001C8, 499, - 0x001C9, 498, - 0x001CB, 499, - 0x001CC, 498, - 0x001CE, 499, - 0x001D0, 499, - 0x001D2, 499, - 0x001D4, 499, - 0x001D6, 499, - 0x001D8, 499, - 0x001DA, 499, - 0x001DC, 499, - 0x001DD, 421, - 0x001DF, 499, - 0x001E1, 499, - 0x001E3, 499, - 0x001E5, 499, - 0x001E7, 499, - 0x001E9, 499, - 0x001EB, 499, - 0x001ED, 499, - 0x001EF, 499, - 0x001F2, 499, - 0x001F3, 498, - 0x001F5, 499, - 0x001F9, 499, - 0x001FB, 499, - 0x001FD, 499, - 0x001FF, 499, - 0x00201, 499, - 0x00203, 499, - 0x00205, 499, - 0x00207, 499, - 0x00209, 499, - 0x0020B, 499, - 0x0020D, 499, - 0x0020F, 499, - 0x00211, 499, - 0x00213, 499, - 0x00215, 499, - 0x00217, 499, - 0x00219, 499, - 0x0021B, 499, - 0x0021D, 499, - 0x0021F, 499, - 0x00223, 499, - 0x00225, 499, - 0x00227, 499, - 0x00229, 499, - 0x0022B, 499, - 0x0022D, 499, - 0x0022F, 499, - 0x00231, 499, - 0x00233, 499, - 0x0023C, 499, - 0x00242, 499, - 0x00247, 499, - 0x00249, 499, - 0x0024B, 499, - 0x0024D, 499, - 0x0024F, 499, - 0x00250, 11283, - 0x00251, 11280, - 0x00252, 11282, - 0x00253, 290, - 0x00254, 294, - 0x00259, 298, - 0x0025B, 297, - 0x0025C, 42819, - 0x00260, 295, - 0x00261, 42815, - 0x00263, 293, - 0x00265, 42780, - 0x00266, 42808, - 0x00268, 291, - 0x00269, 289, - 0x0026A, 42808, - 0x0026B, 11243, - 0x0026C, 42805, - 0x0026F, 289, - 0x00271, 11249, - 0x00272, 287, - 0x00275, 286, - 0x0027D, 11227, - 0x00280, 282, - 0x00282, 42807, - 0x00283, 282, - 0x00287, 42782, - 0x00288, 282, - 0x00289, 431, - 0x0028C, 429, - 0x00292, 281, - 0x0029D, 42761, - 0x0029E, 42758, - 0x00371, 499, - 0x00373, 499, - 0x00377, 499, - 0x003AC, 462, - 0x003C2, 469, - 0x003CC, 436, - 0x003D0, 438, - 0x003D1, 443, - 0x003D5, 453, - 0x003D6, 446, - 0x003D7, 492, - 0x003D9, 499, - 0x003DB, 499, - 0x003DD, 499, - 0x003DF, 499, - 0x003E1, 499, - 0x003E3, 499, - 0x003E5, 499, - 0x003E7, 499, - 0x003E9, 499, - 0x003EB, 499, - 0x003ED, 499, - 0x003EF, 499, - 0x003F0, 414, - 0x003F1, 420, - 0x003F2, 507, - 0x003F3, 384, - 0x003F5, 404, - 0x003F8, 499, - 0x003FB, 499, - 0x00461, 499, - 0x00463, 499, - 0x00465, 499, - 0x00467, 499, - 0x00469, 499, - 0x0046B, 499, - 0x0046D, 499, - 0x0046F, 499, - 0x00471, 499, - 0x00473, 499, - 0x00475, 499, - 0x00477, 499, - 0x00479, 499, - 0x0047B, 499, - 0x0047D, 499, - 0x0047F, 499, - 0x00481, 499, - 0x0048B, 499, - 0x0048D, 499, - 0x0048F, 499, - 0x00491, 499, - 0x00493, 499, - 0x00495, 499, - 0x00497, 499, - 0x00499, 499, - 0x0049B, 499, - 0x0049D, 499, - 0x0049F, 499, - 0x004A1, 499, - 0x004A3, 499, - 0x004A5, 499, - 0x004A7, 499, - 0x004A9, 499, - 0x004AB, 499, - 0x004AD, 499, - 0x004AF, 499, - 0x004B1, 499, - 0x004B3, 499, - 0x004B5, 499, - 0x004B7, 499, - 0x004B9, 499, - 0x004BB, 499, - 0x004BD, 499, - 0x004BF, 499, - 0x004C2, 499, - 0x004C4, 499, - 0x004C6, 499, - 0x004C8, 499, - 0x004CA, 499, - 0x004CC, 499, - 0x004CE, 499, - 0x004CF, 485, - 0x004D1, 499, - 0x004D3, 499, - 0x004D5, 499, - 0x004D7, 499, - 0x004D9, 499, - 0x004DB, 499, - 0x004DD, 499, - 0x004DF, 499, - 0x004E1, 499, - 0x004E3, 499, - 0x004E5, 499, - 0x004E7, 499, - 0x004E9, 499, - 0x004EB, 499, - 0x004ED, 499, - 0x004EF, 499, - 0x004F1, 499, - 0x004F3, 499, - 0x004F5, 499, - 0x004F7, 499, - 0x004F9, 499, - 0x004FB, 499, - 0x004FD, 499, - 0x004FF, 499, - 0x00501, 499, - 0x00503, 499, - 0x00505, 499, - 0x00507, 499, - 0x00509, 499, - 0x0050B, 499, - 0x0050D, 499, - 0x0050F, 499, - 0x00511, 499, - 0x00513, 499, - 0x00515, 499, - 0x00517, 499, - 0x00519, 499, - 0x0051B, 499, - 0x0051D, 499, - 0x0051F, 499, - 0x00521, 499, - 0x00523, 499, - 0x00525, 499, - 0x00527, 499, - 0x00529, 499, - 0x0052B, 499, - 0x0052D, 499, - 0x0052F, 499, - 0x01C80, -5754, - 0x01C81, -5753, - 0x01C82, -5744, - 0x01C85, -5743, - 0x01C86, -5736, - 0x01C87, -5681, - 0x01C88, 35766, - 0x01D79, 35832, - 0x01D7D, 4314, - 0x01D8E, 35884, - 0x01E01, 499, - 0x01E03, 499, - 0x01E05, 499, - 0x01E07, 499, - 0x01E09, 499, - 0x01E0B, 499, - 0x01E0D, 499, - 0x01E0F, 499, - 0x01E11, 499, - 0x01E13, 499, - 0x01E15, 499, - 0x01E17, 499, - 0x01E19, 499, - 0x01E1B, 499, - 0x01E1D, 499, - 0x01E1F, 499, - 0x01E21, 499, - 0x01E23, 499, - 0x01E25, 499, - 0x01E27, 499, - 0x01E29, 499, - 0x01E2B, 499, - 0x01E2D, 499, - 0x01E2F, 499, - 0x01E31, 499, - 0x01E33, 499, - 0x01E35, 499, - 0x01E37, 499, - 0x01E39, 499, - 0x01E3B, 499, - 0x01E3D, 499, - 0x01E3F, 499, - 0x01E41, 499, - 0x01E43, 499, - 0x01E45, 499, - 0x01E47, 499, - 0x01E49, 499, - 0x01E4B, 499, - 0x01E4D, 499, - 0x01E4F, 499, - 0x01E51, 499, - 0x01E53, 499, - 0x01E55, 499, - 0x01E57, 499, - 0x01E59, 499, - 0x01E5B, 499, - 0x01E5D, 499, - 0x01E5F, 499, - 0x01E61, 499, - 0x01E63, 499, - 0x01E65, 499, - 0x01E67, 499, - 0x01E69, 499, - 0x01E6B, 499, - 0x01E6D, 499, - 0x01E6F, 499, - 0x01E71, 499, - 0x01E73, 499, - 0x01E75, 499, - 0x01E77, 499, - 0x01E79, 499, - 0x01E7B, 499, - 0x01E7D, 499, - 0x01E7F, 499, - 0x01E81, 499, - 0x01E83, 499, - 0x01E85, 499, - 0x01E87, 499, - 0x01E89, 499, - 0x01E8B, 499, - 0x01E8D, 499, - 0x01E8F, 499, - 0x01E91, 499, - 0x01E93, 499, - 0x01E95, 499, - 0x01E9B, 441, - 0x01EA1, 499, - 0x01EA3, 499, - 0x01EA5, 499, - 0x01EA7, 499, - 0x01EA9, 499, - 0x01EAB, 499, - 0x01EAD, 499, - 0x01EAF, 499, - 0x01EB1, 499, - 0x01EB3, 499, - 0x01EB5, 499, - 0x01EB7, 499, - 0x01EB9, 499, - 0x01EBB, 499, - 0x01EBD, 499, - 0x01EBF, 499, - 0x01EC1, 499, - 0x01EC3, 499, - 0x01EC5, 499, - 0x01EC7, 499, - 0x01EC9, 499, - 0x01ECB, 499, - 0x01ECD, 499, - 0x01ECF, 499, - 0x01ED1, 499, - 0x01ED3, 499, - 0x01ED5, 499, - 0x01ED7, 499, - 0x01ED9, 499, - 0x01EDB, 499, - 0x01EDD, 499, - 0x01EDF, 499, - 0x01EE1, 499, - 0x01EE3, 499, - 0x01EE5, 499, - 0x01EE7, 499, - 0x01EE9, 499, - 0x01EEB, 499, - 0x01EED, 499, - 0x01EEF, 499, - 0x01EF1, 499, - 0x01EF3, 499, - 0x01EF5, 499, - 0x01EF7, 499, - 0x01EF9, 499, - 0x01EFB, 499, - 0x01EFD, 499, - 0x01EFF, 499, - 0x01F51, 508, - 0x01F53, 508, - 0x01F55, 508, - 0x01F57, 508, - 0x01FB3, 509, - 0x01FBE, -6705, - 0x01FC3, 509, - 0x01FE5, 507, - 0x01FF3, 509, - 0x0214E, 472, - 0x02184, 499, - 0x02C61, 499, - 0x02C65, -10295, - 0x02C66, -10292, - 0x02C68, 499, - 0x02C6A, 499, - 0x02C6C, 499, - 0x02C73, 499, - 0x02C76, 499, - 0x02C81, 499, - 0x02C83, 499, - 0x02C85, 499, - 0x02C87, 499, - 0x02C89, 499, - 0x02C8B, 499, - 0x02C8D, 499, - 0x02C8F, 499, - 0x02C91, 499, - 0x02C93, 499, - 0x02C95, 499, - 0x02C97, 499, - 0x02C99, 499, - 0x02C9B, 499, - 0x02C9D, 499, - 0x02C9F, 499, - 0x02CA1, 499, - 0x02CA3, 499, - 0x02CA5, 499, - 0x02CA7, 499, - 0x02CA9, 499, - 0x02CAB, 499, - 0x02CAD, 499, - 0x02CAF, 499, - 0x02CB1, 499, - 0x02CB3, 499, - 0x02CB5, 499, - 0x02CB7, 499, - 0x02CB9, 499, - 0x02CBB, 499, - 0x02CBD, 499, - 0x02CBF, 499, - 0x02CC1, 499, - 0x02CC3, 499, - 0x02CC5, 499, - 0x02CC7, 499, - 0x02CC9, 499, - 0x02CCB, 499, - 0x02CCD, 499, - 0x02CCF, 499, - 0x02CD1, 499, - 0x02CD3, 499, - 0x02CD5, 499, - 0x02CD7, 499, - 0x02CD9, 499, - 0x02CDB, 499, - 0x02CDD, 499, - 0x02CDF, 499, - 0x02CE1, 499, - 0x02CE3, 499, - 0x02CEC, 499, - 0x02CEE, 499, - 0x02CF3, 499, - 0x02D27, -6764, - 0x02D2D, -6764, - 0x0A641, 499, - 0x0A643, 499, - 0x0A645, 499, - 0x0A647, 499, - 0x0A649, 499, - 0x0A64B, 499, - 0x0A64D, 499, - 0x0A64F, 499, - 0x0A651, 499, - 0x0A653, 499, - 0x0A655, 499, - 0x0A657, 499, - 0x0A659, 499, - 0x0A65B, 499, - 0x0A65D, 499, - 0x0A65F, 499, - 0x0A661, 499, - 0x0A663, 499, - 0x0A665, 499, - 0x0A667, 499, - 0x0A669, 499, - 0x0A66B, 499, - 0x0A66D, 499, - 0x0A681, 499, - 0x0A683, 499, - 0x0A685, 499, - 0x0A687, 499, - 0x0A689, 499, - 0x0A68B, 499, - 0x0A68D, 499, - 0x0A68F, 499, - 0x0A691, 499, - 0x0A693, 499, - 0x0A695, 499, - 0x0A697, 499, - 0x0A699, 499, - 0x0A69B, 499, - 0x0A723, 499, - 0x0A725, 499, - 0x0A727, 499, - 0x0A729, 499, - 0x0A72B, 499, - 0x0A72D, 499, - 0x0A72F, 499, - 0x0A733, 499, - 0x0A735, 499, - 0x0A737, 499, - 0x0A739, 499, - 0x0A73B, 499, - 0x0A73D, 499, - 0x0A73F, 499, - 0x0A741, 499, - 0x0A743, 499, - 0x0A745, 499, - 0x0A747, 499, - 0x0A749, 499, - 0x0A74B, 499, - 0x0A74D, 499, - 0x0A74F, 499, - 0x0A751, 499, - 0x0A753, 499, - 0x0A755, 499, - 0x0A757, 499, - 0x0A759, 499, - 0x0A75B, 499, - 0x0A75D, 499, - 0x0A75F, 499, - 0x0A761, 499, - 0x0A763, 499, - 0x0A765, 499, - 0x0A767, 499, - 0x0A769, 499, - 0x0A76B, 499, - 0x0A76D, 499, - 0x0A76F, 499, - 0x0A77A, 499, - 0x0A77C, 499, - 0x0A77F, 499, - 0x0A781, 499, - 0x0A783, 499, - 0x0A785, 499, - 0x0A787, 499, - 0x0A78C, 499, - 0x0A791, 499, - 0x0A793, 499, - 0x0A794, 548, - 0x0A797, 499, - 0x0A799, 499, - 0x0A79B, 499, - 0x0A79D, 499, - 0x0A79F, 499, - 0x0A7A1, 499, - 0x0A7A3, 499, - 0x0A7A5, 499, - 0x0A7A7, 499, - 0x0A7A9, 499, - 0x0A7B5, 499, - 0x0A7B7, 499, - 0x0A7B9, 499, - 0x0A7BB, 499, - 0x0A7BD, 499, - 0x0A7BF, 499, - 0x0A7C3, 499, - 0x0AB53, -428, + 0x000B5'i32, 1243, + 0x000FF'i32, 621, + 0x00101'i32, 499, + 0x00103'i32, 499, + 0x00105'i32, 499, + 0x00107'i32, 499, + 0x00109'i32, 499, + 0x0010B'i32, 499, + 0x0010D'i32, 499, + 0x0010F'i32, 499, + 0x00111'i32, 499, + 0x00113'i32, 499, + 0x00115'i32, 499, + 0x00117'i32, 499, + 0x00119'i32, 499, + 0x0011B'i32, 499, + 0x0011D'i32, 499, + 0x0011F'i32, 499, + 0x00121'i32, 499, + 0x00123'i32, 499, + 0x00125'i32, 499, + 0x00127'i32, 499, + 0x00129'i32, 499, + 0x0012B'i32, 499, + 0x0012D'i32, 499, + 0x0012F'i32, 499, + 0x00131'i32, 268, + 0x00133'i32, 499, + 0x00135'i32, 499, + 0x00137'i32, 499, + 0x0013A'i32, 499, + 0x0013C'i32, 499, + 0x0013E'i32, 499, + 0x00140'i32, 499, + 0x00142'i32, 499, + 0x00144'i32, 499, + 0x00146'i32, 499, + 0x00148'i32, 499, + 0x0014B'i32, 499, + 0x0014D'i32, 499, + 0x0014F'i32, 499, + 0x00151'i32, 499, + 0x00153'i32, 499, + 0x00155'i32, 499, + 0x00157'i32, 499, + 0x00159'i32, 499, + 0x0015B'i32, 499, + 0x0015D'i32, 499, + 0x0015F'i32, 499, + 0x00161'i32, 499, + 0x00163'i32, 499, + 0x00165'i32, 499, + 0x00167'i32, 499, + 0x00169'i32, 499, + 0x0016B'i32, 499, + 0x0016D'i32, 499, + 0x0016F'i32, 499, + 0x00171'i32, 499, + 0x00173'i32, 499, + 0x00175'i32, 499, + 0x00177'i32, 499, + 0x0017A'i32, 499, + 0x0017C'i32, 499, + 0x0017E'i32, 499, + 0x0017F'i32, 200, + 0x00180'i32, 695, + 0x00183'i32, 499, + 0x00185'i32, 499, + 0x00188'i32, 499, + 0x0018C'i32, 499, + 0x00192'i32, 499, + 0x00195'i32, 597, + 0x00199'i32, 499, + 0x0019A'i32, 663, + 0x0019E'i32, 630, + 0x001A1'i32, 499, + 0x001A3'i32, 499, + 0x001A5'i32, 499, + 0x001A8'i32, 499, + 0x001AD'i32, 499, + 0x001B0'i32, 499, + 0x001B4'i32, 499, + 0x001B6'i32, 499, + 0x001B9'i32, 499, + 0x001BD'i32, 499, + 0x001BF'i32, 556, + 0x001C5'i32, 499, + 0x001C6'i32, 498, + 0x001C8'i32, 499, + 0x001C9'i32, 498, + 0x001CB'i32, 499, + 0x001CC'i32, 498, + 0x001CE'i32, 499, + 0x001D0'i32, 499, + 0x001D2'i32, 499, + 0x001D4'i32, 499, + 0x001D6'i32, 499, + 0x001D8'i32, 499, + 0x001DA'i32, 499, + 0x001DC'i32, 499, + 0x001DD'i32, 421, + 0x001DF'i32, 499, + 0x001E1'i32, 499, + 0x001E3'i32, 499, + 0x001E5'i32, 499, + 0x001E7'i32, 499, + 0x001E9'i32, 499, + 0x001EB'i32, 499, + 0x001ED'i32, 499, + 0x001EF'i32, 499, + 0x001F2'i32, 499, + 0x001F3'i32, 498, + 0x001F5'i32, 499, + 0x001F9'i32, 499, + 0x001FB'i32, 499, + 0x001FD'i32, 499, + 0x001FF'i32, 499, + 0x00201'i32, 499, + 0x00203'i32, 499, + 0x00205'i32, 499, + 0x00207'i32, 499, + 0x00209'i32, 499, + 0x0020B'i32, 499, + 0x0020D'i32, 499, + 0x0020F'i32, 499, + 0x00211'i32, 499, + 0x00213'i32, 499, + 0x00215'i32, 499, + 0x00217'i32, 499, + 0x00219'i32, 499, + 0x0021B'i32, 499, + 0x0021D'i32, 499, + 0x0021F'i32, 499, + 0x00223'i32, 499, + 0x00225'i32, 499, + 0x00227'i32, 499, + 0x00229'i32, 499, + 0x0022B'i32, 499, + 0x0022D'i32, 499, + 0x0022F'i32, 499, + 0x00231'i32, 499, + 0x00233'i32, 499, + 0x0023C'i32, 499, + 0x00242'i32, 499, + 0x00247'i32, 499, + 0x00249'i32, 499, + 0x0024B'i32, 499, + 0x0024D'i32, 499, + 0x0024F'i32, 499, + 0x00250'i32, 11283, + 0x00251'i32, 11280, + 0x00252'i32, 11282, + 0x00253'i32, 290, + 0x00254'i32, 294, + 0x00259'i32, 298, + 0x0025B'i32, 297, + 0x0025C'i32, 42819, + 0x00260'i32, 295, + 0x00261'i32, 42815, + 0x00263'i32, 293, + 0x00265'i32, 42780, + 0x00266'i32, 42808, + 0x00268'i32, 291, + 0x00269'i32, 289, + 0x0026A'i32, 42808, + 0x0026B'i32, 11243, + 0x0026C'i32, 42805, + 0x0026F'i32, 289, + 0x00271'i32, 11249, + 0x00272'i32, 287, + 0x00275'i32, 286, + 0x0027D'i32, 11227, + 0x00280'i32, 282, + 0x00282'i32, 42807, + 0x00283'i32, 282, + 0x00287'i32, 42782, + 0x00288'i32, 282, + 0x00289'i32, 431, + 0x0028C'i32, 429, + 0x00292'i32, 281, + 0x0029D'i32, 42761, + 0x0029E'i32, 42758, + 0x00371'i32, 499, + 0x00373'i32, 499, + 0x00377'i32, 499, + 0x003AC'i32, 462, + 0x003C2'i32, 469, + 0x003CC'i32, 436, + 0x003D0'i32, 438, + 0x003D1'i32, 443, + 0x003D5'i32, 453, + 0x003D6'i32, 446, + 0x003D7'i32, 492, + 0x003D9'i32, 499, + 0x003DB'i32, 499, + 0x003DD'i32, 499, + 0x003DF'i32, 499, + 0x003E1'i32, 499, + 0x003E3'i32, 499, + 0x003E5'i32, 499, + 0x003E7'i32, 499, + 0x003E9'i32, 499, + 0x003EB'i32, 499, + 0x003ED'i32, 499, + 0x003EF'i32, 499, + 0x003F0'i32, 414, + 0x003F1'i32, 420, + 0x003F2'i32, 507, + 0x003F3'i32, 384, + 0x003F5'i32, 404, + 0x003F8'i32, 499, + 0x003FB'i32, 499, + 0x00461'i32, 499, + 0x00463'i32, 499, + 0x00465'i32, 499, + 0x00467'i32, 499, + 0x00469'i32, 499, + 0x0046B'i32, 499, + 0x0046D'i32, 499, + 0x0046F'i32, 499, + 0x00471'i32, 499, + 0x00473'i32, 499, + 0x00475'i32, 499, + 0x00477'i32, 499, + 0x00479'i32, 499, + 0x0047B'i32, 499, + 0x0047D'i32, 499, + 0x0047F'i32, 499, + 0x00481'i32, 499, + 0x0048B'i32, 499, + 0x0048D'i32, 499, + 0x0048F'i32, 499, + 0x00491'i32, 499, + 0x00493'i32, 499, + 0x00495'i32, 499, + 0x00497'i32, 499, + 0x00499'i32, 499, + 0x0049B'i32, 499, + 0x0049D'i32, 499, + 0x0049F'i32, 499, + 0x004A1'i32, 499, + 0x004A3'i32, 499, + 0x004A5'i32, 499, + 0x004A7'i32, 499, + 0x004A9'i32, 499, + 0x004AB'i32, 499, + 0x004AD'i32, 499, + 0x004AF'i32, 499, + 0x004B1'i32, 499, + 0x004B3'i32, 499, + 0x004B5'i32, 499, + 0x004B7'i32, 499, + 0x004B9'i32, 499, + 0x004BB'i32, 499, + 0x004BD'i32, 499, + 0x004BF'i32, 499, + 0x004C2'i32, 499, + 0x004C4'i32, 499, + 0x004C6'i32, 499, + 0x004C8'i32, 499, + 0x004CA'i32, 499, + 0x004CC'i32, 499, + 0x004CE'i32, 499, + 0x004CF'i32, 485, + 0x004D1'i32, 499, + 0x004D3'i32, 499, + 0x004D5'i32, 499, + 0x004D7'i32, 499, + 0x004D9'i32, 499, + 0x004DB'i32, 499, + 0x004DD'i32, 499, + 0x004DF'i32, 499, + 0x004E1'i32, 499, + 0x004E3'i32, 499, + 0x004E5'i32, 499, + 0x004E7'i32, 499, + 0x004E9'i32, 499, + 0x004EB'i32, 499, + 0x004ED'i32, 499, + 0x004EF'i32, 499, + 0x004F1'i32, 499, + 0x004F3'i32, 499, + 0x004F5'i32, 499, + 0x004F7'i32, 499, + 0x004F9'i32, 499, + 0x004FB'i32, 499, + 0x004FD'i32, 499, + 0x004FF'i32, 499, + 0x00501'i32, 499, + 0x00503'i32, 499, + 0x00505'i32, 499, + 0x00507'i32, 499, + 0x00509'i32, 499, + 0x0050B'i32, 499, + 0x0050D'i32, 499, + 0x0050F'i32, 499, + 0x00511'i32, 499, + 0x00513'i32, 499, + 0x00515'i32, 499, + 0x00517'i32, 499, + 0x00519'i32, 499, + 0x0051B'i32, 499, + 0x0051D'i32, 499, + 0x0051F'i32, 499, + 0x00521'i32, 499, + 0x00523'i32, 499, + 0x00525'i32, 499, + 0x00527'i32, 499, + 0x00529'i32, 499, + 0x0052B'i32, 499, + 0x0052D'i32, 499, + 0x0052F'i32, 499, + 0x01C80'i32, -5754, + 0x01C81'i32, -5753, + 0x01C82'i32, -5744, + 0x01C85'i32, -5743, + 0x01C86'i32, -5736, + 0x01C87'i32, -5681, + 0x01C88'i32, 35766, + 0x01D79'i32, 35832, + 0x01D7D'i32, 4314, + 0x01D8E'i32, 35884, + 0x01E01'i32, 499, + 0x01E03'i32, 499, + 0x01E05'i32, 499, + 0x01E07'i32, 499, + 0x01E09'i32, 499, + 0x01E0B'i32, 499, + 0x01E0D'i32, 499, + 0x01E0F'i32, 499, + 0x01E11'i32, 499, + 0x01E13'i32, 499, + 0x01E15'i32, 499, + 0x01E17'i32, 499, + 0x01E19'i32, 499, + 0x01E1B'i32, 499, + 0x01E1D'i32, 499, + 0x01E1F'i32, 499, + 0x01E21'i32, 499, + 0x01E23'i32, 499, + 0x01E25'i32, 499, + 0x01E27'i32, 499, + 0x01E29'i32, 499, + 0x01E2B'i32, 499, + 0x01E2D'i32, 499, + 0x01E2F'i32, 499, + 0x01E31'i32, 499, + 0x01E33'i32, 499, + 0x01E35'i32, 499, + 0x01E37'i32, 499, + 0x01E39'i32, 499, + 0x01E3B'i32, 499, + 0x01E3D'i32, 499, + 0x01E3F'i32, 499, + 0x01E41'i32, 499, + 0x01E43'i32, 499, + 0x01E45'i32, 499, + 0x01E47'i32, 499, + 0x01E49'i32, 499, + 0x01E4B'i32, 499, + 0x01E4D'i32, 499, + 0x01E4F'i32, 499, + 0x01E51'i32, 499, + 0x01E53'i32, 499, + 0x01E55'i32, 499, + 0x01E57'i32, 499, + 0x01E59'i32, 499, + 0x01E5B'i32, 499, + 0x01E5D'i32, 499, + 0x01E5F'i32, 499, + 0x01E61'i32, 499, + 0x01E63'i32, 499, + 0x01E65'i32, 499, + 0x01E67'i32, 499, + 0x01E69'i32, 499, + 0x01E6B'i32, 499, + 0x01E6D'i32, 499, + 0x01E6F'i32, 499, + 0x01E71'i32, 499, + 0x01E73'i32, 499, + 0x01E75'i32, 499, + 0x01E77'i32, 499, + 0x01E79'i32, 499, + 0x01E7B'i32, 499, + 0x01E7D'i32, 499, + 0x01E7F'i32, 499, + 0x01E81'i32, 499, + 0x01E83'i32, 499, + 0x01E85'i32, 499, + 0x01E87'i32, 499, + 0x01E89'i32, 499, + 0x01E8B'i32, 499, + 0x01E8D'i32, 499, + 0x01E8F'i32, 499, + 0x01E91'i32, 499, + 0x01E93'i32, 499, + 0x01E95'i32, 499, + 0x01E9B'i32, 441, + 0x01EA1'i32, 499, + 0x01EA3'i32, 499, + 0x01EA5'i32, 499, + 0x01EA7'i32, 499, + 0x01EA9'i32, 499, + 0x01EAB'i32, 499, + 0x01EAD'i32, 499, + 0x01EAF'i32, 499, + 0x01EB1'i32, 499, + 0x01EB3'i32, 499, + 0x01EB5'i32, 499, + 0x01EB7'i32, 499, + 0x01EB9'i32, 499, + 0x01EBB'i32, 499, + 0x01EBD'i32, 499, + 0x01EBF'i32, 499, + 0x01EC1'i32, 499, + 0x01EC3'i32, 499, + 0x01EC5'i32, 499, + 0x01EC7'i32, 499, + 0x01EC9'i32, 499, + 0x01ECB'i32, 499, + 0x01ECD'i32, 499, + 0x01ECF'i32, 499, + 0x01ED1'i32, 499, + 0x01ED3'i32, 499, + 0x01ED5'i32, 499, + 0x01ED7'i32, 499, + 0x01ED9'i32, 499, + 0x01EDB'i32, 499, + 0x01EDD'i32, 499, + 0x01EDF'i32, 499, + 0x01EE1'i32, 499, + 0x01EE3'i32, 499, + 0x01EE5'i32, 499, + 0x01EE7'i32, 499, + 0x01EE9'i32, 499, + 0x01EEB'i32, 499, + 0x01EED'i32, 499, + 0x01EEF'i32, 499, + 0x01EF1'i32, 499, + 0x01EF3'i32, 499, + 0x01EF5'i32, 499, + 0x01EF7'i32, 499, + 0x01EF9'i32, 499, + 0x01EFB'i32, 499, + 0x01EFD'i32, 499, + 0x01EFF'i32, 499, + 0x01F51'i32, 508, + 0x01F53'i32, 508, + 0x01F55'i32, 508, + 0x01F57'i32, 508, + 0x01FB3'i32, 509, + 0x01FBE'i32, -6705, + 0x01FC3'i32, 509, + 0x01FE5'i32, 507, + 0x01FF3'i32, 509, + 0x0214E'i32, 472, + 0x02184'i32, 499, + 0x02C61'i32, 499, + 0x02C65'i32, -10295, + 0x02C66'i32, -10292, + 0x02C68'i32, 499, + 0x02C6A'i32, 499, + 0x02C6C'i32, 499, + 0x02C73'i32, 499, + 0x02C76'i32, 499, + 0x02C81'i32, 499, + 0x02C83'i32, 499, + 0x02C85'i32, 499, + 0x02C87'i32, 499, + 0x02C89'i32, 499, + 0x02C8B'i32, 499, + 0x02C8D'i32, 499, + 0x02C8F'i32, 499, + 0x02C91'i32, 499, + 0x02C93'i32, 499, + 0x02C95'i32, 499, + 0x02C97'i32, 499, + 0x02C99'i32, 499, + 0x02C9B'i32, 499, + 0x02C9D'i32, 499, + 0x02C9F'i32, 499, + 0x02CA1'i32, 499, + 0x02CA3'i32, 499, + 0x02CA5'i32, 499, + 0x02CA7'i32, 499, + 0x02CA9'i32, 499, + 0x02CAB'i32, 499, + 0x02CAD'i32, 499, + 0x02CAF'i32, 499, + 0x02CB1'i32, 499, + 0x02CB3'i32, 499, + 0x02CB5'i32, 499, + 0x02CB7'i32, 499, + 0x02CB9'i32, 499, + 0x02CBB'i32, 499, + 0x02CBD'i32, 499, + 0x02CBF'i32, 499, + 0x02CC1'i32, 499, + 0x02CC3'i32, 499, + 0x02CC5'i32, 499, + 0x02CC7'i32, 499, + 0x02CC9'i32, 499, + 0x02CCB'i32, 499, + 0x02CCD'i32, 499, + 0x02CCF'i32, 499, + 0x02CD1'i32, 499, + 0x02CD3'i32, 499, + 0x02CD5'i32, 499, + 0x02CD7'i32, 499, + 0x02CD9'i32, 499, + 0x02CDB'i32, 499, + 0x02CDD'i32, 499, + 0x02CDF'i32, 499, + 0x02CE1'i32, 499, + 0x02CE3'i32, 499, + 0x02CEC'i32, 499, + 0x02CEE'i32, 499, + 0x02CF3'i32, 499, + 0x02D27'i32, -6764, + 0x02D2D'i32, -6764, + 0x0A641'i32, 499, + 0x0A643'i32, 499, + 0x0A645'i32, 499, + 0x0A647'i32, 499, + 0x0A649'i32, 499, + 0x0A64B'i32, 499, + 0x0A64D'i32, 499, + 0x0A64F'i32, 499, + 0x0A651'i32, 499, + 0x0A653'i32, 499, + 0x0A655'i32, 499, + 0x0A657'i32, 499, + 0x0A659'i32, 499, + 0x0A65B'i32, 499, + 0x0A65D'i32, 499, + 0x0A65F'i32, 499, + 0x0A661'i32, 499, + 0x0A663'i32, 499, + 0x0A665'i32, 499, + 0x0A667'i32, 499, + 0x0A669'i32, 499, + 0x0A66B'i32, 499, + 0x0A66D'i32, 499, + 0x0A681'i32, 499, + 0x0A683'i32, 499, + 0x0A685'i32, 499, + 0x0A687'i32, 499, + 0x0A689'i32, 499, + 0x0A68B'i32, 499, + 0x0A68D'i32, 499, + 0x0A68F'i32, 499, + 0x0A691'i32, 499, + 0x0A693'i32, 499, + 0x0A695'i32, 499, + 0x0A697'i32, 499, + 0x0A699'i32, 499, + 0x0A69B'i32, 499, + 0x0A723'i32, 499, + 0x0A725'i32, 499, + 0x0A727'i32, 499, + 0x0A729'i32, 499, + 0x0A72B'i32, 499, + 0x0A72D'i32, 499, + 0x0A72F'i32, 499, + 0x0A733'i32, 499, + 0x0A735'i32, 499, + 0x0A737'i32, 499, + 0x0A739'i32, 499, + 0x0A73B'i32, 499, + 0x0A73D'i32, 499, + 0x0A73F'i32, 499, + 0x0A741'i32, 499, + 0x0A743'i32, 499, + 0x0A745'i32, 499, + 0x0A747'i32, 499, + 0x0A749'i32, 499, + 0x0A74B'i32, 499, + 0x0A74D'i32, 499, + 0x0A74F'i32, 499, + 0x0A751'i32, 499, + 0x0A753'i32, 499, + 0x0A755'i32, 499, + 0x0A757'i32, 499, + 0x0A759'i32, 499, + 0x0A75B'i32, 499, + 0x0A75D'i32, 499, + 0x0A75F'i32, 499, + 0x0A761'i32, 499, + 0x0A763'i32, 499, + 0x0A765'i32, 499, + 0x0A767'i32, 499, + 0x0A769'i32, 499, + 0x0A76B'i32, 499, + 0x0A76D'i32, 499, + 0x0A76F'i32, 499, + 0x0A77A'i32, 499, + 0x0A77C'i32, 499, + 0x0A77F'i32, 499, + 0x0A781'i32, 499, + 0x0A783'i32, 499, + 0x0A785'i32, 499, + 0x0A787'i32, 499, + 0x0A78C'i32, 499, + 0x0A791'i32, 499, + 0x0A793'i32, 499, + 0x0A794'i32, 548, + 0x0A797'i32, 499, + 0x0A799'i32, 499, + 0x0A79B'i32, 499, + 0x0A79D'i32, 499, + 0x0A79F'i32, 499, + 0x0A7A1'i32, 499, + 0x0A7A3'i32, 499, + 0x0A7A5'i32, 499, + 0x0A7A7'i32, 499, + 0x0A7A9'i32, 499, + 0x0A7B5'i32, 499, + 0x0A7B7'i32, 499, + 0x0A7B9'i32, 499, + 0x0A7BB'i32, 499, + 0x0A7BD'i32, 499, + 0x0A7BF'i32, 499, + 0x0A7C3'i32, 499, + 0x0AB53'i32, -428, ] toTitleSinglets = [ - 0x001C4, 501, - 0x001C6, 499, - 0x001C7, 501, - 0x001C9, 499, - 0x001CA, 501, - 0x001CC, 499, - 0x001F1, 501, - 0x001F3, 499, + 0x001C4'i32, 501, + 0x001C6'i32, 499, + 0x001C7'i32, 501, + 0x001C9'i32, 499, + 0x001CA'i32, 501, + 0x001CC'i32, 499, + 0x001F1'i32, 501, + 0x001F3'i32, 499, ] alphaRanges = [ - 0x00041, 0x0005A, - 0x00061, 0x0007A, - 0x000C0, 0x000D6, - 0x000D8, 0x000F6, - 0x000F8, 0x002C1, - 0x002C6, 0x002D1, - 0x002E0, 0x002E4, - 0x00370, 0x00374, - 0x00376, 0x00377, - 0x0037A, 0x0037D, - 0x00388, 0x0038A, - 0x0038E, 0x003A1, - 0x003A3, 0x003F5, - 0x003F7, 0x00481, - 0x0048A, 0x0052F, - 0x00531, 0x00556, - 0x00560, 0x00588, - 0x005D0, 0x005EA, - 0x005EF, 0x005F2, - 0x00620, 0x0064A, - 0x0066E, 0x0066F, - 0x00671, 0x006D3, - 0x006E5, 0x006E6, - 0x006EE, 0x006EF, - 0x006FA, 0x006FC, - 0x00712, 0x0072F, - 0x0074D, 0x007A5, - 0x007CA, 0x007EA, - 0x007F4, 0x007F5, - 0x00800, 0x00815, - 0x00840, 0x00858, - 0x00860, 0x0086A, - 0x008A0, 0x008B4, - 0x008B6, 0x008BD, - 0x00904, 0x00939, - 0x00958, 0x00961, - 0x00971, 0x00980, - 0x00985, 0x0098C, - 0x0098F, 0x00990, - 0x00993, 0x009A8, - 0x009AA, 0x009B0, - 0x009B6, 0x009B9, - 0x009DC, 0x009DD, - 0x009DF, 0x009E1, - 0x009F0, 0x009F1, - 0x00A05, 0x00A0A, - 0x00A0F, 0x00A10, - 0x00A13, 0x00A28, - 0x00A2A, 0x00A30, - 0x00A32, 0x00A33, - 0x00A35, 0x00A36, - 0x00A38, 0x00A39, - 0x00A59, 0x00A5C, - 0x00A72, 0x00A74, - 0x00A85, 0x00A8D, - 0x00A8F, 0x00A91, - 0x00A93, 0x00AA8, - 0x00AAA, 0x00AB0, - 0x00AB2, 0x00AB3, - 0x00AB5, 0x00AB9, - 0x00AE0, 0x00AE1, - 0x00B05, 0x00B0C, - 0x00B0F, 0x00B10, - 0x00B13, 0x00B28, - 0x00B2A, 0x00B30, - 0x00B32, 0x00B33, - 0x00B35, 0x00B39, - 0x00B5C, 0x00B5D, - 0x00B5F, 0x00B61, - 0x00B85, 0x00B8A, - 0x00B8E, 0x00B90, - 0x00B92, 0x00B95, - 0x00B99, 0x00B9A, - 0x00B9E, 0x00B9F, - 0x00BA3, 0x00BA4, - 0x00BA8, 0x00BAA, - 0x00BAE, 0x00BB9, - 0x00C05, 0x00C0C, - 0x00C0E, 0x00C10, - 0x00C12, 0x00C28, - 0x00C2A, 0x00C39, - 0x00C58, 0x00C5A, - 0x00C60, 0x00C61, - 0x00C85, 0x00C8C, - 0x00C8E, 0x00C90, - 0x00C92, 0x00CA8, - 0x00CAA, 0x00CB3, - 0x00CB5, 0x00CB9, - 0x00CE0, 0x00CE1, - 0x00CF1, 0x00CF2, - 0x00D05, 0x00D0C, - 0x00D0E, 0x00D10, - 0x00D12, 0x00D3A, - 0x00D54, 0x00D56, - 0x00D5F, 0x00D61, - 0x00D7A, 0x00D7F, - 0x00D85, 0x00D96, - 0x00D9A, 0x00DB1, - 0x00DB3, 0x00DBB, - 0x00DC0, 0x00DC6, - 0x00E01, 0x00E30, - 0x00E32, 0x00E33, - 0x00E40, 0x00E46, - 0x00E81, 0x00E82, - 0x00E86, 0x00E8A, - 0x00E8C, 0x00EA3, - 0x00EA7, 0x00EB0, - 0x00EB2, 0x00EB3, - 0x00EC0, 0x00EC4, - 0x00EDC, 0x00EDF, - 0x00F40, 0x00F47, - 0x00F49, 0x00F6C, - 0x00F88, 0x00F8C, - 0x01000, 0x0102A, - 0x01050, 0x01055, - 0x0105A, 0x0105D, - 0x01065, 0x01066, - 0x0106E, 0x01070, - 0x01075, 0x01081, - 0x010A0, 0x010C5, - 0x010D0, 0x010FA, - 0x010FC, 0x01248, - 0x0124A, 0x0124D, - 0x01250, 0x01256, - 0x0125A, 0x0125D, - 0x01260, 0x01288, - 0x0128A, 0x0128D, - 0x01290, 0x012B0, - 0x012B2, 0x012B5, - 0x012B8, 0x012BE, - 0x012C2, 0x012C5, - 0x012C8, 0x012D6, - 0x012D8, 0x01310, - 0x01312, 0x01315, - 0x01318, 0x0135A, - 0x01380, 0x0138F, - 0x013A0, 0x013F5, - 0x013F8, 0x013FD, - 0x01401, 0x0166C, - 0x0166F, 0x0167F, - 0x01681, 0x0169A, - 0x016A0, 0x016EA, - 0x016F1, 0x016F8, - 0x01700, 0x0170C, - 0x0170E, 0x01711, - 0x01720, 0x01731, - 0x01740, 0x01751, - 0x01760, 0x0176C, - 0x0176E, 0x01770, - 0x01780, 0x017B3, - 0x01820, 0x01878, - 0x01880, 0x01884, - 0x01887, 0x018A8, - 0x018B0, 0x018F5, - 0x01900, 0x0191E, - 0x01950, 0x0196D, - 0x01970, 0x01974, - 0x01980, 0x019AB, - 0x019B0, 0x019C9, - 0x01A00, 0x01A16, - 0x01A20, 0x01A54, - 0x01B05, 0x01B33, - 0x01B45, 0x01B4B, - 0x01B83, 0x01BA0, - 0x01BAE, 0x01BAF, - 0x01BBA, 0x01BE5, - 0x01C00, 0x01C23, - 0x01C4D, 0x01C4F, - 0x01C5A, 0x01C7D, - 0x01C80, 0x01C88, - 0x01C90, 0x01CBA, - 0x01CBD, 0x01CBF, - 0x01CE9, 0x01CEC, - 0x01CEE, 0x01CF3, - 0x01CF5, 0x01CF6, - 0x01D00, 0x01DBF, - 0x01E00, 0x01F15, - 0x01F18, 0x01F1D, - 0x01F20, 0x01F45, - 0x01F48, 0x01F4D, - 0x01F50, 0x01F57, - 0x01F5F, 0x01F7D, - 0x01F80, 0x01FB4, - 0x01FB6, 0x01FBC, - 0x01FC2, 0x01FC4, - 0x01FC6, 0x01FCC, - 0x01FD0, 0x01FD3, - 0x01FD6, 0x01FDB, - 0x01FE0, 0x01FEC, - 0x01FF2, 0x01FF4, - 0x01FF6, 0x01FFC, - 0x02090, 0x0209C, - 0x0210A, 0x02113, - 0x02119, 0x0211D, - 0x0212A, 0x0212D, - 0x0212F, 0x02139, - 0x0213C, 0x0213F, - 0x02145, 0x02149, - 0x02183, 0x02184, - 0x02C00, 0x02C2E, - 0x02C30, 0x02C5E, - 0x02C60, 0x02CE4, - 0x02CEB, 0x02CEE, - 0x02CF2, 0x02CF3, - 0x02D00, 0x02D25, - 0x02D30, 0x02D67, - 0x02D80, 0x02D96, - 0x02DA0, 0x02DA6, - 0x02DA8, 0x02DAE, - 0x02DB0, 0x02DB6, - 0x02DB8, 0x02DBE, - 0x02DC0, 0x02DC6, - 0x02DC8, 0x02DCE, - 0x02DD0, 0x02DD6, - 0x02DD8, 0x02DDE, - 0x03005, 0x03006, - 0x03031, 0x03035, - 0x0303B, 0x0303C, - 0x03041, 0x03096, - 0x0309D, 0x0309F, - 0x030A1, 0x030FA, - 0x030FC, 0x030FF, - 0x03105, 0x0312F, - 0x03131, 0x0318E, - 0x031A0, 0x031BA, - 0x031F0, 0x031FF, - 0x0A000, 0x0A48C, - 0x0A4D0, 0x0A4FD, - 0x0A500, 0x0A60C, - 0x0A610, 0x0A61F, - 0x0A62A, 0x0A62B, - 0x0A640, 0x0A66E, - 0x0A67F, 0x0A69D, - 0x0A6A0, 0x0A6E5, - 0x0A717, 0x0A71F, - 0x0A722, 0x0A788, - 0x0A78B, 0x0A7BF, - 0x0A7C2, 0x0A7C6, - 0x0A7F7, 0x0A801, - 0x0A803, 0x0A805, - 0x0A807, 0x0A80A, - 0x0A80C, 0x0A822, - 0x0A840, 0x0A873, - 0x0A882, 0x0A8B3, - 0x0A8F2, 0x0A8F7, - 0x0A8FD, 0x0A8FE, - 0x0A90A, 0x0A925, - 0x0A930, 0x0A946, - 0x0A960, 0x0A97C, - 0x0A984, 0x0A9B2, - 0x0A9E0, 0x0A9E4, - 0x0A9E6, 0x0A9EF, - 0x0A9FA, 0x0A9FE, - 0x0AA00, 0x0AA28, - 0x0AA40, 0x0AA42, - 0x0AA44, 0x0AA4B, - 0x0AA60, 0x0AA76, - 0x0AA7E, 0x0AAAF, - 0x0AAB5, 0x0AAB6, - 0x0AAB9, 0x0AABD, - 0x0AADB, 0x0AADD, - 0x0AAE0, 0x0AAEA, - 0x0AAF2, 0x0AAF4, - 0x0AB01, 0x0AB06, - 0x0AB09, 0x0AB0E, - 0x0AB11, 0x0AB16, - 0x0AB20, 0x0AB26, - 0x0AB28, 0x0AB2E, - 0x0AB30, 0x0AB5A, - 0x0AB5C, 0x0AB67, - 0x0AB70, 0x0ABE2, - 0x0D7B0, 0x0D7C6, - 0x0D7CB, 0x0D7FB, - 0x0F900, 0x0FA6D, - 0x0FA70, 0x0FAD9, - 0x0FB00, 0x0FB06, - 0x0FB13, 0x0FB17, - 0x0FB1F, 0x0FB28, - 0x0FB2A, 0x0FB36, - 0x0FB38, 0x0FB3C, - 0x0FB40, 0x0FB41, - 0x0FB43, 0x0FB44, - 0x0FB46, 0x0FBB1, - 0x0FBD3, 0x0FD3D, - 0x0FD50, 0x0FD8F, - 0x0FD92, 0x0FDC7, - 0x0FDF0, 0x0FDFB, - 0x0FE70, 0x0FE74, - 0x0FE76, 0x0FEFC, - 0x0FF21, 0x0FF3A, - 0x0FF41, 0x0FF5A, - 0x0FF66, 0x0FFBE, - 0x0FFC2, 0x0FFC7, - 0x0FFCA, 0x0FFCF, - 0x0FFD2, 0x0FFD7, - 0x0FFDA, 0x0FFDC, - 0x10000, 0x1000B, - 0x1000D, 0x10026, - 0x10028, 0x1003A, - 0x1003C, 0x1003D, - 0x1003F, 0x1004D, - 0x10050, 0x1005D, - 0x10080, 0x100FA, - 0x10280, 0x1029C, - 0x102A0, 0x102D0, - 0x10300, 0x1031F, - 0x1032D, 0x10340, - 0x10342, 0x10349, - 0x10350, 0x10375, - 0x10380, 0x1039D, - 0x103A0, 0x103C3, - 0x103C8, 0x103CF, - 0x10400, 0x1049D, - 0x104B0, 0x104D3, - 0x104D8, 0x104FB, - 0x10500, 0x10527, - 0x10530, 0x10563, - 0x10600, 0x10736, - 0x10740, 0x10755, - 0x10760, 0x10767, - 0x10800, 0x10805, - 0x1080A, 0x10835, - 0x10837, 0x10838, - 0x1083F, 0x10855, - 0x10860, 0x10876, - 0x10880, 0x1089E, - 0x108E0, 0x108F2, - 0x108F4, 0x108F5, - 0x10900, 0x10915, - 0x10920, 0x10939, - 0x10980, 0x109B7, - 0x109BE, 0x109BF, - 0x10A10, 0x10A13, - 0x10A15, 0x10A17, - 0x10A19, 0x10A35, - 0x10A60, 0x10A7C, - 0x10A80, 0x10A9C, - 0x10AC0, 0x10AC7, - 0x10AC9, 0x10AE4, - 0x10B00, 0x10B35, - 0x10B40, 0x10B55, - 0x10B60, 0x10B72, - 0x10B80, 0x10B91, - 0x10C00, 0x10C48, - 0x10C80, 0x10CB2, - 0x10CC0, 0x10CF2, - 0x10D00, 0x10D23, - 0x10F00, 0x10F1C, - 0x10F30, 0x10F45, - 0x10FE0, 0x10FF6, - 0x11003, 0x11037, - 0x11083, 0x110AF, - 0x110D0, 0x110E8, - 0x11103, 0x11126, - 0x11150, 0x11172, - 0x11183, 0x111B2, - 0x111C1, 0x111C4, - 0x11200, 0x11211, - 0x11213, 0x1122B, - 0x11280, 0x11286, - 0x1128A, 0x1128D, - 0x1128F, 0x1129D, - 0x1129F, 0x112A8, - 0x112B0, 0x112DE, - 0x11305, 0x1130C, - 0x1130F, 0x11310, - 0x11313, 0x11328, - 0x1132A, 0x11330, - 0x11332, 0x11333, - 0x11335, 0x11339, - 0x1135D, 0x11361, - 0x11400, 0x11434, - 0x11447, 0x1144A, - 0x11480, 0x114AF, - 0x114C4, 0x114C5, - 0x11580, 0x115AE, - 0x115D8, 0x115DB, - 0x11600, 0x1162F, - 0x11680, 0x116AA, - 0x11700, 0x1171A, - 0x11800, 0x1182B, - 0x118A0, 0x118DF, - 0x119A0, 0x119A7, - 0x119AA, 0x119D0, - 0x11A0B, 0x11A32, - 0x11A5C, 0x11A89, - 0x11AC0, 0x11AF8, - 0x11C00, 0x11C08, - 0x11C0A, 0x11C2E, - 0x11C72, 0x11C8F, - 0x11D00, 0x11D06, - 0x11D08, 0x11D09, - 0x11D0B, 0x11D30, - 0x11D60, 0x11D65, - 0x11D67, 0x11D68, - 0x11D6A, 0x11D89, - 0x11EE0, 0x11EF2, - 0x12000, 0x12399, - 0x12480, 0x12543, - 0x13000, 0x1342E, - 0x14400, 0x14646, - 0x16800, 0x16A38, - 0x16A40, 0x16A5E, - 0x16AD0, 0x16AED, - 0x16B00, 0x16B2F, - 0x16B40, 0x16B43, - 0x16B63, 0x16B77, - 0x16B7D, 0x16B8F, - 0x16E40, 0x16E7F, - 0x16F00, 0x16F4A, - 0x16F93, 0x16F9F, - 0x16FE0, 0x16FE1, - 0x18800, 0x18AF2, - 0x1B000, 0x1B11E, - 0x1B150, 0x1B152, - 0x1B164, 0x1B167, - 0x1B170, 0x1B2FB, - 0x1BC00, 0x1BC6A, - 0x1BC70, 0x1BC7C, - 0x1BC80, 0x1BC88, - 0x1BC90, 0x1BC99, - 0x1D400, 0x1D454, - 0x1D456, 0x1D49C, - 0x1D49E, 0x1D49F, - 0x1D4A5, 0x1D4A6, - 0x1D4A9, 0x1D4AC, - 0x1D4AE, 0x1D4B9, - 0x1D4BD, 0x1D4C3, - 0x1D4C5, 0x1D505, - 0x1D507, 0x1D50A, - 0x1D50D, 0x1D514, - 0x1D516, 0x1D51C, - 0x1D51E, 0x1D539, - 0x1D53B, 0x1D53E, - 0x1D540, 0x1D544, - 0x1D54A, 0x1D550, - 0x1D552, 0x1D6A5, - 0x1D6A8, 0x1D6C0, - 0x1D6C2, 0x1D6DA, - 0x1D6DC, 0x1D6FA, - 0x1D6FC, 0x1D714, - 0x1D716, 0x1D734, - 0x1D736, 0x1D74E, - 0x1D750, 0x1D76E, - 0x1D770, 0x1D788, - 0x1D78A, 0x1D7A8, - 0x1D7AA, 0x1D7C2, - 0x1D7C4, 0x1D7CB, - 0x1E100, 0x1E12C, - 0x1E137, 0x1E13D, - 0x1E2C0, 0x1E2EB, - 0x1E800, 0x1E8C4, - 0x1E900, 0x1E943, - 0x1EE00, 0x1EE03, - 0x1EE05, 0x1EE1F, - 0x1EE21, 0x1EE22, - 0x1EE29, 0x1EE32, - 0x1EE34, 0x1EE37, - 0x1EE4D, 0x1EE4F, - 0x1EE51, 0x1EE52, - 0x1EE61, 0x1EE62, - 0x1EE67, 0x1EE6A, - 0x1EE6C, 0x1EE72, - 0x1EE74, 0x1EE77, - 0x1EE79, 0x1EE7C, - 0x1EE80, 0x1EE89, - 0x1EE8B, 0x1EE9B, - 0x1EEA1, 0x1EEA3, - 0x1EEA5, 0x1EEA9, - 0x1EEAB, 0x1EEBB, - 0x2F800, 0x2FA1D, + 0x00041'i32, 0x0005A'i32, + 0x00061'i32, 0x0007A'i32, + 0x000C0'i32, 0x000D6'i32, + 0x000D8'i32, 0x000F6'i32, + 0x000F8'i32, 0x002C1'i32, + 0x002C6'i32, 0x002D1'i32, + 0x002E0'i32, 0x002E4'i32, + 0x00370'i32, 0x00374'i32, + 0x00376'i32, 0x00377'i32, + 0x0037A'i32, 0x0037D'i32, + 0x00388'i32, 0x0038A'i32, + 0x0038E'i32, 0x003A1'i32, + 0x003A3'i32, 0x003F5'i32, + 0x003F7'i32, 0x00481'i32, + 0x0048A'i32, 0x0052F'i32, + 0x00531'i32, 0x00556'i32, + 0x00560'i32, 0x00588'i32, + 0x005D0'i32, 0x005EA'i32, + 0x005EF'i32, 0x005F2'i32, + 0x00620'i32, 0x0064A'i32, + 0x0066E'i32, 0x0066F'i32, + 0x00671'i32, 0x006D3'i32, + 0x006E5'i32, 0x006E6'i32, + 0x006EE'i32, 0x006EF'i32, + 0x006FA'i32, 0x006FC'i32, + 0x00712'i32, 0x0072F'i32, + 0x0074D'i32, 0x007A5'i32, + 0x007CA'i32, 0x007EA'i32, + 0x007F4'i32, 0x007F5'i32, + 0x00800'i32, 0x00815'i32, + 0x00840'i32, 0x00858'i32, + 0x00860'i32, 0x0086A'i32, + 0x008A0'i32, 0x008B4'i32, + 0x008B6'i32, 0x008BD'i32, + 0x00904'i32, 0x00939'i32, + 0x00958'i32, 0x00961'i32, + 0x00971'i32, 0x00980'i32, + 0x00985'i32, 0x0098C'i32, + 0x0098F'i32, 0x00990'i32, + 0x00993'i32, 0x009A8'i32, + 0x009AA'i32, 0x009B0'i32, + 0x009B6'i32, 0x009B9'i32, + 0x009DC'i32, 0x009DD'i32, + 0x009DF'i32, 0x009E1'i32, + 0x009F0'i32, 0x009F1'i32, + 0x00A05'i32, 0x00A0A'i32, + 0x00A0F'i32, 0x00A10'i32, + 0x00A13'i32, 0x00A28'i32, + 0x00A2A'i32, 0x00A30'i32, + 0x00A32'i32, 0x00A33'i32, + 0x00A35'i32, 0x00A36'i32, + 0x00A38'i32, 0x00A39'i32, + 0x00A59'i32, 0x00A5C'i32, + 0x00A72'i32, 0x00A74'i32, + 0x00A85'i32, 0x00A8D'i32, + 0x00A8F'i32, 0x00A91'i32, + 0x00A93'i32, 0x00AA8'i32, + 0x00AAA'i32, 0x00AB0'i32, + 0x00AB2'i32, 0x00AB3'i32, + 0x00AB5'i32, 0x00AB9'i32, + 0x00AE0'i32, 0x00AE1'i32, + 0x00B05'i32, 0x00B0C'i32, + 0x00B0F'i32, 0x00B10'i32, + 0x00B13'i32, 0x00B28'i32, + 0x00B2A'i32, 0x00B30'i32, + 0x00B32'i32, 0x00B33'i32, + 0x00B35'i32, 0x00B39'i32, + 0x00B5C'i32, 0x00B5D'i32, + 0x00B5F'i32, 0x00B61'i32, + 0x00B85'i32, 0x00B8A'i32, + 0x00B8E'i32, 0x00B90'i32, + 0x00B92'i32, 0x00B95'i32, + 0x00B99'i32, 0x00B9A'i32, + 0x00B9E'i32, 0x00B9F'i32, + 0x00BA3'i32, 0x00BA4'i32, + 0x00BA8'i32, 0x00BAA'i32, + 0x00BAE'i32, 0x00BB9'i32, + 0x00C05'i32, 0x00C0C'i32, + 0x00C0E'i32, 0x00C10'i32, + 0x00C12'i32, 0x00C28'i32, + 0x00C2A'i32, 0x00C39'i32, + 0x00C58'i32, 0x00C5A'i32, + 0x00C60'i32, 0x00C61'i32, + 0x00C85'i32, 0x00C8C'i32, + 0x00C8E'i32, 0x00C90'i32, + 0x00C92'i32, 0x00CA8'i32, + 0x00CAA'i32, 0x00CB3'i32, + 0x00CB5'i32, 0x00CB9'i32, + 0x00CE0'i32, 0x00CE1'i32, + 0x00CF1'i32, 0x00CF2'i32, + 0x00D05'i32, 0x00D0C'i32, + 0x00D0E'i32, 0x00D10'i32, + 0x00D12'i32, 0x00D3A'i32, + 0x00D54'i32, 0x00D56'i32, + 0x00D5F'i32, 0x00D61'i32, + 0x00D7A'i32, 0x00D7F'i32, + 0x00D85'i32, 0x00D96'i32, + 0x00D9A'i32, 0x00DB1'i32, + 0x00DB3'i32, 0x00DBB'i32, + 0x00DC0'i32, 0x00DC6'i32, + 0x00E01'i32, 0x00E30'i32, + 0x00E32'i32, 0x00E33'i32, + 0x00E40'i32, 0x00E46'i32, + 0x00E81'i32, 0x00E82'i32, + 0x00E86'i32, 0x00E8A'i32, + 0x00E8C'i32, 0x00EA3'i32, + 0x00EA7'i32, 0x00EB0'i32, + 0x00EB2'i32, 0x00EB3'i32, + 0x00EC0'i32, 0x00EC4'i32, + 0x00EDC'i32, 0x00EDF'i32, + 0x00F40'i32, 0x00F47'i32, + 0x00F49'i32, 0x00F6C'i32, + 0x00F88'i32, 0x00F8C'i32, + 0x01000'i32, 0x0102A'i32, + 0x01050'i32, 0x01055'i32, + 0x0105A'i32, 0x0105D'i32, + 0x01065'i32, 0x01066'i32, + 0x0106E'i32, 0x01070'i32, + 0x01075'i32, 0x01081'i32, + 0x010A0'i32, 0x010C5'i32, + 0x010D0'i32, 0x010FA'i32, + 0x010FC'i32, 0x01248'i32, + 0x0124A'i32, 0x0124D'i32, + 0x01250'i32, 0x01256'i32, + 0x0125A'i32, 0x0125D'i32, + 0x01260'i32, 0x01288'i32, + 0x0128A'i32, 0x0128D'i32, + 0x01290'i32, 0x012B0'i32, + 0x012B2'i32, 0x012B5'i32, + 0x012B8'i32, 0x012BE'i32, + 0x012C2'i32, 0x012C5'i32, + 0x012C8'i32, 0x012D6'i32, + 0x012D8'i32, 0x01310'i32, + 0x01312'i32, 0x01315'i32, + 0x01318'i32, 0x0135A'i32, + 0x01380'i32, 0x0138F'i32, + 0x013A0'i32, 0x013F5'i32, + 0x013F8'i32, 0x013FD'i32, + 0x01401'i32, 0x0166C'i32, + 0x0166F'i32, 0x0167F'i32, + 0x01681'i32, 0x0169A'i32, + 0x016A0'i32, 0x016EA'i32, + 0x016F1'i32, 0x016F8'i32, + 0x01700'i32, 0x0170C'i32, + 0x0170E'i32, 0x01711'i32, + 0x01720'i32, 0x01731'i32, + 0x01740'i32, 0x01751'i32, + 0x01760'i32, 0x0176C'i32, + 0x0176E'i32, 0x01770'i32, + 0x01780'i32, 0x017B3'i32, + 0x01820'i32, 0x01878'i32, + 0x01880'i32, 0x01884'i32, + 0x01887'i32, 0x018A8'i32, + 0x018B0'i32, 0x018F5'i32, + 0x01900'i32, 0x0191E'i32, + 0x01950'i32, 0x0196D'i32, + 0x01970'i32, 0x01974'i32, + 0x01980'i32, 0x019AB'i32, + 0x019B0'i32, 0x019C9'i32, + 0x01A00'i32, 0x01A16'i32, + 0x01A20'i32, 0x01A54'i32, + 0x01B05'i32, 0x01B33'i32, + 0x01B45'i32, 0x01B4B'i32, + 0x01B83'i32, 0x01BA0'i32, + 0x01BAE'i32, 0x01BAF'i32, + 0x01BBA'i32, 0x01BE5'i32, + 0x01C00'i32, 0x01C23'i32, + 0x01C4D'i32, 0x01C4F'i32, + 0x01C5A'i32, 0x01C7D'i32, + 0x01C80'i32, 0x01C88'i32, + 0x01C90'i32, 0x01CBA'i32, + 0x01CBD'i32, 0x01CBF'i32, + 0x01CE9'i32, 0x01CEC'i32, + 0x01CEE'i32, 0x01CF3'i32, + 0x01CF5'i32, 0x01CF6'i32, + 0x01D00'i32, 0x01DBF'i32, + 0x01E00'i32, 0x01F15'i32, + 0x01F18'i32, 0x01F1D'i32, + 0x01F20'i32, 0x01F45'i32, + 0x01F48'i32, 0x01F4D'i32, + 0x01F50'i32, 0x01F57'i32, + 0x01F5F'i32, 0x01F7D'i32, + 0x01F80'i32, 0x01FB4'i32, + 0x01FB6'i32, 0x01FBC'i32, + 0x01FC2'i32, 0x01FC4'i32, + 0x01FC6'i32, 0x01FCC'i32, + 0x01FD0'i32, 0x01FD3'i32, + 0x01FD6'i32, 0x01FDB'i32, + 0x01FE0'i32, 0x01FEC'i32, + 0x01FF2'i32, 0x01FF4'i32, + 0x01FF6'i32, 0x01FFC'i32, + 0x02090'i32, 0x0209C'i32, + 0x0210A'i32, 0x02113'i32, + 0x02119'i32, 0x0211D'i32, + 0x0212A'i32, 0x0212D'i32, + 0x0212F'i32, 0x02139'i32, + 0x0213C'i32, 0x0213F'i32, + 0x02145'i32, 0x02149'i32, + 0x02183'i32, 0x02184'i32, + 0x02C00'i32, 0x02C2E'i32, + 0x02C30'i32, 0x02C5E'i32, + 0x02C60'i32, 0x02CE4'i32, + 0x02CEB'i32, 0x02CEE'i32, + 0x02CF2'i32, 0x02CF3'i32, + 0x02D00'i32, 0x02D25'i32, + 0x02D30'i32, 0x02D67'i32, + 0x02D80'i32, 0x02D96'i32, + 0x02DA0'i32, 0x02DA6'i32, + 0x02DA8'i32, 0x02DAE'i32, + 0x02DB0'i32, 0x02DB6'i32, + 0x02DB8'i32, 0x02DBE'i32, + 0x02DC0'i32, 0x02DC6'i32, + 0x02DC8'i32, 0x02DCE'i32, + 0x02DD0'i32, 0x02DD6'i32, + 0x02DD8'i32, 0x02DDE'i32, + 0x03005'i32, 0x03006'i32, + 0x03031'i32, 0x03035'i32, + 0x0303B'i32, 0x0303C'i32, + 0x03041'i32, 0x03096'i32, + 0x0309D'i32, 0x0309F'i32, + 0x030A1'i32, 0x030FA'i32, + 0x030FC'i32, 0x030FF'i32, + 0x03105'i32, 0x0312F'i32, + 0x03131'i32, 0x0318E'i32, + 0x031A0'i32, 0x031BA'i32, + 0x031F0'i32, 0x031FF'i32, + 0x03400'i32, 0x04DB5'i32, + 0x04E00'i32, 0x09FEF'i32, + 0x0A000'i32, 0x0A48C'i32, + 0x0A4D0'i32, 0x0A4FD'i32, + 0x0A500'i32, 0x0A60C'i32, + 0x0A610'i32, 0x0A61F'i32, + 0x0A62A'i32, 0x0A62B'i32, + 0x0A640'i32, 0x0A66E'i32, + 0x0A67F'i32, 0x0A69D'i32, + 0x0A6A0'i32, 0x0A6E5'i32, + 0x0A717'i32, 0x0A71F'i32, + 0x0A722'i32, 0x0A788'i32, + 0x0A78B'i32, 0x0A7BF'i32, + 0x0A7C2'i32, 0x0A7C6'i32, + 0x0A7F7'i32, 0x0A801'i32, + 0x0A803'i32, 0x0A805'i32, + 0x0A807'i32, 0x0A80A'i32, + 0x0A80C'i32, 0x0A822'i32, + 0x0A840'i32, 0x0A873'i32, + 0x0A882'i32, 0x0A8B3'i32, + 0x0A8F2'i32, 0x0A8F7'i32, + 0x0A8FD'i32, 0x0A8FE'i32, + 0x0A90A'i32, 0x0A925'i32, + 0x0A930'i32, 0x0A946'i32, + 0x0A960'i32, 0x0A97C'i32, + 0x0A984'i32, 0x0A9B2'i32, + 0x0A9E0'i32, 0x0A9E4'i32, + 0x0A9E6'i32, 0x0A9EF'i32, + 0x0A9FA'i32, 0x0A9FE'i32, + 0x0AA00'i32, 0x0AA28'i32, + 0x0AA40'i32, 0x0AA42'i32, + 0x0AA44'i32, 0x0AA4B'i32, + 0x0AA60'i32, 0x0AA76'i32, + 0x0AA7E'i32, 0x0AAAF'i32, + 0x0AAB5'i32, 0x0AAB6'i32, + 0x0AAB9'i32, 0x0AABD'i32, + 0x0AADB'i32, 0x0AADD'i32, + 0x0AAE0'i32, 0x0AAEA'i32, + 0x0AAF2'i32, 0x0AAF4'i32, + 0x0AB01'i32, 0x0AB06'i32, + 0x0AB09'i32, 0x0AB0E'i32, + 0x0AB11'i32, 0x0AB16'i32, + 0x0AB20'i32, 0x0AB26'i32, + 0x0AB28'i32, 0x0AB2E'i32, + 0x0AB30'i32, 0x0AB5A'i32, + 0x0AB5C'i32, 0x0AB67'i32, + 0x0AB70'i32, 0x0ABE2'i32, + 0x0AC00'i32, 0x0D7A3'i32, + 0x0D7B0'i32, 0x0D7C6'i32, + 0x0D7CB'i32, 0x0D7FB'i32, + 0x0F900'i32, 0x0FA6D'i32, + 0x0FA70'i32, 0x0FAD9'i32, + 0x0FB00'i32, 0x0FB06'i32, + 0x0FB13'i32, 0x0FB17'i32, + 0x0FB1F'i32, 0x0FB28'i32, + 0x0FB2A'i32, 0x0FB36'i32, + 0x0FB38'i32, 0x0FB3C'i32, + 0x0FB40'i32, 0x0FB41'i32, + 0x0FB43'i32, 0x0FB44'i32, + 0x0FB46'i32, 0x0FBB1'i32, + 0x0FBD3'i32, 0x0FD3D'i32, + 0x0FD50'i32, 0x0FD8F'i32, + 0x0FD92'i32, 0x0FDC7'i32, + 0x0FDF0'i32, 0x0FDFB'i32, + 0x0FE70'i32, 0x0FE74'i32, + 0x0FE76'i32, 0x0FEFC'i32, + 0x0FF21'i32, 0x0FF3A'i32, + 0x0FF41'i32, 0x0FF5A'i32, + 0x0FF66'i32, 0x0FFBE'i32, + 0x0FFC2'i32, 0x0FFC7'i32, + 0x0FFCA'i32, 0x0FFCF'i32, + 0x0FFD2'i32, 0x0FFD7'i32, + 0x0FFDA'i32, 0x0FFDC'i32, + 0x10000'i32, 0x1000B'i32, + 0x1000D'i32, 0x10026'i32, + 0x10028'i32, 0x1003A'i32, + 0x1003C'i32, 0x1003D'i32, + 0x1003F'i32, 0x1004D'i32, + 0x10050'i32, 0x1005D'i32, + 0x10080'i32, 0x100FA'i32, + 0x10280'i32, 0x1029C'i32, + 0x102A0'i32, 0x102D0'i32, + 0x10300'i32, 0x1031F'i32, + 0x1032D'i32, 0x10340'i32, + 0x10342'i32, 0x10349'i32, + 0x10350'i32, 0x10375'i32, + 0x10380'i32, 0x1039D'i32, + 0x103A0'i32, 0x103C3'i32, + 0x103C8'i32, 0x103CF'i32, + 0x10400'i32, 0x1049D'i32, + 0x104B0'i32, 0x104D3'i32, + 0x104D8'i32, 0x104FB'i32, + 0x10500'i32, 0x10527'i32, + 0x10530'i32, 0x10563'i32, + 0x10600'i32, 0x10736'i32, + 0x10740'i32, 0x10755'i32, + 0x10760'i32, 0x10767'i32, + 0x10800'i32, 0x10805'i32, + 0x1080A'i32, 0x10835'i32, + 0x10837'i32, 0x10838'i32, + 0x1083F'i32, 0x10855'i32, + 0x10860'i32, 0x10876'i32, + 0x10880'i32, 0x1089E'i32, + 0x108E0'i32, 0x108F2'i32, + 0x108F4'i32, 0x108F5'i32, + 0x10900'i32, 0x10915'i32, + 0x10920'i32, 0x10939'i32, + 0x10980'i32, 0x109B7'i32, + 0x109BE'i32, 0x109BF'i32, + 0x10A10'i32, 0x10A13'i32, + 0x10A15'i32, 0x10A17'i32, + 0x10A19'i32, 0x10A35'i32, + 0x10A60'i32, 0x10A7C'i32, + 0x10A80'i32, 0x10A9C'i32, + 0x10AC0'i32, 0x10AC7'i32, + 0x10AC9'i32, 0x10AE4'i32, + 0x10B00'i32, 0x10B35'i32, + 0x10B40'i32, 0x10B55'i32, + 0x10B60'i32, 0x10B72'i32, + 0x10B80'i32, 0x10B91'i32, + 0x10C00'i32, 0x10C48'i32, + 0x10C80'i32, 0x10CB2'i32, + 0x10CC0'i32, 0x10CF2'i32, + 0x10D00'i32, 0x10D23'i32, + 0x10F00'i32, 0x10F1C'i32, + 0x10F30'i32, 0x10F45'i32, + 0x10FE0'i32, 0x10FF6'i32, + 0x11003'i32, 0x11037'i32, + 0x11083'i32, 0x110AF'i32, + 0x110D0'i32, 0x110E8'i32, + 0x11103'i32, 0x11126'i32, + 0x11150'i32, 0x11172'i32, + 0x11183'i32, 0x111B2'i32, + 0x111C1'i32, 0x111C4'i32, + 0x11200'i32, 0x11211'i32, + 0x11213'i32, 0x1122B'i32, + 0x11280'i32, 0x11286'i32, + 0x1128A'i32, 0x1128D'i32, + 0x1128F'i32, 0x1129D'i32, + 0x1129F'i32, 0x112A8'i32, + 0x112B0'i32, 0x112DE'i32, + 0x11305'i32, 0x1130C'i32, + 0x1130F'i32, 0x11310'i32, + 0x11313'i32, 0x11328'i32, + 0x1132A'i32, 0x11330'i32, + 0x11332'i32, 0x11333'i32, + 0x11335'i32, 0x11339'i32, + 0x1135D'i32, 0x11361'i32, + 0x11400'i32, 0x11434'i32, + 0x11447'i32, 0x1144A'i32, + 0x11480'i32, 0x114AF'i32, + 0x114C4'i32, 0x114C5'i32, + 0x11580'i32, 0x115AE'i32, + 0x115D8'i32, 0x115DB'i32, + 0x11600'i32, 0x1162F'i32, + 0x11680'i32, 0x116AA'i32, + 0x11700'i32, 0x1171A'i32, + 0x11800'i32, 0x1182B'i32, + 0x118A0'i32, 0x118DF'i32, + 0x119A0'i32, 0x119A7'i32, + 0x119AA'i32, 0x119D0'i32, + 0x11A0B'i32, 0x11A32'i32, + 0x11A5C'i32, 0x11A89'i32, + 0x11AC0'i32, 0x11AF8'i32, + 0x11C00'i32, 0x11C08'i32, + 0x11C0A'i32, 0x11C2E'i32, + 0x11C72'i32, 0x11C8F'i32, + 0x11D00'i32, 0x11D06'i32, + 0x11D08'i32, 0x11D09'i32, + 0x11D0B'i32, 0x11D30'i32, + 0x11D60'i32, 0x11D65'i32, + 0x11D67'i32, 0x11D68'i32, + 0x11D6A'i32, 0x11D89'i32, + 0x11EE0'i32, 0x11EF2'i32, + 0x12000'i32, 0x12399'i32, + 0x12480'i32, 0x12543'i32, + 0x13000'i32, 0x1342E'i32, + 0x14400'i32, 0x14646'i32, + 0x16800'i32, 0x16A38'i32, + 0x16A40'i32, 0x16A5E'i32, + 0x16AD0'i32, 0x16AED'i32, + 0x16B00'i32, 0x16B2F'i32, + 0x16B40'i32, 0x16B43'i32, + 0x16B63'i32, 0x16B77'i32, + 0x16B7D'i32, 0x16B8F'i32, + 0x16E40'i32, 0x16E7F'i32, + 0x16F00'i32, 0x16F4A'i32, + 0x16F93'i32, 0x16F9F'i32, + 0x16FE0'i32, 0x16FE1'i32, + 0x17000'i32, 0x187F7'i32, + 0x18800'i32, 0x18AF2'i32, + 0x1B000'i32, 0x1B11E'i32, + 0x1B150'i32, 0x1B152'i32, + 0x1B164'i32, 0x1B167'i32, + 0x1B170'i32, 0x1B2FB'i32, + 0x1BC00'i32, 0x1BC6A'i32, + 0x1BC70'i32, 0x1BC7C'i32, + 0x1BC80'i32, 0x1BC88'i32, + 0x1BC90'i32, 0x1BC99'i32, + 0x1D400'i32, 0x1D454'i32, + 0x1D456'i32, 0x1D49C'i32, + 0x1D49E'i32, 0x1D49F'i32, + 0x1D4A5'i32, 0x1D4A6'i32, + 0x1D4A9'i32, 0x1D4AC'i32, + 0x1D4AE'i32, 0x1D4B9'i32, + 0x1D4BD'i32, 0x1D4C3'i32, + 0x1D4C5'i32, 0x1D505'i32, + 0x1D507'i32, 0x1D50A'i32, + 0x1D50D'i32, 0x1D514'i32, + 0x1D516'i32, 0x1D51C'i32, + 0x1D51E'i32, 0x1D539'i32, + 0x1D53B'i32, 0x1D53E'i32, + 0x1D540'i32, 0x1D544'i32, + 0x1D54A'i32, 0x1D550'i32, + 0x1D552'i32, 0x1D6A5'i32, + 0x1D6A8'i32, 0x1D6C0'i32, + 0x1D6C2'i32, 0x1D6DA'i32, + 0x1D6DC'i32, 0x1D6FA'i32, + 0x1D6FC'i32, 0x1D714'i32, + 0x1D716'i32, 0x1D734'i32, + 0x1D736'i32, 0x1D74E'i32, + 0x1D750'i32, 0x1D76E'i32, + 0x1D770'i32, 0x1D788'i32, + 0x1D78A'i32, 0x1D7A8'i32, + 0x1D7AA'i32, 0x1D7C2'i32, + 0x1D7C4'i32, 0x1D7CB'i32, + 0x1E100'i32, 0x1E12C'i32, + 0x1E137'i32, 0x1E13D'i32, + 0x1E2C0'i32, 0x1E2EB'i32, + 0x1E800'i32, 0x1E8C4'i32, + 0x1E900'i32, 0x1E943'i32, + 0x1EE00'i32, 0x1EE03'i32, + 0x1EE05'i32, 0x1EE1F'i32, + 0x1EE21'i32, 0x1EE22'i32, + 0x1EE29'i32, 0x1EE32'i32, + 0x1EE34'i32, 0x1EE37'i32, + 0x1EE4D'i32, 0x1EE4F'i32, + 0x1EE51'i32, 0x1EE52'i32, + 0x1EE61'i32, 0x1EE62'i32, + 0x1EE67'i32, 0x1EE6A'i32, + 0x1EE6C'i32, 0x1EE72'i32, + 0x1EE74'i32, 0x1EE77'i32, + 0x1EE79'i32, 0x1EE7C'i32, + 0x1EE80'i32, 0x1EE89'i32, + 0x1EE8B'i32, 0x1EE9B'i32, + 0x1EEA1'i32, 0x1EEA3'i32, + 0x1EEA5'i32, 0x1EEA9'i32, + 0x1EEAB'i32, 0x1EEBB'i32, + 0x20000'i32, 0x2A6D6'i32, + 0x2A700'i32, 0x2B734'i32, + 0x2B740'i32, 0x2B81D'i32, + 0x2B820'i32, 0x2CEA1'i32, + 0x2CEB0'i32, 0x2EBE0'i32, + 0x2F800'i32, 0x2FA1D'i32, ] alphaSinglets = [ - 0x000AA, - 0x000B5, - 0x000BA, - 0x002EC, - 0x002EE, - 0x0037F, - 0x00386, - 0x0038C, - 0x00559, - 0x006D5, - 0x006FF, - 0x00710, - 0x007B1, - 0x007FA, - 0x0081A, - 0x00824, - 0x00828, - 0x0093D, - 0x00950, - 0x009B2, - 0x009BD, - 0x009CE, - 0x009FC, - 0x00A5E, - 0x00ABD, - 0x00AD0, - 0x00AF9, - 0x00B3D, - 0x00B71, - 0x00B83, - 0x00B9C, - 0x00BD0, - 0x00C3D, - 0x00C80, - 0x00CBD, - 0x00CDE, - 0x00D3D, - 0x00D4E, - 0x00DBD, - 0x00E84, - 0x00EA5, - 0x00EBD, - 0x00EC6, - 0x00F00, - 0x0103F, - 0x01061, - 0x0108E, - 0x010C7, - 0x010CD, - 0x01258, - 0x012C0, - 0x017D7, - 0x017DC, - 0x018AA, - 0x01AA7, - 0x01CFA, - 0x01F59, - 0x01F5B, - 0x01F5D, - 0x01FBE, - 0x02071, - 0x0207F, - 0x02102, - 0x02107, - 0x02115, - 0x02124, - 0x02126, - 0x02128, - 0x0214E, - 0x02D27, - 0x02D2D, - 0x02D6F, - 0x02E2F, - 0x03400, - 0x04DB5, - 0x04E00, - 0x09FEF, - 0x0A8FB, - 0x0A9CF, - 0x0AA7A, - 0x0AAB1, - 0x0AAC0, - 0x0AAC2, - 0x0AC00, - 0x0D7A3, - 0x0FB1D, - 0x0FB3E, - 0x10808, - 0x1083C, - 0x10A00, - 0x10F27, - 0x11144, - 0x11176, - 0x111DA, - 0x111DC, - 0x11288, - 0x1133D, - 0x11350, - 0x1145F, - 0x114C7, - 0x11644, - 0x116B8, - 0x118FF, - 0x119E1, - 0x119E3, - 0x11A00, - 0x11A3A, - 0x11A50, - 0x11A9D, - 0x11C40, - 0x11D46, - 0x11D98, - 0x16F50, - 0x16FE3, - 0x17000, - 0x187F7, - 0x1D4A2, - 0x1D4BB, - 0x1D546, - 0x1E14E, - 0x1E94B, - 0x1EE24, - 0x1EE27, - 0x1EE39, - 0x1EE3B, - 0x1EE42, - 0x1EE47, - 0x1EE49, - 0x1EE4B, - 0x1EE54, - 0x1EE57, - 0x1EE59, - 0x1EE5B, - 0x1EE5D, - 0x1EE5F, - 0x1EE64, - 0x1EE7E, - 0x20000, - 0x2A6D6, - 0x2A700, - 0x2B734, - 0x2B740, - 0x2B81D, - 0x2B820, - 0x2CEA1, - 0x2CEB0, - 0x2EBE0, + 0x000AA'i32, + 0x000B5'i32, + 0x000BA'i32, + 0x002EC'i32, + 0x002EE'i32, + 0x0037F'i32, + 0x00386'i32, + 0x0038C'i32, + 0x00559'i32, + 0x006D5'i32, + 0x006FF'i32, + 0x00710'i32, + 0x007B1'i32, + 0x007FA'i32, + 0x0081A'i32, + 0x00824'i32, + 0x00828'i32, + 0x0093D'i32, + 0x00950'i32, + 0x009B2'i32, + 0x009BD'i32, + 0x009CE'i32, + 0x009FC'i32, + 0x00A5E'i32, + 0x00ABD'i32, + 0x00AD0'i32, + 0x00AF9'i32, + 0x00B3D'i32, + 0x00B71'i32, + 0x00B83'i32, + 0x00B9C'i32, + 0x00BD0'i32, + 0x00C3D'i32, + 0x00C80'i32, + 0x00CBD'i32, + 0x00CDE'i32, + 0x00D3D'i32, + 0x00D4E'i32, + 0x00DBD'i32, + 0x00E84'i32, + 0x00EA5'i32, + 0x00EBD'i32, + 0x00EC6'i32, + 0x00F00'i32, + 0x0103F'i32, + 0x01061'i32, + 0x0108E'i32, + 0x010C7'i32, + 0x010CD'i32, + 0x01258'i32, + 0x012C0'i32, + 0x017D7'i32, + 0x017DC'i32, + 0x018AA'i32, + 0x01AA7'i32, + 0x01CFA'i32, + 0x01F59'i32, + 0x01F5B'i32, + 0x01F5D'i32, + 0x01FBE'i32, + 0x02071'i32, + 0x0207F'i32, + 0x02102'i32, + 0x02107'i32, + 0x02115'i32, + 0x02124'i32, + 0x02126'i32, + 0x02128'i32, + 0x0214E'i32, + 0x02D27'i32, + 0x02D2D'i32, + 0x02D6F'i32, + 0x02E2F'i32, + 0x0A8FB'i32, + 0x0A9CF'i32, + 0x0AA7A'i32, + 0x0AAB1'i32, + 0x0AAC0'i32, + 0x0AAC2'i32, + 0x0FB1D'i32, + 0x0FB3E'i32, + 0x10808'i32, + 0x1083C'i32, + 0x10A00'i32, + 0x10F27'i32, + 0x11144'i32, + 0x11176'i32, + 0x111DA'i32, + 0x111DC'i32, + 0x11288'i32, + 0x1133D'i32, + 0x11350'i32, + 0x1145F'i32, + 0x114C7'i32, + 0x11644'i32, + 0x116B8'i32, + 0x118FF'i32, + 0x119E1'i32, + 0x119E3'i32, + 0x11A00'i32, + 0x11A3A'i32, + 0x11A50'i32, + 0x11A9D'i32, + 0x11C40'i32, + 0x11D46'i32, + 0x11D98'i32, + 0x16F50'i32, + 0x16FE3'i32, + 0x1D4A2'i32, + 0x1D4BB'i32, + 0x1D546'i32, + 0x1E14E'i32, + 0x1E94B'i32, + 0x1EE24'i32, + 0x1EE27'i32, + 0x1EE39'i32, + 0x1EE3B'i32, + 0x1EE42'i32, + 0x1EE47'i32, + 0x1EE49'i32, + 0x1EE4B'i32, + 0x1EE54'i32, + 0x1EE57'i32, + 0x1EE59'i32, + 0x1EE5B'i32, + 0x1EE5D'i32, + 0x1EE5F'i32, + 0x1EE64'i32, + 0x1EE7E'i32, ] spaceRanges = [ - 0x00009, 0x0000D, - 0x00020, 0x00020, - 0x00085, 0x00085, - 0x000A0, 0x000A0, - 0x01680, 0x01680, - 0x02000, 0x0200A, - 0x02028, 0x02029, - 0x0202F, 0x0202F, - 0x0205F, 0x0205F, - 0x03000, 0x03000, + 0x00009'i32, 0x0000D'i32, + 0x00020'i32, 0x00020'i32, + 0x00085'i32, 0x00085'i32, + 0x000A0'i32, 0x000A0'i32, + 0x01680'i32, 0x01680'i32, + 0x02000'i32, 0x0200A'i32, + 0x02028'i32, 0x02029'i32, + 0x0202F'i32, 0x0202F'i32, + 0x0205F'i32, 0x0205F'i32, + 0x03000'i32, 0x03000'i32, ] unicodeSpaces = [ diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index bf13cc83e..10658b78e 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -9,7 +9,7 @@ # This module implements Linux epoll(). -import posix, times, epoll +import std/[posix, times, epoll] # Maximum number of events that can be returned const MAX_EPOLL_EVENTS = 64 @@ -55,7 +55,7 @@ when hasThreadSupport: maxFD: int numFD: int fds: ptr SharedArray[SelectorKey[T]] - count: int + count*: int Selector*[T] = ptr SelectorImpl[T] else: type @@ -64,7 +64,7 @@ else: maxFD: int numFD: int fds: seq[SelectorKey[T]] - count: int + count*: int Selector*[T] = ref SelectorImpl[T] type SelectEventImpl = object @@ -72,16 +72,18 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 # Retrieve the maximum fd count (for current OS) via getrlimit() - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseOSError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand - const numFD = 1024 + let numFD = initialNumFD() - var epollFD = epoll_create(MAX_EPOLL_EVENTS) + var epollFD = epoll_create1(O_CLOEXEC) if epollFD < 0: raiseOSError(osLastError()) @@ -110,10 +112,9 @@ proc close*[T](s: Selector[T]) = raiseIOSelectorsError(osLastError()) proc newSelectEvent*(): SelectEvent = - let fdci = eventfd(0, 0) + let fdci = eventfd(0, O_CLOEXEC or O_NONBLOCK) if fdci == -1: raiseIOSelectorsError(osLastError()) - setNonBlocking(fdci) result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) result.efd = fdci @@ -137,7 +138,7 @@ template checkFd(s, f) = var numFD = s.numFD while numFD <= f: numFD *= 2 when hasThreadSupport: - s.fds = reallocSharedArray(s.fds, numFD) + s.fds = reallocSharedArray(s.fds, s.numFD, numFD) else: s.fds.setLen(numFD) for i in s.numFD ..< numFD: @@ -269,10 +270,9 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, var newTs: Itimerspec oldTs: Itimerspec - let fdi = timerfd_create(CLOCK_MONOTONIC, 0).int + let fdi = timerfd_create(CLOCK_MONOTONIC, O_CLOEXEC or O_NONBLOCK).int if fdi == -1: raiseIOSelectorsError(osLastError()) - setNonBlocking(fdi.cint) s.checkFd(fdi) doAssert(s.fds[fdi].ident == InvalidIdent) @@ -314,10 +314,9 @@ when not defined(android): discard sigaddset(nmask, cint(signal)) blockSignals(nmask, omask) - let fdi = signalfd(-1, nmask, 0).int + let fdi = signalfd(-1, nmask, O_CLOEXEC or O_NONBLOCK).int if fdi == -1: raiseIOSelectorsError(osLastError()) - setNonBlocking(fdi.cint) s.checkFd(fdi) doAssert(s.fds[fdi].ident == InvalidIdent) @@ -341,10 +340,9 @@ when not defined(android): discard sigaddset(nmask, posix.SIGCHLD) blockSignals(nmask, omask) - let fdi = signalfd(-1, nmask, 0).int + let fdi = signalfd(-1, nmask, O_CLOEXEC or O_NONBLOCK).int if fdi == -1: raiseIOSelectorsError(osLastError()) - setNonBlocking(fdi.cint) s.checkFd(fdi) doAssert(s.fds[fdi].ident == InvalidIdent) @@ -518,7 +516,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, let fdi = int(fd) s.checkFd(fdi) if fdi in s: - var value = addr(s.getData(fdi)) + var value = addr(s.fds[fdi].data) body template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, @@ -527,10 +525,10 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, let fdi = int(fd) s.checkFd(fdi) if fdi in s: - var value = addr(s.getData(fdi)) + var value = addr(s.fds[fdi].data) body1 else: body2 proc getFd*[T](s: Selector[T]): int = - return s.epollFd.int + return s.epollFD.int diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 83e15d479..513578eda 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -9,7 +9,7 @@ # This module implements BSD kqueue(). -import posix, times, kqueue +import std/[posix, times, kqueue, nativesockets] const # Maximum number of events that can be returned. @@ -30,7 +30,7 @@ when defined(macosx) or defined(freebsd) or defined(dragonfly): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include <sys/types.h> - #include <sys/sysctl.h>"""} + #include <sys/sysctl.h>""".} elif defined(netbsd) or defined(openbsd): # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using # KERN_MAXFILES, because KERN_MAXFILES is always bigger, @@ -39,7 +39,7 @@ elif defined(netbsd) or defined(openbsd): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include <sys/param.h> - #include <sys/sysctl.h>"""} + #include <sys/sysctl.h>""".} when hasThreadSupport: type @@ -48,7 +48,7 @@ when hasThreadSupport: maxFD: int changes: ptr SharedArray[KEvent] fds: ptr SharedArray[SelectorKey[T]] - count: int + count*: int changesLock: Lock changesSize: int changesLength: int @@ -61,7 +61,7 @@ else: maxFD: int changes: seq[KEvent] fds: seq[SelectorKey[T]] - count: int + count*: int sock: cint Selector*[T] = ref SelectorImpl[T] @@ -76,7 +76,7 @@ type proc getUnique[T](s: Selector[T]): int {.inline.} = # we create duplicated handles to get unique indexes for our `fds` array. - result = posix.fcntl(s.sock, F_DUPFD, s.sock) + result = posix.fcntl(s.sock, F_DUPFD_CLOEXEC, s.sock) if result == -1: raiseIOSelectorsError(osLastError()) @@ -96,8 +96,8 @@ proc newSelector*[T](): owned(Selector[T]) = # we allocating empty socket to duplicate it handle in future, to get unique # indexes for `fds` array. This is needed to properly identify # {Event.Timer, Event.Signal, Event.Process} events. - let usock = posix.socket(posix.AF_INET, posix.SOCK_STREAM, - posix.IPPROTO_TCP).cint + let usock = createNativeSocket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).cint if usock == -1: let err = osLastError() discard posix.close(kqFD) @@ -194,7 +194,9 @@ when hasThreadSupport: if s.changesLength > 0: if kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), nil, 0, nil) == -1: - raiseIOSelectorsError(osLastError()) + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) s.changesLength = 0 else: template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, @@ -211,7 +213,9 @@ else: if length > 0: if kevent(s.kqFD, addr(s.changes[0]), length, nil, 0, nil) == -1: - raiseIOSelectorsError(osLastError()) + let res = osLastError() + if cint(res) != ENOENT: # ignore pipes whose read end is closed + raiseIOSelectorsError(res) s.changes.setLen(0) proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, @@ -616,7 +620,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, let fdi = int(fd) s.checkFd(fdi) if fdi in s: - var value = addr(s.getData(fdi)) + var value = addr(s.fds[fdi].data) body template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, @@ -625,7 +629,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, let fdi = int(fd) s.checkFd(fdi) if fdi in s: - var value = addr(s.getData(fdi)) + var value = addr(s.fds[fdi].data) body1 else: body2 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 1af2a46db..7c5347156 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -9,11 +9,17 @@ # This module implements Posix poll(). -import posix, times +import std/[posix, times] # Maximum number of events that can be returned const MAX_POLL_EVENTS = 64 +const hasEventFds = defined(zephyr) or defined(nimPollHasEventFds) + +when hasEventFds: + proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} + when hasThreadSupport: type SelectorImpl[T] = object @@ -21,7 +27,7 @@ when hasThreadSupport: pollcnt: int fds: ptr SharedArray[SelectorKey[T]] pollfds: ptr SharedArray[TPollFd] - count: int + count*: int lock: Lock Selector*[T] = ptr SelectorImpl[T] else: @@ -31,7 +37,7 @@ else: pollcnt: int fds: seq[SelectorKey[T]] pollfds: seq[TPollFd] - count: int + count*: int Selector*[T] = ref SelectorImpl[T] type @@ -53,10 +59,7 @@ else: body proc newSelector*[T](): Selector[T] = - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseIOSelectorsError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() when hasThreadSupport: result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) @@ -187,14 +190,22 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = s.pollRemove(fdi.cint) proc newSelectEvent*(): SelectEvent = - var fds: array[2, cint] - if posix.pipe(fds) != 0: - raiseIOSelectorsError(osLastError()) - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] + when not hasEventFds: + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + else: + let fdci = eventfd(0, posix.O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fdci + result.wfd = fdci proc trigger*(ev: SelectEvent) = var data: uint64 = 1 @@ -203,13 +214,16 @@ proc trigger*(ev: SelectEvent) = proc close*(ev: SelectEvent) = let res1 = posix.close(ev.rfd) - let res2 = posix.close(ev.wfd) + let res2 = + when hasEventFds: 0 + else: posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey]): int = + results: var openArray[ReadyKey]): int = var maxres = MAX_POLL_EVENTS if maxres > len(results): maxres = len(results) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index b90a01c06..6c516395b 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -9,10 +9,10 @@ # This module implements Posix and Windows select(). -import times, nativesockets +import std/[times, nativesockets] when defined(windows): - import winlean + import std/winlean when defined(gcc): {.passl: "-lws2_32".} elif defined(vcc): @@ -26,27 +26,27 @@ else: #include <sys/types.h> #include <unistd.h>""" type - Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object + FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object var FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint -proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} -proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} -proc IOFD_ZERO(fdset: ptr Fdset) +proc IOFD_ZERO(fdset: ptr FdSet) {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} when defined(windows): - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.stdcall, importc: "select", header: platformHeaders.} else: - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.cdecl, importc: "select", header: platformHeaders.} @@ -58,7 +58,7 @@ when hasThreadSupport: eSet: FdSet maxFD: int fds: ptr SharedArray[SelectorKey[T]] - count: int + count*: int lock: Lock Selector*[T] = ptr SelectorImpl[T] else: @@ -69,7 +69,7 @@ else: eSet: FdSet maxFD: int fds: seq[SelectorKey[T]] - count: int + count*: int Selector*[T] = ref SelectorImpl[T] type @@ -110,11 +110,12 @@ proc close*[T](s: Selector[T]) = when hasThreadSupport: deallocSharedArray(s.fds) deallocShared(cast[pointer](s)) + deinitLock(s.lock) when defined(windows): proc newSelectEvent*(): SelectEvent = - var ssock = newNativeSocket() - var wsock = newNativeSocket() + var ssock = createNativeSocket() + var wsock = createNativeSocket() var rsock: SocketHandle = INVALID_SOCKET var saddr = Sockaddr_in() @@ -304,7 +305,7 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = s.delKey(fd) proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey]): int = + results: var openArray[ReadyKey]): int = var tv = Timeval() var ptv = addr tv var rset, wset, eset: FdSet @@ -312,8 +313,8 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode): - tv.tv_sec = Time(timeout div 1_000) + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): + tv.tv_sec = posix.Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 @@ -397,18 +398,6 @@ proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = if s.fds[i].ident == fdi: return true -when hasThreadSupport: - template withSelectLock[T](s: Selector[T], body: untyped) = - acquire(s.lock) - {.locks: [s.lock].}: - try: - body - finally: - release(s.lock) -else: - template withSelectLock[T](s: Selector[T], body: untyped) = - body - proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = s.withSelectLock(): let fdi = int(fd) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 319a7b7ff..53fa7553a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -14,65 +14,75 @@ ## JSON is based on a subset of the JavaScript Programming Language, ## Standard ECMA-262 3rd Edition - December 1999. ## +## See also +## ======== +## * `std/parsejson <parsejson.html>`_ +## * `std/jsonutils <jsonutils.html>`_ +## * `std/marshal <marshal.html>`_ +## * `std/jscore <jscore.html>`_ +## +## ## Overview ## ======== ## ## Parsing JSON ## ------------ ## -## JSON often arrives into your program (via an API or a file) as a ``string``. +## JSON often arrives into your program (via an API or a file) as a `string`. ## The first step is to change it from its serialized form into a nested object -## structure called a ``JsonNode``. +## structure called a `JsonNode`. ## -## The ``parseJson`` procedure takes a string containing JSON and returns a -## ``JsonNode`` object. This is an object variant and it is either a -## ``JObject``, ``JArray``, ``JString``, ``JInt``, ``JFloat``, ``JBool`` or -## ``JNull``. You check the kind of this object variant by using the ``kind`` +## The `parseJson` procedure takes a string containing JSON and returns a +## `JsonNode` object. This is an object variant and it is either a +## `JObject`, `JArray`, `JString`, `JInt`, `JFloat`, `JBool` or +## `JNull`. You check the kind of this object variant by using the `kind` ## accessor. ## -## For a ``JsonNode`` who's kind is ``JObject``, you can access its fields using -## the ``[]`` operator. The following example shows how to do this: +## For a `JsonNode` who's kind is `JObject`, you can access its fields using +## the `[]` operator. The following example shows how to do this: ## -## .. code-block:: Nim -## import json +## ```Nim +## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode.kind == JObject ## doAssert jsonNode["key"].kind == JFloat +## ``` ## ## Reading values ## -------------- ## -## Once you have a ``JsonNode``, retrieving the values can then be achieved +## Once you have a `JsonNode`, retrieving the values can then be achieved ## by using one of the helper procedures, which include: ## -## * ``getInt`` -## * ``getFloat`` -## * ``getStr`` -## * ``getBool`` +## * `getInt` +## * `getFloat` +## * `getStr` +## * `getBool` ## -## To retrieve the value of ``"key"`` you can do the following: +## To retrieve the value of `"key"` you can do the following: ## -## .. code-block:: Nim -## import json +## ```Nim +## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14}""") ## ## doAssert jsonNode["key"].getFloat() == 3.14 +## ``` ## -## **Important:** The ``[]`` operator will raise an exception when the +## **Important:** The `[]` operator will raise an exception when the ## specified field does not exist. ## ## Handling optional keys ## ---------------------- ## -## By using the ``{}`` operator instead of ``[]``, it will return ``nil`` -## when the field is not found. The ``get``-family of procedures will return a -## type's default value when called on ``nil``. +## By using the `{}` operator instead of `[]`, it will return `nil` +## when the field is not found. The `get`-family of procedures will return a +## type's default value when called on `nil`. ## -## .. code-block:: Nim -## import json +## ```Nim +## import std/json ## ## let jsonNode = parseJson("{}") ## @@ -80,47 +90,57 @@ ## doAssert jsonNode{"nope"}.getFloat() == 0 ## doAssert jsonNode{"nope"}.getStr() == "" ## doAssert jsonNode{"nope"}.getBool() == false +## ``` ## ## Using default values ## -------------------- ## -## The ``get``-family helpers also accept an additional parameter which allow -## you to fallback to a default value should the key's values be ``null``: +## The `get`-family helpers also accept an additional parameter which allow +## you to fallback to a default value should the key's values be `null`: ## -## .. code-block:: Nim -## import json +## ```Nim +## import std/json ## ## let jsonNode = parseJson("""{"key": 3.14, "key2": null}""") ## ## doAssert jsonNode["key"].getFloat(6.28) == 3.14 ## doAssert jsonNode["key2"].getFloat(3.14) == 3.14 ## doAssert jsonNode{"nope"}.getFloat(3.14) == 3.14 # note the {} +## ``` ## ## Unmarshalling ## ------------- ## ## In addition to reading dynamic data, Nim can also unmarshal JSON directly -## into a type with the ``to`` macro. +## into a type with the `to` macro. ## -## .. code-block:: Nim -## import json +## Note: Use `Option <options.html#Option>`_ for keys sometimes missing in json +## responses, and backticks around keys with a reserved keyword as name. +## +## ```Nim +## import std/json +## import std/options ## ## type ## User = object ## name: string ## age: int +## `type`: Option[string] ## ## let userJson = parseJson("""{ "name": "Nim", "age": 12 }""") ## let user = to(userJson, User) +## if user.`type`.isSome(): +## assert user.`type`.get() != "robot" +## ``` ## ## Creating JSON ## ============= ## -## This module can also be used to comfortably create JSON using the ``%*`` +## This module can also be used to comfortably create JSON using the `%*` ## operator: ## -## .. code-block:: nim -## import json +## ```nim +## import std/json ## ## var hisName = "John" ## let herAge = 31 @@ -133,6 +153,10 @@ ## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} ## j2["details"] = %* {"age":35, "pi":3.1415} ## echo j2 +## ``` +## +## See also: std/jsonutils for hookable json serialization/deserialization +## of arbitrary types. runnableExamples: ## Note: for JObject, key ordering is preserved, unlike in some languages, @@ -141,9 +165,13 @@ runnableExamples: a1, a2, a0, a3, a4: int doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" -import - hashes, tables, strutils, lexbase, streams, macros, parsejson, - options +import std/[hashes, tables, strutils, lexbase, streams, macros, parsejson] + +import std/options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, formatfloat] export tables.`$` @@ -151,7 +179,7 @@ export export parsejson.JsonEventKind, parsejson.JsonError, JsonParser, JsonKindError, open, close, str, getInt, getFloat, kind, getColumn, getLine, getFilename, - errorMsg, errorMsgExpected, next, JsonParsingError, raiseParseErr + errorMsg, errorMsgExpected, next, JsonParsingError, raiseParseErr, nimIdentNormalize type JsonNodeKind* = enum ## possible JSON node types @@ -165,6 +193,8 @@ type JsonNode* = ref JsonNodeObj ## JSON node JsonNodeObj* {.acyclic.} = object + isUnquoted: bool # the JString was a number-like token and + # so shouldn't be quoted case kind*: JsonNodeKind of JString: str*: string @@ -181,13 +211,18 @@ type of JArray: elems*: seq[JsonNode] +const DepthLimit = 1000 + proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. result = JsonNode(kind: JString, str: s) -proc newJStringMove(s: string): JsonNode = - result = JsonNode(kind: JString) - shallowCopy(result.str, s) +proc newJRawNumber(s: string): JsonNode = + ## Creates a "raw JS number", that is a number that does not + ## fit into Nim's `BiggestInt` field. This is really a `JString` + ## with the additional information that it should be converted back + ## to the string representation without the quotes. + result = JsonNode(kind: JString, str: s, isUnquoted: true) proc newJInt*(n: BiggestInt): JsonNode = ## Creates a new `JInt JsonNode`. @@ -207,7 +242,7 @@ proc newJNull*(): JsonNode = proc newJObject*(): JsonNode = ## Creates a new `JObject JsonNode` - result = JsonNode(kind: JObject, fields: initOrderedTable[string, JsonNode](4)) + result = JsonNode(kind: JObject, fields: initOrderedTable[string, JsonNode](2)) proc newJArray*(): JsonNode = ## Creates a new `JArray JsonNode` @@ -216,28 +251,28 @@ proc newJArray*(): JsonNode = proc getStr*(n: JsonNode, default: string = ""): string = ## Retrieves the string value of a `JString JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JString``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JString`, or if `n` is nil. if n.isNil or n.kind != JString: return default else: return n.str proc getInt*(n: JsonNode, default: int = 0): int = ## Retrieves the int value of a `JInt JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JInt`, or if `n` is nil. if n.isNil or n.kind != JInt: return default else: return int(n.num) proc getBiggestInt*(n: JsonNode, default: BiggestInt = 0): BiggestInt = ## Retrieves the BiggestInt value of a `JInt JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JInt`, or if `n` is nil. if n.isNil or n.kind != JInt: return default else: return n.num proc getFloat*(n: JsonNode, default: float = 0.0): float = ## Retrieves the float value of a `JFloat JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JFloat`` or ``JInt``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JFloat` or `JInt`, or if `n` is nil. if n.isNil: return default case n.kind of JFloat: return n.fnum @@ -247,23 +282,23 @@ proc getFloat*(n: JsonNode, default: float = 0.0): float = proc getBool*(n: JsonNode, default: bool = false): bool = ## Retrieves the bool value of a `JBool JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JBool``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JBool`, or if `n` is nil. if n.isNil or n.kind != JBool: return default else: return n.bval proc getFields*(n: JsonNode, - default = initOrderedTable[string, JsonNode](4)): + default = initOrderedTable[string, JsonNode](2)): OrderedTable[string, JsonNode] = ## Retrieves the key, value pairs of a `JObject JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JObject``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JObject`, or if `n` is nil. if n.isNil or n.kind != JObject: return default else: return n.fields proc getElems*(n: JsonNode, default: seq[JsonNode] = @[]): seq[JsonNode] = ## Retrieves the array of a `JArray JsonNode`. ## - ## Returns ``default`` if ``n`` is not a ``JArray``, or if ``n`` is nil. + ## Returns `default` if `n` is not a `JArray`, or if `n` is nil. if n.isNil or n.kind != JArray: return default else: return n.elems @@ -283,7 +318,10 @@ proc `%`*(s: string): JsonNode = proc `%`*(n: uint): JsonNode = ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. - result = JsonNode(kind: JInt, num: BiggestInt(n)) + if n > cast[uint](int.high): + result = newJRawNumber($n) + else: + result = JsonNode(kind: JInt, num: BiggestInt(n)) proc `%`*(n: int): JsonNode = ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. @@ -291,7 +329,10 @@ proc `%`*(n: int): JsonNode = proc `%`*(n: BiggestUInt): JsonNode = ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. - result = JsonNode(kind: JInt, num: BiggestInt(n)) + if n > cast[BiggestUInt](BiggestInt.high): + result = newJRawNumber($n) + else: + result = JsonNode(kind: JInt, num: BiggestInt(n)) proc `%`*(n: BiggestInt): JsonNode = ## Generic constructor for JSON data. Creates a new `JInt JsonNode`. @@ -299,7 +340,16 @@ proc `%`*(n: BiggestInt): JsonNode = proc `%`*(n: float): JsonNode = ## Generic constructor for JSON data. Creates a new `JFloat JsonNode`. - result = JsonNode(kind: JFloat, fnum: n) + runnableExamples: + assert $(%[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2]) == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert (%NaN).kind == JString + assert (%0.0).kind == JFloat + # for those special cases, we could also have used `newJRawNumber` but then + # it would've been inconsisten with the case of `parseJson` vs `%` for representing them. + if n != n: newJString("nan") + elif n == Inf: newJString("inf") + elif n == -Inf: newJString("-inf") + else: JsonNode(kind: JFloat, fnum: n) proc `%`*(b: bool): JsonNode = ## Generic constructor for JSON data. Creates a new `JBool JsonNode`. @@ -319,13 +369,13 @@ proc `%`*[T](elements: openArray[T]): JsonNode = for elem in elements: result.add(%elem) proc `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode = - ## Generic constructor for JSON data. Creates a new ``JObject JsonNode``. + ## Generic constructor for JSON data. Creates a new `JObject JsonNode`. result = newJObject() for k, v in table: result[k] = %v proc `%`*[T](opt: Option[T]): JsonNode = - ## Generic constructor for JSON data. Creates a new ``JNull JsonNode`` - ## if ``opt`` is empty, otherwise it delegates to the underlying value. + ## Generic constructor for JSON data. Creates a new `JNull JsonNode` + ## if `opt` is empty, otherwise it delegates to the underlying value. if opt.isSome: %opt.get else: newJNull() when false: @@ -334,8 +384,8 @@ when false: # causing problems later on. proc `%`*(elements: set[bool]): JsonNode = ## Generic constructor for JSON data. Creates a new `JObject JsonNode`. - ## This can only be used with the empty set ``{}`` and is supported - ## to prevent the gotcha ``%*{}`` which used to produce an empty + ## This can only be used with the empty set `{}` and is supported + ## to prevent the gotcha `%*{}` which used to produce an empty ## JSON array. result = newJObject() assert false notin elements, "usage error: only empty sets allowed" @@ -360,23 +410,23 @@ proc `%`*(o: ref object): JsonNode = proc `%`*(o: enum): JsonNode = ## Construct a JsonNode that represents the specified enum value as a - ## string. Creates a new ``JString JsonNode``. + ## string. Creates a new `JString JsonNode`. result = %($o) -proc toJson(x: NimNode): NimNode {.compileTime.} = +proc toJsonImpl(x: NimNode): NimNode = case x.kind of nnkBracket: # array if x.len == 0: return newCall(bindSym"newJArray") result = newNimNode(nnkBracket) for i in 0 ..< x.len: - result.add(toJson(x[i])) + result.add(toJsonImpl(x[i])) result = newCall(bindSym("%", brOpen), result) of nnkTableConstr: # object if x.len == 0: return newCall(bindSym"newJObject") result = newNimNode(nnkTableConstr) for i in 0 ..< x.len: x[i].expectKind nnkExprColonExpr - result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1])) + result.add newTree(nnkExprColonExpr, x[i][0], toJsonImpl(x[i][1])) result = newCall(bindSym("%", brOpen), result) of nnkCurly: # empty object x.expectLen(0) @@ -384,7 +434,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} = of nnkNilLit: result = newCall(bindSym"newJNull") of nnkPar: - if x.len == 1: result = toJson(x[0]) + if x.len == 1: result = toJsonImpl(x[0]) else: result = newCall(bindSym("%", brOpen), x) else: result = newCall(bindSym("%", brOpen), x) @@ -392,9 +442,9 @@ proc toJson(x: NimNode): NimNode {.compileTime.} = macro `%*`*(x: untyped): untyped = ## Convert an expression to a JsonNode directly, without having to specify ## `%` for every element. - result = toJson(x) + result = toJsonImpl(x) -proc `==`*(a, b: JsonNode): bool = +proc `==`*(a, b: JsonNode): bool {.noSideEffect, raises: [].} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -414,19 +464,25 @@ proc `==`*(a, b: JsonNode): bool = of JNull: result = true of JArray: - result = a.elems == b.elems + {.cast(raises: []).}: # bug #19303 + result = a.elems == b.elems of JObject: # we cannot use OrderedTable's equality here as # the order does not matter for equality here. if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - if b.fields[key] != val: return false + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: + if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} -proc hash*(n: JsonNode): Hash = +proc hash*(n: JsonNode): Hash {.noSideEffect.} = ## Compute the hash for a JSON node case n.kind of JArray: @@ -475,6 +531,37 @@ proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = assert(node.kind == JArray) return node.elems[index] +proc `[]`*(node: JsonNode, index: BackwardsIndex): JsonNode {.inline, since: (1, 5, 1).} = + ## Gets the node at `array.len-i` in an array through the `^` operator. + ## + ## i.e. `j[^i]` is a shortcut for `j[j.len-i]`. + runnableExamples: + let + j = parseJson("[1,2,3,4,5]") + + doAssert j[^1].getInt == 5 + doAssert j[^2].getInt == 4 + + `[]`(node, node.len - int(index)) + +proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode = + ## Slice operation for JArray. + ## + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + runnableExamples: + import std/json + let arr = %[0,1,2,3,4,5] + doAssert arr[2..4] == %[2,3,4] + doAssert arr[2..^2] == %[2,3,4] + doAssert arr[^4..^2] == %[2,3,4] + + assert(a.kind == JArray) + result = newJArray() + let xa = (when x.a is BackwardsIndex: a.len - int(x.a) else: int(x.a)) + let L = (when x.b is BackwardsIndex: a.len - int(x.b) else: int(x.b)) - xa + 1 + for i in 0..<L: + result.add(a[i + xa]) + proc hasKey*(node: JsonNode, key: string): bool = ## Checks if `key` exists in `node`. assert(node.kind == JObject) @@ -492,15 +579,16 @@ proc contains*(node: JsonNode, val: JsonNode): bool = proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## keys do not exist, returns `nil`. Also returns `nil` if one of the ## intermediate data structures is not an object. ## ## This proc can be used to create tree structures on the ## fly (sometimes called `autovivification`:idx:): ## - ## .. code-block:: nim - ## myjson{"parent", "child", "grandchild"} = newJInt(1) - ## + runnableExamples: + var myjson = %* {"parent": {"child": {"grandchild": 1}}} + doAssert myjson{"parent", "child", "grandchild"} == newJInt(1) + result = node for key in keys: if isNil(result) or result.kind != JObject: @@ -509,7 +597,7 @@ proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = proc `{}`*(node: JsonNode, index: varargs[int]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## indexes do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## indexes do not exist, returns `nil`. Also returns `nil` if one of the ## intermediate data structures is not an array. result = node for i in index: @@ -523,12 +611,14 @@ proc getOrDefault*(node: JsonNode, key: string): JsonNode = if not isNil(node) and node.kind == JObject: result = node.fields.getOrDefault(key) -template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, - key: string): JsonNode = node.getOrDefault(key) +proc `{}`*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + node.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to ``value``. If any of the keys are missing, they are added. + ## to `value`. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): if not node.hasKey(keys[i]): @@ -537,17 +627,18 @@ proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = node[keys[keys.len-1]] = value proc delete*(obj: JsonNode, key: string) = - ## Deletes ``obj[key]``. + ## Deletes `obj[key]`. assert(obj.kind == JObject) if not obj.fields.hasKey(key): raise newException(KeyError, "key not in object") obj.fields.del(key) proc copy*(p: JsonNode): JsonNode = - ## Performs a deep copy of `a`. + ## Performs a deep copy of `p`. case p.kind of JString: result = newJString(p.str) + result.isUnquoted = p.isUnquoted of JInt: result = newJInt(p.num) of JFloat: @@ -579,14 +670,14 @@ proc nl(s: var string, ml: bool) = proc escapeJsonUnquoted*(s: string; result: var string) = ## Converts a string `s` to its JSON representation without quotes. - ## Appends to ``result``. + ## Appends to `result`. for c in s: case c of '\L': result.add("\\n") of '\b': result.add("\\b") of '\f': result.add("\\f") of '\t': result.add("\\t") - of '\v': result.add("\\v") + of '\v': result.add("\\u000b") of '\r': result.add("\\r") of '"': result.add("\\\"") of '\0'..'\7': result.add("\\u000" & $ord(c)) @@ -601,7 +692,7 @@ proc escapeJsonUnquoted*(s: string): string = proc escapeJson*(s: string; result: var string) = ## Converts a string `s` to its JSON representation with quotes. - ## Appends to ``result``. + ## Appends to `result`. result.add("\"") escapeJsonUnquoted(s, result) result.add("\"") @@ -611,6 +702,47 @@ proc escapeJson*(s: string): string = result = newStringOfCap(s.len + s.len shr 3) escapeJson(s, result) +proc toUgly*(result: var string, node: JsonNode) = + ## Converts `node` to its JSON Representation, without + ## regard for human readability. Meant to improve `$` string + ## conversion performance. + ## + ## JSON representation is stored in the passed `result` + ## + ## This provides higher efficiency than the `pretty` procedure as it + ## does **not** attempt to format the resulting JSON to make it human readable. + var comma = false + case node.kind: + of JArray: + result.add "[" + for child in node.elems: + if comma: result.add "," + else: comma = true + result.toUgly child + result.add "]" + of JObject: + result.add "{" + for key, value in pairs(node.fields): + if comma: result.add "," + else: comma = true + key.escapeJson(result) + result.add ":" + result.toUgly value + result.add "}" + of JString: + if node.isUnquoted: + result.add node.str + else: + escapeJson(node.str, result) + of JInt: + result.addInt(node.num) + of JFloat: + result.addFloat(node.fnum) + of JBool: + result.add(if node.bval: "true" else: "false") + of JNull: + result.add "null" + proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, lstArr = false, currIndent = 0) = case node.kind @@ -638,16 +770,13 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("{}") of JString: if lstArr: result.indent(currIndent) - escapeJson(node.str, result) + toUgly(result, node) of JInt: if lstArr: result.indent(currIndent) - when defined(js): result.add($node.num) - else: result.addInt(node.num) + result.addInt(node.num) of JFloat: if lstArr: result.indent(currIndent) - # Fixme: implement new system.add ops for the JS target - when defined(js): result.add($node.fnum) - else: result.addFloat(node.fnum) + result.addFloat(node.fnum) of JBool: if lstArr: result.indent(currIndent) result.add(if node.bval: "true" else: "false") @@ -692,46 +821,6 @@ proc pretty*(node: JsonNode, indent = 2): string = result = "" toPretty(result, node, indent) -proc toUgly*(result: var string, node: JsonNode) = - ## Converts `node` to its JSON Representation, without - ## regard for human readability. Meant to improve ``$`` string - ## conversion performance. - ## - ## JSON representation is stored in the passed `result` - ## - ## This provides higher efficiency than the ``pretty`` procedure as it - ## does **not** attempt to format the resulting JSON to make it human readable. - var comma = false - case node.kind: - of JArray: - result.add "[" - for child in node.elems: - if comma: result.add "," - else: comma = true - result.toUgly child - result.add "]" - of JObject: - result.add "{" - for key, value in pairs(node.fields): - if comma: result.add "," - else: comma = true - key.escapeJson(result) - result.add ":" - result.toUgly value - result.add "}" - of JString: - node.str.escapeJson(result) - of JInt: - when defined(js): result.add($node.num) - else: result.addInt(node.num) - of JFloat: - when defined(js): result.add($node.fnum) - else: result.addFloat(node.fnum) - of JBool: - result.add(if node.bval: "true" else: "false") - of JNull: - result.add "null" - proc `$`*(node: JsonNode): string = ## Converts `node` to its JSON Representation on one line. result = newStringOfCap(node.len shl 1) @@ -739,49 +828,65 @@ proc `$`*(node: JsonNode): string = iterator items*(node: JsonNode): JsonNode = ## Iterator for the items of `node`. `node` has to be a JArray. - assert node.kind == JArray + assert node.kind == JArray, ": items() can not iterate a JsonNode of kind " & $node.kind for i in items(node.elems): yield i iterator mitems*(node: var JsonNode): var JsonNode = ## Iterator for the items of `node`. `node` has to be a JArray. Items can be ## modified. - assert node.kind == JArray + assert node.kind == JArray, ": mitems() can not iterate a JsonNode of kind " & $node.kind for i in mitems(node.elems): yield i iterator pairs*(node: JsonNode): tuple[key: string, val: JsonNode] = ## Iterator for the child elements of `node`. `node` has to be a JObject. - assert node.kind == JObject + assert node.kind == JObject, ": pairs() can not iterate a JsonNode of kind " & $node.kind for key, val in pairs(node.fields): yield (key, val) iterator keys*(node: JsonNode): string = ## Iterator for the keys in `node`. `node` has to be a JObject. - assert node.kind == JObject + assert node.kind == JObject, ": keys() can not iterate a JsonNode of kind " & $node.kind for key in node.fields.keys: yield key iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = ## Iterator for the child elements of `node`. `node` has to be a JObject. ## Values can be modified - assert node.kind == JObject + assert node.kind == JObject, ": mpairs() can not iterate a JsonNode of kind " & $node.kind for key, val in mpairs(node.fields): yield (key, val) -proc parseJson(p: var JsonParser): JsonNode = +proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool, depth = 0): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: - result = newJStringMove(p.a) - p.a = "" + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + result = JsonNode(kind: JString, str: move p.a) + else: + result = JsonNode(kind: JString) + shallowCopy(result.str, p.a) + p.a = "" discard getTok(p) of tkInt: - result = newJInt(parseBiggestInt(p.a)) + if rawIntegers: + result = newJRawNumber(p.a) + else: + try: + result = newJInt(parseBiggestInt(p.a)) + except ValueError: + result = newJRawNumber(p.a) discard getTok(p) of tkFloat: - result = newJFloat(parseFloat(p.a)) + if rawFloats: + result = newJRawNumber(p.a) + else: + try: + result = newJFloat(parseFloat(p.a)) + except ValueError: + result = newJRawNumber(p.a) discard getTok(p) of tkTrue: result = newJBool(true) @@ -793,6 +898,8 @@ proc parseJson(p: var JsonParser): JsonNode = result = newJNull() discard getTok(p) of tkCurlyLe: + if depth > DepthLimit: + raiseParseErr(p, "}") result = newJObject() discard getTok(p) while p.tok != tkCurlyRi: @@ -801,69 +908,81 @@ proc parseJson(p: var JsonParser): JsonNode = var key = p.a discard getTok(p) eat(p, tkColon) - var val = parseJson(p) + var val = parseJson(p, rawIntegers, rawFloats, depth+1) result[key] = val if p.tok != tkComma: break discard getTok(p) eat(p, tkCurlyRi) of tkBracketLe: + if depth > DepthLimit: + raiseParseErr(p, "]") result = newJArray() discard getTok(p) while p.tok != tkBracketRi: - result.add(parseJson(p)) + result.add(parseJson(p, rawIntegers, rawFloats, depth+1)) if p.tok != tkComma: break discard getTok(p) eat(p, tkBracketRi) of tkError, tkCurlyRi, tkBracketRi, tkColon, tkComma, tkEof: raiseParseErr(p, "{") -iterator parseJsonFragments*(s: Stream, filename: string = ""): JsonNode = +iterator parseJsonFragments*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats = false): JsonNode = ## Parses from a stream `s` into `JsonNodes`. `filename` is only needed ## for nice error messages. ## The JSON fragments are separated by whitespace. This can be substantially ## faster than the comparable loop - ## ``for x in splitWhitespace(s): yield parseJson(x)``. + ## `for x in splitWhitespace(s): yield parseJson(x)`. ## This closes the stream `s` after it's done. + ## If `rawIntegers` is true, integer literals will not be converted to a `JInt` + ## field but kept as raw numbers via `JString`. + ## If `rawFloats` is true, floating point literals will not be converted to a `JFloat` + ## field but kept as raw numbers via `JString`. var p: JsonParser p.open(s, filename) try: discard getTok(p) # read first token while p.tok != tkEof: - yield p.parseJson() + yield p.parseJson(rawIntegers, rawFloats) finally: p.close() -proc parseJson*(s: Stream, filename: string = ""): JsonNode = +proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats = false): JsonNode = ## Parses from a stream `s` into a `JsonNode`. `filename` is only needed ## for nice error messages. ## If `s` contains extra data, it will raise `JsonParsingError`. ## This closes the stream `s` after it's done. + ## If `rawIntegers` is true, integer literals will not be converted to a `JInt` + ## field but kept as raw numbers via `JString`. + ## If `rawFloats` is true, floating point literals will not be converted to a `JFloat` + ## field but kept as raw numbers via `JString`. var p: JsonParser p.open(s, filename) try: discard getTok(p) # read first token - result = p.parseJson() + result = p.parseJson(rawIntegers, rawFloats) eat(p, tkEof) # check if there is no extra data finally: p.close() when defined(js): - from math import `mod` - type - JSObject = object + from std/math import `mod` + from std/jsffi import JsObject, `[]`, to + from std/private/jsutils import getProtoName, isInteger, isSafeInteger - proc parseNativeJson(x: cstring): JSObject {.importc: "JSON.parse".} + proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".} - proc getVarType(x: JSObject): JsonNodeKind = + proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind = result = JNull - proc getProtoName(y: JSObject): cstring - {.importc: "Object.prototype.toString.call".} case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray of "[object Object]": return JObject of "[object Number]": - if cast[float](x) mod 1.0 == 0: - return JInt + if isInteger(x) and 1.0 / cast[float](x) != -Inf: # preserve -0.0 as float + if isSafeInteger(x): + return JInt + else: + isRawNumber = true + return JString else: return JFloat of "[object Boolean]": return JBool @@ -871,48 +990,43 @@ when defined(js): of "[object String]": return JString else: assert false - proc len(x: JSObject): int = - assert x.getVarType == JArray - asm """ + proc len(x: JsObject): int = + {.emit: """ `result` = `x`.length; - """ - - proc `[]`(x: JSObject, y: string): JSObject = - assert x.getVarType == JObject - asm """ - `result` = `x`[`y`]; - """ - - proc `[]`(x: JSObject, y: int): JSObject = - assert x.getVarType == JArray - asm """ - `result` = `x`[`y`]; - """ - - proc convertObject(x: JSObject): JsonNode = - case getVarType(x) + """.} + + proc convertObject(x: JsObject): JsonNode = + var isRawNumber = false + case getVarType(x, isRawNumber) of JArray: result = newJArray() for i in 0 ..< x.len: result.add(x[i].convertObject()) of JObject: result = newJObject() - asm """for (var property in `x`) { + {.emit: """for (var property in `x`) { if (`x`.hasOwnProperty(property)) { - """ + """.} + var nimProperty: cstring - var nimValue: JSObject - asm "`nimProperty` = property; `nimValue` = `x`[property];" + var nimValue: JsObject + {.emit: "`nimProperty` = property; `nimValue` = `x`[property];".} result[$nimProperty] = nimValue.convertObject() - asm "}}" + {.emit: "}}".} of JInt: - result = newJInt(cast[int](x)) + result = newJInt(x.to(int)) of JFloat: - result = newJFloat(cast[float](x)) + result = newJFloat(x.to(float)) of JString: - result = newJString($cast[cstring](x)) + # Dunno what to do with isUnquoted here + if isRawNumber: + var value: cstring + {.emit: "`value` = `x`.toString();".} + result = newJRawNumber($value) + else: + result = newJString($x.to(cstring)) of JBool: - result = newJBool(cast[bool](x)) + result = newJBool(x.to(bool)) of JNull: result = newJNull() @@ -923,10 +1037,14 @@ when defined(js): return parseNativeJson(buffer).convertObject() else: - proc parseJson*(buffer: string): JsonNode = + proc parseJson*(buffer: string; rawIntegers = false, rawFloats = false): JsonNode = ## Parses JSON from `buffer`. ## If `buffer` contains extra data, it will raise `JsonParsingError`. - result = parseJson(newStringStream(buffer), "input") + ## If `rawIntegers` is true, integer literals will not be converted to a `JInt` + ## field but kept as raw numbers via `JString`. + ## If `rawFloats` is true, floating point literals will not be converted to a `JFloat` + ## field but kept as raw numbers via `JString`. + result = parseJson(newStringStream(buffer), "input", rawIntegers, rawFloats) proc parseFile*(filename: string): JsonNode = ## Parses `file` into a `JsonNode`. @@ -934,7 +1052,7 @@ else: var stream = newFileStream(filename, fmRead) if stream == nil: raise newException(IOError, "cannot read from file: " & filename) - result = parseJson(stream, filename) + result = parseJson(stream, filename, rawIntegers=false, rawFloats=false) # -- Json deserialiser. -- @@ -950,286 +1068,306 @@ template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ] raise newException(JsonKindError, msg) -when defined(nimFixedForwardGeneric): - - macro isRefSkipDistinct(arg: typed): untyped = - var impl = getTypeImpl(arg) - if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): - impl = getTypeImpl(impl[1]) - while impl.kind == nnkDistinctTy: - impl = getTypeImpl(impl[0]) - result = newLit(impl.kind == nnkRefTy) - - # The following forward declarations don't work in older versions of Nim - - # forward declare all initFromJson - - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string,T];jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string,T];jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) - - # initFromJson definitions - - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) - # since strings don't have a nil state anymore, this mapping of - # JNull to the default string is questionable. `none(string)` and - # `some("")` have the same potentional json value `JNull`. - if jsonNode.kind == JNull: - dst = "" - else: - dst = jsonNode.str +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) + +# The following forward declarations don't work in older versions of Nim + +# forward declare all initFromJson + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) + +# initFromJson definitions + +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JBool}, jsonPath) - dst = jsonNode.bval +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = - dst = jsonNode.copy +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or int.sizeof == 4: + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) + else: + dst = T(jsonNode.num) + else: verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = T(jsonNode.num) - - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) + dst = cast[T](jsonNode.num) + +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JInt, JFloat, JString}, jsonPath) + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: if jsonNode.kind == JFloat: dst = T(jsonNode.fnum) else: dst = T(jsonNode.num) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString}, jsonPath) - dst = parseEnum[T](jsonNode.getStr) - - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - dst.setLen jsonNode.len - let orignalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen orignalJsonPathLen - - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - let originalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var Table[string,T];jsonNode: JsonNode; jsonPath: var string) = - dst = initTable[string, T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var OrderedTable[string,T];jsonNode: JsonNode; jsonPath: var string) = - dst = initOrderedTable[string,T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) - if jsonNode.kind == JNull: - dst = nil - else: - dst = new(T) - initFromJson(dst[], jsonNode, jsonPath) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) + +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen + +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode != nil and jsonNode.kind != JNull: +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) + else: dst = some(default(T)) - initFromJson(dst.get, jsonNode, jsonPath) - - macro assignDistinctImpl[T : distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = - let typInst = getTypeInst(dst) - let typImpl = getTypeImpl(dst) - let baseTyp = typImpl[0] - - result = quote do: + initFromJson(dst.get, jsonNode, jsonPath) + +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] + + result = quote do: + initFromJson(`baseTyp`(`dst`), `jsonNode`, `jsonPath`) + +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) + +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) + +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` when nimvm: - # workaround #12282 - var tmp: `baseTyp` - initFromJson( tmp, `jsonNode`, `jsonPath`) - `dst` = `typInst`(tmp) - else: - initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - - proc initFromJson[T : distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignDistinctImpl(dst, jsonNode, jsonPath) - - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode): void = - if typeExpr.kind == nnkTupleConstr: - error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - - proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode): void {.compileTime.} = - case typeNode.kind - of nnkEmpty: - discard - of nnkRecList, nnkTupleTy: - for it in typeNode: - foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkIdentDefs: - typeNode.expectLen 3 - let fieldSym = typeNode[0] - let fieldNameLit = newLit(fieldSym.strVal) - let fieldPathLit = newLit("." & fieldSym.strVal) - let fieldType = typeNode[1] - - # Detecting incompatiple tuple types in `assignObjectImpl` only - # would be much cleaner, but the ast for tuple types does not - # contain usable type information. - detectIncompatibleType(fieldType, fieldSym) - - dst.add quote do: - jsonPath.add `fieldPathLit` - when nimvm: - when isRefSkipDistinct(`tmpSym`.`fieldSym`): - # workaround #12489 - var tmp: `fieldType` - initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - `tmpSym`.`fieldSym` = tmp - else: - initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp else: initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - - of nnkRecCase: - let kindSym = typeNode[0][0] - let kindNameLit = newLit(kindSym.strVal) - let kindPathLit = newLit("." & kindSym.strVal) - let kindType = typeNode[0][1] - let kindOffsetLit = newLit(uint(getOffset(kindSym))) - dst.add quote do: - var kindTmp: `kindType` - jsonPath.add `kindPathLit` - initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - when defined js: + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: `tmpSym`.`kindSym` = kindTmp else: - when nimvm: - `tmpSym`.`kindSym` = kindTmp - else: - # fuck it, assign kind field anyway - ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp - dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) - for i in 1 ..< typeNode.len: - foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkOfBranch, nnkElse: - let ofBranch = newNimNode(typeNode.kind) - for i in 0 ..< typeNode.len-1: - ofBranch.add copyNimTree(typeNode[i]) - let dstInner = newNimNode(nnkStmtListExpr) - foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - # resOuter now contains the inner stmtList - ofBranch.add dstInner - dst[^1].expectKind nnkCaseStmt - dst[^1].add ofBranch - - of nnkObjectTy: - typeNode[0].expectKind nnkEmpty - typeNode[1].expectKind {nnkEmpty, nnkOfInherit} - if typeNode[1].kind == nnkOfInherit: - let base = typeNode[1][0] - var impl = getTypeImpl(base) - while impl.kind in {nnkRefTy, nnkPtrTy}: - impl = getTypeImpl(impl[0]) - foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - let body = typeNode[2] - foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch + + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - else: - error("unhandled kind: " & $typeNode.kind, typeNode) - - - macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - let typeSym = getTypeInst(dst) - let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") - result = newStmtList() - result.add quote do: - let `originalJsonPathLen` = len(`jsonPath`) - if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: - # both, `dst` and `typeSym` don't have good lineinfo. But nothing - # else is available here. - detectIncompatibleType(typeSym, dst) - foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) - else: - foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) - - proc initFromJson[T : object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignObjectImpl(dst, jsonNode, jsonPath) - - proc to*[T](node: JsonNode, t: typedesc[T]): T = - ## `Unmarshals`:idx: the specified node into the object type specified. - ## - ## Known limitations: - ## - ## * Heterogeneous arrays are not supported. - ## * Sets in object variants are not supported. - ## * Not nil annotations are not supported. - ## - ## Example: - ## - ## .. code-block:: Nim - ## let jsonNode = parseJson(""" - ## { - ## "person": { - ## "name": "Nimmer", - ## "age": 21 - ## }, - ## "list": [1, 2, 3, 4] - ## } - ## """) - ## - ## type - ## Person = object - ## name: string - ## age: int - ## - ## Data = object - ## person: Person - ## list: seq[int] - ## - ## var data = to(jsonNode, Data) - ## doAssert data.person.name == "Nimmer" - ## doAssert data.person.age == 21 - ## doAssert data.list == @[1, 2, 3, 4] - - var jsonPath = "" - initFromJson(result, node, jsonPath) + else: + error("unhandled kind: " & $typeNode.kind, typeNode) + +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) + +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) + +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) + + type + Person = object + name: string + age: int + + Data = object + person: Person + list: seq[int] + + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] + + var jsonPath = "" + result = default(T) + initFromJson(result, node, jsonPath) when false: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: JsonParser @@ -1254,235 +1392,3 @@ when false: # { "json": 5 } # To get that we shall use, obj["json"] - -when isMainModule: - # Note: Macro tests are in tests/stdlib/tjsonmacro.nim - - let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" - # nil passthrough - doAssert(testJson{"doesnt_exist"}{"anything"}.isNil) - testJson{["e", "f"]} = %true - doAssert(testJson["e"]["f"].bval) - - # make sure UTF-16 decoding works. - doAssert(testJson["c"].str == "🎃") - doAssert(testJson["d"].str == "æ") - - # make sure no memory leek when parsing invalid string - let startMemory = getOccupiedMem() - for i in 0 .. 10000: - try: - discard parseJson"""{ invalid""" - except: - discard - # memory diff should less than 4M - doAssert(abs(getOccupiedMem() - startMemory) < 4 * 1024 * 1024) - - - # test `$` - let stringified = $testJson - let parsedAgain = parseJson(stringified) - doAssert(parsedAgain["b"].str == "asd") - - parsedAgain["abc"] = %5 - doAssert parsedAgain["abc"].num == 5 - - # Bounds checking - when compileOption("boundChecks"): - try: - let a = testJson["a"][9] - doAssert(false, "IndexError not thrown") - except IndexError: - discard - try: - let a = testJson["a"][-1] - doAssert(false, "IndexError not thrown") - except IndexError: - discard - try: - doAssert(testJson["a"][0].num == 1, "Index doesn't correspond to its value") - except: - doAssert(false, "IndexError thrown for valid index") - - doAssert(testJson{"b"}.getStr() == "asd", "Couldn't fetch a singly nested key with {}") - doAssert(isNil(testJson{"nonexistent"}), "Non-existent keys should return nil") - doAssert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") - doAssert(isNil(testJson{"a", "b"}), "Indexing through a list should return nil") - doAssert(testJson{"a"} == parseJson"[1, 2, 3, 4]", "Didn't return a non-JObject when there was one to be found") - doAssert(isNil(parseJson("[1, 2, 3]"){"foo"}), "Indexing directly into a list should return nil") - - # Generator: - var j = %* [{"name": "John", "age": 30}, {"name": "Susan", "age": 31}] - doAssert j == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] - - var j2 = %* - [ - { - "name": "John", - "age": 30 - }, - { - "name": "Susan", - "age": 31 - } - ] - doAssert j2 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] - - var name = "John" - let herAge = 30 - const hisAge = 31 - - var j3 = %* - [ {"name": "John" - , "age": herAge - } - , {"name": "Susan" - , "age": hisAge - } - ] - doAssert j3 == %[%{"name": %"John", "age": %30}, %{"name": %"Susan", "age": %31}] - - var j4 = %*{"test": nil} - doAssert j4 == %{"test": newJNull()} - - let seqOfNodes = @[%1, %2] - let jSeqOfNodes = %seqOfNodes - doAssert(jSeqOfNodes[1].num == 2) - - type MyObj = object - a, b: int - s: string - f32: float32 - f64: float64 - next: ref MyObj - var m: MyObj - m.s = "hi" - m.a = 5 - let jMyObj = %m - doAssert(jMyObj["a"].num == 5) - doAssert(jMyObj["s"].str == "hi") - - # Test loading of file. - when not defined(js): - var parsed = parseFile("tests/testdata/jsontest.json") - - try: - discard parsed["key2"][12123] - doAssert(false) - except IndexError: doAssert(true) - - var parsed2 = parseFile("tests/testdata/jsontest2.json") - doAssert(parsed2{"repository", "description"}.str == - "IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") - - doAssert escapeJsonUnquoted("\10Foo🎃barÄ") == "\\nFoo🎃barÄ" - doAssert escapeJsonUnquoted("\0\7\20") == "\\u0000\\u0007\\u0014" # for #7887 - doAssert escapeJson("\10Foo🎃barÄ") == "\"\\nFoo🎃barÄ\"" - doAssert escapeJson("\0\7\20") == "\"\\u0000\\u0007\\u0014\"" # for #7887 - - # Test with extra data - when not defined(js): - try: - discard parseJson("123 456") - doAssert(false) - except JsonParsingError: - doAssert getCurrentExceptionMsg().contains(errorMessages[errEofExpected]) - - try: - discard parseFile("tests/testdata/jsonwithextradata.json") - doAssert(false) - except JsonParsingError: - doAssert getCurrentExceptionMsg().contains(errorMessages[errEofExpected]) - - # bug #6438 - doAssert($ %*[] == "[]") - doAssert($ %*{} == "{}") - - doAssert(not compiles(%{"error": "No messages"})) - - # bug #9111 - block: - type - Bar = string - Foo = object - a: int - b: Bar - - let - js = """{"a": 123, "b": "abc"}""".parseJson - foo = js.to Foo - - doAssert(foo.b == "abc") - - # Generate constructors for range[T] types - block: - type - Q1 = range[0'u8 .. 50'u8] - Q2 = range[0'u16 .. 50'u16] - Q3 = range[0'u32 .. 50'u32] - Q4 = range[0'i8 .. 50'i8] - Q5 = range[0'i16 .. 50'i16] - Q6 = range[0'i32 .. 50'i32] - Q7 = range[0'f32 .. 50'f32] - Q8 = range[0'f64 .. 50'f64] - Q9 = range[0 .. 50] - - X = object - m1: Q1 - m2: Q2 - m3: Q3 - m4: Q4 - m5: Q5 - m6: Q6 - m7: Q7 - m8: Q8 - m9: Q9 - - let obj = X( - m1: Q1(42), - m2: Q2(42), - m3: Q3(42), - m4: Q4(42), - m5: Q5(42), - m6: Q6(42), - m7: Q7(42), - m8: Q8(42), - m9: Q9(42) - ) - - doAssert(obj == to(%obj, type(obj))) - - when not defined(js): - const fragments = """[1,2,3] {"hi":3} 12 [] """ - var res = "" - for x in parseJsonFragments(newStringStream(fragments)): - res.add($x) - res.add " " - doAssert res == fragments - - - # test isRefSkipDistinct - type - MyRef = ref object - MyObject = object - MyDistinct = distinct MyRef - MyOtherDistinct = distinct MyRef - - var x0: ref int - var x1: MyRef - var x2: MyObject - var x3: MyDistinct - var x4: MyOtherDistinct - - doAssert isRefSkipDistinct(x0) - doAssert isRefSkipDistinct(x1) - doAssert not isRefSkipDistinct(x2) - doAssert isRefSkipDistinct(x3) - doAssert isRefSkipDistinct(x4) - - - doAssert isRefSkipDistinct(ref int) - doAssert isRefSkipDistinct(MyRef) - doAssert not isRefSkipDistinct(MyObject) - doAssert isRefSkipDistinct(MyDistinct) - doAssert isRefSkipDistinct(MyOtherDistinct) diff --git a/lib/pure/lenientops.nim b/lib/pure/lenientops.nim index cdea780a2..a8fc78e39 100644 --- a/lib/pure/lenientops.nim +++ b/lib/pure/lenientops.nim @@ -8,51 +8,49 @@ # ## This module offers implementations of common binary operations -## like ``+``, ``-``, ``*``, ``/`` and comparison operations, +## like `+`, `-`, `*`, `/` and comparison operations, ## which work for mixed float/int operands. ## All operations convert the integer operand into the ## type of the float operand. For numerical expressions, the return ## type is always the type of the float involved in the expression, ## i.e., there is no auto conversion from float32 to float64. ## -## Note: In general, auto-converting from int to float loses +## **Note:** In general, auto-converting from int to float loses ## information, which is why these operators live in a separate ## module. Use with care. ## ## Regarding binary comparison, this module only provides unequal operators. -## The equality operator ``==`` is omitted, because depending on the use case +## The equality operator `==` is omitted, because depending on the use case ## either casting to float or rounding to int might be preferred, and users ## should make an explicit choice. -import typetraits - -proc `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `+`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) + f -proc `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `+`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f + F(i) -proc `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `-`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) - f -proc `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `-`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f - F(i) -proc `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `*`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) * f -proc `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `*`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f * F(i) -proc `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.noSideEffect, inline.} = +func `/`*[I: SomeInteger, F: SomeFloat](i: I, f: F): F {.inline.} = F(i) / f -proc `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.noSideEffect, inline.} = +func `/`*[I: SomeInteger, F: SomeFloat](f: F, i: I): F {.inline.} = f / F(i) -proc `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = +func `<`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.inline.} = F(i) < f -proc `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = +func `<`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.inline.} = f < F(i) -proc `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.noSideEffect, inline.} = +func `<=`*[I: SomeInteger, F: SomeFloat](i: I, f: F): bool {.inline.} = F(i) <= f -proc `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.noSideEffect, inline.} = +func `<=`*[I: SomeInteger, F: SomeFloat](f: F, i: I): bool {.inline.} = f <= F(i) # Note that we must not defined `>=` and `>`, because system.nim already has a diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 8bc96c82c..1efd97b24 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2009 Andreas Rumpf # # See the file "copying.txt", included in this @@ -12,7 +12,10 @@ ## needs refilling. import - strutils, streams + std/[strutils, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions const EndOfFile* = '\0' ## end of file marker @@ -33,7 +36,7 @@ type lineNumber*: int ## the current line number sentinel: int lineStart: int # index of last line start in buffer - offsetBase*: int # use ``offsetBase + bufpos`` to get the offset + offsetBase*: int # use `offsetBase + bufpos` to get the offset refillChars: set[char] proc close*(L: var BaseLexer) = @@ -52,7 +55,8 @@ proc fillBuffer(L: var BaseLexer) = toCopy = L.buf.len - (L.sentinel + 1) assert(toCopy >= 0) if toCopy > 0: - when defined(js): + when defined(js) or defined(nimscript): + # nimscript has to be here to avoid compiling other branch (moveMem) for i in 0 ..< toCopy: L.buf[i] = L.buf[L.sentinel + 1 + i] else: @@ -100,9 +104,9 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int = result = 0 proc handleCR*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\c' in the buffer; it returns the + ## Call this if you scanned over `'\c'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\c'. + ## of the `'\c'`. assert(L.buf[pos] == '\c') inc(L.lineNumber) result = fillBaseLexer(L, pos) @@ -111,9 +115,9 @@ proc handleCR*(L: var BaseLexer, pos: int): int = L.lineStart = result proc handleLF*(L: var BaseLexer, pos: int): int = - ## Call this if you scanned over '\L' in the buffer; it returns the + ## Call this if you scanned over `'\L'` in the buffer; it returns the ## position to continue the scanning from. `pos` must be the position - ## of the '\L'. + ## of the `'\L'`. assert(L.buf[pos] == '\L') inc(L.lineNumber) result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result; diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 2637fdf9d..c30f68af8 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -17,10 +17,11 @@ ## ## To get started, first create a logger: ## -## .. code-block:: -## import logging +## ```Nim +## import std/logging ## ## var logger = newConsoleLogger() +## ``` ## ## The logger that was created above logs to the console, but this module ## also provides loggers that log to files, such as the @@ -30,9 +31,10 @@ ## Once a logger has been created, call its `log proc ## <#log.e,ConsoleLogger,Level,varargs[string,]>`_ to log a message: ## -## .. code-block:: +## ```Nim ## logger.log(lvlInfo, "a log message") ## # Output: INFO a log message +## ``` ## ## The ``INFO`` within the output is the result of a format string being ## prepended to the message, and it will differ depending on the message's @@ -45,11 +47,10 @@ ## ``levelThreshold`` field and the global log filter. The latter can be changed ## with the `setLogFilter proc<#setLogFilter,Level>`_. ## -## **Warning:** -## * For loggers that log to a console or to files, only error and fatal -## messages will cause their output buffers to be flushed immediately. -## Use the `flushFile proc <io.html#flushFile,File>`_ to flush the buffer -## manually if needed. +## .. warning:: +## For loggers that log to a console or to files, only error and fatal +## messages will cause their output buffers to be flushed immediately by default. +## set ``flushThreshold`` when creating the logger to change this. ## ## Handlers ## -------- @@ -59,8 +60,8 @@ ## used with the `addHandler proc<#addHandler,Logger>`_, which is demonstrated ## in the following example: ## -## .. code-block:: -## import logging +## ```Nim +## import std/logging ## ## var consoleLog = newConsoleLogger() ## var fileLog = newFileLogger("errors.log", levelThreshold=lvlError) @@ -69,17 +70,19 @@ ## addHandler(consoleLog) ## addHandler(fileLog) ## addHandler(rollingLog) +## ``` ## ## After doing this, use either the `log template ## <#log.t,Level,varargs[string,]>`_ or one of the level-specific templates, ## such as the `error template<#error.t,varargs[string,]>`_, to log messages ## to all registered handlers at once. ## -## .. code-block:: +## ```Nim ## # This example uses the loggers created above ## log(lvlError, "an error occurred") ## error("an error occurred") # Equivalent to the above line ## info("something normal happened") # Will not be written to errors.log +## ``` ## ## Note that a message's level is still checked against each handler's ## ``levelThreshold`` and the global log filter. @@ -117,12 +120,13 @@ ## ## The following example illustrates how to use format strings: ## -## .. code-block:: -## import logging +## ```Nim +## import std/logging ## ## var logger = newConsoleLogger(fmtStr="[$time] - $levelname: ") ## logger.log(lvlInfo, "this is a message") ## # Output: [19:50:13] - INFO: this is a message +## ``` ## ## Notes when using multiple threads ## --------------------------------- @@ -142,9 +146,12 @@ ## * `strscans module<strscans.html>`_ for ``scanf`` and ``scanp`` macros, which ## offer easier substring extraction than regular expressions -import strutils, times +import std/[strutils, times] when not defined(js): - import os + import std/os + +when defined(nimPreviewSlimSystem): + import std/syncio type Level* = enum ## \ @@ -197,6 +204,15 @@ const ## If a different format string is preferred, refer to the ## `documentation about format strings<#basic-usage-format-strings>`_ ## for more information, including a list of available variables. + defaultFlushThreshold = when NimMajor >= 2: + when defined(nimV1LogFlushBehavior): lvlError else: lvlAll + else: + when defined(nimFlushAllLogs): lvlAll else: lvlError + ## The threshold above which log messages to file-like loggers + ## are automatically flushed. + ## + ## By default, only error and fatal messages are logged, + ## but defining ``-d:nimFlushAllLogs`` will make all levels be flushed type Logger* = ref object of RootObj @@ -225,6 +241,8 @@ type ## * `FileLogger<#FileLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ useStderr*: bool ## If true, writes to stderr; otherwise, writes to stdout + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately when not defined(js): type @@ -240,13 +258,15 @@ when not defined(js): ## * `ConsoleLogger<#ConsoleLogger>`_ ## * `RollingFileLogger<#RollingFileLogger>`_ file*: File ## The wrapped file + flushThreshold*: Level ## Only messages that are at or above this + ## threshold will be flushed immediately RollingFileLogger* = ref object of FileLogger ## A logger that writes log messages to a file while performing log ## rotation. ## ## Create a new ``RollingFileLogger`` with the `newRollingFileLogger proc - ## <#newRollingFileLogger,FileMode,int,int>`_. + ## <#newRollingFileLogger,FileMode,Positive,int>`_. ## ## **Note:** This logger is not available for the JavaScript backend. ## @@ -287,6 +307,7 @@ proc substituteLog*(frmt: string, level: Level, runnableExamples: doAssert substituteLog(defaultFmtStr, lvlInfo, "a message") == "INFO a message" doAssert substituteLog("$levelid - ", lvlError, "an error") == "E - an error" + doAssert substituteLog("$levelid", lvlDebug, "error") == "Derror" var msgLen = 0 for arg in args: msgLen += arg.len @@ -300,7 +321,7 @@ proc substituteLog*(frmt: string, level: Level, inc(i) var v = "" let app = when defined(js): "" else: getAppFilename() - while frmt[i] in IdentChars: + while i < frmt.len and frmt[i] in IdentChars: v.add(toLowerAscii(frmt[i])) inc(i) case v @@ -344,8 +365,8 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## `setLogFilter proc<#setLogFilter,Level>`_. ## ## **Note:** Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <io.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## ## See also: ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ @@ -356,27 +377,33 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var consoleLog = newConsoleLogger() ## consoleLog.log(lvlInfo, "this is a message") ## consoleLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: let ln = substituteLog(logger.fmtStr, level, args) when defined(js): - let cln: cstring = ln - {.emit: "console.log(`cln`);".} + let cln = ln.cstring + case level + of lvlDebug: {.emit: "console.debug(`cln`);".} + of lvlInfo: {.emit: "console.info(`cln`);".} + of lvlWarn: {.emit: "console.warn(`cln`);".} + of lvlError: {.emit: "console.error(`cln`);".} + else: {.emit: "console.log(`cln`);".} else: try: var handle = stdout if logger.useStderr: handle = stderr writeLine(handle, ln) - if level in {lvlError, lvlFatal}: flushFile(handle) + if level >= logger.flushThreshold: flushFile(handle) except IOError: discard proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, - useStderr = false): ConsoleLogger = + useStderr = false, flushThreshold = defaultFlushThreshold): ConsoleLogger = ## Creates a new `ConsoleLogger<#ConsoleLogger>`_. ## ## By default, log messages are written to ``stdout``. If ``useStderr`` is @@ -389,17 +416,19 @@ proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr, ## * `newFileLogger proc<#newFileLogger,File>`_ that uses a file handle ## * `newFileLogger proc<#newFileLogger,FileMode,int>`_ ## that accepts a filename - ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,int,int>`_ + ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,Positive,int>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newConsoleLogger() ## var formatLog = newConsoleLogger(fmtStr=verboseFmtStr) ## var errorLog = newConsoleLogger(levelThreshold=lvlError, useStderr=true) + ## ``` new result result.fmtStr = fmtStr result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.useStderr = useStderr when not defined(js): @@ -415,8 +444,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <io.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -428,13 +457,14 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var fileLog = newFileLogger("messages.log") ## fileLog.log(lvlInfo, "this is a message") ## fileLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) proc defaultFilename*(): string = ## Returns the filename that is used by default when naming log files. @@ -445,7 +475,8 @@ when not defined(js): proc newFileLogger*(file: File, levelThreshold = lvlAll, - fmtStr = defaultFmtStr): FileLogger = + fmtStr = defaultFmtStr, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that uses the given file handle. ## ## **Note:** This proc is not available for the JavaScript backend. @@ -454,11 +485,11 @@ when not defined(js): ## * `newConsoleLogger proc<#newConsoleLogger>`_ ## * `newFileLogger proc<#newFileLogger,FileMode,int>`_ ## that accepts a filename - ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,int,int>`_ + ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,Positive,int>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var messages = open("messages.log", fmWrite) ## var formatted = open("formatted.log", fmWrite) ## var errors = open("errors.log", fmWrite) @@ -466,16 +497,19 @@ when not defined(js): ## var normalLog = newFileLogger(messages) ## var formatLog = newFileLogger(formatted, fmtStr=verboseFmtStr) ## var errorLog = newFileLogger(errors, levelThreshold=lvlError) + ## ``` new(result) result.file = file result.levelThreshold = levelThreshold + result.flushThreshold = flushThreshold result.fmtStr = fmtStr proc newFileLogger*(filename = defaultFilename(), mode: FileMode = fmAppend, levelThreshold = lvlAll, fmtStr = defaultFmtStr, - bufSize: int = -1): FileLogger = + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): FileLogger = ## Creates a new `FileLogger<#FileLogger>`_ that logs to a file with the ## given filename. ## @@ -490,21 +524,21 @@ when not defined(js): ## See also: ## * `newConsoleLogger proc<#newConsoleLogger>`_ ## * `newFileLogger proc<#newFileLogger,File>`_ that uses a file handle - ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,int,int>`_ + ## * `newRollingFileLogger proc<#newRollingFileLogger,FileMode,Positive,int>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newFileLogger("messages.log") ## var formatLog = newFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var errorLog = newFileLogger("errors.log", levelThreshold=lvlError) + ## ``` let file = open(filename, mode, bufSize = bufSize) - newFileLogger(file, levelThreshold, fmtStr) + newFileLogger(file, levelThreshold, fmtStr, flushThreshold) # ------ proc countLogLines(logger: RollingFileLogger): int = - result = 0 let fp = open(logger.baseName, fmRead) for line in fp.lines(): result.inc() @@ -531,8 +565,9 @@ when not defined(js): mode: FileMode = fmReadWrite, levelThreshold = lvlAll, fmtStr = defaultFmtStr, - maxLines = 1000, - bufSize: int = -1): RollingFileLogger = + maxLines: Positive = 1000, + bufSize: int = -1, + flushThreshold = defaultFlushThreshold): RollingFileLogger = ## Creates a new `RollingFileLogger<#RollingFileLogger>`_. ## ## Once the current log file being written to contains ``maxLines`` lines, @@ -554,11 +589,12 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var normalLog = newRollingFileLogger("messages.log") ## var formatLog = newRollingFileLogger("formatted.log", fmtStr=verboseFmtStr) ## var shortLog = newRollingFileLogger("short.log", maxLines=200) ## var errorLog = newRollingFileLogger("errors.log", levelThreshold=lvlError) + ## ``` new(result) result.levelThreshold = levelThreshold result.fmtStr = fmtStr @@ -568,6 +604,7 @@ when not defined(js): result.curLine = 0 result.baseName = filename result.baseMode = mode + result.flushThreshold = flushThreshold result.logFiles = countFiles(filename) @@ -594,8 +631,8 @@ when not defined(js): ## ## **Notes:** ## * Only error and fatal messages will cause the output buffer - ## to be flushed immediately. Use the `flushFile proc - ## <io.html#flushFile,File>`_ to flush the buffer manually if needed. + ## to be flushed immediately by default. Set ``flushThreshold`` when creating + ## the logger to change this. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -607,10 +644,11 @@ when not defined(js): ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var rollingLog = newRollingFileLogger("messages.log") ## rollingLog.log(lvlInfo, "this is a message") ## rollingLog.log(lvlError, "error code is: ", 404) + ## ``` if level >= logging.level and level >= logger.levelThreshold: if logger.curLine >= logger.maxLines: logger.file.close() @@ -621,7 +659,7 @@ when not defined(js): bufSize = logger.bufSize) writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) + if level >= logger.flushThreshold: flushFile(logger.file) logger.curLine.inc # -------- @@ -640,11 +678,12 @@ template log*(level: Level, args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## log(lvlInfo, "This is an example.") + ## ``` ## ## See also: ## * `debug template<#debug.t,varargs[string,]>`_ @@ -669,11 +708,12 @@ template debug*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## debug("myProc called with arguments: foo, 5") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -690,11 +730,12 @@ template info*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## info("Application started successfully.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -711,11 +752,12 @@ template notice*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## notice("An important operation has completed.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -731,11 +773,12 @@ template warn*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## warn("The previous operation took too long to process.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -753,11 +796,12 @@ template error*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## error("An exception occurred while processing the form.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -774,11 +818,12 @@ template fatal*(args: varargs[string, `$`]) = ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## var logger = newConsoleLogger() ## addHandler(logger) ## ## fatal("Can't open database -- exiting.") + ## ``` ## ## See also: ## * `log template<#log.t,Level,varargs[string,]>`_ @@ -789,11 +834,12 @@ template fatal*(args: varargs[string, `$`]) = proc addHandler*(handler: Logger) = ## Adds a logger to the list of registered handlers. ## - ## **Warning:** The list of handlers is a thread-local variable. If the given - ## handler will be used in multiple threads, this proc should be called in - ## each of those threads. + ## .. warning:: The list of handlers is a thread-local variable. If the given + ## handler will be used in multiple threads, this proc should be called in + ## each of those threads. ## ## See also: + ## * `removeHandler proc`_ ## * `getHandlers proc<#getHandlers>`_ runnableExamples: var logger = newConsoleLogger() @@ -801,6 +847,16 @@ proc addHandler*(handler: Logger) = doAssert logger in getHandlers() handlers.add(handler) +proc removeHandler*(handler: Logger) = + ## Removes a logger from the list of registered handlers. + ## + ## Note that for n times a logger is registered, n calls to this proc + ## are required to remove that logger. + for i, hnd in handlers: + if hnd == handler: + handlers.delete(i) + return + proc getHandlers*(): seq[Logger] = ## Returns a list of all the registered handlers. ## @@ -815,10 +871,10 @@ proc setLogFilter*(lvl: Level) = ## individual logger's ``levelThreshold``. By default, all messages are ## logged. ## - ## **Warning:** The global log filter is a thread-local variable. If logging - ## is being performed in multiple threads, this proc should be called in each - ## thread unless it is intended that different threads should log at different - ## logging levels. + ## .. warning:: The global log filter is a thread-local variable. If logging + ## is being performed in multiple threads, this proc should be called in each + ## thread unless it is intended that different threads should log at different + ## logging levels. ## ## See also: ## * `getLogFilter proc<#getLogFilter>`_ diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index cf262b5f0..f9b3d3e4c 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -10,7 +10,7 @@ ## This module contains procs for `serialization`:idx: and `deserialization`:idx: ## of arbitrary Nim data structures. The serialization format uses `JSON`:idx:. ## -## **Restriction**: For objects their type is **not** serialized. This means +## **Restriction:** For objects, their type is **not** serialized. This means ## essentially that it does not work if the object has some other runtime ## type than its compiletime type. ## @@ -18,31 +18,24 @@ ## Basic usage ## =========== ## -## .. code-block:: nim -## -## type -## A = object of RootObj -## B = object of A -## f: int -## -## var -## a: ref A -## b: ref B -## -## new(b) -## a = b -## echo($$a[]) # produces "{}", not "{f: 0}" -## -## # unmarshal -## let c = to[B]("""{"f": 2}""") -## assert typeof(c) is B -## assert c.f == 2 -## -## # marshal -## let s = $$c -## assert s == """{"f": 2}""" -## -## **Note**: The ``to`` and ``$$`` operations are available at compile-time! +runnableExamples: + type + A = object of RootObj + B = object of A + f: int + + let a: ref A = new(B) + assert $$a[] == "{}" # not "{f: 0}" + + # unmarshal + let c = to[B]("""{"f": 2}""") + assert typeof(c) is B + assert c.f == 2 + + # marshal + assert $$c == """{"f": 2}""" + +## **Note:** The `to` and `$$` operations are available at compile-time! ## ## ## See also @@ -50,8 +43,21 @@ ## * `streams module <streams.html>`_ ## * `json module <json.html>`_ +const unsupportedPlatform = + when defined(js): "javascript" + elif defined(nimscript): "nimscript" + else: "" + +when unsupportedPlatform != "": + {.error: "marshal module is not supported in " & unsupportedPlatform & """. +Please use alternative packages for serialization. +It is possible to reimplement this module using generics and type traits. +Please contribute a new implementation.""".} + +import std/[streams, typeinfo, json, intsets, tables, unicode] -import streams, typeinfo, json, intsets, tables, unicode +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] proc ptrToInt(x: pointer): int {.inline.} = result = cast[int](x) # don't skip alignment @@ -160,7 +166,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akSequence: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + invokeNewSeq(a, 0) + else: + setPointer(a, nil) next(p) of jsonArrayStart: next(p) @@ -201,7 +210,8 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = setPointer(a, nil) next(p) of jsonInt: - setPointer(a, t.getOrDefault(p.getInt)) + var raw = t.getOrDefault(p.getInt) + setPointer(a, addr raw) next(p) of jsonArrayStart: next(p) @@ -227,7 +237,10 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of akString: case p.kind of jsonNull: - setPointer(a, nil) + when defined(nimSeqsV2): + setString(a, "") + else: + setPointer(a, nil) next(p) of jsonString: setString(a, p.str) @@ -269,7 +282,8 @@ proc loadAny(s: Stream, a: Any, t: var Table[BiggestInt, pointer]) = proc load*[T](s: Stream, data: var T) = ## Loads `data` from the stream `s`. Raises `IOError` in case of an error. runnableExamples: - import marshal, streams + import std/streams + var s = newStringStream("[1, 3, 5]") var a: array[3, int] load(s, a) @@ -278,10 +292,11 @@ proc load*[T](s: Stream, data: var T) = var tab = initTable[BiggestInt, pointer]() loadAny(s, toAny(data), tab) -proc store*[T](s: Stream, data: T) = +proc store*[T](s: Stream, data: sink T) = ## Stores `data` into the stream `s`. Raises `IOError` in case of an error. runnableExamples: - import marshal, streams + import std/streams + var s = newStringStream("") var a = [1, 3, 5] store(s, a) @@ -290,13 +305,20 @@ proc store*[T](s: Stream, data: T) = var stored = initIntSet() var d: T - shallowCopy(d, data) + when defined(gcArc) or defined(gcOrc)or defined(gcAtomicArc): + d = data + else: + shallowCopy(d, data) storeAny(s, toAny(d), stored) -proc `$$`*[T](x: T): string = +proc loadVM[T](typ: typedesc[T], x: T): string = + discard "the implementation is in the compiler/vmops" + +proc `$$`*[T](x: sink T): string = ## Returns a string representation of `x` (serialization, marshalling). ## - ## **Note:** to serialize `x` to JSON use `$(%x)` from the ``json`` module. + ## **Note:** to serialize `x` to JSON use `%x` from the `json` module + ## or `jsonutils.toJson(x)`. runnableExamples: type Foo = object @@ -307,15 +329,24 @@ proc `$$`*[T](x: T): string = let y = $$x assert y == """{"id": 1, "bar": "baz"}""" - var stored = initIntSet() - var d: T - shallowCopy(d, x) - var s = newStringStream() - storeAny(s, toAny(d), stored) - result = s.data + when nimvm: + result = loadVM(T, x) + else: + var stored = initIntSet() + var d: T + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + d = x + else: + shallowCopy(d, x) + var s = newStringStream() + storeAny(s, toAny(d), stored) + result = s.data + +proc toVM[T](typ: typedesc[T], data: string): T = + discard "the implementation is in the compiler/vmops" proc to*[T](data: string): T = - ## Reads data and transforms it to a type ``T`` (deserialization, unmarshalling). + ## Reads data and transforms it to a type `T` (deserialization, unmarshalling). runnableExamples: type Foo = object @@ -329,79 +360,8 @@ proc to*[T](data: string): T = assert z.id == 1 assert z.bar == "baz" - var tab = initTable[BiggestInt, pointer]() - loadAny(newStringStream(data), toAny(result), tab) - - -when not defined(testing) and isMainModule: - template testit(x: untyped) = echo($$to[type(x)]($$x)) - - var x: array[0..4, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], - ["test", "1", "2", "3", "4"]] - testit(x) - var test2: tuple[name: string, s: uint] = ("tuple test", 56u) - testit(test2) - - type - TE = enum - blah, blah2 - - TestObj = object - test, asd: int - case test2: TE - of blah: - help: string - else: - nil - - PNode = ref Node - Node = object - next, prev: PNode - data: string - - proc buildList(): PNode = - new(result) - new(result.next) - new(result.prev) - result.data = "middle" - result.next.data = "next" - result.prev.data = "prev" - result.next.next = result.prev - result.next.prev = result - result.prev.next = result - result.prev.prev = result.next - - var test3: TestObj - test3.test = 42 - test3 = TestObj(test2: blah) - testit(test3) - - var test4: ref tuple[a, b: string] - new(test4) - test4.a = "ref string test: A" - test4.b = "ref string test: B" - testit(test4) - - var test5 = @[(0, 1), (2, 3), (4, 5)] - testit(test5) - - var test6: set[char] = {'A'..'Z', '_'} - testit(test6) - - var test7 = buildList() - echo($$test7) - testit(test7) - - type - A {.inheritable.} = object - B = object of A - f: int - - var - a: ref A - b: ref B - new(b) - a = b - echo($$a[]) # produces "{}", not "{f: 0}" + when nimvm: + result = toVM(T, data) + else: + var tab = initTable[BiggestInt, pointer]() + loadAny(newStringStream(data), toAny(result), tab) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index c8a433559..ed7d2382f 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -12,80 +12,137 @@ ## Basic math routines for Nim. ## ## Note that the trigonometric functions naturally operate on radians. -## The helper functions `degToRad<#degToRad,T>`_ and `radToDeg<#radToDeg,T>`_ +## The helper functions `degToRad <#degToRad,T>`_ and `radToDeg <#radToDeg,T>`_ ## provide conversion between radians and degrees. -## -## .. code-block:: -## -## import math -## from sequtils import map -## -## let a = [0.0, PI/6, PI/4, PI/3, PI/2] -## -## echo a.map(sin) -## # @[0.0, 0.499…, 0.707…, 0.866…, 1.0] -## -## echo a.map(tan) -## # @[0.0, 0.577…, 0.999…, 1.732…, 1.633…e+16] -## -## echo cos(degToRad(180.0)) -## # -1.0 -## -## echo sqrt(-1.0) -## # nan (use `complex` module) -## + +runnableExamples: + from std/fenv import epsilon + from std/random import rand + + proc generateGaussianNoise(mu: float = 0.0, sigma: float = 1.0): (float, float) = + # Generates values from a normal distribution. + # Translated from https://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform#Implementation. + var u1: float + var u2: float + while true: + u1 = rand(1.0) + u2 = rand(1.0) + if u1 > epsilon(float): break + let mag = sigma * sqrt(-2 * ln(u1)) + let z0 = mag * cos(2 * PI * u2) + mu + let z1 = mag * sin(2 * PI * u2) + mu + (z0, z1) + + echo generateGaussianNoise() + ## This module is available for the `JavaScript target ## <backends.html#backends-the-javascript-target>`_. ## -## **See also:** -## * `complex module<complex.html>`_ for complex numbers and their +## See also +## ======== +## * `complex module <complex.html>`_ for complex numbers and their ## mathematical operations -## * `rationals module<rationals.html>`_ for rational numbers and their +## * `rationals module <rationals.html>`_ for rational numbers and their ## mathematical operations -## * `fenv module<fenv.html>`_ for handling of floating-point rounding +## * `fenv module <fenv.html>`_ for handling of floating-point rounding ## and exceptions (overflow, zero-divide, etc.) -## * `random module<random.html>`_ for fast and tiny random number generator -## * `mersenne module<mersenne.html>`_ for Mersenne twister random number generator -## * `stats module<stats.html>`_ for statistical analysis -## * `strformat module<strformat.html>`_ for formatting floats for print -## * `system module<system.html>`_ Some very basic and trivial math operators -## are on system directly, to name a few ``shr``, ``shl``, ``xor``, ``clamp``, etc. +## * `random module <random.html>`_ for a fast and tiny random number generator +## * `stats module <stats.html>`_ for statistical analysis +## * `strformat module <strformat.html>`_ for formatting floats for printing +## * `system module <system.html>`_ for some very basic and trivial math operators +## (`shr`, `shl`, `xor`, `clamp`, etc.) -include "system/inclrtl" +import std/private/since {.push debugger: off.} # the user does not want to trace a part # of the standard library! -import bitops +import std/[bitops, fenv] +import system/countbits_impl + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when not defined(js) and not defined(nimscript): # C + proc c_isnan(x: float): bool {.importc: "isnan", header: "<math.h>".} + # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. + + proc c_copysign(x, y: cfloat): cfloat {.importc: "copysignf", header: "<math.h>".} + proc c_copysign(x, y: cdouble): cdouble {.importc: "copysign", header: "<math.h>".} + + proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "<math.h>".} + + # don't export `c_frexp` in the future and remove `c_frexp2`. + func c_frexp2(x: cfloat, exponent: var cint): cfloat {. + importc: "frexpf", header: "<math.h>".} + func c_frexp2(x: cdouble, exponent: var cint): cdouble {. + importc: "frexp", header: "<math.h>".} + + type + div_t {.importc, header: "<stdlib.h>".} = object + quot: cint + rem: cint + ldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clong + rem: clong + lldiv_t {.importc, header: "<stdlib.h>".} = object + quot: clonglong + rem: clonglong + + when cint isnot clong: + func divmod_c(x, y: cint): div_t {.importc: "div", header: "<stdlib.h>".} + when clong isnot clonglong: + func divmod_c(x, y: clonglong): lldiv_t {.importc: "lldiv", header: "<stdlib.h>".} + func divmod_c(x, y: clong): ldiv_t {.importc: "ldiv", header: "<stdlib.h>".} + func divmod*[T: SomeInteger](x, y: T): (T, T) {.inline.} = + ## Specialized instructions for computing both division and modulus. + ## Return structure is: (quotient, remainder) + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + when T is cint | clong | clonglong: + when compileOption("overflowChecks"): + if y == 0: + raise new(DivByZeroDefect) + elif (x == T.low and y == -1.T): + raise new(OverflowDefect) + let res = divmod_c(x, y) + result[0] = res.quot + result[1] = res.rem + else: + result[0] = x div y + result[1] = x mod y -proc binom*(n, k: int): int {.noSideEffect.} = - ## Computes the `binomial coefficient <https://en.wikipedia.org/wiki/Binomial_coefficient>`_. +func binom*(n, k: int): int = + ## Computes the [binomial coefficient](https://en.wikipedia.org/wiki/Binomial_coefficient). runnableExamples: - doAssert binom(6, 2) == binom(6, 4) doAssert binom(6, 2) == 15 doAssert binom(-6, 2) == 1 doAssert binom(6, 0) == 1 + if k <= 0: return 1 - if 2*k > n: return binom(n, n-k) + if 2 * k > n: return binom(n, n - k) result = n for i in countup(2, k): result = (result * (n + 1 - i)) div i -proc createFactTable[N: static[int]]: array[N, int] = +func createFactTable[N: static[int]]: array[N, int] = result[0] = 1 for i in 1 ..< N: result[i] = result[i - 1] * i -proc fac*(n: int): int = - ## Computes the `factorial <https://en.wikipedia.org/wiki/Factorial>`_ of - ## a non-negative integer ``n``. +func fac*(n: int): int = + ## Computes the [factorial](https://en.wikipedia.org/wiki/Factorial) of + ## a non-negative integer `n`. ## - ## See also: - ## * `prod proc <#prod,openArray[T]>`_ + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ runnableExamples: - doAssert fac(3) == 6 + doAssert fac(0) == 1 doAssert fac(4) == 24 doAssert fac(10) == 3628800 + const factTable = when sizeof(int) == 2: createFactTable[5]() @@ -99,91 +156,199 @@ proc fac*(n: int): int = {.push checks: off, line_dir: off, stack_trace: off.} -when defined(Posix) and not defined(genode): +when defined(posix) and not defined(genode) and not defined(macosx): {.passl: "-lm".} const - PI* = 3.1415926535897932384626433 ## The circle constant PI (Ludolph's number) - TAU* = 2.0 * PI ## The circle constant TAU (= 2 * PI) - E* = 2.71828182845904523536028747 ## Euler's number + PI* = 3.1415926535897932384626433 ## The circle constant PI (Ludolph's number). + TAU* = 2.0 * PI ## The circle constant TAU (= 2 * PI). + E* = 2.71828182845904523536028747 ## Euler's number. MaxFloat64Precision* = 16 ## Maximum number of meaningful digits ## after the decimal point for Nim's - ## ``float64`` type. + ## `float64` type. MaxFloat32Precision* = 8 ## Maximum number of meaningful digits ## after the decimal point for Nim's - ## ``float32`` type. + ## `float32` type. MaxFloatPrecision* = MaxFloat64Precision ## Maximum number of ## meaningful digits ## after the decimal point - ## for Nim's ``float`` type. + ## for Nim's `float` type. MinFloatNormal* = 2.225073858507201e-308 ## Smallest normal number for Nim's - ## ``float`` type. (= 2^-1022). - RadPerDeg = PI / 180.0 ## Number of radians per degree + ## `float` type (= 2^-1022). + RadPerDeg = PI / 180.0 ## Number of radians per degree. type FloatClass* = enum ## Describes the class a floating point value belongs to. - ## This is the type that is returned by - ## `classify proc <#classify,float>`_. + ## This is the type that is returned by the + ## `classify func <#classify,float>`_. fcNormal, ## value is an ordinary nonzero floating point value fcSubnormal, ## value is a subnormal (a very small) floating point value fcZero, ## value is zero fcNegZero, ## value is the negative zero - fcNan, ## value is Not-A-Number (NAN) + fcNan, ## value is Not a Number (NaN) fcInf, ## value is positive infinity fcNegInf ## value is negative infinity -proc classify*(x: float): FloatClass = +func isNaN*(x: SomeFloat): bool {.inline, since: (1,5,1).} = + ## Returns whether `x` is a `NaN`, more efficiently than via `classify(x) == fcNan`. + ## Works even with `--passc:-ffast-math`. + runnableExamples: + doAssert NaN.isNaN + doAssert not Inf.isNaN + doAssert not isNaN(3.1415926) + + template fn: untyped = result = x != x + when nimvm: fn() + else: + when defined(js) or defined(nimscript): fn() + else: result = c_isnan(x) + +when defined(js): + import std/private/jsutils + + proc toBitsImpl(x: float): array[2, uint32] = + let buffer = newArrayBuffer(8) + let a = newFloat64Array(buffer) + let b = newUint32Array(buffer) + a[0] = x + {.emit: "`result` = `b`;".} + # result = cast[array[2, uint32]](b) + + proc jsSetSign(x: float, sgn: bool): float = + let buffer = newArrayBuffer(8) + let a = newFloat64Array(buffer) + let b = newUint32Array(buffer) + a[0] = x + {.emit: """ + function updateBit(num, bitPos, bitVal) { + return (num & ~(1 << bitPos)) | (bitVal << bitPos); + } + `b`[1] = updateBit(`b`[1], 31, `sgn`); + `result` = `a`[0]; + """.} + +proc signbit*(x: SomeFloat): bool {.inline, since: (1, 5, 1).} = + ## Returns true if `x` is negative, false otherwise. + runnableExamples: + doAssert not signbit(0.0) + doAssert signbit(-0.0) + doAssert signbit(-0.1) + doAssert not signbit(0.1) + + when defined(js): + let uintBuffer = toBitsImpl(x) + result = (uintBuffer[1] shr 31) != 0 + else: + result = c_signbit(x) != 0 + +func copySign*[T: SomeFloat](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Returns a value with the magnitude of `x` and the sign of `y`; + ## this works even if x or y are NaN, infinity or zero, all of which can carry a sign. + runnableExamples: + doAssert copySign(10.0, 1.0) == 10.0 + doAssert copySign(10.0, -1.0) == -10.0 + doAssert copySign(-Inf, -0.0) == -Inf + doAssert copySign(NaN, 1.0).isNaN + doAssert copySign(1.0, copySign(NaN, -1.0)) == -1.0 + + # TODO: use signbit for examples + when defined(js): + let uintBuffer = toBitsImpl(y) + let sgn = (uintBuffer[1] shr 31) != 0 + result = jsSetSign(x, sgn) + else: + when nimvm: # not exact but we have a vmops for recent enough nim + if y > 0.0 or (y == 0.0 and 1.0 / y > 0.0): + result = abs(x) + elif y <= 0.0: + result = -abs(x) + else: # must be NaN + result = abs(x) + else: result = c_copysign(x, y) + +func classify*(x: float): FloatClass = ## Classifies a floating point value. ## - ## Returns ``x``'s class as specified by `FloatClass enum<#FloatClass>`_. + ## Returns `x`'s class as specified by the `FloatClass enum<#FloatClass>`_. runnableExamples: doAssert classify(0.3) == fcNormal doAssert classify(0.0) == fcZero - doAssert classify(0.3/0.0) == fcInf - doAssert classify(-0.3/0.0) == fcNegInf + doAssert classify(0.3 / 0.0) == fcInf + doAssert classify(-0.3 / 0.0) == fcNegInf doAssert classify(5.0e-324) == fcSubnormal # JavaScript and most C compilers have no classify: + if isNan(x): return fcNan if x == 0.0: - if 1.0/x == Inf: + if 1.0 / x == Inf: return fcZero else: return fcNegZero - if x*0.5 == x: + if x * 0.5 == x: if x > 0.0: return fcInf else: return fcNegInf - if x != x: return fcNan if abs(x) < MinFloatNormal: return fcSubnormal return fcNormal -proc isPowerOfTwo*(x: int): bool {.noSideEffect.} = - ## Returns ``true``, if ``x`` is a power of two, ``false`` otherwise. +func almostEqual*[T: SomeFloat](x, y: T; unitsInLastPlace: Natural = 4): bool {. + since: (1, 5), inline.} = + ## Checks if two float values are almost equal, using the + ## [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon). + ## + ## `unitsInLastPlace` is the max number of + ## [units in the last place](https://en.wikipedia.org/wiki/Unit_in_the_last_place) + ## difference tolerated when comparing two numbers. The larger the value, the + ## more error is allowed. A `0` value means that two numbers must be exactly the + ## same to be considered equal. + ## + ## The machine epsilon has to be scaled to the magnitude of the values used + ## and multiplied by the desired precision in ULPs unless the difference is + ## subnormal. + ## + # taken from: https://en.cppreference.com/w/cpp/types/numeric_limits/epsilon + runnableExamples: + doAssert almostEqual(PI, 3.14159265358979) + doAssert almostEqual(Inf, Inf) + doAssert not almostEqual(NaN, NaN) + + if x == y: + # short circuit exact equality -- needed to catch two infinities of + # the same sign. And perhaps speeds things up a bit sometimes. + return true + let diff = abs(x - y) + result = diff <= epsilon(T) * abs(x + y) * T(unitsInLastPlace) or + diff < minimumPositiveValue(T) + +func isPowerOfTwo*(x: int): bool = + ## Returns `true`, if `x` is a power of two, `false` otherwise. ## ## Zero and negative numbers are not a power of two. ## - ## See also: - ## * `nextPowerOfTwo proc<#nextPowerOfTwo,int>`_ + ## **See also:** + ## * `nextPowerOfTwo func <#nextPowerOfTwo,int>`_ runnableExamples: - doAssert isPowerOfTwo(16) == true - doAssert isPowerOfTwo(5) == false - doAssert isPowerOfTwo(0) == false - doAssert isPowerOfTwo(-16) == false + doAssert isPowerOfTwo(16) + doAssert not isPowerOfTwo(5) + doAssert not isPowerOfTwo(0) + doAssert not isPowerOfTwo(-16) + return (x > 0) and ((x and (x - 1)) == 0) -proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = - ## Returns ``x`` rounded up to the nearest power of two. +func nextPowerOfTwo*(x: int): int = + ## Returns `x` rounded up to the nearest power of two. ## ## Zero and negative numbers get rounded up to 1. ## - ## See also: - ## * `isPowerOfTwo proc<#isPowerOfTwo,int>`_ + ## **See also:** + ## * `isPowerOfTwo func <#isPowerOfTwo,int>`_ runnableExamples: doAssert nextPowerOfTwo(16) == 16 doAssert nextPowerOfTwo(5) == 8 doAssert nextPowerOfTwo(0) == 1 doAssert nextPowerOfTwo(-16) == 1 + result = x - 1 when defined(cpu64): result = result or (result shr 32) @@ -196,476 +361,348 @@ proc nextPowerOfTwo*(x: int): int {.noSideEffect.} = result = result or (result shr 1) result += 1 + ord(x <= 0) -proc countBits32*(n: int32): int {.noSideEffect, deprecated: - "Deprecated since v0.20.0; use 'bitops.countSetBits' instead".} = - runnableExamples: - doAssert countBits32(7) == 3 - doAssert countBits32(8) == 1 - doAssert countBits32(15) == 4 - doAssert countBits32(16) == 1 - doAssert countBits32(17) == 2 - bitops.countSetBits(n) -proc sum*[T](x: openArray[T]): T {.noSideEffect.} = - ## Computes the sum of the elements in ``x``. - ## - ## If ``x`` is empty, 0 is returned. - ## - ## See also: - ## * `prod proc <#prod,openArray[T]>`_ - ## * `cumsum proc <#cumsum,openArray[T]>`_ - ## * `cumsummed proc <#cumsummed,openArray[T]>`_ - runnableExamples: - doAssert sum([1, 2, 3, 4]) == 10 - doAssert sum([-1.5, 2.7, -0.1]) == 1.1 - for i in items(x): result = result + i -proc prod*[T](x: openArray[T]): T {.noSideEffect.} = - ## Computes the product of the elements in ``x``. - ## - ## If ``x`` is empty, 1 is returned. - ## - ## See also: - ## * `sum proc <#sum,openArray[T]>`_ - ## * `fac proc <#fac,int>`_ - runnableExamples: - doAssert prod([1, 2, 3, 4]) == 24 - doAssert prod([-4, 3, 5]) == -60 - result = 1.T - for i in items(x): result = result * i - -proc cumsummed*[T](x: openArray[T]): seq[T] = - ## Return cumulative (aka prefix) summation of ``x``. - ## - ## See also: - ## * `sum proc <#sum,openArray[T]>`_ - ## * `cumsum proc <#cumsum,openArray[T]>`_ for the in-place version - runnableExamples: - let a = [1, 2, 3, 4] - doAssert cumsummed(a) == @[1, 3, 6, 10] - result.setLen(x.len) - result[0] = x[0] - for i in 1 ..< x.len: result[i] = result[i-1] + x[i] - -proc cumsum*[T](x: var openArray[T]) = - ## Transforms ``x`` in-place (must be declared as `var`) into its - ## cumulative (aka prefix) summation. - ## - ## See also: - ## * `sum proc <#sum,openArray[T]>`_ - ## * `cumsummed proc <#cumsummed,openArray[T]>`_ for a version which - ## returns cumsummed sequence - runnableExamples: - var a = [1, 2, 3, 4] - cumsum(a) - doAssert a == @[1, 3, 6, 10] - for i in 1 ..< x.len: x[i] = x[i-1] + x[i] - -{.push noSideEffect.} when not defined(js): # C - proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} - proc sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} - ## Computes the square root of ``x``. - ## - ## See also: - ## * `cbrt proc <#cbrt,float64>`_ for cubic root - ## - ## .. code-block:: nim - ## echo sqrt(4.0) ## 2.0 - ## echo sqrt(1.44) ## 1.2 - ## echo sqrt(-4.0) ## nan - proc cbrt*(x: float32): float32 {.importc: "cbrtf", header: "<math.h>".} - proc cbrt*(x: float64): float64 {.importc: "cbrt", header: "<math.h>".} - ## Computes the cubic root of ``x``. + func sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".} + func sqrt*(x: float64): float64 {.importc: "sqrt", header: "<math.h>".} = + ## Computes the square root of `x`. ## - ## See also: - ## * `sqrt proc <#sqrt,float64>`_ for square root + ## **See also:** + ## * `cbrt func <#cbrt,float64>`_ for the cube root + runnableExamples: + doAssert almostEqual(sqrt(4.0), 2.0) + doAssert almostEqual(sqrt(1.44), 1.2) + func cbrt*(x: float32): float32 {.importc: "cbrtf", header: "<math.h>".} + func cbrt*(x: float64): float64 {.importc: "cbrt", header: "<math.h>".} = + ## Computes the cube root of `x`. ## - ## .. code-block:: nim - ## echo cbrt(8.0) ## 2.0 - ## echo cbrt(2.197) ## 1.3 - ## echo cbrt(-27.0) ## -3.0 - proc ln*(x: float32): float32 {.importc: "logf", header: "<math.h>".} - proc ln*(x: float64): float64 {.importc: "log", header: "<math.h>".} - ## Computes the `natural logarithm <https://en.wikipedia.org/wiki/Natural_logarithm>`_ - ## of ``x``. + ## **See also:** + ## * `sqrt func <#sqrt,float64>`_ for the square root + runnableExamples: + doAssert almostEqual(cbrt(8.0), 2.0) + doAssert almostEqual(cbrt(2.197), 1.3) + doAssert almostEqual(cbrt(-27.0), -3.0) + func ln*(x: float32): float32 {.importc: "logf", header: "<math.h>".} + func ln*(x: float64): float64 {.importc: "log", header: "<math.h>".} = + ## Computes the [natural logarithm](https://en.wikipedia.org/wiki/Natural_logarithm) + ## of `x`. ## - ## See also: - ## * `log proc <#log,T,T>`_ - ## * `log10 proc <#log10,float64>`_ - ## * `log2 proc <#log2,float64>`_ - ## * `exp proc <#exp,float64>`_ - ## - ## .. code-block:: nim - ## echo ln(exp(4.0)) ## 4.0 - ## echo ln(1.0)) ## 0.0 - ## echo ln(0.0) ## -inf - ## echo ln(-7.0) ## nan + ## **See also:** + ## * `log func <#log,T,T>`_ + ## * `log10 func <#log10,float64>`_ + ## * `log2 func <#log2,float64>`_ + ## * `exp func <#exp,float64>`_ + runnableExamples: + doAssert almostEqual(ln(exp(4.0)), 4.0) + doAssert almostEqual(ln(1.0), 0.0) + doAssert almostEqual(ln(0.0), -Inf) + doAssert ln(-7.0).isNaN else: # JS - proc sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} - proc sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} + func sqrt*(x: float32): float32 {.importc: "Math.sqrt", nodecl.} + func sqrt*(x: float64): float64 {.importc: "Math.sqrt", nodecl.} - proc cbrt*(x: float32): float32 {.importc: "Math.cbrt", nodecl.} - proc cbrt*(x: float64): float64 {.importc: "Math.cbrt", nodecl.} + func cbrt*(x: float32): float32 {.importc: "Math.cbrt", nodecl.} + func cbrt*(x: float64): float64 {.importc: "Math.cbrt", nodecl.} - proc ln*(x: float32): float32 {.importc: "Math.log", nodecl.} - proc ln*(x: float64): float64 {.importc: "Math.log", nodecl.} + func ln*(x: float32): float32 {.importc: "Math.log", nodecl.} + func ln*(x: float64): float64 {.importc: "Math.log", nodecl.} -proc log*[T: SomeFloat](x, base: T): T = - ## Computes the logarithm of ``x`` to base ``base``. - ## - ## See also: - ## * `ln proc <#ln,float64>`_ - ## * `log10 proc <#log10,float64>`_ - ## * `log2 proc <#log2,float64>`_ - ## * `exp proc <#exp,float64>`_ +func log*[T: SomeFloat](x, base: T): T = + ## Computes the logarithm of `x` to base `base`. ## - ## .. code-block:: nim - ## echo log(9.0, 3.0) ## 2.0 - ## echo log(32.0, 2.0) ## 5.0 - ## echo log(0.0, 2.0) ## -inf - ## echo log(-7.0, 4.0) ## nan - ## echo log(8.0, -2.0) ## nan + ## **See also:** + ## * `ln func <#ln,float64>`_ + ## * `log10 func <#log10,float64>`_ + ## * `log2 func <#log2,float64>`_ + runnableExamples: + doAssert almostEqual(log(9.0, 3.0), 2.0) + doAssert almostEqual(log(0.0, 2.0), -Inf) + doAssert log(-7.0, 4.0).isNaN + doAssert log(8.0, -2.0).isNaN + ln(x) / ln(base) when not defined(js): # C - proc log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} - proc log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} - ## Computes the common logarithm (base 10) of ``x``. - ## - ## See also: - ## * `ln proc <#ln,float64>`_ - ## * `log proc <#log,T,T>`_ - ## * `log2 proc <#log2,float64>`_ - ## * `exp proc <#exp,float64>`_ - ## - ## .. code-block:: nim - ## echo log10(100.0) ## 2.0 - ## echo log10(0.0) ## nan - ## echo log10(-100.0) ## -inf - proc exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} - proc exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} - ## Computes the exponential function of ``x`` (e^x). + func log10*(x: float32): float32 {.importc: "log10f", header: "<math.h>".} + func log10*(x: float64): float64 {.importc: "log10", header: "<math.h>".} = + ## Computes the common logarithm (base 10) of `x`. ## - ## See also: - ## * `ln proc <#ln,float64>`_ - ## * `log proc <#log,T,T>`_ - ## * `log10 proc <#log10,float64>`_ - ## * `log2 proc <#log2,float64>`_ + ## **See also:** + ## * `ln func <#ln,float64>`_ + ## * `log func <#log,T,T>`_ + ## * `log2 func <#log2,float64>`_ + runnableExamples: + doAssert almostEqual(log10(100.0) , 2.0) + doAssert almostEqual(log10(0.0), -Inf) + doAssert log10(-100.0).isNaN + func exp*(x: float32): float32 {.importc: "expf", header: "<math.h>".} + func exp*(x: float64): float64 {.importc: "exp", header: "<math.h>".} = + ## Computes the exponential function of `x` (`e^x`). ## - ## .. code-block:: nim - ## echo exp(1.0) ## 2.718281828459045 - ## echo ln(exp(4.0)) ## 4.0 - ## echo exp(0.0) ## 1.0 - ## echo exp(-1.0) ## 0.3678794411714423 - proc sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} - proc sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} - ## Computes the sine of ``x``. + ## **See also:** + ## * `ln func <#ln,float64>`_ + runnableExamples: + doAssert almostEqual(exp(1.0), E) + doAssert almostEqual(ln(exp(4.0)), 4.0) + doAssert almostEqual(exp(0.0), 1.0) + func sin*(x: float32): float32 {.importc: "sinf", header: "<math.h>".} + func sin*(x: float64): float64 {.importc: "sin", header: "<math.h>".} = + ## Computes the sine of `x`. ## - ## See also: - ## * `cos proc <#cos,float64>`_ - ## * `tan proc <#tan,float64>`_ - ## * `arcsin proc <#arcsin,float64>`_ - ## * `sinh proc <#sinh,float64>`_ + ## **See also:** + ## * `arcsin func <#arcsin,float64>`_ + runnableExamples: + doAssert almostEqual(sin(PI / 6), 0.5) + doAssert almostEqual(sin(degToRad(90.0)), 1.0) + func cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} + func cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} = + ## Computes the cosine of `x`. ## - ## .. code-block:: nim - ## echo sin(PI / 6) ## 0.4999999999999999 - ## echo sin(degToRad(90.0)) ## 1.0 - proc cos*(x: float32): float32 {.importc: "cosf", header: "<math.h>".} - proc cos*(x: float64): float64 {.importc: "cos", header: "<math.h>".} - ## Computes the cosine of ``x``. + ## **See also:** + ## * `arccos func <#arccos,float64>`_ + runnableExamples: + doAssert almostEqual(cos(2 * PI), 1.0) + doAssert almostEqual(cos(degToRad(60.0)), 0.5) + func tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} + func tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} = + ## Computes the tangent of `x`. ## - ## See also: - ## * `sin proc <#sin,float64>`_ - ## * `tan proc <#tan,float64>`_ - ## * `arccos proc <#arccos,float64>`_ - ## * `cosh proc <#cosh,float64>`_ + ## **See also:** + ## * `arctan func <#arctan,float64>`_ + runnableExamples: + doAssert almostEqual(tan(degToRad(45.0)), 1.0) + doAssert almostEqual(tan(PI / 4), 1.0) + func sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} + func sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} = + ## Computes the [hyperbolic sine](https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions) of `x`. ## - ## .. code-block:: nim - ## echo cos(2 * PI) ## 1.0 - ## echo cos(degToRad(60.0)) ## 0.5000000000000001 - proc tan*(x: float32): float32 {.importc: "tanf", header: "<math.h>".} - proc tan*(x: float64): float64 {.importc: "tan", header: "<math.h>".} - ## Computes the tangent of ``x``. + ## **See also:** + ## * `arcsinh func <#arcsinh,float64>`_ + runnableExamples: + doAssert almostEqual(sinh(0.0), 0.0) + doAssert almostEqual(sinh(1.0), 1.175201193643801) + func cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} + func cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} = + ## Computes the [hyperbolic cosine](https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions) of `x`. ## - ## See also: - ## * `sin proc <#sin,float64>`_ - ## * `cos proc <#cos,float64>`_ - ## * `arctan proc <#arctan,float64>`_ - ## * `tanh proc <#tanh,float64>`_ + ## **See also:** + ## * `arccosh func <#arccosh,float64>`_ + runnableExamples: + doAssert almostEqual(cosh(0.0), 1.0) + doAssert almostEqual(cosh(1.0), 1.543080634815244) + func tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} + func tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} = + ## Computes the [hyperbolic tangent](https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions) of `x`. ## - ## .. code-block:: nim - ## echo tan(degToRad(45.0)) ## 0.9999999999999999 - ## echo tan(PI / 4) ## 0.9999999999999999 - proc sinh*(x: float32): float32 {.importc: "sinhf", header: "<math.h>".} - proc sinh*(x: float64): float64 {.importc: "sinh", header: "<math.h>".} - ## Computes the `hyperbolic sine <https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions>`_ of ``x``. + ## **See also:** + ## * `arctanh func <#arctanh,float64>`_ + runnableExamples: + doAssert almostEqual(tanh(0.0), 0.0) + doAssert almostEqual(tanh(1.0), 0.7615941559557649) + func arcsin*(x: float32): float32 {.importc: "asinf", header: "<math.h>".} + func arcsin*(x: float64): float64 {.importc: "asin", header: "<math.h>".} = + ## Computes the arc sine of `x`. ## - ## See also: - ## * `cosh proc <#cosh,float64>`_ - ## * `tanh proc <#tanh,float64>`_ - ## * `arcsinh proc <#arcsinh,float64>`_ - ## * `sin proc <#sin,float64>`_ + ## **See also:** + ## * `sin func <#sin,float64>`_ + runnableExamples: + doAssert almostEqual(radToDeg(arcsin(0.0)), 0.0) + doAssert almostEqual(radToDeg(arcsin(1.0)), 90.0) + func arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} + func arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} = + ## Computes the arc cosine of `x`. ## - ## .. code-block:: nim - ## echo sinh(0.0) ## 0.0 - ## echo sinh(1.0) ## 1.175201193643801 - ## echo sinh(degToRad(90.0)) ## 2.301298902307295 - proc cosh*(x: float32): float32 {.importc: "coshf", header: "<math.h>".} - proc cosh*(x: float64): float64 {.importc: "cosh", header: "<math.h>".} - ## Computes the `hyperbolic cosine <https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions>`_ of ``x``. + ## **See also:** + ## * `cos func <#cos,float64>`_ + runnableExamples: + doAssert almostEqual(radToDeg(arccos(0.0)), 90.0) + doAssert almostEqual(radToDeg(arccos(1.0)), 0.0) + func arctan*(x: float32): float32 {.importc: "atanf", header: "<math.h>".} + func arctan*(x: float64): float64 {.importc: "atan", header: "<math.h>".} = + ## Calculate the arc tangent of `x`. ## - ## See also: - ## * `sinh proc <#sinh,float64>`_ - ## * `tanh proc <#tanh,float64>`_ - ## * `arccosh proc <#arccosh,float64>`_ - ## * `cos proc <#cos,float64>`_ + ## **See also:** + ## * `arctan2 func <#arctan2,float64,float64>`_ + ## * `tan func <#tan,float64>`_ + runnableExamples: + doAssert almostEqual(arctan(1.0), 0.7853981633974483) + doAssert almostEqual(radToDeg(arctan(1.0)), 45.0) + func arctan2*(y, x: float32): float32 {.importc: "atan2f", header: "<math.h>".} + func arctan2*(y, x: float64): float64 {.importc: "atan2", header: "<math.h>".} = + ## Calculate the arc tangent of `y/x`. ## - ## .. code-block:: nim - ## echo cosh(0.0) ## 1.0 - ## echo cosh(1.0) ## 1.543080634815244 - ## echo cosh(degToRad(90.0)) ## 2.509178478658057 - proc tanh*(x: float32): float32 {.importc: "tanhf", header: "<math.h>".} - proc tanh*(x: float64): float64 {.importc: "tanh", header: "<math.h>".} - ## Computes the `hyperbolic tangent <https://en.wikipedia.org/wiki/Hyperbolic_function#Definitions>`_ of ``x``. - ## - ## See also: - ## * `sinh proc <#sinh,float64>`_ - ## * `cosh proc <#cosh,float64>`_ - ## * `arctanh proc <#arctanh,float64>`_ - ## * `tan proc <#tan,float64>`_ - ## - ## .. code-block:: nim - ## echo tanh(0.0) ## 0.0 - ## echo tanh(1.0) ## 0.7615941559557649 - ## echo tanh(degToRad(90.0)) ## 0.9171523356672744 - - proc arccos*(x: float32): float32 {.importc: "acosf", header: "<math.h>".} - proc arccos*(x: float64): float64 {.importc: "acos", header: "<math.h>".} - ## Computes the arc cosine of ``x``. - ## - ## See also: - ## * `arcsin proc <#arcsin,float64>`_ - ## * `arctan proc <#arctan,float64>`_ - ## * `arctan2 proc <#arctan2,float64,float64>`_ - ## * `cos proc <#cos,float64>`_ - ## - ## .. code-block:: nim - ## echo radToDeg(arccos(0.0)) ## 90.0 - ## echo radToDeg(arccos(1.0)) ## 0.0 - proc arcsin*(x: float32): float32 {.importc: "asinf", header: "<math.h>".} - proc arcsin*(x: float64): float64 {.importc: "asin", header: "<math.h>".} - ## Computes the arc sine of ``x``. - ## - ## See also: - ## * `arccos proc <#arccos,float64>`_ - ## * `arctan proc <#arctan,float64>`_ - ## * `arctan2 proc <#arctan2,float64,float64>`_ - ## * `sin proc <#sin,float64>`_ - ## - ## .. code-block:: nim - ## echo radToDeg(arcsin(0.0)) ## 0.0 - ## echo radToDeg(arcsin(1.0)) ## 90.0 - proc arctan*(x: float32): float32 {.importc: "atanf", header: "<math.h>".} - proc arctan*(x: float64): float64 {.importc: "atan", header: "<math.h>".} - ## Calculate the arc tangent of ``x``. - ## - ## See also: - ## * `arcsin proc <#arcsin,float64>`_ - ## * `arccos proc <#arccos,float64>`_ - ## * `arctan2 proc <#arctan2,float64,float64>`_ - ## * `tan proc <#tan,float64>`_ + ## It produces correct results even when the resulting angle is near + ## `PI/2` or `-PI/2` (`x` near 0). ## - ## .. code-block:: nim - ## echo arctan(1.0) ## 0.7853981633974483 - ## echo radToDeg(arctan(1.0)) ## 45.0 - proc arctan2*(y, x: float32): float32 {.importc: "atan2f", - header: "<math.h>".} - proc arctan2*(y, x: float64): float64 {.importc: "atan2", header: "<math.h>".} - ## Calculate the arc tangent of ``y`` / ``x``. + ## **See also:** + ## * `arctan func <#arctan,float64>`_ + runnableExamples: + doAssert almostEqual(arctan2(1.0, 0.0), PI / 2.0) + doAssert almostEqual(radToDeg(arctan2(1.0, 0.0)), 90.0) + func arcsinh*(x: float32): float32 {.importc: "asinhf", header: "<math.h>".} + func arcsinh*(x: float64): float64 {.importc: "asinh", header: "<math.h>".} + ## Computes the inverse hyperbolic sine of `x`. ## - ## It produces correct results even when the resulting angle is near - ## pi/2 or -pi/2 (``x`` near 0). + ## **See also:** + ## * `sinh func <#sinh,float64>`_ + func arccosh*(x: float32): float32 {.importc: "acoshf", header: "<math.h>".} + func arccosh*(x: float64): float64 {.importc: "acosh", header: "<math.h>".} + ## Computes the inverse hyperbolic cosine of `x`. ## - ## See also: - ## * `arcsin proc <#arcsin,float64>`_ - ## * `arccos proc <#arccos,float64>`_ - ## * `arctan proc <#arctan,float64>`_ - ## * `tan proc <#tan,float64>`_ + ## **See also:** + ## * `cosh func <#cosh,float64>`_ + func arctanh*(x: float32): float32 {.importc: "atanhf", header: "<math.h>".} + func arctanh*(x: float64): float64 {.importc: "atanh", header: "<math.h>".} + ## Computes the inverse hyperbolic tangent of `x`. ## - ## .. code-block:: nim - ## echo arctan2(1.0, 0.0) ## 1.570796326794897 - ## echo radToDeg(arctan2(1.0, 0.0)) ## 90.0 - proc arcsinh*(x: float32): float32 {.importc: "asinhf", header: "<math.h>".} - proc arcsinh*(x: float64): float64 {.importc: "asinh", header: "<math.h>".} - ## Computes the inverse hyperbolic sine of ``x``. - proc arccosh*(x: float32): float32 {.importc: "acoshf", header: "<math.h>".} - proc arccosh*(x: float64): float64 {.importc: "acosh", header: "<math.h>".} - ## Computes the inverse hyperbolic cosine of ``x``. - proc arctanh*(x: float32): float32 {.importc: "atanhf", header: "<math.h>".} - proc arctanh*(x: float64): float64 {.importc: "atanh", header: "<math.h>".} - ## Computes the inverse hyperbolic tangent of ``x``. + ## **See also:** + ## * `tanh func <#tanh,float64>`_ else: # JS - proc log10*(x: float32): float32 {.importc: "Math.log10", nodecl.} - proc log10*(x: float64): float64 {.importc: "Math.log10", nodecl.} - proc log2*(x: float32): float32 {.importc: "Math.log2", nodecl.} - proc log2*(x: float64): float64 {.importc: "Math.log2", nodecl.} - proc exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} - proc exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} - - proc sin*[T: float32|float64](x: T): T {.importc: "Math.sin", nodecl.} - proc cos*[T: float32|float64](x: T): T {.importc: "Math.cos", nodecl.} - proc tan*[T: float32|float64](x: T): T {.importc: "Math.tan", nodecl.} - - proc sinh*[T: float32|float64](x: T): T {.importc: "Math.sinh", nodecl.} - proc cosh*[T: float32|float64](x: T): T {.importc: "Math.cosh", nodecl.} - proc tanh*[T: float32|float64](x: T): T {.importc: "Math.tanh", nodecl.} - - proc arcsin*[T: float32|float64](x: T): T {.importc: "Math.asin", nodecl.} - proc arccos*[T: float32|float64](x: T): T {.importc: "Math.acos", nodecl.} - proc arctan*[T: float32|float64](x: T): T {.importc: "Math.atan", nodecl.} - proc arctan2*[T: float32|float64](y, x: T): T {.importc: "Math.atan2", nodecl.} - - proc arcsinh*[T: float32|float64](x: T): T {.importc: "Math.asinh", nodecl.} - proc arccosh*[T: float32|float64](x: T): T {.importc: "Math.acosh", nodecl.} - proc arctanh*[T: float32|float64](x: T): T {.importc: "Math.atanh", nodecl.} - -proc cot*[T: float32|float64](x: T): T = 1.0 / tan(x) - ## Computes the cotangent of ``x`` (1 / tan(x)). -proc sec*[T: float32|float64](x: T): T = 1.0 / cos(x) - ## Computes the secant of ``x`` (1 / cos(x)). -proc csc*[T: float32|float64](x: T): T = 1.0 / sin(x) - ## Computes the cosecant of ``x`` (1 / sin(x)). - -proc coth*[T: float32|float64](x: T): T = 1.0 / tanh(x) - ## Computes the hyperbolic cotangent of ``x`` (1 / tanh(x)). -proc sech*[T: float32|float64](x: T): T = 1.0 / cosh(x) - ## Computes the hyperbolic secant of ``x`` (1 / cosh(x)). -proc csch*[T: float32|float64](x: T): T = 1.0 / sinh(x) - ## Computes the hyperbolic cosecant of ``x`` (1 / sinh(x)). - -proc arccot*[T: float32|float64](x: T): T = arctan(1.0 / x) - ## Computes the inverse cotangent of ``x``. -proc arcsec*[T: float32|float64](x: T): T = arccos(1.0 / x) - ## Computes the inverse secant of ``x``. -proc arccsc*[T: float32|float64](x: T): T = arcsin(1.0 / x) - ## Computes the inverse cosecant of ``x``. - -proc arccoth*[T: float32|float64](x: T): T = arctanh(1.0 / x) - ## Computes the inverse hyperbolic cotangent of ``x``. -proc arcsech*[T: float32|float64](x: T): T = arccosh(1.0 / x) - ## Computes the inverse hyperbolic secant of ``x``. -proc arccsch*[T: float32|float64](x: T): T = arcsinh(1.0 / x) - ## Computes the inverse hyperbolic cosecant of ``x``. + func log10*(x: float32): float32 {.importc: "Math.log10", nodecl.} + func log10*(x: float64): float64 {.importc: "Math.log10", nodecl.} + func log2*(x: float32): float32 {.importc: "Math.log2", nodecl.} + func log2*(x: float64): float64 {.importc: "Math.log2", nodecl.} + func exp*(x: float32): float32 {.importc: "Math.exp", nodecl.} + func exp*(x: float64): float64 {.importc: "Math.exp", nodecl.} + + func sin*[T: float32|float64](x: T): T {.importc: "Math.sin", nodecl.} + func cos*[T: float32|float64](x: T): T {.importc: "Math.cos", nodecl.} + func tan*[T: float32|float64](x: T): T {.importc: "Math.tan", nodecl.} + + func sinh*[T: float32|float64](x: T): T {.importc: "Math.sinh", nodecl.} + func cosh*[T: float32|float64](x: T): T {.importc: "Math.cosh", nodecl.} + func tanh*[T: float32|float64](x: T): T {.importc: "Math.tanh", nodecl.} + + func arcsin*[T: float32|float64](x: T): T {.importc: "Math.asin", nodecl.} + # keep this as generic or update test in `tvmops.nim` to make sure we + # keep testing that generic importc procs work + func arccos*[T: float32|float64](x: T): T {.importc: "Math.acos", nodecl.} + func arctan*[T: float32|float64](x: T): T {.importc: "Math.atan", nodecl.} + func arctan2*[T: float32|float64](y, x: T): T {.importc: "Math.atan2", nodecl.} + + func arcsinh*[T: float32|float64](x: T): T {.importc: "Math.asinh", nodecl.} + func arccosh*[T: float32|float64](x: T): T {.importc: "Math.acosh", nodecl.} + func arctanh*[T: float32|float64](x: T): T {.importc: "Math.atanh", nodecl.} + +func cot*[T: float32|float64](x: T): T = 1.0 / tan(x) + ## Computes the cotangent of `x` (`1/tan(x)`). +func sec*[T: float32|float64](x: T): T = 1.0 / cos(x) + ## Computes the secant of `x` (`1/cos(x)`). +func csc*[T: float32|float64](x: T): T = 1.0 / sin(x) + ## Computes the cosecant of `x` (`1/sin(x)`). + +func coth*[T: float32|float64](x: T): T = 1.0 / tanh(x) + ## Computes the hyperbolic cotangent of `x` (`1/tanh(x)`). +func sech*[T: float32|float64](x: T): T = 1.0 / cosh(x) + ## Computes the hyperbolic secant of `x` (`1/cosh(x)`). +func csch*[T: float32|float64](x: T): T = 1.0 / sinh(x) + ## Computes the hyperbolic cosecant of `x` (`1/sinh(x)`). + +func arccot*[T: float32|float64](x: T): T = arctan(1.0 / x) + ## Computes the inverse cotangent of `x` (`arctan(1/x)`). +func arcsec*[T: float32|float64](x: T): T = arccos(1.0 / x) + ## Computes the inverse secant of `x` (`arccos(1/x)`). +func arccsc*[T: float32|float64](x: T): T = arcsin(1.0 / x) + ## Computes the inverse cosecant of `x` (`arcsin(1/x)`). + +func arccoth*[T: float32|float64](x: T): T = arctanh(1.0 / x) + ## Computes the inverse hyperbolic cotangent of `x` (`arctanh(1/x)`). +func arcsech*[T: float32|float64](x: T): T = arccosh(1.0 / x) + ## Computes the inverse hyperbolic secant of `x` (`arccosh(1/x)`). +func arccsch*[T: float32|float64](x: T): T = arcsinh(1.0 / x) + ## Computes the inverse hyperbolic cosecant of `x` (`arcsinh(1/x)`). const windowsCC89 = defined(windows) and defined(bcc) when not defined(js): # C - proc hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} - proc hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} - ## Computes the hypotenuse of a right-angle triangle with ``x`` and - ## ``y`` as its base and height. Equivalent to ``sqrt(x*x + y*y)``. + func hypot*(x, y: float32): float32 {.importc: "hypotf", header: "<math.h>".} + func hypot*(x, y: float64): float64 {.importc: "hypot", header: "<math.h>".} = + ## Computes the length of the hypotenuse of a right-angle triangle with + ## `x` as its base and `y` as its height. Equivalent to `sqrt(x*x + y*y)`. + runnableExamples: + doAssert almostEqual(hypot(3.0, 4.0), 5.0) + func pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} + func pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} = + ## Computes `x` raised to the power of `y`. ## - ## .. code-block:: nim - ## echo hypot(4.0, 3.0) ## 5.0 - proc pow*(x, y: float32): float32 {.importc: "powf", header: "<math.h>".} - proc pow*(x, y: float64): float64 {.importc: "pow", header: "<math.h>".} - ## Computes x to power raised of y. + ## To compute the power between integers (e.g. 2^6), + ## use the `^ func <#^,T,Natural>`_. ## - ## To compute power between integers (e.g. 2^6), use `^ proc<#^,T,Natural>`_. - ## - ## See also: - ## * `^ proc<#^,T,Natural>`_ - ## * `sqrt proc <#sqrt,float64>`_ - ## * `cbrt proc <#cbrt,float64>`_ - ## - ## .. code-block:: nim - ## echo pow(100, 1.5) ## 1000.0 - ## echo pow(16.0, 0.5) ## 4.0 + ## **See also:** + ## * `^ func <#^,T,Natural>`_ + ## * `sqrt func <#sqrt,float64>`_ + ## * `cbrt func <#cbrt,float64>`_ + runnableExamples: + doAssert almostEqual(pow(100, 1.5), 1000.0) + doAssert almostEqual(pow(16.0, 0.5), 4.0) # TODO: add C89 version on windows when not windowsCC89: - proc erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} - proc erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} - ## Computes the `error function <https://en.wikipedia.org/wiki/Error_function>`_ for ``x``. - ## - ## Note: Not available for JS backend. - proc erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} - proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} - ## Computes the `complementary error function <https://en.wikipedia.org/wiki/Error_function#Complementary_error_function>`_ for ``x``. + func erf*(x: float32): float32 {.importc: "erff", header: "<math.h>".} + func erf*(x: float64): float64 {.importc: "erf", header: "<math.h>".} + ## Computes the [error function](https://en.wikipedia.org/wiki/Error_function) for `x`. ## - ## Note: Not available for JS backend. - proc gamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} - proc gamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} - ## Computes the the `gamma function <https://en.wikipedia.org/wiki/Gamma_function>`_ for ``x``. + ## **Note:** Not available for the JS backend. + func erfc*(x: float32): float32 {.importc: "erfcf", header: "<math.h>".} + func erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".} + ## Computes the [complementary error function](https://en.wikipedia.org/wiki/Error_function#Complementary_error_function) for `x`. ## - ## Note: Not available for JS backend. + ## **Note:** Not available for the JS backend. + func gamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".} + func gamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".} = + ## Computes the [gamma function](https://en.wikipedia.org/wiki/Gamma_function) for `x`. ## - ## See also: - ## * `lgamma proc <#lgamma,float64>`_ for a natural log of gamma function + ## **Note:** Not available for the JS backend. ## - ## .. code-block:: Nim - ## echo gamma(1.0) # 1.0 - ## echo gamma(4.0) # 6.0 - ## echo gamma(11.0) # 3628800.0 - ## echo gamma(-1.0) # nan - proc tgamma*(x: float32): float32 - {.deprecated: "Deprecated since v0.19.0; use 'gamma' instead", - importc: "tgammaf", header: "<math.h>".} - proc tgamma*(x: float64): float64 - {.deprecated: "Deprecated since v0.19.0; use 'gamma' instead", - importc: "tgamma", header: "<math.h>".} - ## The gamma function - proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} - proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} - ## Computes the natural log of the gamma function for ``x``. + ## **See also:** + ## * `lgamma func <#lgamma,float64>`_ for the natural logarithm of the gamma function + runnableExamples: + doAssert almostEqual(gamma(1.0), 1.0) + doAssert almostEqual(gamma(4.0), 6.0) + doAssert almostEqual(gamma(11.0), 3628800.0) + func lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".} + func lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".} = + ## Computes the natural logarithm of the gamma function for `x`. ## - ## Note: Not available for JS backend. + ## **Note:** Not available for the JS backend. ## - ## See also: - ## * `gamma proc <#gamma,float64>`_ for gamma function - ## - ## .. code-block:: Nim - ## echo lgamma(1.0) # 1.0 - ## echo lgamma(4.0) # 1.791759469228055 - ## echo lgamma(11.0) # 15.10441257307552 - ## echo lgamma(-1.0) # inf - - proc floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} - proc floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} - ## Computes the floor function (i.e., the largest integer not greater than ``x``). - ## - ## See also: - ## * `ceil proc <#ceil,float64>`_ - ## * `round proc <#round,float64>`_ - ## * `trunc proc <#trunc,float64>`_ - ## - ## .. code-block:: nim - ## echo floor(2.1) ## 2.0 - ## echo floor(2.9) ## 2.0 - ## echo floor(-3.5) ## -4.0 - - proc ceil*(x: float32): float32 {.importc: "ceilf", header: "<math.h>".} - proc ceil*(x: float64): float64 {.importc: "ceil", header: "<math.h>".} - ## Computes the ceiling function (i.e., the smallest integer not smaller - ## than ``x``). + ## **See also:** + ## * `gamma func <#gamma,float64>`_ for gamma function + + func floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".} + func floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".} = + ## Computes the floor function (i.e. the largest integer not greater than `x`). ## - ## See also: - ## * `floor proc <#floor,float64>`_ - ## * `round proc <#round,float64>`_ - ## * `trunc proc <#trunc,float64>`_ + ## **See also:** + ## * `ceil func <#ceil,float64>`_ + ## * `round func <#round,float64>`_ + ## * `trunc func <#trunc,float64>`_ + runnableExamples: + doAssert floor(2.1) == 2.0 + doAssert floor(2.9) == 2.0 + doAssert floor(-3.5) == -4.0 + + func ceil*(x: float32): float32 {.importc: "ceilf", header: "<math.h>".} + func ceil*(x: float64): float64 {.importc: "ceil", header: "<math.h>".} = + ## Computes the ceiling function (i.e. the smallest integer not smaller + ## than `x`). ## - ## .. code-block:: nim - ## echo ceil(2.1) ## 3.0 - ## echo ceil(2.9) ## 3.0 - ## echo ceil(-2.1) ## -2.0 + ## **See also:** + ## * `floor func <#floor,float64>`_ + ## * `round func <#round,float64>`_ + ## * `trunc func <#trunc,float64>`_ + runnableExamples: + doAssert ceil(2.1) == 3.0 + doAssert ceil(2.9) == 3.0 + doAssert ceil(-2.1) == -2.0 when windowsCC89: # MSVC 2010 don't have trunc/truncf # this implementation was inspired by Go-lang Math.Trunc - proc truncImpl(f: float64): float64 = + func truncImpl(f: float64): float64 = const mask: uint64 = 0x7FF shift: uint64 = 64 - 12 @@ -680,12 +717,12 @@ when not defined(js): # C let e = (x shr shift) and mask - bias # Keep the top 12+e bits, the integer part; clear the rest. - if e < 64-12: - x = x and (not (1'u64 shl (64'u64-12'u64-e) - 1'u64)) + if e < 64 - 12: + x = x and (not (1'u64 shl (64'u64 - 12'u64 - e) - 1'u64)) result = cast[float64](x) - proc truncImpl(f: float32): float32 = + func truncImpl(f: float32): float32 = const mask: uint32 = 0xFF shift: uint32 = 32 - 9 @@ -700,240 +737,338 @@ when not defined(js): # C let e = (x shr shift) and mask - bias # Keep the top 9+e bits, the integer part; clear the rest. - if e < 32-9: - x = x and (not (1'u32 shl (32'u32-9'u32-e) - 1'u32)) + if e < 32 - 9: + x = x and (not (1'u32 shl (32'u32 - 9'u32 - e) - 1'u32)) result = cast[float32](x) - proc trunc*(x: float64): float64 = + func trunc*(x: float64): float64 = if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x result = truncImpl(x) - proc trunc*(x: float32): float32 = + func trunc*(x: float32): float32 = if classify(x) in {fcZero, fcNegZero, fcNan, fcInf, fcNegInf}: return x result = truncImpl(x) - proc round*[T: float32|float64](x: T): T = + func round*[T: float32|float64](x: T): T = ## Windows compilers prior to MSVC 2012 do not implement 'round', ## 'roundl' or 'roundf'. result = if x < 0.0: ceil(x - T(0.5)) else: floor(x + T(0.5)) else: - proc round*(x: float32): float32 {.importc: "roundf", header: "<math.h>".} - proc round*(x: float64): float64 {.importc: "round", header: "<math.h>".} + func round*(x: float32): float32 {.importc: "roundf", header: "<math.h>".} + func round*(x: float64): float64 {.importc: "round", header: "<math.h>".} = ## Rounds a float to zero decimal places. ## - ## Used internally by the `round proc <#round,T,int>`_ + ## Used internally by the `round func <#round,T,int>`_ ## when the specified number of places is 0. ## - ## See also: - ## * `round proc <#round,T,int>`_ for rounding to the specific + ## **See also:** + ## * `round func <#round,T,int>`_ for rounding to the specific ## number of decimal places - ## * `floor proc <#floor,float64>`_ - ## * `ceil proc <#ceil,float64>`_ - ## * `trunc proc <#trunc,float64>`_ - ## - ## .. code-block:: nim - ## echo round(3.4) ## 3.0 - ## echo round(3.5) ## 4.0 - ## echo round(4.5) ## 5.0 - - proc trunc*(x: float32): float32 {.importc: "truncf", header: "<math.h>".} - proc trunc*(x: float64): float64 {.importc: "trunc", header: "<math.h>".} - ## Truncates ``x`` to the decimal point. - ## - ## See also: - ## * `floor proc <#floor,float64>`_ - ## * `ceil proc <#ceil,float64>`_ - ## * `round proc <#round,float64>`_ + ## * `floor func <#floor,float64>`_ + ## * `ceil func <#ceil,float64>`_ + ## * `trunc func <#trunc,float64>`_ + runnableExamples: + doAssert round(3.4) == 3.0 + doAssert round(3.5) == 4.0 + doAssert round(4.5) == 5.0 + + func trunc*(x: float32): float32 {.importc: "truncf", header: "<math.h>".} + func trunc*(x: float64): float64 {.importc: "trunc", header: "<math.h>".} = + ## Truncates `x` to the decimal point. ## - ## .. code-block:: nim - ## echo trunc(PI) # 3.0 - ## echo trunc(-1.85) # -1.0 - - proc fmod*(x, y: float32): float32 {.deprecated: "Deprecated since v0.19.0; use 'mod' instead", - importc: "fmodf", header: "<math.h>".} - proc fmod*(x, y: float64): float64 {.deprecated: "Deprecated since v0.19.0; use 'mod' instead", - importc: "fmod", header: "<math.h>".} - ## Computes the remainder of ``x`` divided by ``y``. - - proc `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} - proc `mod`*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} - ## Computes the modulo operation for float values (the remainder of ``x`` divided by ``y``). + ## **See also:** + ## * `floor func <#floor,float64>`_ + ## * `ceil func <#ceil,float64>`_ + ## * `round func <#round,float64>`_ + runnableExamples: + doAssert trunc(PI) == 3.0 + doAssert trunc(-1.85) == -1.0 + + func `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".} + func `mod`*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".} = + ## Computes the modulo operation for float values (the remainder of `x` divided by `y`). ## - ## See also: - ## * `floorMod proc <#floorMod,T,T>`_ for Python-like (% operator) behavior - ## - ## .. code-block:: nim - ## ( 6.5 mod 2.5) == 1.5 - ## (-6.5 mod 2.5) == -1.5 - ## ( 6.5 mod -2.5) == 1.5 - ## (-6.5 mod -2.5) == -1.5 + ## **See also:** + ## * `floorMod func <#floorMod,T,T>`_ for Python-like (`%` operator) behavior + runnableExamples: + doAssert 6.5 mod 2.5 == 1.5 + doAssert -6.5 mod 2.5 == -1.5 + doAssert 6.5 mod -2.5 == 1.5 + doAssert -6.5 mod -2.5 == -1.5 else: # JS - proc hypot*(x, y: float32): float32 {.importc: "Math.hypot", varargs, nodecl.} - proc hypot*(x, y: float64): float64 {.importc: "Math.hypot", varargs, nodecl.} - proc pow*(x, y: float32): float32 {.importc: "Math.pow", nodecl.} - proc pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} - proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} - proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} - proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} - proc ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} - proc round*(x: float): float {.importc: "Math.round", nodecl.} - proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} - proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} - - proc `mod`*(x, y: float32): float32 {.importcpp: "# % #".} - proc `mod`*(x, y: float64): float64 {.importcpp: "# % #".} - ## Computes the modulo operation for float values (the remainder of ``x`` divided by ``y``). - ## - ## .. code-block:: nim - ## ( 6.5 mod 2.5) == 1.5 - ## (-6.5 mod 2.5) == -1.5 - ## ( 6.5 mod -2.5) == 1.5 - ## (-6.5 mod -2.5) == -1.5 - -proc round*[T: float32|float64](x: T, places: int): T {. - deprecated: "use strformat module instead".} = + func hypot*(x, y: float32): float32 {.importc: "Math.hypot", varargs, nodecl.} + func hypot*(x, y: float64): float64 {.importc: "Math.hypot", varargs, nodecl.} + func pow*(x, y: float32): float32 {.importc: "Math.pow", nodecl.} + func pow*(x, y: float64): float64 {.importc: "Math.pow", nodecl.} + func floor*(x: float32): float32 {.importc: "Math.floor", nodecl.} + func floor*(x: float64): float64 {.importc: "Math.floor", nodecl.} + func ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.} + func ceil*(x: float64): float64 {.importc: "Math.ceil", nodecl.} + + when (NimMajor, NimMinor) < (1, 5) or defined(nimLegacyJsRound): + func round*(x: float): float {.importc: "Math.round", nodecl.} + else: + func jsRound(x: float): float {.importc: "Math.round", nodecl.} + func round*[T: float64 | float32](x: T): T = + if x >= 0: result = jsRound(x) + else: + result = ceil(x) + if result - x >= T(0.5): + result -= T(1.0) + func trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.} + func trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.} + + func `mod`*(x, y: float32): float32 {.importjs: "(# % #)".} + func `mod`*(x, y: float64): float64 {.importjs: "(# % #)".} = + ## Computes the modulo operation for float values (the remainder of `x` divided by `y`). + runnableExamples: + doAssert 6.5 mod 2.5 == 1.5 + doAssert -6.5 mod 2.5 == -1.5 + doAssert 6.5 mod -2.5 == 1.5 + doAssert -6.5 mod -2.5 == -1.5 + + func divmod*[T:SomeInteger](num, denom: T): (T, T) = + runnableExamples: + doAssert divmod(5, 2) == (2, 1) + doAssert divmod(5, -3) == (-1, 2) + result[0] = num div denom + result[1] = num mod denom + + +func round*[T: float32|float64](x: T, places: int): T = ## Decimal rounding on a binary floating point number. ## ## This function is NOT reliable. Floating point numbers cannot hold - ## non integer decimals precisely. If ``places`` is 0 (or omitted), + ## non integer decimals precisely. If `places` is 0 (or omitted), ## round to the nearest integral value following normal mathematical - ## rounding rules (e.g. ``round(54.5) -> 55.0``). If ``places`` is + ## rounding rules (e.g. `round(54.5) -> 55.0`). If `places` is ## greater than 0, round to the given number of decimal places, - ## e.g. ``round(54.346, 2) -> 54.350000000000001421…``. If ``places`` is negative, round - ## to the left of the decimal place, e.g. ``round(537.345, -1) -> - ## 540.0`` - ## - ## .. code-block:: Nim - ## echo round(PI, 2) ## 3.14 - ## echo round(PI, 4) ## 3.1416 + ## e.g. `round(54.346, 2) -> 54.350000000000001421…`. If `places` is negative, round + ## to the left of the decimal place, e.g. `round(537.345, -1) -> 540.0`. + runnableExamples: + doAssert round(PI, 2) == 3.14 + doAssert round(PI, 4) == 3.1416 + if places == 0: result = round(x) else: - var mult = pow(10.0, places.T) - result = round(x*mult)/mult + var mult = pow(10.0, T(places)) + result = round(x * mult) / mult -proc floorDiv*[T: SomeInteger](x, y: T): T = - ## Floor division is conceptually defined as ``floor(x / y)``. +func floorDiv*[T: SomeInteger](x, y: T): T = + ## Floor division is conceptually defined as `floor(x / y)`. ## ## This is different from the `system.div <system.html#div,int,int>`_ - ## operator, which is defined as ``trunc(x / y)``. - ## That is, ``div`` rounds towards ``0`` and ``floorDiv`` rounds down. + ## operator, which is defined as `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `floorDiv` rounds down. ## - ## See also: + ## **See also:** ## * `system.div proc <system.html#div,int,int>`_ for integer division - ## * `floorMod proc <#floorMod,T,T>`_ for Python-like (% operator) behavior - ## - ## .. code-block:: nim - ## echo floorDiv( 13, 3) # 4 - ## echo floorDiv(-13, 3) # -5 - ## echo floorDiv( 13, -3) # -5 - ## echo floorDiv(-13, -3) # 4 + ## * `floorMod func <#floorMod,T,T>`_ for Python-like (`%` operator) behavior + runnableExamples: + doAssert floorDiv( 13, 3) == 4 + doAssert floorDiv(-13, 3) == -5 + doAssert floorDiv( 13, -3) == -5 + doAssert floorDiv(-13, -3) == 4 + result = x div y let r = x mod y if (r > 0 and y < 0) or (r < 0 and y > 0): result.dec 1 -proc floorMod*[T: SomeNumber](x, y: T): T = - ## Floor modulus is conceptually defined as ``x - (floorDiv(x, y) * y)``. +func floorMod*[T: SomeNumber](x, y: T): T = + ## Floor modulo is conceptually defined as `x - (floorDiv(x, y) * y)`. ## - ## This proc behaves the same as the ``%`` operator in Python. + ## This func behaves the same as the `%` operator in Python. ## - ## See also: - ## * `mod proc <#mod,float64,float64>`_ - ## * `floorDiv proc <#floorDiv,T,T>`_ - ## - ## .. code-block:: nim - ## echo floorMod( 13, 3) # 1 - ## echo floorMod(-13, 3) # 2 - ## echo floorMod( 13, -3) # -2 - ## echo floorMod(-13, -3) # -1 + ## **See also:** + ## * `mod func <#mod,float64,float64>`_ + ## * `floorDiv func <#floorDiv,T,T>`_ + runnableExamples: + doAssert floorMod( 13, 3) == 1 + doAssert floorMod(-13, 3) == 2 + doAssert floorMod( 13, -3) == -2 + doAssert floorMod(-13, -3) == -1 + result = x mod y if (result > 0 and y < 0) or (result < 0 and y > 0): result += y -when not defined(js): - proc c_frexp*(x: float32, exponent: var int32): float32 {. - importc: "frexp", header: "<math.h>".} - proc c_frexp*(x: float64, exponent: var int32): float64 {. - importc: "frexp", header: "<math.h>".} - proc frexp*[T, U](x: T, exponent: var U): T = - ## Split a number into mantissa and exponent. - ## - ## ``frexp`` calculates the mantissa m (a float greater than or equal to 0.5 - ## and less than 1) and the integer value n such that ``x`` (the original - ## float value) equals ``m * 2**n``. frexp stores n in `exponent` and returns - ## m. - ## - ## .. code-block:: nim - ## var x: int - ## echo frexp(5.0, x) # 0.625 - ## echo x # 3 - var exp: int32 - result = c_frexp(x, exp) - exponent = exp +func euclDiv*[T: SomeInteger](x, y: T): T {.since: (1, 5, 1).} = + ## Returns euclidean division of `x` by `y`. + runnableExamples: + doAssert euclDiv(13, 3) == 4 + doAssert euclDiv(-13, 3) == -5 + doAssert euclDiv(13, -3) == -4 + doAssert euclDiv(-13, -3) == 5 + + result = x div y + if x mod y < 0: + if y > 0: + dec result + else: + inc result + +func euclMod*[T: SomeNumber](x, y: T): T {.since: (1, 5, 1).} = + ## Returns euclidean modulo of `x` by `y`. + ## `euclMod(x, y)` is non-negative. + runnableExamples: + doAssert euclMod(13, 3) == 1 + doAssert euclMod(-13, 3) == 2 + doAssert euclMod(13, -3) == 1 + doAssert euclMod(-13, -3) == 2 + result = x mod y + if result < 0: + result += abs(y) + +func ceilDiv*[T: SomeInteger](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Ceil division is conceptually defined as `ceil(x / y)`. + ## + ## Assumes `x >= 0` and `y > 0` (and `x + y - 1 <= high(T)` if T is SomeUnsignedInt). + ## + ## This is different from the `system.div <system.html#div,int,int>`_ + ## operator, which works like `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `ceilDiv` rounds up. + ## + ## This function has the above input limitation, because that allows the + ## compiler to generate faster code and it is rarely used with + ## negative values or unsigned integers close to `high(T)/2`. + ## If you need a `ceilDiv` that works with any input, see: + ## https://github.com/demotomohiro/divmath. + ## + ## **See also:** + ## * `system.div proc <system.html#div,int,int>`_ for integer division + ## * `floorDiv func <#floorDiv,T,T>`_ for integer division which rounds down. + runnableExamples: + assert ceilDiv(12, 3) == 4 + assert ceilDiv(13, 3) == 5 + + when sizeof(T) == 8: + type UT = uint64 + elif sizeof(T) == 4: + type UT = uint32 + elif sizeof(T) == 2: + type UT = uint16 + elif sizeof(T) == 1: + type UT = uint8 + else: + {.fatal: "Unsupported int type".} + + assert x >= 0 and y > 0 + when T is SomeUnsignedInt: + assert x + y - 1 >= x + + # If the divisor is const, the backend C/C++ compiler generates code without a `div` + # instruction, as it is slow on most CPUs. + # If the divisor is a power of 2 and a const unsigned integer type, the + # compiler generates faster code. + # If the divisor is const and a signed integer, generated code becomes slower + # than the code with unsigned integers, because division with signed integers + # need to works for both positive and negative value without `idiv`/`sdiv`. + # That is why this code convert parameters to unsigned. + # This post contains a comparison of the performance of signed/unsigned integers: + # https://github.com/nim-lang/Nim/pull/18596#issuecomment-894420984. + # If signed integer arguments were not converted to unsigned integers, + # `ceilDiv` wouldn't work for any positive signed integer value, because + # `x + (y - 1)` can overflow. + ((x.UT + (y.UT - 1.UT)) div y.UT).T + +func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = + ## Splits `x` into a normalized fraction `frac` and an integral power of 2 `exp`, + ## such that `abs(frac) in 0.5..<1` and `x == frac * 2 ^ exp`, except for special + ## cases shown below. + runnableExamples: + doAssert frexp(8.0) == (0.5, 4) + doAssert frexp(-8.0) == (-0.5, 4) + doAssert frexp(0.0) == (0.0, 0) + + # special cases: + when sizeof(int) == 8: + doAssert frexp(-0.0).frac.signbit # signbit preserved for +-0 + doAssert frexp(Inf).frac == Inf # +- Inf preserved + doAssert frexp(NaN).frac.isNaN + + when not defined(js): + var exp: cint + result.frac = c_frexp2(x, exp) + result.exp = exp + else: + if x == 0.0: + # reuse signbit implementation + let uintBuffer = toBitsImpl(x) + if (uintBuffer[1] shr 31) != 0: + # x is -0.0 + result = (-0.0, 0) + else: + result = (0.0, 0) + elif x < 0.0: + result = frexp(-x) + result.frac = -result.frac + else: + var ex = trunc(log2(x)) + result.exp = int(ex) + result.frac = x / pow(2.0, ex) + if abs(result.frac) >= 1: + inc(result.exp) + result.frac = result.frac / 2 + if result.exp == 1024 and result.frac == 0.0: + result.frac = 0.99999999999999988898 + +func frexp*[T: float32|float64](x: T, exponent: var int): T {.inline.} = + ## Overload of `frexp` that calls `(result, exponent) = frexp(x)`. + runnableExamples: + var x: int + doAssert frexp(5.0, x) == 0.625 + doAssert x == 3 + + (result, exponent) = frexp(x) + + +when not defined(js): when windowsCC89: # taken from Go-lang Math.Log2 const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 template log2Impl[T](x: T): T = - var exp: int32 + var exp: int var frac = frexp(x, exp) # Make sure exact powers of two give an exact answer. # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. if frac == 0.5: return T(exp - 1) - log10(frac)*(1/ln2) + T(exp) + log10(frac) * (1 / ln2) + T(exp) - proc log2*(x: float32): float32 = log2Impl(x) - proc log2*(x: float64): float64 = log2Impl(x) + func log2*(x: float32): float32 = log2Impl(x) + func log2*(x: float64): float64 = log2Impl(x) ## Log2 returns the binary logarithm of x. ## The special cases are the same as for Log. else: - proc log2*(x: float32): float32 {.importc: "log2f", header: "<math.h>".} - proc log2*(x: float64): float64 {.importc: "log2", header: "<math.h>".} - ## Computes the binary logarithm (base 2) of ``x``. - ## - ## See also: - ## * `log proc <#log,T,T>`_ - ## * `log10 proc <#log10,float64>`_ - ## * `ln proc <#ln,float64>`_ - ## * `exp proc <#exp,float64>`_ + func log2*(x: float32): float32 {.importc: "log2f", header: "<math.h>".} + func log2*(x: float64): float64 {.importc: "log2", header: "<math.h>".} = + ## Computes the binary logarithm (base 2) of `x`. ## - ## .. code-block:: Nim - ## echo log2(8.0) # 3.0 - ## echo log2(1.0) # 0.0 - ## echo log2(0.0) # -inf - ## echo log2(-2.0) # nan - -else: - proc frexp*[T: float32|float64](x: T, exponent: var int): T = - if x == 0.0: - exponent = 0 - result = 0.0 - elif x < 0.0: - result = -frexp(-x, exponent) - else: - var ex = trunc(log2(x)) - exponent = int(ex) - result = x / pow(2.0, ex) - if abs(result) >= 1: - inc(exponent) - result = result / 2 - if exponent == 1024 and result == 0.0: - result = 0.99999999999999988898 - -proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = - ## Breaks ``x`` into an integer and a fractional part. + ## **See also:** + ## * `log func <#log,T,T>`_ + ## * `log10 func <#log10,float64>`_ + ## * `ln func <#ln,float64>`_ + runnableExamples: + doAssert almostEqual(log2(8.0), 3.0) + doAssert almostEqual(log2(1.0), 0.0) + doAssert almostEqual(log2(0.0), -Inf) + doAssert log2(-2.0).isNaN + +func splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = + ## Breaks `x` into an integer and a fractional part. ## - ## Returns a tuple containing ``intpart`` and ``floatpart`` representing - ## the integer part and the fractional part respectively. + ## Returns a tuple containing `intpart` and `floatpart`, representing + ## the integer part and the fractional part, respectively. ## - ## Both parts have the same sign as ``x``. Analogous to the ``modf`` + ## Both parts have the same sign as `x`. Analogous to the `modf` ## function in C. - ## - ## .. code-block:: nim - ## echo splitDecimal(5.25) # (intpart: 5.0, floatpart: 0.25) - ## echo splitDecimal(-2.73) # (intpart: -2.0, floatpart: -0.73) + runnableExamples: + doAssert splitDecimal(5.25) == (intpart: 5.0, floatpart: 0.25) + doAssert splitDecimal(-2.73) == (intpart: -2.0, floatpart: -0.73) + var absolute: T absolute = abs(x) @@ -943,63 +1078,122 @@ proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] = result.intpart = -result.intpart result.floatpart = -result.floatpart -{.pop.} -proc degToRad*[T: float32|float64](d: T): T {.inline.} = - ## Convert from degrees to radians. - ## - ## See also: - ## * `radToDeg proc <#radToDeg,T>`_ +func degToRad*[T: float32|float64](d: T): T {.inline.} = + ## Converts from degrees to radians. ## - ## .. code-block:: nim - ## echo degToRad(180.0) # 3.141592653589793 - result = T(d) * RadPerDeg + ## **See also:** + ## * `radToDeg func <#radToDeg,T>`_ + runnableExamples: + doAssert almostEqual(degToRad(180.0), PI) -proc radToDeg*[T: float32|float64](d: T): T {.inline.} = - ## Convert from radians to degrees. - ## - ## See also: - ## * `degToRad proc <#degToRad,T>`_ + result = d * T(RadPerDeg) + +func radToDeg*[T: float32|float64](d: T): T {.inline.} = + ## Converts from radians to degrees. ## - ## .. code-block:: nim - ## echo degToRad(2 * PI) # 360.0 - result = T(d) / RadPerDeg + ## **See also:** + ## * `degToRad func <#degToRad,T>`_ + runnableExamples: + doAssert almostEqual(radToDeg(2 * PI), 360.0) -proc sgn*[T: SomeNumber](x: T): int {.inline.} = + result = d / T(RadPerDeg) + +func sgn*[T: SomeNumber](x: T): int {.inline.} = ## Sign function. ## ## Returns: - ## * `-1` for negative numbers and ``NegInf``, - ## * `1` for positive numbers and ``Inf``, - ## * `0` for positive zero, negative zero and ``NaN`` - ## - ## .. code-block:: nim - ## echo sgn(5) # 1 - ## echo sgn(0) # 0 - ## echo sgn(-4.1) # -1 + ## * `-1` for negative numbers and `NegInf`, + ## * `1` for positive numbers and `Inf`, + ## * `0` for positive zero, negative zero and `NaN` + runnableExamples: + doAssert sgn(5) == 1 + doAssert sgn(0) == 0 + doAssert sgn(-4.1) == -1 + ord(T(0) < x) - ord(x < T(0)) {.pop.} {.pop.} -proc `^`*[T](x: T, y: Natural): T = - ## Computes ``x`` to the power ``y``. +func sum*[T](x: openArray[T]): T = + ## Computes the sum of the elements in `x`. ## - ## Exponent ``y`` must be non-negative, use - ## `pow proc <#pow,float64,float64>`_ for negative exponents. + ## If `x` is empty, 0 is returned. ## - ## See also: - ## * `pow proc <#pow,float64,float64>`_ for negative exponent or - ## floats - ## * `sqrt proc <#sqrt,float64>`_ - ## * `cbrt proc <#cbrt,float64>`_ + ## **See also:** + ## * `prod func <#prod,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ + runnableExamples: + doAssert sum([1, 2, 3, 4]) == 10 + doAssert sum([-4, 3, 5]) == 4 + + for i in items(x): result = result + i + +func prod*[T](x: openArray[T]): T = + ## Computes the product of the elements in `x`. + ## + ## If `x` is empty, 1 is returned. ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `fac func <#fac,int>`_ runnableExamples: - assert -3.0^0 == 1.0 - assert -3^1 == -3 - assert -3^2 == 9 - assert -3.0^3 == -27.0 - assert -3.0^4 == 81.0 + doAssert prod([1, 2, 3, 4]) == 24 + doAssert prod([-4, 3, 5]) == -60 + + result = T(1) + for i in items(x): result = result * i + +func cumsummed*[T](x: openArray[T]): seq[T] = + ## Returns the cumulative (aka prefix) summation of `x`. + ## + ## If `x` is empty, `@[]` is returned. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsum func <#cumsum,openArray[T]>`_ for the in-place version + runnableExamples: + doAssert cumsummed([1, 2, 3, 4]) == @[1, 3, 6, 10] + + let xLen = x.len + if xLen == 0: + return @[] + result.setLen(xLen) + result[0] = x[0] + for i in 1 ..< xLen: result[i] = result[i - 1] + x[i] + +func cumsum*[T](x: var openArray[T]) = + ## Transforms `x` in-place (must be declared as `var`) into its + ## cumulative (aka prefix) summation. + ## + ## **See also:** + ## * `sum func <#sum,openArray[T]>`_ + ## * `cumsummed func <#cumsummed,openArray[T]>`_ for a version which + ## returns a cumsummed sequence + runnableExamples: + var a = [1, 2, 3, 4] + cumsum(a) + doAssert a == @[1, 3, 6, 10] + + for i in 1 ..< x.len: x[i] = x[i - 1] + x[i] + +func `^`*[T: SomeNumber](x: T, y: Natural): T = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be non-negative, use + ## `pow <#pow,float64,float64>`_ for negative exponents. + ## + ## **See also:** + ## * `pow func <#pow,float64,float64>`_ for negative exponent or + ## floats + ## * `sqrt func <#sqrt,float64>`_ + ## * `cbrt func <#cbrt,float64>`_ + runnableExamples: + doAssert -3 ^ 0 == 1 + doAssert -3 ^ 1 == -3 + doAssert -3 ^ 2 == 9 case y of 0: result = 1 @@ -1017,229 +1211,104 @@ proc `^`*[T](x: T, y: Natural): T = break x *= x -proc gcd*[T](x, y: T): T = - ## Computes the greatest common (positive) divisor of ``x`` and ``y``. +func gcd*[T](x, y: T): T = + ## Computes the greatest common (positive) divisor of `x` and `y`. ## ## Note that for floats, the result cannot always be interpreted as - ## "greatest decimal `z` such that ``z*N == x and z*M == y`` - ## where N and M are positive integers." + ## "greatest decimal `z` such that `z*N == x and z*M == y` + ## where N and M are positive integers". ## - ## See also: - ## * `gcd proc <#gcd,SomeInteger,SomeInteger>`_ for integer version - ## * `lcm proc <#lcm,T,T>`_ + ## **See also:** + ## * `gcd func <#gcd,SomeInteger,SomeInteger>`_ for an integer version + ## * `lcm func <#lcm,T,T>`_ runnableExamples: doAssert gcd(13.5, 9.0) == 4.5 + var (x, y) = (x, y) while y != 0: x = x mod y swap x, y abs x -proc gcd*(x, y: SomeInteger): SomeInteger = - ## Computes the greatest common (positive) divisor of ``x`` and ``y``, - ## using binary GCD (aka Stein's) algorithm. - ## - ## See also: - ## * `gcd proc <#gcd,T,T>`_ for floats version - ## * `lcm proc <#lcm,T,T>`_ - runnableExamples: - doAssert gcd(12, 8) == 4 - doAssert gcd(17, 63) == 1 - when x is SomeSignedInt: - var x = abs(x) - else: - var x = x - when y is SomeSignedInt: - var y = abs(y) - else: - var y = y - - if x == 0: - return y - if y == 0: - return x - - let shift = countTrailingZeroBits(x or y) - y = y shr countTrailingZeroBits(y) - while x != 0: - x = x shr countTrailingZeroBits(x) - if y > x: - swap y, x - x -= y - y shl shift - -proc gcd*[T](x: openArray[T]): T {.since: (1, 1).} = - ## Computes the greatest common (positive) divisor of the elements of ``x``. +when useBuiltins: + ## this func uses bitwise comparisons from C compilers, which are not always available. + func gcd*(x, y: SomeInteger): SomeInteger = + ## Computes the greatest common (positive) divisor of `x` and `y`, + ## using the binary GCD (aka Stein's) algorithm. + ## + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ for a float version + ## * `lcm func <#lcm,T,T>`_ + runnableExamples: + doAssert gcd(12, 8) == 4 + doAssert gcd(17, 63) == 1 + + when x is SomeSignedInt: + var x = abs(x) + else: + var x = x + when y is SomeSignedInt: + var y = abs(y) + else: + var y = y + + if x == 0: + return y + if y == 0: + return x + + let shift = countTrailingZeroBits(x or y) + y = y shr countTrailingZeroBits(y) + while x != 0: + x = x shr countTrailingZeroBits(x) + if y > x: + swap y, x + x -= y + y shl shift + +func gcd*[T](x: openArray[T]): T {.since: (1, 1).} = + ## Computes the greatest common (positive) divisor of the elements of `x`. ## - ## See also: - ## * `gcd proc <#gcd,T,T>`_ for integer version + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ for a version with two arguments runnableExamples: doAssert gcd(@[13.5, 9.0]) == 4.5 + result = x[0] - var i = 1 - while i < x.len: + for i in 1 ..< x.len: result = gcd(result, x[i]) - inc(i) -proc lcm*[T](x, y: T): T = - ## Computes the least common multiple of ``x`` and ``y``. +func lcm*[T](x, y: T): T = + ## Computes the least common multiple of `x` and `y`. ## - ## See also: - ## * `gcd proc <#gcd,T,T>`_ + ## **See also:** + ## * `gcd func <#gcd,T,T>`_ runnableExamples: doAssert lcm(24, 30) == 120 doAssert lcm(13, 39) == 39 + x div gcd(x, y) * y -proc lcm*[T](x: openArray[T]): T {.since: (1, 1).} = - ## Computes the least common multiple of the elements of ``x``. +func clamp*[T](val: T, bounds: Slice[T]): T {.since: (1, 5), inline.} = + ## Like `system.clamp`, but takes a slice, so you can easily clamp within a range. + runnableExamples: + assert clamp(10, 1 .. 5) == 5 + assert clamp(1, 1 .. 3) == 1 + type A = enum a0, a1, a2, a3, a4, a5 + assert a1.clamp(a2..a4) == a2 + assert clamp((3, 0), (1, 0) .. (2, 9)) == (2, 9) + doAssertRaises(AssertionDefect): discard clamp(1, 3..2) # invalid bounds + assert bounds.a <= bounds.b, $(bounds.a, bounds.b) + clamp(val, bounds.a, bounds.b) + +func lcm*[T](x: openArray[T]): T {.since: (1, 1).} = + ## Computes the least common multiple of the elements of `x`. ## - ## See also: - ## * `gcd proc <#gcd,T,T>`_ for integer version + ## **See also:** + ## * `lcm func <#lcm,T,T>`_ for a version with two arguments runnableExamples: doAssert lcm(@[24, 30]) == 120 + result = x[0] - var i = 1 - while i < x.len: + for i in 1 ..< x.len: result = lcm(result, x[i]) - inc(i) - -when isMainModule and not defined(js) and not windowsCC89: - # Check for no side effect annotation - proc mySqrt(num: float): float {.noSideEffect.} = - return sqrt(num) - - # check gamma function - assert(gamma(5.0) == 24.0) # 4! - assert($tgamma(5.0) == $24.0) # 4! - assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0 - assert(erf(6.0) > erf(5.0)) - assert(erfc(6.0) < erfc(5.0)) - -when isMainModule: - # Function for approximate comparison of floats - proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9) - - block: # prod - doAssert prod([1, 2, 3, 4]) == 24 - doAssert prod([1.5, 3.4]) == 5.1 - let x: seq[float] = @[] - doAssert prod(x) == 1.0 - - block: # round() tests - # Round to 0 decimal places - doAssert round(54.652) ==~ 55.0 - doAssert round(54.352) ==~ 54.0 - doAssert round(-54.652) ==~ -55.0 - doAssert round(-54.352) ==~ -54.0 - doAssert round(0.0) ==~ 0.0 - # Round to positive decimal places - doAssert round(-547.652, 1) ==~ -547.7 - doAssert round(547.652, 1) ==~ 547.7 - doAssert round(-547.652, 2) ==~ -547.65 - doAssert round(547.652, 2) ==~ 547.65 - # Round to negative decimal places - doAssert round(547.652, -1) ==~ 550.0 - doAssert round(547.652, -2) ==~ 500.0 - doAssert round(547.652, -3) ==~ 1000.0 - doAssert round(547.652, -4) ==~ 0.0 - doAssert round(-547.652, -1) ==~ -550.0 - doAssert round(-547.652, -2) ==~ -500.0 - doAssert round(-547.652, -3) ==~ -1000.0 - doAssert round(-547.652, -4) ==~ 0.0 - - block: # splitDecimal() tests - doAssert splitDecimal(54.674).intpart ==~ 54.0 - doAssert splitDecimal(54.674).floatpart ==~ 0.674 - doAssert splitDecimal(-693.4356).intpart ==~ -693.0 - doAssert splitDecimal(-693.4356).floatpart ==~ -0.4356 - doAssert splitDecimal(0.0).intpart ==~ 0.0 - doAssert splitDecimal(0.0).floatpart ==~ 0.0 - - block: # trunc tests for vcc - doAssert(trunc(-1.1) == -1) - doAssert(trunc(1.1) == 1) - doAssert(trunc(-0.1) == -0) - doAssert(trunc(0.1) == 0) - - #special case - doAssert(classify(trunc(1e1000000)) == fcInf) - doAssert(classify(trunc(-1e1000000)) == fcNegInf) - doAssert(classify(trunc(0.0/0.0)) == fcNan) - doAssert(classify(trunc(0.0)) == fcZero) - - #trick the compiler to produce signed zero - let - f_neg_one = -1.0 - f_zero = 0.0 - f_nan = f_zero / f_zero - - doAssert(classify(trunc(f_neg_one*f_zero)) == fcNegZero) - - doAssert(trunc(-1.1'f32) == -1) - doAssert(trunc(1.1'f32) == 1) - doAssert(trunc(-0.1'f32) == -0) - doAssert(trunc(0.1'f32) == 0) - doAssert(classify(trunc(1e1000000'f32)) == fcInf) - doAssert(classify(trunc(-1e1000000'f32)) == fcNegInf) - doAssert(classify(trunc(f_nan.float32)) == fcNan) - doAssert(classify(trunc(0.0'f32)) == fcZero) - - block: # sgn() tests - assert sgn(1'i8) == 1 - assert sgn(1'i16) == 1 - assert sgn(1'i32) == 1 - assert sgn(1'i64) == 1 - assert sgn(1'u8) == 1 - assert sgn(1'u16) == 1 - assert sgn(1'u32) == 1 - assert sgn(1'u64) == 1 - assert sgn(-12342.8844'f32) == -1 - assert sgn(123.9834'f64) == 1 - assert sgn(0'i32) == 0 - assert sgn(0'f32) == 0 - assert sgn(NegInf) == -1 - assert sgn(Inf) == 1 - assert sgn(NaN) == 0 - - block: # fac() tests - try: - discard fac(-1) - except AssertionError: - discard - - doAssert fac(0) == 1 - doAssert fac(1) == 1 - doAssert fac(2) == 2 - doAssert fac(3) == 6 - doAssert fac(4) == 24 - - block: # floorMod/floorDiv - doAssert floorDiv(8, 3) == 2 - doAssert floorMod(8, 3) == 2 - - doAssert floorDiv(8, -3) == -3 - doAssert floorMod(8, -3) == -1 - - doAssert floorDiv(-8, 3) == -3 - doAssert floorMod(-8, 3) == 1 - - doAssert floorDiv(-8, -3) == 2 - doAssert floorMod(-8, -3) == -2 - - doAssert floorMod(8.0, -3.0) ==~ -1.0 - doAssert floorMod(-8.5, 3.0) ==~ 0.5 - - block: # log - doAssert log(4.0, 3.0) ==~ ln(4.0) / ln(3.0) - doAssert log2(8.0'f64) == 3.0'f64 - doAssert log2(4.0'f64) == 2.0'f64 - doAssert log2(2.0'f64) == 1.0'f64 - doAssert log2(1.0'f64) == 0.0'f64 - doAssert classify(log2(0.0'f64)) == fcNegInf - - doAssert log2(8.0'f32) == 3.0'f32 - doAssert log2(4.0'f32) == 2.0'f32 - doAssert log2(2.0'f32) == 1.0'f32 - doAssert log2(1.0'f32) == 0.0'f32 - doAssert classify(log2(0.0'f32)) == fcNegInf diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 1ea71afd2..9c3f6d51b 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -7,14 +7,19 @@ # distribution, for details about the copyright. # -## Module for computing `MD5 checksums <https://en.wikipedia.org/wiki/MD5>`_. +## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). ## -## **See also:** -## * `base64 module<base64.html>`_ implements a base64 encoder and decoder -## * `std/sha1 module <sha1.html>`_ for a sha1 encoder and decoder +## This module also works at compile time and in JavaScript. +## +## See also +## ======== +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder +## * `sha1 module <sha1.html>`_ for the SHA-1 checksum algorithm ## * `hashes module<hashes.html>`_ for efficient computations of hash values ## for diverse Nim types +{.deprecated: "use command `nimble install checksums` and import `checksums/md5` instead".} + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -22,8 +27,8 @@ type MD5State = array[0..3, uint32] MD5Block = array[0..15, uint32] MD5CBits = array[0..7, uint8] - MD5Digest* = array[0..15, uint8] ## \ - ## MD5 checksum of a string, obtained with `toMD5 proc <#toMD5,string>`_. + MD5Digest* = array[0..15, uint8] + ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. MD5Buffer = array[0..63, uint8] MD5Context* {.final.} = object state: MD5State @@ -31,15 +36,16 @@ type buffer: MD5Buffer const - padding: cstring = "\x80\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0" + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) @@ -76,7 +82,7 @@ proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = rot(a, s) a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: openArray[uint8]) = var j = 0 for i in 0..high(dest): dest[i] = uint32(ord(src[j])) or @@ -94,10 +100,35 @@ proc decode(dest: var openArray[uint8], src: openArray[uint32]) = dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +template slice(s: cstring, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + toOpenArrayByte($s, a, b) + else: + when defined(js): + # toOpenArrayByte for cstring is not implemented in JS + toOpenArrayByte($s, a, b) + else: + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + s.toOpenArray(a, b) + +const useMem = declared(copyMem) + +template memOrNot(withMem, withoutMem): untyped = + when nimvm: + withoutMem + else: + when useMem: + withMem + else: + withoutMem + +proc transform(buffer: openArray[uint8], state: var MD5State) = var myBlock: MD5Block - encode(myBlock, cast[cstring](buffer)) + encode(myBlock, buffer) var a = state[0] var b = state[1] var c = state[2] @@ -172,15 +203,23 @@ proc transform(buffer: pointer, state: var MD5State) = state[3] = state[3] + d proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], tags: [], gcsafe.} proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + proc toMD5*(s: string): MD5Digest = ## Computes the `MD5Digest` value for a string `s`. ## - ## See also: + ## **See also:** ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation ## of the `MD5Digest` ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string @@ -189,7 +228,7 @@ proc toMD5*(s: string): MD5Digest = var c: MD5Context md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, result) proc `$`*(d: MD5Digest): string = @@ -202,10 +241,8 @@ proc `$`*(d: MD5Digest): string = proc getMD5*(s: string): string = ## Computes an MD5 value of `s` and returns its string representation. - ## .. note:: - ## available at compile time ## - ## See also: + ## **See also:** ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string runnableExamples: assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" @@ -214,7 +251,7 @@ proc getMD5*(s: string): string = c: MD5Context d: MD5Digest md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, d) result = $d @@ -225,10 +262,16 @@ proc `==`*(D1, D2: MD5Digest): bool = return true +proc clearBuffer(c: var MD5Context) {.inline.} = + memOrNot: + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + do: + reset(c.buffer) + proc md5Init*(c: var MD5Context) = - ## Initializes a `MD5Context`. + ## Initializes an `MD5Context`. ## - ## If you use `toMD5 proc <#toMD5,string>`_ there's no need to call this + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. c.state[0] = 0x67452301'u32 c.state[1] = 0xEFCDAB89'u32 @@ -236,34 +279,44 @@ proc md5Init*(c: var MD5Context) = c.state[3] = 0x10325476'u32 c.count[0] = 0'u32 c.count[1] = 0'u32 - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + clearBuffer(c) -proc md5Update*(c: var MD5Context, input: cstring, len: int) = - ## Updates the `MD5Context` with the `input` data of length `len`. +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + memOrNot: + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + do: + # cannot use system.`[]=` for arrays and openarrays as + # it can raise RangeDefect which gets tracked + for i in 0..<len: + c.buffer[index + i] = input[inputIndex + i] + +proc md5Update*(c: var MD5Context, input: openArray[uint8]) = + ## Updates the `MD5Context` with the `input` data. ## - ## If you use `toMD5 proc <#toMD5,string>`_ there's no need to call this + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. - var input = input var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(len) shl 3) - if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(len) shr 29) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) var PartLen = 64 - Index - if len >= PartLen: - copyMem(addr(c.buffer[Index]), input, PartLen) - transform(addr(c.buffer), c.state) + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) var i = PartLen - while i + 63 < len: - transform(addr(input[i]), c.state) + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) inc(i, 64) - copyMem(addr(c.buffer[0]), addr(input[i]), len-i) - else: - copyMem(addr(c.buffer[Index]), addr(input[0]), len) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## Finishes the `MD5Context` and stores the result in `digest`. ## - ## If you use `toMD5 proc <#toMD5,string>`_ there's no need to call this + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this ## function explicitly. var Bits: MD5CBits @@ -272,18 +325,11 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = var Index = int((c.count[0] shr 3) and 0x3F) if Index < 56: PadLen = 56 - Index else: PadLen = 120 - Index - md5Update(c, padding, PadLen) - md5Update(c, cast[cstring](addr(Bits)), 8) + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) decode(digest, c.state) - zeroMem(addr(c), sizeof(MD5Context)) - + clearBuffer(c) -when isMainModule: - assert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "a3cca2b2aa1e3b5b3b5aad99a8529074") - assert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "7e716d0e702df0505fc72e2b89467910") - assert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") when defined(nimHasStyleChecks): - {.pop.} #{.push styleChecks: off.} + {.pop.} #{.push styleChecks: off.} \ No newline at end of file diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 152b03ec6..8eec551c4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -16,18 +16,54 @@ ## other "line-like", variable length, delimited records). when defined(windows): - import winlean + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs elif defined(posix): - import posix + import std/posix else: {.error: "the memfiles module is not supported on your operating system!".} -import os, streams +import std/streams +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1, + ## allocating | freeing space from the file system. This routine returns the + ## last OSErrorCode found rather than raising to support old rollback/clean-up + ## code style. [ Should maybe move to std/osfiles. ] + if newFileSize < 0 or newFileSize == oldSize: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + if newFileSize > oldSize: # grow the file + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable BUT allows SEGV + elif e != 0: + result = osLastError() + else: # shrink the file + if ftruncate(fh.cint, newFileSize) == -1: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -48,8 +84,8 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0, mapFlags = cint(-1)): pointer = ## returns a pointer to a mapped portion of MemFile `m` ## - ## ``mappedSize`` of ``-1`` maps to the whole file, and - ## ``offset`` must be multiples of the PAGE SIZE of your OS + ## `mappedSize` of `-1` maps to the whole file, and + ## `offset` must be multiples of the PAGE SIZE of your OS if mode == fmAppend: raise newEIO("The append mode is not supported.") @@ -83,12 +119,12 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead, proc unmapMem*(f: var MemFile, p: pointer, size: int) = - ## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`. + ## unmaps the memory region `(p, <p+size)` of the mapped file `f`. ## All changes are written back to the file system, if `f` was opened ## with write access. ## - ## ``size`` must be of exactly the size that was requested - ## via ``mapMem``. + ## `size` must be of exactly the size that was requested + ## via `mapMem`. when defined(windows): if unmapViewOfFile(p) == 0: raiseOSError(osLastError()) else: @@ -98,27 +134,27 @@ proc unmapMem*(f: var MemFile, p: pointer, size: int) = proc open*(filename: string, mode: FileMode = fmRead, mappedSize = -1, offset = 0, newFileSize = -1, allowRemap = false, mapFlags = cint(-1)): MemFile = - ## opens a memory mapped file. If this fails, ``OSError`` is raised. + ## opens a memory mapped file. If this fails, `OSError` is raised. ## - ## ``newFileSize`` can only be set if the file does not exist and is opened + ## `newFileSize` can only be set if the file does not exist and is opened ## with write access (e.g., with fmReadWrite). ## - ##``mappedSize`` and ``offset`` + ##`mappedSize` and `offset` ## can be used to map only a slice of the file. ## - ## ``offset`` must be multiples of the PAGE SIZE of your OS + ## `offset` must be multiples of the PAGE SIZE of your OS ## (usually 4K or 8K but is unique to your OS) ## - ## ``allowRemap`` only needs to be true if you want to call ``mapMem`` on + ## `allowRemap` only needs to be true if you want to call `mapMem` on ## the resulting MemFile; else file handles are not kept open. ## - ## ``mapFlags`` allows callers to override default choices for memory mapping + ## `mapFlags` allows callers to override default choices for memory mapping ## flags with a bitwise mask of a variety of likely platform-specific flags ## which may be ignored or even cause `open` to fail if misspecified. ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## var ## mm, mm_full, mm_half: MemFile ## @@ -130,6 +166,7 @@ proc open*(filename: string, mode: FileMode = fmRead, ## ## # Read the first 512 bytes ## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512) + ## ``` # The file can be resized only when write mode is used: if mode == fmAppend: @@ -167,25 +204,13 @@ proc open*(filename: string, mode: FileMode = fmRead, else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -219,7 +244,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -228,48 +253,37 @@ proc open*(filename: string, mode: FileMode = fmRead, if result.handle != -1: discard close(result.handle) raiseOSError(errCode) - var flags = if readonly: O_RDONLY else: O_RDWR + var flags = (if readonly: O_RDONLY else: O_RDWR) or O_CLOEXEC if newFileSize != -1: flags = flags or O_CREAT or O_TRUNC - var permissions_mode = S_IRUSR or S_IWUSR - result.handle = open(filename, flags, permissions_mode) + var permissionsMode = S_IRUSR or S_IWUSR + result.handle = open(filename, flags, permissionsMode) + if result.handle != -1: + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") else: result.handle = open(filename, flags) if result.handle == -1: - # XXX: errno is supposed to be set here - # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") - - if mappedSize != -1: - result.size = mappedSize - else: - var stat: Stat + if mappedSize != -1: #XXX Logic here differs from `when windows` branch .. + result.size = mappedSize #.. which always fstats&Uses min(mappedSize, st). + else: # if newFileSize!=-1: result.size=newFileSize # if trust setFileSize + var stat: Stat #^^.. BUT some FSes (eg. Linux HugeTLBfs) round to 2MiB. if fstat(result.handle, stat) != -1: - # XXX: Hmm, this could be unsafe - # Why is mmap taking int anyway? - result.size = int(stat.st_size) + result.size = stat.st_size.int # int may be 32-bit-unsafe for 2..<4 GiB else: fail(osLastError(), "error getting file size") result.flags = if mapFlags == cint(-1): MAP_SHARED else: mapFlags - #Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set + # Ensure exactly one of MAP_PRIVATE cr MAP_SHARED is set if int(result.flags and MAP_PRIVATE) == 0: result.flags = result.flags or MAP_SHARED - result.mem = mmap( - nil, - result.size, - if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - result.flags, - result.handle, - offset) - + let pr = if readonly: PROT_READ else: PROT_READ or PROT_WRITE + result.mem = mmap(nil, result.size, pr, result.flags, result.handle, offset) if result.mem == cast[pointer](MAP_FAILED): fail(osLastError(), "file mapping failed") @@ -299,34 +313,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an ``allowRemap MemFile``. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of ``.mem`` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize; flags: cint): - pointer {.importc: "mremap", header: "<sys/mman.h>".} - let newAddr = mremap(f.mem, csize(f.size), csize(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize, f.size); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "<sys/mman.h>".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the @@ -374,10 +412,10 @@ proc `==`*(x, y: MemSlice): bool = proc `$`*(ms: MemSlice): string {.inline.} = ## Return a Nim string built from a MemSlice. result.setLen(ms.size) - copyMem(addr(result[0]), ms.data, ms.size) + copyMem(result.cstring, ms.data, ms.size) iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline.} = - ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`. + ## Iterates over \[optional `eat`] `delim`-delimited slices in MemFile `mfile`. ## ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l) ## style on on a line-by-line basis. I.e., not every line needs the same ending. @@ -400,15 +438,15 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ## functions, not str* functions). ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## var count = 0 ## for slice in memSlices(memfiles.open("foo")): ## if slice.size > 0 and cast[cstring](slice.data)[0] != '#': ## inc(count) ## echo count + ## ``` - proc c_memchr(cstr: pointer, c: char, n: csize): pointer {. + proc c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q) var ms: MemSlice @@ -416,7 +454,7 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ms.data = mfile.mem var remaining = mfile.size while remaining > 0: - ending = c_memchr(ms.data, delim, remaining) + ending = c_memchr(ms.data, delim, csize_t(remaining)) if ending == nil: # unterminated final slice ms.size = remaining # Weird case..check eat? yield ms @@ -428,39 +466,39 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline ms.data = cast[pointer](cast[int](ending) +% 1) # skip delim remaining = mfile.size - (ms.data -! mfile.mem) -iterator lines*(mfile: MemFile, buf: var TaintedString, delim = '\l', - eat = '\r'): TaintedString {.inline.} = +iterator lines*(mfile: MemFile, buf: var string, delim = '\l', + eat = '\r'): string {.inline.} = ## Replace contents of passed buffer with each new line, like - ## `readLine(File) <io.html#readLine,File,TaintedString>`_. + ## `readLine(File) <syncio.html#readLine,File,string>`_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim - ## var buffer: TaintedString = "" + ## ```nim + ## var buffer: string = "" ## for line in lines(memfiles.open("foo"), buffer): ## echo line + ## ``` for ms in memSlices(mfile, delim, eat): - setLen(buf.string, ms.size) + setLen(buf, ms.size) if ms.size > 0: copyMem(addr buf[0], ms.data, ms.size) yield buf -iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): TaintedString {.inline.} = +iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## Return each line in a file as a Nim string, like - ## `lines(File) <io.html#lines.i,File>`_. + ## `lines(File) <syncio.html#lines.i,File>`_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## ## Example: - ## - ## .. code-block:: nim + ## ```nim ## for line in lines(memfiles.open("foo")): ## echo line + ## ``` - var buf = TaintedString(newStringOfCap(80)) + var buf = newStringOfCap(80) for line in lines(mfile, buf, delim, eat): yield buf @@ -488,8 +526,8 @@ proc mmsSetPosition(s: Stream, pos: int) = proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) - let p = cast[ByteAddress](MemMapFileStream(s).pos) + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) let l = min(bufLen, MemMapFileStream(s).mf.size - p) moveMem(buffer, cast[pointer](startAddress + p), l) result = l @@ -504,8 +542,8 @@ proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = let size = MemMapFileStream(s).mf.size if MemMapFileStream(s).pos + bufLen > size: raise newEIO("cannot write to stream") - let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + - cast[ByteAddress](MemMapFileStream(s).pos) + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) moveMem(cast[pointer](p), buffer, bufLen) inc(MemMapFileStream(s).pos, bufLen) @@ -514,7 +552,7 @@ proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, ## creates a new stream from the file named `filename` with the mode `mode`. ## Raises ## `OSError` if the file cannot be opened. See the `system ## <system.html>`_ module for a list of available FileMode enums. - ## ``fileSize`` can only be set if the file does not exist and is opened + ## `fileSize` can only be set if the file does not exist and is opened ## with write access (e.g., with fmReadWrite). var mf: MemFile = open(filename, mode, newFileSize = fileSize) new(result) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index f3f6e9324..ff639e8e5 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -8,1856 +8,734 @@ # ## This module implements a mimetypes database -import strtabs -from strutils import startsWith, toLowerAscii, strip + +runnableExamples: + var m = newMimetypes() + doAssert m.getMimetype("mp4") == "video/mp4" + doAssert m.getExt("text/html") == "html" + ## Values can be uppercase too. + doAssert m.getMimetype("MP4") == "video/mp4" + doAssert m.getExt("TEXT/HTML") == "html" + ## If values are invalid then `default` is returned. + doAssert m.getMimetype("INVALID") == "text/plain" + doAssert m.getExt("INVALID/NONEXISTENT") == "txt" + doAssert m.getMimetype("") == "text/plain" + doAssert m.getExt("") == "txt" + ## Register new Mimetypes. + m.register(ext = "fakext", mimetype = "text/fakelang") + doAssert m.getMimetype("fakext") == "text/fakelang" + doAssert m.getMimetype("FaKeXT") == "text/fakelang" + +import std/tables +from std/strutils import startsWith, toLowerAscii, strip + +when defined(nimPreviewSlimSystem): + import std/assertions + type MimeDB* = object - mimes: StringTableRef + mimes: OrderedTable[string, string] const mimes* = { - "123": "application/vnd.lotus-1-2-3", - "1km": "application/vnd.1000minds.decision-model+xml", - "323": "text/h323", - "3dm": "text/vnd.in3d.3dml", - "3dmf": "x-world/x-3dmf", - "3dml": "text/vnd.in3d.3dml", - "3ds": "image/x-3ds", - "3g2": "video/3gpp2", - "3gp": "video/3gpp", - "3gpp": "audio/3gpp", - "3gpp2": "video/3gpp2", - "3mf": "application/vnd.ms-3mfdocument", - "669": "audio/x-mod", - "726": "audio/32kadpcm", - "7z": "application/x-7z-compressed", - "a": "text/plain", - "a2l": "application/a2l", - "aa3": "audio/atrac3", - "aab": "application/x-authorware-bin", - "aac": "audio/x-aac", - "aal": "audio/atrac-advanced-lossless", - "aam": "application/x-authorware-map", - "aas": "application/x-authorware-seg", - "abc": "text/vnd.abc", - "abw": "application/x-abiword", + "ez": "application/andrew-inset", + "aw": "application/applixware", + "atom": "application/atom+xml", + "atomcat": "application/atomcat+xml", + "atomsvc": "application/atomsvc+xml", + "ccxml": "application/ccxml+xml", + "cdmia": "application/cdmi-capability", + "cdmic": "application/cdmi-container", + "cdmid": "application/cdmi-domain", + "cdmio": "application/cdmi-object", + "cdmiq": "application/cdmi-queue", + "cu": "application/cu-seeme", + "davmount": "application/davmount+xml", + "dbk": "application/docbook+xml", + "dssc": "application/dssc+der", + "xdssc": "application/dssc+xml", + "ecma": "application/ecmascript", + "emma": "application/emma+xml", + "epub": "application/epub+zip", + "exi": "application/exi", + "pfr": "application/font-tdpfr", + "gml": "application/gml+xml", + "gpx": "application/gpx+xml", + "gxf": "application/gxf", + "stk": "application/hyperstudio", + "ink": "application/inkml+xml", + "inkml": "application/inkml+xml", + "ipfix": "application/ipfix", + "jar": "application/java-archive", + "ser": "application/java-serialized-object", + "class": "application/java-vm", + "json": "application/json", + "jsonml": "application/jsonml+json", + "lostxml": "application/lost+xml", + "hqx": "application/mac-binhex40", + "cpt": "application/mac-compactpro", + "mads": "application/mads+xml", + "mrc": "application/marc", + "mrcx": "application/marcxml+xml", + "ma": "application/mathematica", + "nb": "application/mathematica", + "mb": "application/mathematica", + "mathml": "application/mathml+xml", + "mbox": "application/mbox", + "mscml": "application/mediaservercontrol+xml", + "metalink": "application/metalink+xml", + "meta4": "application/metalink4+xml", + "mets": "application/mets+xml", + "mods": "application/mods+xml", + "m21": "application/mp21", + "mp21": "application/mp21", + "mp4s": "application/mp4", + "doc": "application/msword", + "dot": "application/msword", + "mxf": "application/mxf", + "bin": "application/octet-stream", + "dms": "application/octet-stream", + "lrf": "application/octet-stream", + "mar": "application/octet-stream", + "so": "application/octet-stream", + "dist": "application/octet-stream", + "distz": "application/octet-stream", + "pkg": "application/octet-stream", + "bpk": "application/octet-stream", + "dump": "application/octet-stream", + "elc": "application/octet-stream", + "deploy": "application/octet-stream", + "oda": "application/oda", + "opf": "application/oebps-package+xml", + "ogx": "application/ogg", + "omdoc": "application/omdoc+xml", + "onetoc": "application/onenote", + "onetoc2": "application/onenote", + "onetmp": "application/onenote", + "onepkg": "application/onenote", + "oxps": "application/oxps", + "xer": "application/patch-ops-error+xml", + "pdf": "application/pdf", + "pgp": "application/pgp-encrypted", + "asc": "application/pgp-signature", + "sig": "application/pgp-signature", + "prf": "application/pics-rules", + "p10": "application/pkcs10", + "p7m": "application/pkcs7-mime", + "p7c": "application/pkcs7-mime", + "p7s": "application/pkcs7-signature", + "p8": "application/pkcs8", "ac": "application/pkix-attr-cert", - "ac3": "audio/ac3", - "acc": "application/vnd.americandynamics.acc", - "ace": "application/x-ace-compressed", - "acn": "audio/asc", - "acu": "application/vnd.acucobol", - "acutc": "application/vnd.acucorp", - "acx": "application/internet-property-stream", - "adp": "audio/adpcm", - "aep": "application/vnd.audiograph", - "afl": "video/animaflex", - "afm": "application/x-font-type1", - "afp": "application/vnd.ibm.modcap", - "ahead": "application/vnd.ahead.space", + "cer": "application/pkix-cert", + "crl": "application/pkix-crl", + "pkipath": "application/pkix-pkipath", + "pki": "application/pkixcmp", + "pls": "application/pls+xml", "ai": "application/postscript", - "aif": "audio/x-aiff", - "aifc": "audio/x-aiff", - "aiff": "audio/x-aiff", - "aim": "application/x-aim", - "aip": "text/x-audiosoft-intra", - "air": "application/vnd.adobe.air-application-installer-package+zip", - "ait": "application/vnd.dvb.ait", - "alc": "chemical/x-alchemy", - "ami": "application/vnd.amiga.ami", - "aml": "application/aml", - "amr": "audio/amr", - "ani": "application/x-navi-animation", - "anx": "application/x-annodex", - "aos": "application/x-nokia-9000-communicator-add-on-software", - "apinotes": "text/apinotes", - "apk": "application/vnd.android.package-archive", - "apkg": "application/vnd.anki", - "apng": "image/apng", - "appcache": "text/cache-manifest", - "appimage": "application/appimage", - "application": "application/x-ms-application", - "apr": "application/vnd.lotus-approach", - "aps": "application/mime", - "apxml": "application/auth-policy+xml", - "arc": "application/x-freearc", - "arj": "application/x-arj", - "art": "message/rfc822", - "asar": "binary/asar", - "asc": "text/plain", - "ascii": "text/vnd.ascii-art", - "asf": "application/vnd.ms-asf", - "asice": "application/vnd.etsi.asic-e+zip", - "asics": "application/vnd.etsi.asic-s+zip", - "asm": "text/x-asm", - "asn": "chemical/x-ncbi-asn1-spec", + "eps": "application/postscript", + "ps": "application/postscript", + "cww": "application/prs.cww", + "pskcxml": "application/pskc+xml", + "rdf": "application/rdf+xml", + "rif": "application/reginfo+xml", + "rnc": "application/relax-ng-compact-syntax", + "rl": "application/resource-lists+xml", + "rld": "application/resource-lists-diff+xml", + "rs": "application/rls-services+xml", + "gbr": "application/rpki-ghostbusters", + "mft": "application/rpki-manifest", + "roa": "application/rpki-roa", + "rsd": "application/rsd+xml", + "rss": "application/rss+xml", + "rtf": "application/rtf", + "sbml": "application/sbml+xml", + "scq": "application/scvp-cv-request", + "scs": "application/scvp-cv-response", + "spq": "application/scvp-vp-request", + "spp": "application/scvp-vp-response", + "sdp": "application/sdp", + "setpay": "application/set-payment-initiation", + "setreg": "application/set-registration-initiation", + "shf": "application/shf+xml", + "smi": "application/smil+xml", + "smil": "application/smil+xml", + "rq": "application/sparql-query", + "srx": "application/sparql-results+xml", + "gram": "application/srgs", + "grxml": "application/srgs+xml", + "sru": "application/sru+xml", + "ssdl": "application/ssdl+xml", + "ssml": "application/ssml+xml", + "tei": "application/tei+xml", + "teicorpus": "application/tei+xml", + "tfi": "application/thraud+xml", + "tsd": "application/timestamped-data", + "plb": "application/vnd.3gpp.pic-bw-large", + "psb": "application/vnd.3gpp.pic-bw-small", + "pvb": "application/vnd.3gpp.pic-bw-var", + "tcap": "application/vnd.3gpp2.tcap", + "pwn": "application/vnd.3m.post-it-notes", "aso": "application/vnd.accpac.simply.aso", - "asp": "text/asp", - "asr": "video/x-ms-asf", - "asx": "video/x-ms-asf", - "at3": "audio/atrac3", + "imp": "application/vnd.accpac.simply.imp", + "acu": "application/vnd.acucobol", "atc": "application/vnd.acucorp", - "atf": "application/atf", - "atfx": "application/atfx", - "atom": "application/atom+xml", - "atomcat": "application/atomcat+xml", - "atomdeleted": "application/atomdeleted+xml", - "atomsrv": "application/atomserv+xml", - "atomsvc": "application/atomsvc+xml", - "atx": "application/vnd.antix.game-component", - "atxml": "application/atxml", - "au": "audio/basic", - "auc": "application/tamp-apex-update-confirm", - "avi": "video/x-msvideo", - "avs": "video/avs-video", - "aw": "application/applixware", - "awb": "audio/amr-wb", - "axa": "audio/x-annodex", - "axs": "application/olescript", - "axv": "video/x-annodex", + "acutc": "application/vnd.acucorp", + "air": "application/vnd.adobe.air-application-installer-package+zip", + "fcdt": "application/vnd.adobe.formscentral.fcdt", + "fxp": "application/vnd.adobe.fxp", + "fxpl": "application/vnd.adobe.fxp", + "xdp": "application/vnd.adobe.xdp+xml", + "xfdf": "application/vnd.adobe.xfdf", + "ahead": "application/vnd.ahead.space", "azf": "application/vnd.airzip.filesecure.azf", "azs": "application/vnd.airzip.filesecure.azs", - "azv": "image/vnd.airzip.accelerator.azv", "azw": "application/vnd.amazon.ebook", - "azw3": "application/vnd.amazon.mobi8-ebook", - "b": "chemical/x-molconn-Z", - "bak": "application/x-trash", - "bar": "application/vnd.qualcomm.brew-app-res", - "bas": "text/plain", - "bash": "text/shell", - "bat": "application/x-msdos-program", - "bcpio": "application/x-bcpio", - "bdf": "application/x-font-bdf", - "bdm": "application/vnd.syncml.dm+wbxml", - "bdoc": "application/bdoc", - "bed": "application/vnd.realvnc.bed", - "bh2": "application/vnd.fujitsu.oasysprs", - "bib": "text/x-bibtex", - "bik": "video/vnd.radgamettools.bink", - "bin": "application/octet-stream", - "bk2": "video/vnd.radgamettools.bink", - "bkm": "application/vnd.nervana", - "blb": "application/x-blorb", - "blend": "binary/blender", - "blorb": "application/x-blorb", - "bm": "image/bmp", - "bmed": "multipart/vnd.bint.med-plus", + "acc": "application/vnd.americandynamics.acc", + "ami": "application/vnd.amiga.ami", + "apk": "application/vnd.android.package-archive", + "cii": "application/vnd.anser-web-certificate-issue-initiation", + "fti": "application/vnd.anser-web-funds-transfer-initiation", + "atx": "application/vnd.antix.game-component", + "mpkg": "application/vnd.apple.installer+xml", + "m3u8": "application/vnd.apple.mpegurl", + "swi": "application/vnd.aristanetworks.swi", + "iota": "application/vnd.astraea-software.iota", + "aep": "application/vnd.audiograph", + "mpm": "application/vnd.blueice.multipass", "bmi": "application/vnd.bmi", - "bmml": "application/vnd.balsamiq.bmml+xml", - "bmp": "image/bmp", - "bmpr": "application/vnd.balsamiq.bmpr", - "boo": "application/book", - "book": "application/book", - "box": "application/vnd.previewsystems.box", - "boz": "application/x-bzip2", - "bpd": "application/vnd.hbci", - "bpk": "application/octet-stream", - "brf": "text/plain", - "bsd": "chemical/x-crossfire", - "bsh": "application/x-bsh", - "bsp": "model/vnd.valve.source.compiled-map", - "btf": "image/prs.btif", - "btif": "image/prs.btif", - "bz": "application/x-bzip", - "bz2": "application/x-bzip2", - "c": "text/x-csrc", - "c++": "text/x-c++src", - "c11amc": "application/vnd.cluetrust.cartomobile-config", - "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", - "c3d": "chemical/x-chem3d", - "c3ex": "application/cccex", + "rep": "application/vnd.businessobjects", + "cdxml": "application/vnd.chemdraw+xml", + "mmd": "application/vnd.chipnuts.karaoke-mmd", + "cdy": "application/vnd.cinderella", + "cla": "application/vnd.claymore", + "rp9": "application/vnd.cloanto.rp9", + "c4g": "application/vnd.clonk.c4group", "c4d": "application/vnd.clonk.c4group", "c4f": "application/vnd.clonk.c4group", - "c4g": "application/vnd.clonk.c4group", "c4p": "application/vnd.clonk.c4group", "c4u": "application/vnd.clonk.c4group", - "cab": "application/vnd.ms-cab-compressed", - "cac": "chemical/x-cache", - "cache": "application/x-cache", - "caf": "audio/x-caf", - "cap": "application/vnd.tcpdump.pcap", - "car": "application/vnd.curl.car", - "cascii": "chemical/x-cactvs-binary", - "cat": "application/vnd.ms-pki.seccat", - "cb7": "application/x-cbr", - "cba": "application/x-cbr", - "cbin": "chemical/x-cactvs-binary", - "cbor": "application/cbor", - "cbr": "application/x-cbr", - "cbt": "application/x-cbr", - "cbz": "application/vnd.comicbook+zip", - "cc": "text/plain", - "ccad": "application/clariscad", - "ccc": "text/vnd.net2phone.commcenter.command", - "ccmp": "application/ccmp+xml", - "cco": "application/x-cocoa", - "cct": "application/x-director", - "ccxml": "application/ccxml+xml", - "cda": "application/x-cdf", + "c11amc": "application/vnd.cluetrust.cartomobile-config", + "c11amz": "application/vnd.cluetrust.cartomobile-config-pkg", + "csp": "application/vnd.commonspace", "cdbcmsg": "application/vnd.contact.cmsg", - "cdf": "application/x-netcdf", - "cdfx": "application/cdfx+xml", - "cdkey": "application/vnd.mediastation.cdkey", - "cdmia": "application/cdmi-capability", - "cdmic": "application/cdmi-container", - "cdmid": "application/cdmi-domain", - "cdmio": "application/cdmi-object", - "cdmiq": "application/cdmi-queue", - "cdr": "image/x-coreldraw", - "cdt": "image/x-coreldrawtemplate", - "cdx": "chemical/x-cdx", - "cdxml": "application/vnd.chemdraw+xml", - "cdy": "application/vnd.cinderella", - "cea": "application/cea", - "cef": "chemical/x-cxf", - "cellml": "application/cellml+xml", - "cer": "application/pkix-cert", - "cfg": "text/cfg", - "cfs": "application/x-cfs-compressed", - "cgm": "image/cgm", - "cha": "application/x-chat", - "chat": "application/x-chat", - "chm": "application/vnd.ms-htmlhelp", - "chrt": "application/vnd.kde.kchart", - "cif": "chemical/x-cif", - "cii": "application/vnd.anser-web-certificate-issue-initiation", - "cil": "application/vnd.ms-artgalry", - "cl": "application/simple-filter+xml", - "cla": "application/vnd.claymore", - "class": "application/java-vm", + "cmc": "application/vnd.cosmocaller", + "clkx": "application/vnd.crick.clicker", "clkk": "application/vnd.crick.clicker.keyboard", "clkp": "application/vnd.crick.clicker.palette", "clkt": "application/vnd.crick.clicker.template", "clkw": "application/vnd.crick.clicker.wordbank", - "clkx": "application/vnd.crick.clicker", - "clp": "application/x-msclip", - "cls": "text/x-tex", - "clue": "application/clue_info+xml", - "cmake": "text/cmake", - "cmc": "application/vnd.cosmocaller", - "cmdf": "chemical/x-cmdf", - "cml": "chemical/x-cml", - "cmp": "application/vnd.yellowriver-custom-menu", - "cmsc": "application/cms", - "cmx": "image/x-cmx", - "cnd": "text/jcr-cnd", - "cnf": "text/cnf", - "cod": "application/vnd.rim.cod", - "coffee": "application/vnd.coffeescript", - "com": "application/x-msdos-program", - "conf": "text/plain", - "copyright": "text/vnd.debian.copyright", - "cpa": "chemical/x-compass", - "cpio": "application/x-cpio", - "cpkg": "application/vnd.xmpie.cpkg", - "cpl": "application/cpl+xml", - "cpp": "text/x-c++src", - "cpt": "application/mac-compactpro", - "cr2": "image/x-canon-cr2", - "crd": "application/x-mscardfile", - "crl": "application/pkix-crl", - "crt": "application/x-x509-ca-cert", - "crtr": "application/vnd.multiad.creator", - "crw": "image/x-canon-crw", - "crx": "application/x-chrome-extension", - "cryptonote": "application/vnd.rig.cryptonote", - "cs": "text/c#", - "csf": "chemical/x-cache-csf", - "csh": "application/x-csh", - "csl": "application/vnd.citationstyles.style+xml", - "csm": "chemical/x-csml", - "csml": "chemical/x-csml", - "cson": "text/cson", - "csp": "application/vnd.commonspace", - "csrattrs": "application/csrattrs", - "css": "text/css", - "cst": "application/vnd.commonspace", - "csv": "text/csv", - "csvs": "text/csv-schema", - "ctab": "chemical/x-cactvs-binary", - "ctx": "chemical/x-ctx", - "cu": "application/cu-seeme", - "cub": "chemical/x-gaussian-cube", - "cuc": "application/tamp-community-update-confirm", - "curl": "text/vnd.curl", - "cw": "application/prs.cww", - "cww": "application/prs.cww", - "cxf": "chemical/x-cxf", - "cxt": "application/x-director", - "cxx": "text/plain", - "d": "text/x-dsrc", - "dae": "model/vnd.collada+xml", - "daf": "application/vnd.mobius.daf", + "wbs": "application/vnd.criticaltools.wbs+xml", + "pml": "application/vnd.ctc-posml", + "ppd": "application/vnd.cups-ppd", + "car": "application/vnd.curl.car", + "pcurl": "application/vnd.curl.pcurl", "dart": "application/vnd.dart", - "dat": "application/x-ns-proxy-autoconfig", - "dataless": "application/vnd.fdsn.seed", - "davmount": "application/davmount+xml", - "dbk": "application/docbook+xml", - "dcd": "application/dcd", - "dcf": "application/vnd.oma.drm.content", - "dcm": "application/dicom", - "dcr": "application/x-director", - "dcurl": "text/vnd.curl.dcurl", - "dd": "application/vnd.oma.dd+xml", - "dd2": "application/vnd.oma.dd2+xml", - "ddd": "application/vnd.fujixerox.ddd", - "ddf": "application/vnd.syncml.dmddf+xml", - "deb": "application/vnd.debian.binary-package", - "deepv": "application/x-deepv", - "def": "text/plain", - "deploy": "application/octet-stream", - "der": "application/x-x509-ca-cert", - "dfac": "application/vnd.dreamfactory", - "dgc": "application/x-dgc-compressed", - "dib": "image/bmp", - "dic": "text/x-c", - "dif": "video/x-dv", - "diff": "text/x-diff", - "dii": "application/dii", - "dim": "application/vnd.fastcopy-disk-image", - "dir": "application/x-director", - "dis": "application/vnd.mobius.dis", - "disposition-notification": "message/disposition-notification", - "dist": "application/vnd.apple.installer+xml", - "distz": "application/vnd.apple.installer+xml", - "dit": "application/dit", - "djv": "image/vnd.djvu", - "djvu": "image/vnd.djvu", - "dl": "video/dl", - "dll": "application/x-msdos-program", - "dls": "audio/dls", - "dm": "application/vnd.oma.drm.message", - "dmg": "application/x-apple-diskimage", - "dmp": "application/vnd.tcpdump.pcap", - "dms": "text/vnd.dmclientscript", + "rdz": "application/vnd.data-vision.rdz", + "uvf": "application/vnd.dece.data", + "uvvf": "application/vnd.dece.data", + "uvd": "application/vnd.dece.data", + "uvvd": "application/vnd.dece.data", + "uvt": "application/vnd.dece.ttml+xml", + "uvvt": "application/vnd.dece.ttml+xml", + "uvx": "application/vnd.dece.unspecified", + "uvvx": "application/vnd.dece.unspecified", + "uvz": "application/vnd.dece.zip", + "uvvz": "application/vnd.dece.zip", + "fe_launch": "application/vnd.denovo.fcselayout-link", "dna": "application/vnd.dna", - "doc": "application/msword", - "docjson": "application/vnd.document+json", - "docm": "application/vnd.ms-word.document.macroenabled.12", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "dor": "model/vnd.gdl", - "dot": "text/vnd.graphviz", - "dotm": "application/vnd.ms-word.template.macroenabled.12", - "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", - "dp": "application/vnd.osgi.dp", + "mlp": "application/vnd.dolby.mlp", "dpg": "application/vnd.dpgraph", - "dpgraph": "application/vnd.dpgraph", - "dpkg": "application/vnd.xmpie.dpkg", - "dr": "application/vnd.oma.drm.rights+xml", - "dra": "audio/vnd.dra", - "drc": "application/vnd.oma.drm.rights+wbxml", - "drle": "image/dicom-rle", - "drw": "application/drafting", - "dsc": "text/prs.lines.tag", - "dsm": "application/vnd.desmume.movie", - "dssc": "application/dssc+der", - "dtb": "application/x-dtbook+xml", - "dtd": "application/xml-dtd", - "dts": "audio/vnd.dts", - "dtshd": "audio/vnd.dts.hd", - "dump": "application/octet-stream", - "dv": "video/x-dv", - "dvb": "video/vnd.dvb.file", - "dvc": "application/dvcs", - "dvi": "application/x-dvi", - "dwf": "model/vnd.dwf", - "dwg": "image/vnd.dwg", - "dx": "chemical/x-jcamp-dx", - "dxf": "image/vnd.dxf", - "dxp": "application/vnd.spotfire.dxp", - "dxr": "application/x-director", - "dzr": "application/vnd.dzr", - "ear": "binary/zip", - "ecelp4800": "audio/vnd.nuera.ecelp4800", - "ecelp7470": "audio/vnd.nuera.ecelp7470", - "ecelp9600": "audio/vnd.nuera.ecelp9600", - "ecig": "application/vnd.evolv.ecig.settings", - "ecigprofile": "application/vnd.evolv.ecig.profile", - "ecigtheme": "application/vnd.evolv.ecig.theme", - "ecma": "application/ecmascript", - "edm": "application/vnd.novadigm.edm", - "edx": "application/vnd.novadigm.edx", - "efi": "application/efi", - "efif": "application/vnd.picsel", - "ei6": "application/vnd.pg.osasli", - "ejs": "text/ejs", - "el": "text/plain", - "elc": "application/x-bytecode.elisp", - "emb": "chemical/x-embl-dl-nucleotide", - "embl": "chemical/x-embl-dl-nucleotide", - "emf": "image/emf", - "eml": "message/rfc822", - "emm": "application/vnd.ibm.electronic-media", - "emma": "application/emma+xml", - "emotionml": "application/emotionml+xml", - "emz": "application/x-msmetafile", - "ent": "text/xml-external-parsed-entity", - "entity": "application/vnd.nervana", - "env": "application/x-envoy", - "enw": "audio/evrcnw", - "eol": "audio/vnd.digital-winds", - "eot": "application/vnd.ms-fontobject", - "ep": "application/vnd.bluetooth.ep.oob", - "eps": "application/postscript", - "eps2": "application/postscript", - "eps3": "application/postscript", - "epsf": "application/postscript", - "epsi": "application/postscript", - "epub": "application/epub+zip", - "erb": "text/erb", - "erf": "image/x-epson-erf", - "es": "application/ecmascript", - "es3": "application/vnd.eszigno3+xml", - "esa": "application/vnd.osgi.subsystem", - "escn": "text/godot", + "dfac": "application/vnd.dreamfactory", + "kpxx": "application/vnd.ds-keypoint", + "ait": "application/vnd.dvb.ait", + "svc": "application/vnd.dvb.service", + "geo": "application/vnd.dynageo", + "mag": "application/vnd.ecowin.chart", + "nml": "application/vnd.enliven", "esf": "application/vnd.epson.esf", - "espass": "application/vnd.espass-espass+zip", + "msf": "application/vnd.epson.msf", + "qam": "application/vnd.epson.quickanime", + "slt": "application/vnd.epson.salt", + "ssf": "application/vnd.epson.ssf", + "es3": "application/vnd.eszigno3+xml", "et3": "application/vnd.eszigno3+xml", - "etx": "text/x-setext", - "eva": "application/x-eva", - "evb": "audio/evrcb", - "evc": "audio/evrc", - "evw": "audio/evrcwb", - "evy": "application/x-envoy", - "exe": "application/x-msdos-program", - "exi": "application/exi", - "exr": "image/aces", - "ext": "application/vnd.novadigm.ext", - "eyaml": "text/yaml", - "ez": "application/andrew-inset", "ez2": "application/vnd.ezpix-album", "ez3": "application/vnd.ezpix-package", - "f": "text/x-fortran", - "f4v": "video/x-f4v", - "f77": "text/x-fortran", - "f90": "text/plain", - "fb": "application/x-maker", - "fbdoc": "application/x-maker", - "fbs": "image/vnd.fastbidsheet", - "fbx": "model/filmbox", - "fcdt": "application/vnd.adobe.formscentral.fcdt", - "fch": "chemical/x-gaussian-checkpoint", - "fchk": "chemical/x-gaussian-checkpoint", - "fcs": "application/vnd.isac.fcs", "fdf": "application/vnd.fdf", - "fdt": "application/fdt+xml", - "fe_launch": "application/vnd.denovo.fcselayout-link", - "feature": "text/gherkin", - "fg5": "application/vnd.fujitsu.oasysgp", - "fgd": "application/x-director", - "fh": "image/x-freehand", - "fh4": "image/x-freehand", - "fh5": "image/x-freehand", - "fh7": "image/x-freehand", - "fhc": "image/x-freehand", - "fif": "image/fif", - "fig": "application/x-xfig", - "finf": "application/fastinfoset", - "fish": "text/fish", - "fit": "image/fits", - "fits": "image/fits", - "fla": "application/vnd.dtg.local.flash", - "flac": "audio/x-flac", - "fli": "video/x-fli", - "flo": "application/vnd.micrografx.flo", - "flr": "x-world/x-vrml", - "flv": "video/x-flv", - "flw": "application/vnd.kde.kivio", - "flx": "text/vnd.fmi.flexstor", - "fly": "text/vnd.fly", + "mseed": "application/vnd.fdsn.mseed", + "seed": "application/vnd.fdsn.seed", + "dataless": "application/vnd.fdsn.seed", + "gph": "application/vnd.flographit", + "ftc": "application/vnd.fluxtime.clip", "fm": "application/vnd.framemaker", - "fmf": "video/x-atomic3d-feature", - "fnc": "application/vnd.frogans.fnc", - "fo": "application/vnd.software602.filler.form+xml", - "for": "text/x-fortran", - "fpx": "image/vnd.fpx", "frame": "application/vnd.framemaker", - "frl": "application/freeloader", - "frm": "application/vnd.ufdl", + "maker": "application/vnd.framemaker", + "book": "application/vnd.framemaker", + "fnc": "application/vnd.frogans.fnc", + "ltf": "application/vnd.frogans.ltf", "fsc": "application/vnd.fsc.weblaunch", - "fst": "image/vnd.fst", - "ftc": "application/vnd.fluxtime.clip", - "fti": "application/vnd.anser-web-funds-transfer-initiation", - "fts": "image/fits", - "funk": "audio/make", - "fvt": "video/vnd.fvt", - "fxm": "video/x-javafx", - "fxp": "application/vnd.adobe.fxp", - "fxpl": "application/vnd.adobe.fxp", + "oas": "application/vnd.fujitsu.oasys", + "oa2": "application/vnd.fujitsu.oasys2", + "oa3": "application/vnd.fujitsu.oasys3", + "fg5": "application/vnd.fujitsu.oasysgp", + "bh2": "application/vnd.fujitsu.oasysprs", + "ddd": "application/vnd.fujixerox.ddd", + "xdw": "application/vnd.fujixerox.docuworks", + "xbd": "application/vnd.fujixerox.docuworks.binder", "fzs": "application/vnd.fuzzysheet", - "g": "text/plain", - "g2w": "application/vnd.geoplan", - "g3": "image/g3fax", - "g3w": "application/vnd.geospace", - "gac": "application/vnd.groove-account", - "gal": "chemical/x-gaussian-log", - "gam": "application/x-tads", - "gamin": "chemical/x-gamess-input", - "gau": "chemical/x-gaussian-input", - "gbr": "application/rpki-ghostbusters", - "gca": "application/x-gca-compressed", - "gcd": "text/x-pcs-gcd", - "gcf": "application/x-graphing-calculator", - "gcg": "chemical/x-gcg8-sequence", - "gdl": "model/vnd.gdl", - "gdoc": "application/vnd.google-apps.document", - "gemspec": "text/ruby", - "gen": "chemical/x-genbank", - "geo": "application/vnd.dynageo", - "geojson": "application/geo+json", - "gex": "application/vnd.geometry-explorer", - "gf": "application/x-tex-gf", + "txd": "application/vnd.genomatix.tuxedo", "ggb": "application/vnd.geogebra.file", + "ggs": "application/vnd.geogebra.slides", "ggt": "application/vnd.geogebra.tool", - "ghf": "application/vnd.groove-help", - "gif": "image/gif", - "gim": "application/vnd.groove-identity-message", - "gjc": "chemical/x-gaussian-input", - "gjf": "chemical/x-gaussian-input", - "gl": "video/gl", - "glb": "model/gltf-binary", - "gltf": "model/gltf+json", - "gml": "application/gml+xml", + "gex": "application/vnd.geometry-explorer", + "gre": "application/vnd.geometry-explorer", + "gxt": "application/vnd.geonext", + "g2w": "application/vnd.geoplan", + "g3w": "application/vnd.geospace", "gmx": "application/vnd.gmx", - "gnumeric": "application/x-gnumeric", - "go": "text/go", - "gotmpl": "text/gotmpl", - "gph": "application/vnd.flographit", - "gpt": "chemical/x-mopac-graph", - "gpx": "application/gpx+xml", + "kml": "application/vnd.google-earth.kml+xml", + "kmz": "application/vnd.google-earth.kmz", "gqf": "application/vnd.grafeq", "gqs": "application/vnd.grafeq", - "gradle": "text/groovy", - "gram": "application/srgs", - "gramps": "application/x-gramps-xml", - "gre": "application/vnd.geometry-explorer", - "groovy": "text/groovy", + "gac": "application/vnd.groove-account", + "ghf": "application/vnd.groove-help", + "gim": "application/vnd.groove-identity-message", "grv": "application/vnd.groove-injector", - "grxml": "application/srgs+xml", - "gsd": "audio/x-gsm", - "gsf": "application/x-font-ghostscript", - "gsheet": "application/vnd.google-apps.spreadsheet", - "gslides": "application/vnd.google-apps.presentation", - "gsm": "model/vnd.gdl", - "gsp": "application/x-gsp", - "gss": "application/x-gss", - "gtar": "application/x-gtar", "gtm": "application/vnd.groove-tool-message", - "gtw": "model/vnd.gtw", - "gv": "text/vnd.graphviz", - "gxf": "application/gxf", - "gxt": "application/vnd.geonext", - "gyb": "text/gyb", - "gyp": "text/gyp", - "gypi": "text/gyp", - "gz": "application/gzip", - "h": "text/x-chdr", - "h++": "text/x-c++hdr", - "h261": "video/h261", - "h263": "video/h263", - "h264": "video/h264", + "tpl": "application/vnd.groove-tool-template", + "vcg": "application/vnd.groove-vcard", "hal": "application/vnd.hal+xml", - "hbc": "application/vnd.hbci", + "zmm": "application/vnd.handheld-entertainment+xml", "hbci": "application/vnd.hbci", - "hbs": "text/x-handlebars-template", - "hdd": "application/x-virtualbox-hdd", - "hdf": "application/x-hdf", - "hdr": "image/vnd.radiance", - "hdt": "application/vnd.hdt", - "heic": "image/heic", - "heics": "image/heic-sequence", - "heif": "image/heif", - "heifs": "image/heif-sequence", - "help": "application/x-helpfile", - "hgl": "application/vnd.hp-hpgl", - "hh": "text/plain", - "hin": "chemical/x-hin", - "hjson": "application/hjson", - "hlb": "text/x-script", - "hlp": "application/winhlp", - "hpg": "application/vnd.hp-hpgl", + "les": "application/vnd.hhe.lesson-player", "hpgl": "application/vnd.hp-hpgl", - "hpi": "application/vnd.hp-hpid", "hpid": "application/vnd.hp-hpid", - "hpp": "text/x-c++hdr", "hps": "application/vnd.hp-hps", - "hpub": "application/prs.hpub+zip", - "hqx": "application/mac-binhex40", - "hs": "text/x-haskell", - "hta": "application/hta", - "htc": "text/x-component", - "htke": "application/vnd.kenameaapp", - "html": "text/html", - "htt": "text/webviewhtml", - "hvd": "application/vnd.yamaha.hv-dic", - "hvp": "application/vnd.yamaha.hv-voice", - "hvs": "application/vnd.yamaha.hv-script", - "hx": "text/haxe", - "hxml": "text/haxe", - "hxx": "text/plain", - "i2g": "application/vnd.intergeo", - "ic0": "application/vnd.commerce-battelle", - "ic1": "application/vnd.commerce-battelle", - "ic2": "application/vnd.commerce-battelle", - "ic3": "application/vnd.commerce-battelle", - "ic4": "application/vnd.commerce-battelle", - "ic5": "application/vnd.commerce-battelle", - "ic6": "application/vnd.commerce-battelle", - "ic7": "application/vnd.commerce-battelle", - "ic8": "application/vnd.commerce-battelle", - "ica": "application/vnd.commerce-battelle", + "jlt": "application/vnd.hp-jlyt", + "pcl": "application/vnd.hp-pcl", + "pclxl": "application/vnd.hp-pclxl", + "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "mpy": "application/vnd.ibm.minipay", + "afp": "application/vnd.ibm.modcap", + "listafp": "application/vnd.ibm.modcap", + "list3820": "application/vnd.ibm.modcap", + "irm": "application/vnd.ibm.rights-management", + "sc": "application/vnd.ibm.secure-container", "icc": "application/vnd.iccprofile", - "icd": "application/vnd.commerce-battelle", - "ice": "x-conference/x-cooltalk", - "icf": "application/vnd.commerce-battelle", "icm": "application/vnd.iccprofile", - "icns": "binary/icns", - "ico": "image/x-icon", - "ics": "text/calendar", - "icz": "text/calendar", - "idc": "text/plain", - "idl": "text/idl", - "ief": "image/ief", - "iefs": "image/ief", - "ifb": "text/calendar", - "ifm": "application/vnd.shana.informed.formdata", - "iges": "model/iges", "igl": "application/vnd.igloader", - "igm": "application/vnd.insors.igm", - "ign": "application/vnd.coreos.ignition+json", - "ignition": "application/vnd.coreos.ignition+json", - "igs": "model/iges", - "igx": "application/vnd.micrografx.igx", - "iif": "application/vnd.shana.informed.interchange", - "iii": "application/x-iphone", - "ima": "application/x-ima", - "imap": "application/x-httpd-imap", - "imf": "application/vnd.imagemeter.folder+zip", - "img": "application/octet-stream", - "imgcal": "application/vnd.3lightssoftware.imagescal", - "imi": "application/vnd.imagemeter.image+zip", - "imp": "application/vnd.accpac.simply.imp", - "ims": "application/vnd.ms-ims", - "imscc": "application/vnd.ims.imsccv1p1", - "in": "text/plain", - "inc": "text/inc", - "inf": "application/inf", - "info": "application/x-info", - "ini": "text/ini", - "ink": "application/inkml+xml", - "inkml": "application/inkml+xml", - "inp": "chemical/x-gamess-input", - "ins": "application/x-internet-signup", - "install": "application/x-install-instructions", - "iota": "application/vnd.astraea-software.iota", - "ip": "application/x-ip2", - "ipfix": "application/ipfix", - "ipk": "application/vnd.shana.informed.package", - "irm": "application/vnd.ibm.rights-management", - "irp": "application/vnd.irepository.package+xml", - "ism": "model/vnd.gdl", - "iso": "application/x-iso9660-image", - "isp": "application/x-internet-signup", - "ist": "chemical/x-isostar", - "istr": "chemical/x-isostar", - "isu": "video/x-isvideo", - "it": "audio/it", - "itp": "application/vnd.shana.informed.formtemplate", - "its": "application/its+xml", - "iv": "application/x-inventor", "ivp": "application/vnd.immervision-ivp", - "ivr": "i-world/i-vrml", "ivu": "application/vnd.immervision-ivu", - "ivy": "application/x-livescreen", - "j2": "text/jinja", - "jad": "text/vnd.sun.j2me.app-descriptor", - "jade": "text/jade", + "igm": "application/vnd.insors.igm", + "xpw": "application/vnd.intercon.formnet", + "xpx": "application/vnd.intercon.formnet", + "i2g": "application/vnd.intergeo", + "qbo": "application/vnd.intu.qbo", + "qfx": "application/vnd.intu.qfx", + "rcprofile": "application/vnd.ipunplugged.rcprofile", + "irp": "application/vnd.irepository.package+xml", + "xpr": "application/vnd.is-xpr", + "fcs": "application/vnd.isac.fcs", "jam": "application/vnd.jam", - "jar": "application/x-java-archive", - "jardiff": "application/x-java-archive-diff", - "java": "text/x-java-source", - "jcm": "application/x-java-commerce", - "jdx": "chemical/x-jcamp-dx", - "jenkinsfile": "text/groovy", - "jfif": "image/jpeg", - "jinja": "text/jinja", - "jinja2": "text/jinja", + "rms": "application/vnd.jcp.javame.midlet-rms", "jisp": "application/vnd.jisp", - "jls": "image/jls", - "jlt": "application/vnd.hp-jlyt", - "jl": "text/julia", - "jmz": "application/x-jmol", - "jng": "image/x-jng", - "jnlp": "application/x-java-jnlp-file", "joda": "application/vnd.joost.joda-archive", - "jp2": "image/jp2", - "jpe": "image/jpeg", - "jpeg": "image/jpeg", - "jpf": "image/jpx", - "jpg": "image/jpeg", - "jpg2": "image/jp2", - "jpgm": "image/jpm", - "jpgv": "video/jpeg", - "jpm": "image/jpm", - "jps": "image/x-jps", - "jpx": "image/jpx", - "jrd": "application/jrd+json", - "js": "application/javascript", - "json": "application/json", - "json-patch": "application/json-patch+json", - "json5": "application/json5", - "jsonld": "application/ld+json", - "jsonml": "application/jsonml+json", - "jsx": "text/jsx", - "jtd": "text/vnd.esmertec.theme-descriptor", - "jut": "image/jutvision", - "kar": "audio/midi", + "ktz": "application/vnd.kahootz", + "ktr": "application/vnd.kahootz", "karbon": "application/vnd.kde.karbon", - "kcm": "application/vnd.nervana", - "key": "application/pgp-keys", - "keynote": "application/vnd.apple.keynote", + "chrt": "application/vnd.kde.kchart", "kfo": "application/vnd.kde.kformula", - "kia": "application/vnd.kidspiration", - "kil": "application/x-killustrator", - "kin": "chemical/x-kinemage", - "kml": "application/vnd.google-earth.kml+xml", - "kmz": "application/vnd.google-earth.kmz", - "kne": "application/vnd.kinar", - "knp": "application/vnd.kinar", - "kom": "application/vnd.hbci", + "flw": "application/vnd.kde.kivio", "kon": "application/vnd.kde.kontour", - "koz": "audio/vnd.audikoz", "kpr": "application/vnd.kde.kpresenter", "kpt": "application/vnd.kde.kpresenter", - "kpxx": "application/vnd.ds-keypoint", - "ksh": "application/x-ksh", "ksp": "application/vnd.kde.kspread", - "kt": "text/kotlin", - "ktr": "application/vnd.kahootz", - "ktx": "image/ktx", - "ktz": "application/vnd.kahootz", "kwd": "application/vnd.kde.kword", "kwt": "application/vnd.kde.kword", - "l16": "audio/l16", - "la": "audio/nspaudio", - "lam": "audio/x-liveaudio", - "lasjson": "application/vnd.las.las+json", + "htke": "application/vnd.kenameaapp", + "kia": "application/vnd.kidspiration", + "kne": "application/vnd.kinar", + "knp": "application/vnd.kinar", + "skp": "application/vnd.koan", + "skd": "application/vnd.koan", + "skt": "application/vnd.koan", + "skm": "application/vnd.koan", + "sse": "application/vnd.kodak-descriptor", "lasxml": "application/vnd.las.las+xml", - "latex": "application/x-latex", - "lbc": "audio/ilbc", "lbd": "application/vnd.llamagraphics.life-balance.desktop", "lbe": "application/vnd.llamagraphics.life-balance.exchange+xml", - "le": "application/vnd.bluetooth.le.oob", - "les": "application/vnd.hhe.lesson-player", - "less": "text/less", - "lgr": "application/lgr+xml", - "lha": "application/octet-stream", - "lhs": "text/x-literate-haskell", - "lhx": "application/octet-stream", - "lin": "application/bbolin", - "link66": "application/vnd.route66.link66+xml", - "list": "text/plain", - "list3820": "application/vnd.ibm.modcap", - "listafp": "application/vnd.ibm.modcap", - "lmp": "model/vnd.gdl", - "lnk": "application/x-ms-shortcut", - "log": "text/plain", - "lostsyncxml": "application/lostsync+xml", - "lostxml": "application/lost+xml", - "lrf": "application/octet-stream", - "lrm": "application/vnd.ms-lrm", - "lsf": "video/x-la-asf", - "lsp": "text/x-script.lisp", - "lst": "text/plain", - "lsx": "video/x-la-asf", - "ltf": "application/vnd.frogans.ltf", - "ltx": "application/x-latex", - "lua": "text/x-lua", - "luac": "application/x-lua-bytecode", - "lvp": "audio/vnd.lucent.voice", + "123": "application/vnd.lotus-1-2-3", + "apr": "application/vnd.lotus-approach", + "pre": "application/vnd.lotus-freelance", + "nsf": "application/vnd.lotus-notes", + "org": "application/vnd.lotus-organizer", + "scm": "application/vnd.lotus-screencam", "lwp": "application/vnd.lotus-wordpro", - "lxf": "application/lxf", - "lyx": "application/x-lyx", - "lzh": "application/octet-stream", - "lzx": "application/x-lzx", - "m": "application/vnd.wolfram.mathematica.package", - "m13": "application/x-msmediaview", - "m14": "application/x-msmediaview", - "m15": "audio/x-mod", - "m1v": "video/mpeg", - "m21": "application/mp21", - "m2a": "audio/mpeg", - "m2v": "video/mpeg", - "m3a": "audio/mpeg", - "m3g": "application/m3g", - "m3u": "audio/x-mpegurl", - "m3u8": "application/vnd.apple.mpegurl", - "m4a": "audio/x-m4a", - "m4s": "video/iso.segment", - "m4u": "video/vnd.mpegurl", - "m4v": "video/x-m4v", - "ma": "application/mathematica", - "mads": "application/mads+xml", - "mag": "application/vnd.ecowin.chart", - "mail": "message/rfc822", - "maker": "application/vnd.framemaker", - "man": "application/x-troff-man", - "manifest": "text/cache-manifest", - "map": "application/x-navimap", - "mar": "text/plain", - "markdown": "text/markdown", - "mathml": "application/mathml+xml", - "mb": "application/mathematica", - "mbd": "application/mbedlet", - "mbk": "application/vnd.mobius.mbk", - "mbox": "application/mbox", - "mc$": "application/x-magic-cap-package-1.0", - "mc1": "application/vnd.medcalcdata", + "portpkg": "application/vnd.macports.portpkg", "mcd": "application/vnd.mcd", - "mcf": "image/vasa", - "mcif": "chemical/x-mmcif", - "mcm": "chemical/x-macmolecule", - "mcp": "application/netmc", - "mcurl": "text/vnd.curl.mcurl", - "md": "text/markdown", - "mdb": "application/x-msaccess", - "mdc": "application/vnd.marlin.drm.mdcf", - "mdi": "image/vnd.ms-modi", - "me": "application/x-troff-me", - "med": "audio/x-mod", - "mesh": "model/mesh", - "meta4": "application/metalink4+xml", - "metalink": "application/metalink+xml", - "mets": "application/mets+xml", - "mf4": "application/mf4", + "mc1": "application/vnd.medcalcdata", + "cdkey": "application/vnd.mediastation.cdkey", + "mwf": "application/vnd.mfer", "mfm": "application/vnd.mfmp", - "mft": "application/rpki-manifest", - "mgp": "application/vnd.osgeo.mapguide.package", - "mgz": "application/vnd.proteus.magazine", - "mht": "message/rfc822", - "mhtml": "message/rfc822", - "mib": "text/mib", - "mid": "audio/midi", - "midi": "audio/midi", - "mie": "application/x-mie", - "mif": "application/x-mif", - "mime": "message/rfc822", - "miz": "text/mizar", - "mj2": "video/mj2", - "mjf": "audio/x-vnd.audioexplosion.mjuicemediafile", - "mjp2": "video/mj2", - "mjpg": "video/x-motion-jpeg", - "mjs": "application/javascript", - "mk": "text/makefile", - "mk3d": "video/x-matroska-3d", - "mka": "audio/x-matroska", - "mkd": "text/x-markdown", - "mks": "video/x-matroska", - "mkv": "video/x-matroska", - "mlp": "application/vnd.dolby.mlp", - "mm": "application/x-freemind", - "mmd": "application/vnd.chipnuts.karaoke-mmd", - "mmdb": "application/vnd.maxmind.maxmind-db", - "mme": "application/base64", - "mmf": "application/vnd.smaf", - "mml": "text/mathml", - "mmod": "chemical/x-macromodel-input", - "mmr": "image/vnd.fujixerox.edmics-mmr", - "mms": "application/vnd.wap.mms-message", - "mng": "video/x-mng", - "mny": "application/x-msmoney", - "mobi": "application/x-mobipocket-ebook", - "moc": "text/x-moc", - "mod": "audio/x-mod", - "model-inter": "application/vnd.vd-study", - "mods": "application/mods+xml", - "modulemap": "text/modulemap", - "mol": "chemical/x-mdl-molfile", - "mol2": "chemical/x-mol2", - "moml": "model/vnd.moml+xml", - "moo": "chemical/x-mopac-out", - "moov": "video/quicktime", - "mop": "chemical/x-mopac-input", - "mopcrt": "chemical/x-mopac-input", - "mov": "video/quicktime", - "movie": "video/x-sgi-movie", - "mp1": "audio/mpeg", - "mp2": "audio/mpeg", - "mp21": "application/mp21", - "mp2a": "audio/mpeg", - "mp3": "audio/mp3", - "mp4": "video/mp4", - "mp4a": "audio/mp4", - "mp4s": "application/mp4", - "mp4v": "video/mp4", - "mpa": "video/mpeg", - "mpc": "application/vnd.mophun.certificate", - "mpd": "application/dash+xml", - "mpdd": "application/dashdelta", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpega": "audio/mpeg", - "mpf": "text/vnd.ms-mediapackage", - "mpg": "video/mpeg", - "mpg4": "video/mp4", - "mpga": "audio/mpeg", - "mpkg": "application/vnd.apple.installer+xml", - "mpm": "application/vnd.blueice.multipass", + "flo": "application/vnd.micrografx.flo", + "igx": "application/vnd.micrografx.igx", + "mif": "application/vnd.mif", + "daf": "application/vnd.mobius.daf", + "dis": "application/vnd.mobius.dis", + "mbk": "application/vnd.mobius.mbk", + "mqy": "application/vnd.mobius.mqy", + "msl": "application/vnd.mobius.msl", + "plc": "application/vnd.mobius.plc", + "txf": "application/vnd.mobius.txf", "mpn": "application/vnd.mophun.application", + "mpc": "application/vnd.mophun.certificate", + "xul": "application/vnd.mozilla.xul+xml", + "cil": "application/vnd.ms-artgalry", + "cab": "application/vnd.ms-cab-compressed", + "xls": "application/vnd.ms-excel", + "xlm": "application/vnd.ms-excel", + "xla": "application/vnd.ms-excel", + "xlc": "application/vnd.ms-excel", + "xlt": "application/vnd.ms-excel", + "xlw": "application/vnd.ms-excel", + "xlam": "application/vnd.ms-excel.addin.macroenabled.12", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", + "xltm": "application/vnd.ms-excel.template.macroenabled.12", + "eot": "application/vnd.ms-fontobject", + "chm": "application/vnd.ms-htmlhelp", + "ims": "application/vnd.ms-ims", + "lrm": "application/vnd.ms-lrm", + "thmx": "application/vnd.ms-officetheme", + "cat": "application/vnd.ms-pki.seccat", + "stl": "application/vnd.ms-pki.stl", + "ppt": "application/vnd.ms-powerpoint", + "pps": "application/vnd.ms-powerpoint", + "pot": "application/vnd.ms-powerpoint", + "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", + "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", + "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", + "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", "mpp": "application/vnd.ms-project", "mpt": "application/vnd.ms-project", - "mpv": "application/x-project", - "mpv2": "video/mpeg", - "mpx": "application/x-project", - "mpy": "application/vnd.ibm.minipay", - "mqy": "application/vnd.mobius.mqy", - "mrc": "application/marc", - "mrcx": "application/marcxml+xml", - "ms": "application/x-troff-ms", - "msa": "application/vnd.msa-disk-image", - "mscml": "application/mediaservercontrol+xml", - "msd": "application/vnd.fdsn.mseed", - "mseed": "application/vnd.fdsn.mseed", + "docm": "application/vnd.ms-word.document.macroenabled.12", + "dotm": "application/vnd.ms-word.template.macroenabled.12", + "wps": "application/vnd.ms-works", + "wks": "application/vnd.ms-works", + "wcm": "application/vnd.ms-works", + "wdb": "application/vnd.ms-works", + "wpl": "application/vnd.ms-wpl", + "xps": "application/vnd.ms-xpsdocument", "mseq": "application/vnd.mseq", - "msf": "application/vnd.epson.msf", - "msg": "application/vnd.ms-outlook", - "msh": "model/mesh", - "msi": "application/x-msi", - "msl": "application/vnd.mobius.msl", - "msm": "model/vnd.gdl", - "msty": "application/vnd.muvee.style", - "mtm": "audio/x-mod", - "mts": "model/vnd.mts", - "multitrack": "audio/vnd.presonus.multitrack", "mus": "application/vnd.musician", - "musd": "application/mmt-usd+xml", - "musicxml": "application/vnd.recordare.musicxml+xml", - "mv": "video/x-sgi-movie", - "mvb": "application/x-msmediaview", - "mvt": "application/vnd.mapbox-vector-tile", - "mwc": "application/vnd.dpgraph", - "mwf": "application/vnd.mfer", - "mxf": "application/mxf", - "mxi": "application/vnd.vd-study", - "mxl": "application/vnd.recordare.musicxml", - "mxmf": "audio/mobile-xmf", - "mxml": "application/xv+xml", - "mxs": "application/vnd.triscape.mxs", - "mxu": "video/vnd.mpegurl", - "my": "audio/make", - "mzz": "application/x-vnd.audioexplosion.mzz", - "n-gage": "application/vnd.nokia.n-gage.symbian.install", - "n3": "text/n3", - "nap": "image/naplps", - "naplps": "image/naplps", - "nb": "application/mathematica", - "nbp": "application/vnd.wolfram.player", - "nc": "application/x-netcdf", - "ncm": "application/vnd.nokia.configuration-message", - "ncx": "application/x-dtbncx+xml", - "ndc": "application/vnd.osa.netdeploy", - "ndjson": "application/json", - "ndl": "application/vnd.lotus-notes", - "nds": "application/vnd.nintendo.nitro.rom", - "nef": "image/x-nikon-nef", - "nfo": "text/x-nfo", - "ngdat": "application/vnd.nokia.n-gage.data", - "ngdoc": "text/ngdoc", - "nif": "image/x-niff", - "niff": "image/x-niff", + "msty": "application/vnd.muvee.style", + "taglet": "application/vnd.mynfc", + "nlu": "application/vnd.neurolanguage.nlu", "nim": "text/nim", "nimble": "text/nimble", "nimf": "text/nim", "nims": "text/nim", + "ntf": "application/vnd.nitf", "nitf": "application/vnd.nitf", - "nix": "application/x-mix-transfer", - "nlu": "application/vnd.neurolanguage.nlu", - "nml": "application/vnd.enliven", "nnd": "application/vnd.noblenet-directory", "nns": "application/vnd.noblenet-sealer", "nnw": "application/vnd.noblenet-web", - "notebook": "application/vnd.smart.notebook", - "npx": "image/vnd.net-fpx", - "nq": "application/n-quads", - "ns2": "application/vnd.lotus-notes", - "ns3": "application/vnd.lotus-notes", - "ns4": "application/vnd.lotus-notes", - "nsc": "application/x-conference", - "nsf": "application/vnd.lotus-notes", - "nsg": "application/vnd.lotus-notes", - "nsh": "application/vnd.lotus-notes", - "nt": "application/n-triples", - "ntf": "application/vnd.lotus-notes", - "numbers": "application/vnd.apple.numbers", - "nvd": "application/x-navidoc", - "nwc": "application/x-nwc", - "nws": "message/rfc822", - "nzb": "application/x-nzb", - "o": "application/x-object", - "o4a": "application/vnd.oma.drm.dcf", - "o4v": "application/vnd.oma.drm.dcf", - "oa2": "application/vnd.fujitsu.oasys2", - "oa3": "application/vnd.fujitsu.oasys3", - "oas": "application/vnd.fujitsu.oasys", - "obd": "application/x-msbinder", - "obg": "application/vnd.openblox.game-binary", - "obgx": "application/vnd.openblox.game+xml", - "obj": "application/x-tgif", - "oda": "application/oda", - "odb": "application/vnd.oasis.opendocument.database", + "ngdat": "application/vnd.nokia.n-gage.data", + "n-gage": "application/vnd.nokia.n-gage.symbian.install", + "rpst": "application/vnd.nokia.radio-preset", + "rpss": "application/vnd.nokia.radio-presets", + "edm": "application/vnd.novadigm.edm", + "edx": "application/vnd.novadigm.edx", + "ext": "application/vnd.novadigm.ext", "odc": "application/vnd.oasis.opendocument.chart", - "odd": "application/tei+xml", + "otc": "application/vnd.oasis.opendocument.chart-template", + "odb": "application/vnd.oasis.opendocument.database", "odf": "application/vnd.oasis.opendocument.formula", "odft": "application/vnd.oasis.opendocument.formula-template", "odg": "application/vnd.oasis.opendocument.graphics", - "odi": "application/vnd.oasis.opendocument.image", - "odm": "application/vnd.oasis.opendocument.text-master", - "odp": "application/vnd.oasis.opendocument.presentation", - "ods": "application/vnd.oasis.opendocument.spreadsheet", - "odt": "application/vnd.oasis.opendocument.text", - "odx": "application/odx", - "oeb": "application/vnd.openeye.oeb", - "oga": "audio/ogg", - "ogex": "model/vnd.opengex", - "ogg": "audio/ogg", - "ogv": "video/ogg", - "ogx": "application/ogg", - "old": "application/x-trash", - "omc": "application/x-omc", - "omcd": "application/x-omcdatamaker", - "omcr": "application/x-omcregerator", - "omdoc": "application/omdoc+xml", - "omg": "audio/atrac3", - "onepkg": "application/onenote", - "onetmp": "application/onenote", - "onetoc": "application/onenote", - "onetoc2": "application/onenote", - "opf": "application/oebps-package+xml", - "opml": "text/x-opml", - "oprc": "application/vnd.palm", - "opus": "audio/ogg", - "or2": "application/vnd.lotus-organizer", - "or3": "application/vnd.lotus-organizer", - "orf": "image/x-olympus-orf", - "org": "text/x-org", - "orq": "application/ocsp-request", - "ors": "application/ocsp-response", - "osf": "application/vnd.yamaha.openscoreformat", - "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", - "osm": "application/vnd.openstreetmap.data+xml", - "otc": "application/vnd.oasis.opendocument.chart-template", - "otf": "font/otf", "otg": "application/vnd.oasis.opendocument.graphics-template", - "oth": "application/vnd.oasis.opendocument.text-web", + "odi": "application/vnd.oasis.opendocument.image", "oti": "application/vnd.oasis.opendocument.image-template", + "odp": "application/vnd.oasis.opendocument.presentation", "otp": "application/vnd.oasis.opendocument.presentation-template", + "ods": "application/vnd.oasis.opendocument.spreadsheet", "ots": "application/vnd.oasis.opendocument.spreadsheet-template", + "odt": "application/vnd.oasis.opendocument.text", + "odm": "application/vnd.oasis.opendocument.text-master", "ott": "application/vnd.oasis.opendocument.text-template", - "ova": "application/x-virtualbox-ova", - "ovf": "application/x-virtualbox-ovf", - "owx": "application/owl+xml", - "oxlicg": "application/vnd.oxli.countgraph", - "oxps": "application/oxps", + "oth": "application/vnd.oasis.opendocument.text-web", + "xo": "application/vnd.olpc-sugar", + "dd2": "application/vnd.oma.dd2+xml", "oxt": "application/vnd.openofficeorg.extension", - "oza": "application/x-oz-application", - "p": "text/x-pascal", - "p10": "application/pkcs10", - "p12": "application/pkcs12", - "p2p": "application/vnd.wfa.p2p", - "p7a": "application/x-pkcs7-signature", - "p7b": "application/x-pkcs7-certificates", - "p7c": "application/pkcs7-mime", - "p7m": "application/pkcs7-mime", - "p7r": "application/x-pkcs7-certreqresp", - "p7s": "application/pkcs7-signature", - "p8": "application/pkcs8", - "pac": "application/x-ns-proxy-autoconfig", - "pack": "application/x-java-pack200", - "package": "application/vnd.autopackage", - "pages": "application/vnd.apple.pages", - "par": "text/plain-bas", - "part": "application/pro_eng", - "pas": "text/pascal", - "pat": "image/x-coreldrawpattern", - "patch": "text/x-diff", - "paw": "application/vnd.pawaafile", - "pbd": "application/vnd.powerbuilder6", - "pbm": "image/x-portable-bitmap", - "pcap": "application/vnd.tcpdump.pcap", - "pcf": "application/x-font-pcf", - "pcl": "application/vnd.hp-pcl", - "pclxl": "application/vnd.hp-pclxl", - "pct": "image/x-pict", - "pcurl": "application/vnd.curl.pcurl", - "pcx": "image/x-pcx", - "pdb": "application/vnd.palm", - "pde": "text/x-processing", - "pdf": "application/pdf", - "pdx": "application/pdx", - "pem": "text/pem", - "pfa": "application/x-font-type1", - "pfb": "application/x-font-type1", - "pfm": "application/x-font-type1", - "pfr": "application/font-tdpfr", - "pfunk": "audio/make", - "pfx": "application/pkcs12", - "pgb": "image/vnd.globalgraphics.pgb", - "pgm": "image/x-portable-graymap", - "pgn": "application/x-chess-pgn", - "pgp": "application/pgp-encrypted", - "php": "application/x-httpd-php", - "php3": "application/x-httpd-php3", - "php3p": "application/x-httpd-php3-preprocessed", - "php4": "application/x-httpd-php4", - "php5": "application/x-httpd-php5", - "phps": "application/x-httpd-php-source", - "pht": "application/x-httpd-php", - "phtml": "application/x-httpd-php", - "pic": "image/pict", - "pict": "image/pict", - "pil": "application/vnd.piaccess.application-license", - "pk": "application/x-tex-pk", - "pkd": "application/vnd.hbci", - "pkg": "application/vnd.apple.installer+xml", - "pki": "application/pkixcmp", - "pkipath": "application/pkix-pkipath", - "pko": "application/ynd.ms-pkipko", - "pkpass": "application/vnd.apple.pkpass", - "pl": "application/x-perl", - "plantuml": "text/plantuml", - "plb": "application/vnd.3gpp.pic-bw-large", - "plc": "application/vnd.mobius.plc", - "plf": "application/vnd.pocketlearn", - "plj": "audio/vnd.everad.plj", - "plp": "application/vnd.panoply", - "pls": "application/pls+xml", - "plx": "application/x-pixclscript", - "ply": "model/stanford", - "pm": "text/plain", - "pm4": "application/x-pagemaker", - "pm5": "application/x-pagemaker", - "pma": "application/x-perfmon", - "pmc": "application/x-perfmon", - "pml": "application/vnd.ctc-posml", - "pmr": "application/x-perfmon", - "pmw": "application/x-perfmon", - "png": "image/png", - "pnm": "image/x-portable-anymap", - "po": "text/pofile", - "pod": "text/x-pod", - "portpkg": "application/vnd.macports.portpkg", - "pot": "application/vnd.ms-powerpoint", - "potm": "application/vnd.ms-powerpoint.template.macroenabled.12", - "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", - "pov": "model/x-pov", - "pp": "text/puppet", - "ppa": "application/vnd.ms-powerpoint", - "ppam": "application/vnd.ms-powerpoint.addin.macroenabled.12", - "ppd": "application/vnd.cups-ppd", - "ppkg": "application/vnd.xmpie.ppkg", - "ppm": "image/x-portable-pixmap", - "pps": "application/vnd.ms-powerpoint", - "ppsm": "application/vnd.ms-powerpoint.slideshow.macroenabled.12", - "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - "ppt": "application/vnd.ms-powerpoint", - "pptm": "application/vnd.ms-powerpoint.presentation.macroenabled.12", "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "ppz": "application/mspowerpoint", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "mgp": "application/vnd.osgeo.mapguide.package", + "dp": "application/vnd.osgi.dp", + "esa": "application/vnd.osgi.subsystem", + "pdb": "application/vnd.palm", "pqa": "application/vnd.palm", - "prc": "application/vnd.palm", - "pre": "application/vnd.lotus-freelance", - "preminet": "application/vnd.preminet", - "prf": "application/pics-rules", - "proto": "text/proto", - "provn": "text/provenance-notation", - "provx": "application/provenance+xml", - "prt": "application/pro_eng", - "prz": "application/vnd.lotus-freelance", - "ps": "application/postscript", - "psb": "application/vnd.3gpp.pic-bw-small", - "psd": "image/vnd.adobe.photoshop", - "pseg3820": "application/vnd.ibm.modcap", - "psf": "application/x-font-linux-psf", - "psid": "audio/prs.sid", - "pskcxml": "application/pskc+xml", - "pti": "image/prs.pti", - "ptid": "application/vnd.pvi.ptid1", - "pub": "application/x-mspublisher", - "purs": "text/purescript", - "pvb": "application/vnd.3gpp.pic-bw-var", - "pvu": "paleovu/x-pv", - "pwn": "application/vnd.3m.post-it-notes", - "pwz": "application/vnd.ms-powerpoint", - "pxd": "text/cython", - "pxi": "text/cython", - "py": "text/x-script.phyton", - "pya": "audio/vnd.ms-playready.media.pya", - "pyc": "application/x-python-code", - "pyi": "text/pyi", - "pyo": "application/x-python-code", - "pyv": "video/vnd.ms-playready.media.pyv", - "pyx": "text/cython", - "qam": "application/vnd.epson.quickanime", - "qbo": "application/vnd.intu.qbo", - "qca": "application/vnd.ericsson.quickcall", - "qcall": "application/vnd.ericsson.quickcall", - "qcp": "audio/qcelp", - "qd3": "x-world/x-3dmf", - "qd3d": "x-world/x-3dmf", - "qfx": "application/vnd.intu.qfx", - "qgs": "application/x-qgis", - "qif": "image/x-quicktime", + "oprc": "application/vnd.palm", + "paw": "application/vnd.pawaafile", + "str": "application/vnd.pg.format", + "ei6": "application/vnd.pg.osasli", + "efif": "application/vnd.picsel", + "wg": "application/vnd.pmi.widget", + "plf": "application/vnd.pocketlearn", + "pbd": "application/vnd.powerbuilder6", + "box": "application/vnd.previewsystems.box", + "mgz": "application/vnd.proteus.magazine", "qps": "application/vnd.publishare-delta-tree", - "qt": "video/quicktime", - "qtc": "video/x-qtc", - "qti": "image/x-quicktime", - "qtif": "image/x-quicktime", - "qtl": "application/x-quicktimeplayer", - "quiz": "application/vnd.quobject-quoxdocument", - "quox": "application/vnd.quobject-quoxdocument", - "qvd": "application/vnd.theqvd", + "ptid": "application/vnd.pvi.ptid1", + "qxd": "application/vnd.quark.quarkxpress", + "qxt": "application/vnd.quark.quarkxpress", "qwd": "application/vnd.quark.quarkxpress", "qwt": "application/vnd.quark.quarkxpress", - "qxb": "application/vnd.quark.quarkxpress", - "qxd": "application/vnd.quark.quarkxpress", "qxl": "application/vnd.quark.quarkxpress", - "qxt": "application/vnd.quark.quarkxpress", - "r": "text/r", - "ra": "audio/x-realaudio", - "ram": "audio/x-pn-realaudio", - "raml": "application/raml+yaml", - "rapd": "application/route-apd+xml", - "rar": "application/x-rar-compressed", - "ras": "image/x-cmu-raster", - "rast": "image/cmu-raster", - "rb": "application/x-ruby", - "rcprofile": "application/vnd.ipunplugged.rcprofile", - "rct": "application/prs.nprend", - "rd": "chemical/x-mdl-rdfile", - "rda": "text/r", - "rdata": "text/r", - "rds": "text/r", - "rdf": "application/rdf+xml", - "rdf-crypt": "application/prs.rdf-xml-crypt", - "rdz": "application/vnd.data-vision.rdz", - "relo": "application/p2p-overlay+xml", - "rep": "application/vnd.businessobjects", - "request": "application/vnd.nervana", - "res": "application/x-dtbresource+xml", - "rexx": "text/x-script.rexx", - "rf": "image/vnd.rn-realflash", - "rfcxml": "application/rfc+xml", - "rgb": "image/x-rgb", - "rgbe": "image/vnd.radiance", - "rhtml": "application/x-httpd-eruby", - "rif": "application/reginfo+xml", - "rip": "audio/vnd.rip", - "ris": "application/x-research-info-systems", - "rl": "application/resource-lists+xml", - "rlc": "image/vnd.fujixerox.edmics-rlc", - "rld": "application/resource-lists-diff+xml", - "rlib": "text/rust", + "qxb": "application/vnd.quark.quarkxpress", + "bed": "application/vnd.realvnc.bed", + "mxl": "application/vnd.recordare.musicxml", + "musicxml": "application/vnd.recordare.musicxml+xml", + "cryptonote": "application/vnd.rig.cryptonote", + "cod": "application/vnd.rim.cod", "rm": "application/vnd.rn-realmedia", - "rmi": "audio/mid", - "rmm": "audio/x-pn-realaudio", - "rmp": "audio/x-pn-realaudio-plugin", - "rms": "application/vnd.jcp.javame.midlet-rms", "rmvb": "application/vnd.rn-realmedia-vbr", - "rnc": "application/relax-ng-compact-syntax", - "rnd": "application/prs.nprend", - "rng": "text/xml", - "rnx": "application/vnd.rn-realplayer", - "roa": "application/rpki-roa", - "roff": "text/troff", - "ros": "chemical/x-rosdal", - "rp": "image/vnd.rn-realpix", - "rp9": "application/vnd.cloanto.rp9", - "rpm": "application/x-redhat-package-manager", - "rpss": "application/vnd.nokia.radio-presets", - "rpst": "application/vnd.nokia.radio-preset", - "rq": "application/sparql-query", - "rs": "application/rls-services+xml", - "rsd": "application/rsd+xml", - "rsheet": "application/urc-ressheet+xml", - "rsm": "model/vnd.gdl", - "rss": "application/rss+xml", - "rst": "text/prs.fallenstein.rst", - "rt": "text/richtext", - "rtf": "text/rtf", - "rtx": "text/richtext", - "run": "application/x-makeself", - "rusd": "application/route-usd+xml", - "rv": "video/vnd.rn-realvideo", - "rxn": "chemical/x-mdl-rxnfile", - "s": "text/x-asm", - "s11": "video/vnd.sealed.mpeg1", - "s14": "video/vnd.sealed.mpeg4", - "s1a": "application/vnd.sealedmedia.softseal.pdf", - "s1e": "application/vnd.sealed.xls", - "s1g": "image/vnd.sealedmedia.softseal.gif", - "s1h": "application/vnd.sealedmedia.softseal.html", - "s1j": "image/vnd.sealedmedia.softseal.jpg", - "s1m": "audio/vnd.sealedmedia.softseal.mpeg", - "s1n": "image/vnd.sealed.png", - "s1p": "application/vnd.sealed.ppt", - "s1q": "video/vnd.sealedmedia.softseal.mov", - "s1w": "application/vnd.sealed.doc", - "s3df": "application/vnd.sealed.3df", - "s3m": "audio/s3m", - "sac": "application/tamp-sequence-adjust-confirm", - "saf": "application/vnd.yamaha.smaf-audio", - "sam": "application/vnd.lotus-wordpro", - "sandboxed": "text/html-sandboxed", - "sass": "text/x-sass", - "saveme": "application/octet-stream", - "sbk": "application/x-tbook", - "sbml": "application/sbml+xml", - "sc": "application/vnd.ibm.secure-container", - "scala": "text/x-scala", - "scd": "application/x-msschedule", - "sce": "application/vnd.etsi.asic-e+zip", - "scim": "application/scim+json", - "scld": "application/vnd.doremir.scorecloud-binary-document", - "scm": "application/vnd.lotus-screencam", - "scq": "application/scvp-cv-request", - "scr": "application/x-silverlight", - "scs": "application/scvp-cv-response", - "scsf": "application/vnd.sealed.csf", - "scss": "text/x-scss", - "sct": "text/scriptlet", - "scurl": "text/vnd.curl.scurl", - "sd": "chemical/x-mdl-sdfile", - "sd2": "audio/x-sd2", - "sda": "application/vnd.stardivision.draw", - "sdc": "application/vnd.stardivision.calc", - "sdd": "application/vnd.stardivision.impress", - "sdf": "application/vnd.kinar", - "sdkd": "application/vnd.solent.sdkm+xml", - "sdkm": "application/vnd.solent.sdkm+xml", - "sdml": "text/plain", - "sdo": "application/vnd.sealed.doc", - "sdoc": "application/vnd.sealed.doc", - "sdp": "application/sdp", - "sdr": "application/sounder", - "sdw": "application/vnd.stardivision.writer", - "sea": "application/x-sea", + "link66": "application/vnd.route66.link66+xml", + "st": "application/vnd.sailingtracker.track", "see": "application/vnd.seemail", - "seed": "application/vnd.fdsn.seed", - "sem": "application/vnd.sealed.eml", "sema": "application/vnd.sema", "semd": "application/vnd.semd", "semf": "application/vnd.semf", - "seml": "application/vnd.sealed.eml", - "ser": "application/java-serialized-object", - "set": "application/set", - "setpay": "application/set-payment-initiation", - "setreg": "application/set-registration-initiation", - "sfc": "application/vnd.nintendo.snes.rom", - "sfd": "application/vnd.font-fontforge-sfd", - "sfd-hdstx": "application/vnd.hydrostatix.sof-data", + "ifm": "application/vnd.shana.informed.formdata", + "itp": "application/vnd.shana.informed.formtemplate", + "iif": "application/vnd.shana.informed.interchange", + "ipk": "application/vnd.shana.informed.package", + "twd": "application/vnd.simtech-mindmapper", + "twds": "application/vnd.simtech-mindmapper", + "mmf": "application/vnd.smaf", + "teacher": "application/vnd.smart.teacher", + "sdkm": "application/vnd.solent.sdkm+xml", + "sdkd": "application/vnd.solent.sdkm+xml", + "dxp": "application/vnd.spotfire.dxp", "sfs": "application/vnd.spotfire.sfs", - "sfv": "text/x-sfv", - "sgf": "application/x-go-sgf", - "sgi": "image/sgi", - "sgif": "image/vnd.sealedmedia.softseal.gif", - "sgl": "application/vnd.stardivision.writer-global", - "sgm": "text/sgml", - "sgml": "text/sgml", - "sh": "application/x-sh", - "shar": "application/x-shar", - "shex": "text/shex", - "shf": "application/shf+xml", - "shp": "application/x-qgis", - "shx": "application/x-qgis", - "si": "text/vnd.wap.si", - "sic": "application/vnd.wap.sic", - "sid": "image/x-mrsid-image", - "sieve": "application/sieve", - "sig": "application/pgp-signature", - "sik": "application/x-trash", - "sil": "audio/silk", - "silo": "model/mesh", - "sis": "application/vnd.symbian.install", - "sisx": "x-epoc/x-sisx-app", - "sit": "application/x-stuffit", - "sitx": "application/x-stuffitx", - "siv": "application/sieve", - "sjp": "image/vnd.sealedmedia.softseal.jpg", - "sjpg": "image/vnd.sealedmedia.softseal.jpg", - "skd": "application/vnd.koan", - "skm": "application/vnd.koan", - "skp": "application/vnd.koan", - "skt": "application/vnd.koan", - "sl": "text/vnd.wap.sl", - "sla": "application/vnd.scribus", - "slaz": "application/vnd.scribus", - "slc": "application/vnd.wap.slc", - "sldm": "application/vnd.ms-powerpoint.slide.macroenabled.12", - "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", - "sls": "application/route-s-tsid+xml", - "slt": "application/vnd.epson.salt", - "sm": "application/vnd.stepmania.stepchart", - "smc": "application/vnd.nintendo.snes.rom", + "sdc": "application/vnd.stardivision.calc", + "sda": "application/vnd.stardivision.draw", + "sdd": "application/vnd.stardivision.impress", "smf": "application/vnd.stardivision.math", - "smh": "application/vnd.sealed.mht", - "smht": "application/vnd.sealed.mht", - "smi": "application/smil+xml", - "smil": "application/smil+xml", - "smk": "video/vnd.radgamettools.smacker", - "sml": "application/smil+xml", - "smo": "video/vnd.sealedmedia.softseal.mov", - "smov": "video/vnd.sealedmedia.softseal.mov", - "smp": "audio/vnd.sealedmedia.softseal.mpeg", - "smp3": "audio/vnd.sealedmedia.softseal.mpeg", - "smpg": "video/vnd.sealed.mpeg1", - "sms": "application/vnd.3gpp2.sms", - "smv": "video/x-smv", + "sdw": "application/vnd.stardivision.writer", + "vor": "application/vnd.stardivision.writer", + "sgl": "application/vnd.stardivision.writer-global", "smzip": "application/vnd.stepmania.package", - "snd": "audio/basic", - "snf": "application/x-font-snf", - "so": "application/octet-stream", - "soa": "text/dns", - "soc": "application/sgml-open-catalog", - "sol": "application/solids", - "spc": "text/x-speech", - "spd": "application/vnd.sealedmedia.softseal.pdf", - "spdf": "application/vnd.sealedmedia.softseal.pdf", - "spec": "text/spec", - "spf": "application/vnd.yamaha.smaf-phrase", - "spl": "application/x-futuresplash", - "spn": "image/vnd.sealed.png", - "spng": "image/vnd.sealed.png", - "spo": "text/vnd.in3d.spot", - "spot": "text/vnd.in3d.spot", - "spp": "application/scvp-vp-response", - "sppt": "application/vnd.sealed.ppt", - "spq": "application/scvp-vp-request", - "spr": "application/x-sprite", - "sprite": "application/x-sprite", - "spx": "audio/ogg", - "sql": "application/x-sql", - "sr": "application/vnd.sigrok.session", - "src": "application/x-wais-source", - "srt": "application/x-subrip", - "sru": "application/sru+xml", - "srx": "application/sparql-results+xml", - "ssdl": "application/ssdl+xml", - "sse": "application/vnd.kodak-descriptor", - "ssf": "application/vnd.epson.ssf", - "ssi": "text/x-server-parsed-html", - "ssm": "application/streamingmedia", - "ssml": "application/ssml+xml", - "sst": "application/vnd.ms-pki.certstore", - "ssw": "video/vnd.sealed.swf", - "sswf": "video/vnd.sealed.swf", - "st": "application/vnd.sailingtracker.track", + "sm": "application/vnd.stepmania.stepchart", + "sxc": "application/vnd.sun.xml.calc", "stc": "application/vnd.sun.xml.calc.template", + "sxd": "application/vnd.sun.xml.draw", "std": "application/vnd.sun.xml.draw.template", - "step": "application/step", - "stf": "application/vnd.wt.stf", + "sxi": "application/vnd.sun.xml.impress", "sti": "application/vnd.sun.xml.impress.template", - "stif": "application/vnd.sealed.tiff", - "stk": "application/hyperstudio", - "stl": "application/vnd.ms-pki.stl", - "stm": "audio/x-stm", - "stml": "application/vnd.sealedmedia.softseal.html", - "stp": "application/step", - "str": "application/vnd.pg.format", - "study-inter": "application/vnd.vd-study", + "sxm": "application/vnd.sun.xml.math", + "sxw": "application/vnd.sun.xml.writer", + "sxg": "application/vnd.sun.xml.writer.global", "stw": "application/vnd.sun.xml.writer.template", - "sty": "text/x-tex", - "styl": "text/stylus", - "sub": "text/vnd.dvb.subtitle", "sus": "application/vnd.sus-calendar", "susp": "application/vnd.sus-calendar", - "sv4cpio": "application/x-sv4cpio", - "sv4crc": "application/x-sv4crc", - "svc": "application/vnd.dvb.service", "svd": "application/vnd.svd", - "svf": "image/x-dwg", - "svg": "image/svg+xml", - "svgz": "image/svg+xml", - "sw": "chemical/x-swissprot", - "swa": "application/x-director", - "swf": "application/x-shockwave-flash", - "swfl": "application/x-shockwave-flash", - "swi": "application/vnd.aristanetworks.swi", - "swift": "text/swift", - "swiftdeps": "text/swiftdeps", - "sxc": "application/vnd.sun.xml.calc", - "sxd": "application/vnd.sun.xml.draw", - "sxg": "application/vnd.sun.xml.writer.global", - "sxi": "application/vnd.sun.xml.impress", - "sxl": "application/vnd.sealed.xls", - "sxls": "application/vnd.sealed.xls", - "sxm": "application/vnd.sun.xml.math", - "sxw": "application/vnd.sun.xml.writer", - "t": "text/troff", - "t3": "application/x-t3vm-image", - "t38": "image/t38", - "tac": "text/twisted", - "tag": "text/prs.lines.tag", - "taglet": "application/vnd.mynfc", - "talk": "text/x-speech", - "tam": "application/vnd.onepager", - "tamp": "application/vnd.onepagertamp", - "tamx": "application/vnd.onepagertamx", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "xsm": "application/vnd.syncml+xml", + "bdm": "application/vnd.syncml.dm+wbxml", + "xdm": "application/vnd.syncml.dm+xml", "tao": "application/vnd.tao.intent-module-archive", - "tap": "image/vnd.tencent.tap", - "tar": "application/x-tar", - "tat": "application/vnd.onepagertat", - "tatp": "application/vnd.onepagertatp", - "tatx": "application/vnd.onepagertatx", - "tau": "application/tamp-apex-update", - "taz": "application/x-gtar", - "tbk": "application/toolbook", - "tcap": "application/vnd.3gpp2.tcap", - "tcl": "application/x-tcl", - "tcsh": "text/x-script.tcsh", - "tcu": "application/tamp-community-update", - "td": "application/urc-targetdesc+xml", - "teacher": "application/vnd.smart.teacher", - "tei": "application/tei+xml", - "teicorpus": "application/tei+xml", - "ter": "application/tamp-error", - "tex": "application/x-tex", - "texi": "application/x-texinfo", - "texinfo": "application/x-texinfo", - "text": "text/plain", - "tf": "text/terraform", - "tfi": "application/thraud+xml", - "tfm": "application/x-tex-tfm", - "tfx": "image/tiff-fx", - "tga": "image/x-tga", - "tgf": "chemical/x-mdl-tgf", - "tgz": "application/gzip", - "thmx": "application/vnd.ms-officetheme", - "thrift": "text/thrift", - "tif": "image/tiff", - "tiff": "image/tiff", - "tk": "text/x-tcl", - "tlclient": "application/vnd.cendio.thinlinc.clientconf", - "tm": "text/texmacs", + "pcap": "application/vnd.tcpdump.pcap", + "cap": "application/vnd.tcpdump.pcap", + "dmp": "application/vnd.tcpdump.pcap", "tmo": "application/vnd.tmobile-livetv", - "tnef": "application/vnd.ms-tnef", - "tnf": "application/vnd.ms-tnef", - "toml": "text/toml", - "torrent": "application/x-bittorrent", - "tpl": "application/vnd.groove-tool-template", "tpt": "application/vnd.trid.tpt", - "tr": "text/troff", + "mxs": "application/vnd.triscape.mxs", "tra": "application/vnd.trueapp", - "tree": "application/vnd.rainstor.data", - "trig": "application/trig", - "trm": "application/x-msterminal", - "ts": "video/mp2t", - "tsa": "application/tamp-sequence-adjust", - "tscn": "text/godot", - "tsd": "application/timestamped-data", - "tsi": "audio/tsp-audio", - "tsp": "audio/tsplayer", - "tsq": "application/timestamp-query", - "tsr": "application/timestamp-reply", - "tst": "application/vnd.etsi.timestamp-token", - "tsv": "text/tab-separated-values", - "tsx": "text/tsx", - "ttc": "font/collection", - "ttf": "font/ttf", - "ttl": "text/turtle", - "ttml": "application/ttml+xml", - "tuc": "application/tamp-update-confirm", - "tur": "application/tamp-update", - "turbot": "image/florian", - "twd": "application/vnd.simtech-mindmapper", - "twds": "application/vnd.simtech-mindmapper", - "txd": "application/vnd.genomatix.tuxedo", - "txf": "application/vnd.mobius.txf", - "txt": "text/plain", - "u32": "application/x-authorware-bin", - "u8dsn": "message/global-delivery-status", - "u8hdr": "message/global-headers", - "u8mdn": "message/global-disposition-notification", - "u8msg": "message/global", - "udeb": "application/vnd.debian.binary-package", "ufd": "application/vnd.ufdl", "ufdl": "application/vnd.ufdl", - "uil": "text/x-uil", - "uis": "application/urc-uisocketdesc+xml", - "uls": "text/iuls", - "ult": "audio/x-mod", - "ulx": "application/x-glulx", + "utz": "application/vnd.uiq.theme", "umj": "application/vnd.umajin", - "uni": "audio/x-mod", - "unis": "text/uri-list", "unityweb": "application/vnd.unity", - "unv": "application/i-deas", - "uo": "application/vnd.uoml+xml", "uoml": "application/vnd.uoml+xml", - "upa": "application/vnd.hbci", - "uri": "text/uri-list", - "uric": "text/vnd.si.uricatalogue", - "urim": "application/vnd.uri-map", - "urimap": "application/vnd.uri-map", - "uris": "text/uri-list", - "urls": "text/uri-list", - "ustar": "application/x-ustar", - "utz": "application/vnd.uiq.theme", - "uu": "text/x-uuencode", - "uue": "text/x-uuencode", - "uva": "audio/vnd.dece.audio", - "uvd": "application/vnd.dece.data", - "uvf": "application/vnd.dece.data", - "uvg": "image/vnd.dece.graphic", - "uvh": "video/vnd.dece.hd", - "uvi": "image/vnd.dece.graphic", - "uvm": "video/vnd.dece.mobile", - "uvp": "video/vnd.dece.pd", - "uvs": "video/vnd.dece.sd", - "uvt": "application/vnd.dece.ttml+xml", - "uvu": "video/vnd.dece.mp4", - "uvv": "video/vnd.dece.video", - "uvva": "audio/vnd.dece.audio", - "uvvd": "application/vnd.dece.data", - "uvvf": "application/vnd.dece.data", - "uvvg": "image/vnd.dece.graphic", - "uvvh": "video/vnd.dece.hd", - "uvvi": "image/vnd.dece.graphic", - "uvvm": "video/vnd.dece.mobile", - "uvvp": "video/vnd.dece.pd", - "uvvs": "video/vnd.dece.sd", - "uvvt": "application/vnd.dece.ttml+xml", - "uvvu": "video/vnd.dece.mp4", - "uvvv": "video/vnd.dece.video", - "uvvx": "application/vnd.dece.unspecified", - "uvvz": "application/vnd.dece.zip", - "uvx": "application/vnd.dece.unspecified", - "uvz": "application/vnd.dece.zip", - "val": "chemical/x-ncbi-asn1-binary", - "vbk": "audio/vnd.nortel.vbk", - "vbox": "application/x-virtualbox-vbox", - "vbox-extpack": "application/x-virtualbox-vbox-extpack", - "vcard": "text/vcard", - "vcd": "application/x-cdlink", - "vcf": "text/x-vcard", - "vcg": "application/vnd.groove-vcard", - "vcs": "text/x-vcalendar", "vcx": "application/vnd.vcx", - "vda": "application/vda", - "vdi": "application/x-virtualbox-vdi", - "vdo": "video/vdo", - "vdx": "text/vdx", - "vew": "application/vnd.lotus-approach", - "vfr": "application/vnd.tml", - "vhd": "application/x-virtualbox-vhd", - "viaframe": "application/vnd.tml", - "vim": "text/vim", - "vis": "application/vnd.visionary", - "viv": "video/vnd.vivo", - "vivo": "video/vivo", - "vmd": "application/vocaltec-media-desc", - "vmdk": "application/x-virtualbox-vmdk", - "vmf": "application/vocaltec-media-file", - "vms": "chemical/x-vamas-iso14976", - "vmt": "application/vnd.valve.source.material", - "vob": "video/x-ms-vob", - "voc": "audio/voc", - "vor": "application/vnd.stardivision.writer", - "vos": "video/vosaic", - "vox": "audio/voxware", - "vpm": "multipart/voice-message", - "vqe": "audio/x-twinvq-plugin", - "vqf": "audio/x-twinvq", - "vql": "audio/x-twinvq-plugin", - "vrm": "x-world/x-vrml", - "vrml": "model/vrml", - "vrt": "x-world/x-vrt", - "vsc": "application/vnd.vidsoft.vidconference", "vsd": "application/vnd.visio", - "vsf": "application/vnd.vsf", - "vss": "application/vnd.visio", "vst": "application/vnd.visio", + "vss": "application/vnd.visio", "vsw": "application/vnd.visio", - "vtf": "image/vnd.valve.source.texture", - "vtt": "text/vtt", - "vtu": "model/vnd.vtu", - "vue": "text/vue", - "vwx": "application/vnd.vectorworks", - "vxml": "application/voicexml+xml", - "w3d": "application/x-director", - "w60": "application/wordperfect6.0", - "w61": "application/wordperfect6.1", - "w6w": "application/msword", - "wad": "application/x-doom", - "wadl": "application/vnd.sun.wadl+xml", - "war": "binary/zip", - "wasm": "application/wasm", - "wav": "audio/wave", - "wax": "audio/x-ms-wax", - "wb1": "application/x-qpro", - "wbmp": "image/vnd.wap.wbmp", - "wbs": "application/vnd.criticaltools.wbs+xml", + "vis": "application/vnd.visionary", + "vsf": "application/vnd.vsf", "wbxml": "application/vnd.wap.wbxml", - "wcm": "application/vnd.ms-works", - "wdb": "application/vnd.ms-works", - "wdp": "image/vnd.ms-photo", - "web": "application/vnd.xara", - "weba": "audio/webm", - "webapp": "application/x-web-app-manifest+json", - "webm": "video/webm", - "webmanifest": "application/manifest+json", - "webp": "image/webp", - "wg": "application/vnd.pmi.widget", - "wgt": "application/widget", - "whl": "binary/wheel", - "wif": "application/watcherinfo+xml", - "win": "model/vnd.gdl", - "wiz": "application/msword", - "wk": "application/x-123", - "wk1": "application/vnd.lotus-1-2-3", - "wk3": "application/vnd.lotus-1-2-3", - "wk4": "application/vnd.lotus-1-2-3", - "wks": "application/vnd.ms-works", - "wkt": "text/wkt", - "wlnk": "application/link-format", - "wm": "video/x-ms-wm", - "wma": "audio/x-ms-wma", - "wmc": "application/vnd.wmc", - "wmd": "application/x-ms-wmd", - "wmf": "image/wmf", - "wml": "text/vnd.wap.wml", "wmlc": "application/vnd.wap.wmlc", - "wmls": "text/vnd.wap.wmlscript", "wmlsc": "application/vnd.wap.wmlscriptc", - "wmv": "video/x-ms-wmv", - "wmx": "video/x-ms-wmx", - "wmz": "application/x-ms-wmz", - "woff": "font/woff", - "woff2": "font/woff2", - "word": "application/msword", - "wp": "application/wordperfect", - "wp5": "application/wordperfect", - "wp6": "application/wordperfect", + "wtb": "application/vnd.webturbo", + "nbp": "application/vnd.wolfram.player", "wpd": "application/vnd.wordperfect", - "wpl": "application/vnd.ms-wpl", - "wps": "application/vnd.ms-works", - "wq1": "application/x-lotus", "wqd": "application/vnd.wqd", - "wri": "application/x-mswrite", - "wrl": "model/vrml", - "wrz": "model/vrml", - "wsc": "message/vnd.wfa.wsc", + "stf": "application/vnd.wt.stf", + "xar": "application/vnd.xara", + "xfdl": "application/vnd.xfdl", + "hvd": "application/vnd.yamaha.hv-dic", + "hvs": "application/vnd.yamaha.hv-script", + "hvp": "application/vnd.yamaha.hv-voice", + "osf": "application/vnd.yamaha.openscoreformat", + "osfpvg": "application/vnd.yamaha.openscoreformat.osfpvg+xml", + "saf": "application/vnd.yamaha.smaf-audio", + "spf": "application/vnd.yamaha.smaf-phrase", + "cmp": "application/vnd.yellowriver-custom-menu", + "zir": "application/vnd.zul", + "zirz": "application/vnd.zul", + "zaz": "application/vnd.zzazz.deck+xml", + "vxml": "application/voicexml+xml", + "wasm": "application/wasm", + "wgt": "application/widget", + "hlp": "application/winhlp", "wsdl": "application/wsdl+xml", - "wsgi": "text/wsgi", "wspolicy": "application/wspolicy+xml", - "wsrc": "application/x-wais-source", - "wtb": "application/vnd.webturbo", - "wtk": "application/x-wintalk", - "wv": "application/vnd.wv.csp+wbxml", - "wvx": "video/x-ms-wvx", - "wz": "application/x-wingz", - "x-png": "image/png", + "7z": "application/x-7z-compressed", + "abw": "application/x-abiword", + "ace": "application/x-ace-compressed", + "dmg": "application/x-apple-diskimage", + "aab": "application/x-authorware-bin", "x32": "application/x-authorware-bin", - "x3d": "application/vnd.hzn-3d-crossword", - "x3db": "model/x3d+xml", - "x3dbz": "model/x3d+binary", - "x3dv": "model/x3d-vrml", - "x3dvz": "model/x3d-vrml", - "x3dz": "model/x3d+xml", - "x_b": "model/vnd.parasolid.transmit.binary", - "x_t": "model/vnd.parasolid.transmit.text", - "xaf": "x-world/x-vrml", - "xaml": "application/xaml+xml", - "xap": "application/x-silverlight-app", - "xar": "application/vnd.xara", - "xav": "application/xcap-att+xml", + "u32": "application/x-authorware-bin", + "vox": "application/x-authorware-bin", + "aam": "application/x-authorware-map", + "aas": "application/x-authorware-seg", + "bcpio": "application/x-bcpio", + "torrent": "application/x-bittorrent", + "blb": "application/x-blorb", + "blorb": "application/x-blorb", + "bz": "application/x-bzip", + "bz2": "application/x-bzip2", + "boz": "application/x-bzip2", + "cbr": "application/x-cbr", + "cba": "application/x-cbr", + "cbt": "application/x-cbr", + "cbz": "application/x-cbr", + "cb7": "application/x-cbr", + "vcd": "application/x-cdlink", + "cfs": "application/x-cfs-compressed", + "chat": "application/x-chat", + "pgn": "application/x-chess-pgn", + "nsc": "application/x-conference", + "cpio": "application/x-cpio", + "csh": "application/x-csh", + "deb": "application/x-debian-package", + "udeb": "application/x-debian-package", + "dgc": "application/x-dgc-compressed", + "dir": "application/x-director", + "dcr": "application/x-director", + "dxr": "application/x-director", + "cst": "application/x-director", + "cct": "application/x-director", + "cxt": "application/x-director", + "w3d": "application/x-director", + "fgd": "application/x-director", + "swa": "application/x-director", + "wad": "application/x-doom", + "ncx": "application/x-dtbncx+xml", + "dtb": "application/x-dtbook+xml", + "res": "application/x-dtbresource+xml", + "dvi": "application/x-dvi", + "evy": "application/x-envoy", + "eva": "application/x-eva", + "bdf": "application/x-font-bdf", + "gsf": "application/x-font-ghostscript", + "psf": "application/x-font-linux-psf", + "pcf": "application/x-font-pcf", + "snf": "application/x-font-snf", + "pfa": "application/x-font-type1", + "pfb": "application/x-font-type1", + "pfm": "application/x-font-type1", + "afm": "application/x-font-type1", + "arc": "application/x-freearc", + "spl": "application/x-futuresplash", + "gca": "application/x-gca-compressed", + "ulx": "application/x-glulx", + "gnumeric": "application/x-gnumeric", + "gramps": "application/x-gramps-xml", + "gtar": "application/x-gtar", + "hdf": "application/x-hdf", + "install": "application/x-install-instructions", + "iso": "application/x-iso9660-image", + "jnlp": "application/x-java-jnlp-file", + "latex": "application/x-latex", + "lzh": "application/x-lzh-compressed", + "lha": "application/x-lzh-compressed", + "mie": "application/x-mie", + "prc": "application/x-mobipocket-ebook", + "mobi": "application/x-mobipocket-ebook", + "application": "application/x-ms-application", + "lnk": "application/x-ms-shortcut", + "wmd": "application/x-ms-wmd", + "wmz": "application/x-ms-wmz", "xbap": "application/x-ms-xbap", - "xbd": "application/vnd.fujixerox.docuworks.binder", - "xbm": "image/x-xbitmap", - "xca": "application/xcap-caps+xml", - "xcf": "application/x-xcf", - "xcs": "application/calendar+xml", - "xct": "application/vnd.fujixerox.docuworks.container", - "xdd": "application/bacnet-xdd+zip", - "xdf": "application/xcap-diff+xml", - "xdm": "application/vnd.syncml.dm+xml", - "xdp": "application/vnd.adobe.xdp+xml", - "xdr": "video/x-amt-demorun", - "xdssc": "application/dssc+xml", - "xdw": "application/vnd.fujixerox.docuworks", - "xel": "application/xcap-el+xml", - "xenc": "application/xenc+xml", - "xer": "application/patch-ops-error+xml", - "xfd": "application/vnd.xfdl", - "xfdf": "application/vnd.adobe.xfdf", - "xfdl": "application/vnd.xfdl", - "xgz": "xgl/drawing", - "xht": "application/xhtml+xml", - "xhtm": "application/xhtml+xml", - "xhtml": "application/xhtml+xml", - "xhvml": "application/xv+xml", - "xif": "image/vnd.xiff", - "xl": "application/excel", - "xla": "application/vnd.ms-excel", - "xlam": "application/vnd.ms-excel.addin.macroenabled.12", - "xlb": "application/vndms-excel", - "xlc": "application/vnd.ms-excel", + "mdb": "application/x-msaccess", + "obd": "application/x-msbinder", + "crd": "application/x-mscardfile", + "clp": "application/x-msclip", + "exe": "application/x-msdownload", + "dll": "application/x-msdownload", + "com": "application/x-msdownload", + "bat": "application/x-msdownload", + "msi": "application/x-msdownload", + "mvb": "application/x-msmediaview", + "m13": "application/x-msmediaview", + "m14": "application/x-msmediaview", + "wmf": "application/x-msmetafile", + "wmz": "application/x-msmetafile", + "emf": "application/x-msmetafile", + "emz": "application/x-msmetafile", + "mny": "application/x-msmoney", + "pub": "application/x-mspublisher", + "scd": "application/x-msschedule", + "trm": "application/x-msterminal", + "wri": "application/x-mswrite", + "nc": "application/x-netcdf", + "cdf": "application/x-netcdf", + "nzb": "application/x-nzb", + "p12": "application/x-pkcs12", + "pfx": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "spc": "application/x-pkcs7-certificates", + "p7r": "application/x-pkcs7-certreqresp", + "rar": "application/x-rar-compressed", + "ris": "application/x-research-info-systems", + "sh": "application/x-sh", + "shar": "application/x-shar", + "swf": "application/x-shockwave-flash", + "xap": "application/x-silverlight-app", + "sql": "application/x-sql", + "sit": "application/x-stuffit", + "sitx": "application/x-stuffitx", + "srt": "application/x-subrip", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "t3": "application/x-t3vm-image", + "gam": "application/x-tads", + "tar": "application/x-tar", + "tcl": "application/x-tcl", + "tex": "application/x-tex", + "tfm": "application/x-tex-tfm", + "texinfo": "application/x-texinfo", + "texi": "application/x-texinfo", + "obj": "application/x-tgif", + "ustar": "application/x-ustar", + "src": "application/x-wais-source", + "der": "application/x-x509-ca-cert", + "crt": "application/x-x509-ca-cert", + "fig": "application/x-xfig", "xlf": "application/x-xliff+xml", - "xlim": "application/vnd.xmpie.xlim", - "xlm": "application/vnd.ms-excel", - "xls": "application/vnd.ms-excel", - "xlsb": "application/vnd.ms-excel.sheet.binary.macroenabled.12", - "xlsm": "application/vnd.ms-excel.sheet.macroenabled.12", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xlt": "application/vnd.ms-excel", - "xltm": "application/vnd.ms-excel.template.macroenabled.12", - "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", - "xlw": "application/vnd.ms-excel", - "xm": "audio/xm", - "xml": "text/xml", - "xmls": "application/dskpp+xml", - "xmt_bin": "model/vnd.parasolid.transmit.binary", - "xmt_txt": "model/vnd.parasolid.transmit.text", - "xmz": "xgl/movie", - "xns": "application/xcap-ns+xml", - "xo": "application/vnd.olpc-sugar", - "xof": "x-world/x-vrml", - "xop": "application/xop+xml", - "xpdl": "application/xml", "xpi": "application/x-xpinstall", - "xpix": "application/x-vnd.ls-xpix", - "xpl": "application/xproc+xml", - "xpm": "image/x-xpixmap", - "xpr": "application/vnd.is-xpr", - "xps": "application/vnd.ms-xpsdocument", - "xpw": "application/vnd.intercon.formnet", - "xpx": "application/vnd.intercon.formnet", - "xq": "text/xquery", - "xql": "text/xquery", - "xqm": "text/xquery", - "xqu": "text/xquery", - "xquery": "text/xquery", - "xqy": "text/xquery", - "xsd": "text/xml", - "xsf": "application/prs.xsf+xml", - "xsl": "application/xslt+xml", - "xslt": "application/xslt+xml", - "xsm": "application/vnd.syncml+xml", - "xspf": "application/xspf+xml", - "xsr": "video/x-amt-showrun", - "xtel": "chemical/x-xtel", - "xul": "application/vnd.mozilla.xul+xml", - "xvm": "application/xv+xml", - "xvml": "application/xv+xml", - "xwd": "image/x-xwindowdump", - "xyz": "chemical/x-xyz", - "xyze": "image/vnd.radiance", "xz": "application/x-xz", - "yaml": "text/yaml", - "yang": "application/yang", - "yin": "application/yin+xml", - "yme": "application/vnd.yaoweme", - "yml": "text/yaml", - "ymp": "text/x-suse-ymp", "z1": "application/x-zmachine", "z2": "application/x-zmachine", "z3": "application/x-zmachine", @@ -1866,31 +744,306 @@ const mimes* = { "z6": "application/x-zmachine", "z7": "application/x-zmachine", "z8": "application/x-zmachine", - "zaz": "application/vnd.zzazz.deck+xml", - "zfc": "application/vnd.filmit.zfc", - "zfo": "application/vnd.software602.filler.form-xml-zip", - "zig": "text/zig", + "xaml": "application/xaml+xml", + "xdf": "application/xcap-diff+xml", + "xenc": "application/xenc+xml", + "xhtml": "application/xhtml+xml", + "xht": "application/xhtml+xml", + "xml": "application/xml", + "xsl": "application/xml", + "dtd": "application/xml-dtd", + "xop": "application/xop+xml", + "xpl": "application/xproc+xml", + "xslt": "application/xslt+xml", + "xspf": "application/xspf+xml", + "mxml": "application/xv+xml", + "xhvml": "application/xv+xml", + "xvml": "application/xv+xml", + "xvm": "application/xv+xml", + "yang": "application/yang", + "yin": "application/yin+xml", "zip": "application/zip", - "zir": "application/vnd.zul", - "zirz": "application/vnd.zul", - "zmm": "application/vnd.handheld-entertainment+xml", - "zmt": "chemical/x-mopac-input", - "zone": "text/dns", - "zoo": "application/octet-stream", - "zsh": "text/x-script.zsh", - "~": "application/x-trash" + "adp": "audio/adpcm", + "au": "audio/basic", + "snd": "audio/basic", + "mid": "audio/midi", + "midi": "audio/midi", + "kar": "audio/midi", + "rmi": "audio/midi", + "m4a": "audio/mp4", + "mp4a": "audio/mp4", + "mpga": "audio/mpeg", + "mp2": "audio/mpeg", + "mp2a": "audio/mpeg", + "mp3": "audio/mpeg", + "m2a": "audio/mpeg", + "m3a": "audio/mpeg", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "spx": "audio/ogg", + "opus": "audio/ogg", + "s3m": "audio/s3m", + "sil": "audio/silk", + "uva": "audio/vnd.dece.audio", + "uvva": "audio/vnd.dece.audio", + "eol": "audio/vnd.digital-winds", + "dra": "audio/vnd.dra", + "dts": "audio/vnd.dts", + "dtshd": "audio/vnd.dts.hd", + "lvp": "audio/vnd.lucent.voice", + "pya": "audio/vnd.ms-playready.media.pya", + "ecelp4800": "audio/vnd.nuera.ecelp4800", + "ecelp7470": "audio/vnd.nuera.ecelp7470", + "ecelp9600": "audio/vnd.nuera.ecelp9600", + "rip": "audio/vnd.rip", + "weba": "audio/webm", + "aac": "audio/x-aac", + "aif": "audio/x-aiff", + "aiff": "audio/x-aiff", + "aifc": "audio/x-aiff", + "caf": "audio/x-caf", + "flac": "audio/x-flac", + "mka": "audio/x-matroska", + "m3u": "audio/x-mpegurl", + "wax": "audio/x-ms-wax", + "wma": "audio/x-ms-wma", + "ram": "audio/x-pn-realaudio", + "ra": "audio/x-pn-realaudio", + "rmp": "audio/x-pn-realaudio-plugin", + "wav": "audio/x-wav", + "xm": "audio/xm", + "cdx": "chemical/x-cdx", + "cif": "chemical/x-cif", + "cmdf": "chemical/x-cmdf", + "cml": "chemical/x-cml", + "csml": "chemical/x-csml", + "xyz": "chemical/x-xyz", + "ttc": "font/collection", + "otf": "font/otf", + "ttf": "font/ttf", + "woff": "font/woff", + "woff2": "font/woff2", + "bmp": "image/bmp", + "cgm": "image/cgm", + "g3": "image/g3fax", + "gif": "image/gif", + "ief": "image/ief", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "jpe": "image/jpeg", + "ktx": "image/ktx", + "png": "image/png", + "btif": "image/prs.btif", + "sgi": "image/sgi", + "svg": "image/svg+xml", + "svgz": "image/svg+xml", + "tiff": "image/tiff", + "tif": "image/tiff", + "psd": "image/vnd.adobe.photoshop", + "uvi": "image/vnd.dece.graphic", + "uvvi": "image/vnd.dece.graphic", + "uvg": "image/vnd.dece.graphic", + "uvvg": "image/vnd.dece.graphic", + "djvu": "image/vnd.djvu", + "djv": "image/vnd.djvu", + "sub": "image/vnd.dvb.subtitle", + "dwg": "image/vnd.dwg", + "dxf": "image/vnd.dxf", + "fbs": "image/vnd.fastbidsheet", + "fpx": "image/vnd.fpx", + "fst": "image/vnd.fst", + "mmr": "image/vnd.fujixerox.edmics-mmr", + "rlc": "image/vnd.fujixerox.edmics-rlc", + "mdi": "image/vnd.ms-modi", + "wdp": "image/vnd.ms-photo", + "npx": "image/vnd.net-fpx", + "wbmp": "image/vnd.wap.wbmp", + "xif": "image/vnd.xiff", + "webp": "image/webp", + "3ds": "image/x-3ds", + "ras": "image/x-cmu-raster", + "cmx": "image/x-cmx", + "fh": "image/x-freehand", + "fhc": "image/x-freehand", + "fh4": "image/x-freehand", + "fh5": "image/x-freehand", + "fh7": "image/x-freehand", + "ico": "image/x-icon", + "sid": "image/x-mrsid-image", + "pcx": "image/x-pcx", + "pic": "image/x-pict", + "pct": "image/x-pict", + "pnm": "image/x-portable-anymap", + "pbm": "image/x-portable-bitmap", + "pgm": "image/x-portable-graymap", + "ppm": "image/x-portable-pixmap", + "rgb": "image/x-rgb", + "tga": "image/x-tga", + "xbm": "image/x-xbitmap", + "xpm": "image/x-xpixmap", + "xwd": "image/x-xwindowdump", + "eml": "message/rfc822", + "mime": "message/rfc822", + "igs": "model/iges", + "iges": "model/iges", + "msh": "model/mesh", + "mesh": "model/mesh", + "silo": "model/mesh", + "dae": "model/vnd.collada+xml", + "dwf": "model/vnd.dwf", + "gdl": "model/vnd.gdl", + "gtw": "model/vnd.gtw", + "mts": "model/vnd.mts", + "vtu": "model/vnd.vtu", + "wrl": "model/vrml", + "vrml": "model/vrml", + "x3db": "model/x3d+binary", + "x3dbz": "model/x3d+binary", + "x3dv": "model/x3d+vrml", + "x3dvz": "model/x3d+vrml", + "x3d": "model/x3d+xml", + "x3dz": "model/x3d+xml", + "appcache": "text/cache-manifest", + "ics": "text/calendar", + "ifb": "text/calendar", + "css": "text/css", + "csv": "text/csv", + "html": "text/html", + "htm": "text/html", + "js": "text/javascript", + "mjs": "text/javascript", + "n3": "text/n3", + "txt": "text/plain", + "text": "text/plain", + "conf": "text/plain", + "def": "text/plain", + "list": "text/plain", + "log": "text/plain", + "in": "text/plain", + "dsc": "text/prs.lines.tag", + "rtx": "text/richtext", + "sgml": "text/sgml", + "sgm": "text/sgml", + "tsv": "text/tab-separated-values", + "t": "text/troff", + "tr": "text/troff", + "roff": "text/troff", + "man": "text/troff", + "me": "text/troff", + "ms": "text/troff", + "ttl": "text/turtle", + "uri": "text/uri-list", + "uris": "text/uri-list", + "urls": "text/uri-list", + "vcard": "text/vcard", + "curl": "text/vnd.curl", + "dcurl": "text/vnd.curl.dcurl", + "mcurl": "text/vnd.curl.mcurl", + "scurl": "text/vnd.curl.scurl", + "sub": "text/vnd.dvb.subtitle", + "fly": "text/vnd.fly", + "flx": "text/vnd.fmi.flexstor", + "gv": "text/vnd.graphviz", + "3dml": "text/vnd.in3d.3dml", + "spot": "text/vnd.in3d.spot", + "jad": "text/vnd.sun.j2me.app-descriptor", + "wml": "text/vnd.wap.wml", + "wmls": "text/vnd.wap.wmlscript", + "s": "text/x-asm", + "asm": "text/x-asm", + "c": "text/x-c", + "cc": "text/x-c", + "cxx": "text/x-c", + "cpp": "text/x-c", + "h": "text/x-c", + "hh": "text/x-c", + "dic": "text/x-c", + "f": "text/x-fortran", + "for": "text/x-fortran", + "f77": "text/x-fortran", + "f90": "text/x-fortran", + "java": "text/x-java-source", + "nfo": "text/x-nfo", + "opml": "text/x-opml", + "p": "text/x-pascal", + "pas": "text/x-pascal", + "etx": "text/x-setext", + "sfv": "text/x-sfv", + "uu": "text/x-uuencode", + "vcs": "text/x-vcalendar", + "vcf": "text/x-vcard", + "3gp": "video/3gpp", + "3g2": "video/3gpp2", + "h261": "video/h261", + "h263": "video/h263", + "h264": "video/h264", + "jpgv": "video/jpeg", + "jpm": "video/jpm", + "jpgm": "video/jpm", + "mj2": "video/mj2", + "mjp2": "video/mj2", + "mp4": "video/mp4", + "mp4v": "video/mp4", + "mpg4": "video/mp4", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpe": "video/mpeg", + "m1v": "video/mpeg", + "m2v": "video/mpeg", + "ogv": "video/ogg", + "qt": "video/quicktime", + "mov": "video/quicktime", + "uvh": "video/vnd.dece.hd", + "uvvh": "video/vnd.dece.hd", + "uvm": "video/vnd.dece.mobile", + "uvvm": "video/vnd.dece.mobile", + "uvp": "video/vnd.dece.pd", + "uvvp": "video/vnd.dece.pd", + "uvs": "video/vnd.dece.sd", + "uvvs": "video/vnd.dece.sd", + "uvv": "video/vnd.dece.video", + "uvvv": "video/vnd.dece.video", + "dvb": "video/vnd.dvb.file", + "fvt": "video/vnd.fvt", + "mxu": "video/vnd.mpegurl", + "m4u": "video/vnd.mpegurl", + "pyv": "video/vnd.ms-playready.media.pyv", + "uvu": "video/vnd.uvvu.mp4", + "uvvu": "video/vnd.uvvu.mp4", + "viv": "video/vnd.vivo", + "webm": "video/webm", + "f4v": "video/x-f4v", + "fli": "video/x-fli", + "flv": "video/x-flv", + "m4v": "video/x-m4v", + "mkv": "video/x-matroska", + "mk3d": "video/x-matroska", + "mks": "video/x-matroska", + "mng": "video/x-mng", + "asf": "video/x-ms-asf", + "asx": "video/x-ms-asf", + "vob": "video/x-ms-vob", + "wm": "video/x-ms-wm", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wvx": "video/x-ms-wvx", + "avi": "video/x-msvideo", + "movie": "video/x-sgi-movie", + "smv": "video/x-smv", + "ice": "x-conference/x-cooltalk", } func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. - result.mimes = mimes.newStringTable() + {.cast(noSideEffect).}: + result.mimes = mimes.toOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = - ## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext`` - ## could not be found. ``ext`` can start with an optional dot which is ignored. - ## ``ext`` is lowercased before querying ``mimedb``. + ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` + ## could not be found. `ext` can start with an optional dot which is ignored. + ## `ext` is lowercased before querying `mimedb`. if ext.startsWith("."): result = mimedb.mimes.getOrDefault(ext.toLowerAscii.substr(1)) else: @@ -1899,9 +1052,9 @@ func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = return default func getExt*(mimedb: MimeDB, mimetype: string, default = "txt"): string = - ## Gets extension which corresponds to ``mimetype``. Returns ``default`` if - ## ``mimetype`` could not be found. Extensions are returned without the - ## leading dot. ``mimetype`` is lowercased before querying ``mimedb``. + ## Gets extension which corresponds to `mimetype`. Returns `default` if + ## `mimetype` could not be found. Extensions are returned without the + ## leading dot. `mimetype` is lowercased before querying `mimedb`. result = default let mimeLowered = mimetype.toLowerAscii() for e, m in mimedb.mimes: @@ -1910,25 +1063,9 @@ func getExt*(mimedb: MimeDB, mimetype: string, default = "txt"): string = break func register*(mimedb: var MimeDB, ext: string, mimetype: string) = - ## Adds ``mimetype`` to the ``mimedb``. - ## ``mimetype`` and ``ext`` are lowercased before registering on ``mimedb``. + ## Adds `mimetype` to the `mimedb`. + ## `mimetype` and `ext` are lowercased before registering on `mimedb`. assert ext.strip.len > 0, "ext argument can not be empty string" assert mimetype.strip.len > 0, "mimetype argument can not be empty string" - mimedb.mimes[ext.toLowerAscii()] = mimetype.toLowerAscii() - -runnableExamples: - var m = newMimetypes() - assert m.getMimetype("mp4") == "video/mp4" - assert m.getExt("text/html") == "html" - ## Values can be uppercase too. - assert m.getMimetype("MP4") == "video/mp4" - assert m.getExt("TEXT/HTML") == "html" - ## If values are invalid then ``default`` is returned. - assert m.getMimetype("INVALID") == "text/plain" - assert m.getExt("INVALID/NONEXISTENT") == "txt" - assert m.getMimetype("") == "text/plain" - assert m.getExt("") == "txt" - ## Register new Mimetypes. - m.register(ext = "fakext", mimetype = "text/fakelang") - assert m.getMimetype("fakext") == "text/fakelang" - assert m.getMimetype("FaKeXT") == "text/fakelang" + {.noSideEffect.}: + mimedb.mimes[ext.toLowerAscii()] = mimetype.toLowerAscii() diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index d9256a921..656c98a20 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -8,33 +8,43 @@ # ## This module implements a low-level cross-platform sockets interface. Look -## at the ``net`` module for the higher-level version. +## at the `net` module for the higher-level version. # TODO: Clean up the exports a bit and everything else in general. -import os, options +import std/[os, options] +import std/private/since +import std/strbasics + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] when hostOS == "solaris": {.passl: "-lsocket -lnsl".} -const useWinVersion = defined(Windows) or defined(nimdoc) +const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: - import winlean + import std/winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, WSANOTINITIALISED, WSAENOTSOCK, WSAEINPROGRESS, WSAEINTR, WSAEDISCON, ERROR_NETNAME_DELETED else: - import posix + import std/posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, EINTR, EINPROGRESS, ECONNRESET, EPIPE, ENETRESET, EBADF export Sockaddr_storage, Sockaddr_un, Sockaddr_un_path_length export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, Sockaddr_storage, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, + recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo +when not useNimNetLite: + export inet_ntoa + export SO_ERROR, SOL_SOCKET, @@ -57,7 +67,7 @@ type ## some procedures, such as getaddrinfo) AF_UNIX = 1, ## for local socket (using a file). Unsupported on Windows. AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = when defined(macosx): 30 else: 23 ## for network protocol IPv6. + AF_INET6 = when defined(macosx): 30 elif defined(windows): 23 else: 10 ## for network protocol IPv6. SockType* = enum ## second argument to `socket` proc SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets @@ -68,11 +78,11 @@ type Protocol* = enum ## third argument to `socket` proc IPPROTO_TCP = 6, ## Transmission control protocol. IPPROTO_UDP = 17, ## User datagram protocol. - IPPROTO_IP, ## Internet protocol. Unsupported on Windows. - IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. + IPPROTO_IP, ## Internet protocol. + IPPROTO_IPV6, ## Internet Protocol Version 6. IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. - IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. - IPPROTO_ICMPV6 ## Control message protocol for IPv6. Unsupported on Windows. + IPPROTO_ICMP ## Internet Control message protocol. + IPPROTO_ICMPV6 ## Internet Control message protocol for IPv6. Servent* = object ## information about a service name*: string @@ -87,6 +97,8 @@ type length*: int addrList*: seq[string] +const IPPROTO_NONE* = IPPROTO_IP ## Use this if your socket type requires a protocol value of zero (e.g. Unix sockets). + when useWinVersion: let osInvalidSocket* = winlean.INVALID_SOCKET @@ -110,19 +122,19 @@ else: nativeAfUnix = posix.AF_UNIX proc `==`*(a, b: Port): bool {.borrow.} - ## ``==`` for ports. + ## `==` for ports. proc `$`*(p: Port): string {.borrow.} - ## returns the port number as a string + ## Returns the port number as a string proc toInt*(domain: Domain): cint - ## Converts the Domain enum to a platform-dependent ``cint``. + ## Converts the Domain enum to a platform-dependent `cint`. proc toInt*(typ: SockType): cint - ## Converts the SockType enum to a platform-dependent ``cint``. + ## Converts the SockType enum to a platform-dependent `cint`. proc toInt*(p: Protocol): cint - ## Converts the Protocol enum to a platform-dependent ``cint``. + ## Converts the Protocol enum to a platform-dependent `cint`. when not useWinVersion: proc toInt(domain: Domain): cint = @@ -133,8 +145,8 @@ when not useWinVersion: of AF_INET6: result = posix.AF_INET6.cint proc toKnownDomain*(family: cint): Option[Domain] = - ## Converts the platform-dependent ``cint`` to the Domain or none(), - ## if the ``cint`` is not known. + ## Converts the platform-dependent `cint` to the Domain or none(), + ## if the `cint` is not known. result = if family == posix.AF_UNSPEC: some(Domain.AF_UNSPEC) elif family == posix.AF_UNIX: some(Domain.AF_UNIX) elif family == posix.AF_INET: some(Domain.AF_INET) @@ -160,11 +172,11 @@ when not useWinVersion: else: proc toInt(domain: Domain): cint = - result = toU32(ord(domain)).cint + result = cast[cint](uint32(ord(domain))) proc toKnownDomain*(family: cint): Option[Domain] = - ## Converts the platform-dependent ``cint`` to the Domain or none(), - ## if the ``cint`` is not known. + ## Converts the platform-dependent `cint` to the Domain or none(), + ## if the `cint` is not known. result = if family == winlean.AF_UNSPEC: some(Domain.AF_UNSPEC) elif family == winlean.AF_INET: some(Domain.AF_INET) elif family == winlean.AF_INET6: some(Domain.AF_INET6) @@ -174,7 +186,21 @@ else: result = cint(ord(typ)) proc toInt(p: Protocol): cint = - result = cint(ord(p)) + case p + of IPPROTO_IP: + result = 0.cint + of IPPROTO_ICMP: + result = 1.cint + of IPPROTO_TCP: + result = 6.cint + of IPPROTO_UDP: + result = 17.cint + of IPPROTO_IPV6: + result = 41.cint + of IPPROTO_ICMPV6: + result = 58.cint + else: + result = cint(ord(p)) proc toSockType*(protocol: Protocol): SockType = result = case protocol @@ -185,38 +211,20 @@ proc toSockType*(protocol: Protocol): SockType = of IPPROTO_IP, IPPROTO_IPV6, IPPROTO_RAW, IPPROTO_ICMP, IPPROTO_ICMPV6: SOCK_RAW -proc createNativeSocket*(domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): SocketHandle = - ## Creates a new socket; returns `osInvalidSocket` if an error occurs. - socket(toInt(domain), toInt(sockType), toInt(protocol)) +proc getProtoByName*(name: string): int {.since: (1, 3, 5).} = + ## Returns a protocol code from the database that matches the protocol `name`. + when useWinVersion: + let protoent = winlean.getprotobyname(name.cstring) + else: + let protoent = posix.getprotobyname(name.cstring) -proc createNativeSocket*(domain: cint, sockType: cint, - protocol: cint): SocketHandle = - ## Creates a new socket; returns `osInvalidSocket` if an error occurs. - ## - ## Use this overload if one of the enums specified above does - ## not contain what you need. - socket(domain, sockType, protocol) + if protoent == nil: + raise newException(OSError, "protocol not found: " & name) -proc newNativeSocket*(domain: Domain = AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): SocketHandle - {.deprecated: "deprecated since v0.18.0; use 'createNativeSocket' instead".} = - ## Creates a new socket; returns `osInvalidSocket` if an error occurs. - createNativeSocket(domain, sockType, protocol) - -proc newNativeSocket*(domain: cint, sockType: cint, - protocol: cint): SocketHandle - {.deprecated: "deprecated since v0.18.0; use 'createNativeSocket' instead".} = - ## Creates a new socket; returns `osInvalidSocket` if an error occurs. - ## - ## Use this overload if one of the enums specified above does - ## not contain what you need. - createNativeSocket(domain, sockType, protocol) + result = protoent.p_proto.int proc close*(socket: SocketHandle) = - ## closes a socket. + ## Closes a socket. when useWinVersion: discard winlean.closesocket(socket) else: @@ -224,14 +232,53 @@ proc close*(socket: SocketHandle) = # TODO: These values should not be discarded. An OSError should be raised. # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times +when declared(setInheritable) or defined(nimdoc): + proc setInheritable*(s: SocketHandle, inheritable: bool): bool {.inline.} = + ## Set whether a socket is inheritable by child processes. Returns `true` + ## on success. + ## + ## This function is not implemented on all platform, test for availability + ## with `declared() <system.html#declared,untyped>`. + setInheritable(FileHandle s, inheritable) + +proc createNativeSocket*(domain: cint, sockType: cint, protocol: cint, + inheritable: bool = defined(nimInheritHandles)): SocketHandle = + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. + ## + ## `inheritable` decides if the resulting SocketHandle can be inherited + ## by child processes. + ## + ## Use this overload if one of the enums specified above does + ## not contain what you need. + let sockType = + when (defined(linux) or defined(bsd)) and not defined(nimdoc): + if inheritable: sockType and not SOCK_CLOEXEC else: sockType or SOCK_CLOEXEC + else: + sockType + result = socket(domain, sockType, protocol) + when declared(setInheritable) and not (defined(linux) or defined(bsd)): + if not setInheritable(result, inheritable): + close result + return osInvalidSocket + +proc createNativeSocket*(domain: Domain = AF_INET, + sockType: SockType = SOCK_STREAM, + protocol: Protocol = IPPROTO_TCP, + inheritable: bool = defined(nimInheritHandles)): SocketHandle = + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. + ## + ## `inheritable` decides if the resulting SocketHandle can be inherited + ## by child processes. + createNativeSocket(toInt(domain), toInt(sockType), toInt(protocol), inheritable) + proc bindAddr*(socket: SocketHandle, name: ptr SockAddr, namelen: SockLen): cint = result = bindSocket(socket, name, namelen) proc listen*(socket: SocketHandle, backlog = SOMAXCONN): cint {.tags: [ ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the + ## Marks `socket` as accepting connections. + ## `Backlog` specifies the maximum length of the ## queue of pending connections. when useWinVersion: result = winlean.listen(socket, cint(backlog)) @@ -243,7 +290,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, protocol: Protocol = IPPROTO_TCP): ptr AddrInfo = ## ## - ## **Warning**: The resulting ``ptr AddrInfo`` must be freed using ``freeAddrInfo``! + ## .. warning:: The resulting `ptr AddrInfo` must be freed using `freeAddrInfo`! var hints: AddrInfo result = nil hints.ai_family = toInt(domain) @@ -258,9 +305,9 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED let socketPort = if sockType == SOCK_RAW: "" else: $port - var gaiResult = getaddrinfo(address, socketPort, addr(hints), result) + var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion: + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) @@ -294,258 +341,396 @@ template htons*(x: uint16): untyped = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by ``name`` matches the s_name member - ## and the protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by ``port`` matches the s_port member and the - ## protocol name specified by ``proto`` matches the s_proto member. - ## - ## On posix this will search through the ``/etc/services`` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = - when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) - else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).SockLen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostname*(): string {.tags: [ReadIOEffect].} = - ## Returns the local hostname (not the FQDN) - # https://tools.ietf.org/html/rfc1035#section-2.3.1 - # https://tools.ietf.org/html/rfc2181#section-11 - const size = 64 - result = newString(size) - when useWinVersion: - let success = winlean.gethostname(result, size) - else: - # Posix - let success = posix.gethostname(result, size) - if success != 0.cint: - raiseOSError(osLastError()) - let x = len(cstring(result)) - result.setLen(x) - proc getSockDomain*(socket: SocketHandle): Domain = - ## returns the socket's domain (AF_INET or AF_INET6). + ## Returns the socket's domain (AF_INET or AF_INET6). var name: Sockaddr_in6 var namelen = sizeof(name).SockLen if getsockname(socket, cast[ptr SockAddr](addr(name)), addr(namelen)) == -1'i32: raiseOSError(osLastError()) - try: - result = toKnownDomain(name.sin6_family.cint).get() - except UnpackError: + let knownDomain = toKnownDomain(name.sin6_family.cint) + if knownDomain.isSome: + result = knownDomain.get() + else: raise newException(IOError, "Unknown socket family in getSockDomain") -proc getAddrString*(sockAddr: ptr SockAddr): string = - ## return the string representation of address within sockAddr - if sockAddr.sa_family.cint == nativeAfInet: - result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN - else: 46 # it's actually 46 in both cases - result = newString(addrLen) - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result = result.substr("::ffff:".len) +when not useNimNetLite: + proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the service name specified by `name` matches the s_name member + ## and the protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. + when useWinVersion: + var s = winlean.getservbyname(name, proto) else: - if winlean.inet_ntop(winlean.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - setLen(result, len(cstring(result))) - else: - when defined(posix) and not defined(nimdoc): - if sockAddr.sa_family.cint == nativeAfUnix: - return "unix" - raise newException(IOError, "Unknown socket family in getAddrString") - -when defined(posix) and not defined(nimdoc): - proc makeUnixAddr*(path: string): Sockaddr_un = - result.sun_family = AF_UNIX.TSa_Family - if path.len >= Sockaddr_un_path_length: - raise newException(ValueError, "socket path too long") - copyMem(addr result.sun_path, path.cstring, path.len + 1) - -proc getSockName*(socket: SocketHandle): Port = - ## returns the socket's associated port number. - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(nativesockets.ntohs(name.sin_port)) + var s = posix.getservbyname(name, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = + ## Searches the database from the beginning and finds the first entry for + ## which the port specified by `port` matches the s_port member and the + ## protocol name specified by `proto` matches the s_proto member. + ## + ## On posix this will search through the `/etc/services` file. + when useWinVersion: + var s = winlean.getservbyport(uint16(port).cint, proto) + else: + var s = posix.getservbyport(uint16(port).cint, proto) + if s == nil: raiseOSError(osLastError(), "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = Port(s.s_port) + result.proto = $s.s_proto + + proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the hostname of an IP Address. + var + addrInfo = getAddrInfo(ip, Port(0), AF_UNSPEC) + myAddr: pointer + addrLen = 0 + family = 0 + + defer: freeAddrInfo(addrInfo) + + if addrInfo.ai_addr.sa_family.cint == nativeAfInet: + family = nativeAfInet + myAddr = addr cast[ptr Sockaddr_in](addrInfo.ai_addr).sin_addr + addrLen = 4 + elif addrInfo.ai_addr.sa_family.cint == nativeAfInet6: + family = nativeAfInet6 + myAddr = addr cast[ptr Sockaddr_in6](addrInfo.ai_addr).sin6_addr + addrLen = 16 + else: + raise newException(IOError, "Unknown socket family in `getHostByAddr()`") -proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## returns the socket's local address and port number. - ## - ## Similar to POSIX's `getsockname`:idx:. - case domain - of AF_INET: - var name: Sockaddr_in when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) + var s = winlean.gethostbyaddr(cast[ptr InAddr](myAddr), addrLen.cuint, + cint(family)) + if s == nil: raiseOSError(osLastError()) else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 + var s = + when defined(android4): + posix.gethostbyaddr(cast[cstring](myAddr), addrLen.cint, + cint(family)) + else: + posix.gethostbyaddr(myAddr, addrLen.SockLen, + cint(family)) + if s == nil: + raiseOSError(osLastError(), $hstrerror(h_errno)) + + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + result.addrtype = Domain(s.h_addrtype) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + let strAddrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 + var i = 0 + while not isNil(s.h_addr_list[i]): + var ipStr = newString(strAddrLen) + if inet_ntop(nativeAfInet6, cast[pointer](s.h_addr_list[i]), + cstring(ipStr), len(ipStr).int32) == nil: + raiseOSError(osLastError()) + when not useWinVersion: + if posix.IN6_IS_ADDR_V4MAPPED(cast[ptr In6Addr](s.h_addr_list[i])) != 0: + ipStr.setSlice("::ffff:".len..<strAddrLen) + setLen(ipStr, len(cstring(ipStr))) + result.addrList.add(ipStr) + inc(i) + result.length = int(s.h_length) + + proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = + ## This function will lookup the IP address of a hostname. + when useWinVersion: + var s = winlean.gethostbyname(name) + else: + var s = posix.gethostbyname(name) + if s == nil: raiseOSError(osLastError()) + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when useWinVersion: + result.addrtype = Domain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raiseOSError(osLastError(), "unknown h_addrtype") + if result.addrtype == AF_INET: + result.addrList = @[] + var i = 0 + while not isNil(s.h_addr_list[i]): + var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) + result.addrList.add($inet_ntoa(inaddrPtr[])) + inc(i) + else: + result.addrList = cstringArrayToSeq(s.h_addr_list) + result.length = int(s.h_length) + + proc getHostname*(): string {.tags: [ReadIOEffect].} = + ## Returns the local hostname (not the FQDN) + # https://tools.ietf.org/html/rfc1035#section-2.3.1 + # https://tools.ietf.org/html/rfc2181#section-11 + const size = 256 + result = newString(size) + when useWinVersion: + let success = winlean.gethostname(result.cstring, size) + else: + # Posix + let success = posix.gethostname(result.cstring, size) + if success != 0.cint: raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") - -proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## returns the socket's peer address and port number. - ## - ## Similar to POSIX's `getpeername`:idx: - case domain - of AF_INET: + let x = len(cstring(result)) + result.setLen(x) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + ## Returns the string representation of address within sockAddr + if sockAddr.sa_family.cint == nativeAfInet: + result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int + else: 46 # it's actually 46 in both cases + result = newString(addrLen) + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr result[0]), + result.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + result.setSlice("::ffff:".len..<addrLen) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr result[0]), + result.len.int32) == nil: + raiseOSError(osLastError()) + setLen(result, len(cstring(result))) + else: + when defined(posix) and not defined(nimdoc): + if sockAddr.sa_family.cint == nativeAfUnix: + return "unix" + raise newException(IOError, "Unknown socket family in getAddrString") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) = + ## Stores in `strAddress` the string representation of the address inside + ## `sockAddr` + ## + ## **Note** + ## * `strAddress` must be initialized to 46 in length. + const length = 46 + assert(length == len(strAddress), + "`strAddress` was not initialized correctly. 46 != `len(strAddress)`") + if sockAddr.sa_family.cint == nativeAfInet: + let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET, addr4, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + if winlean.inet_ntop(winlean.AF_INET, addr4, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + strAddress.setSlice("::ffff:".len..<length) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, cast[cstring](addr strAddress[0]), + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + raise newException(IOError, "Unknown socket family in getAddrString") + setLen(strAddress, len(cstring(strAddress))) + + when defined(posix) and not defined(nimdoc): + proc makeUnixAddr*(path: string): Sockaddr_un = + result.sun_family = AF_UNIX.TSa_Family + if path.len >= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + + proc getSockName*(socket: SocketHandle): Port = + ## Returns the socket's associated port number. var name: Sockaddr_in when useWinVersion: name.sin_family = uint16(ord(AF_INET)) else: name.sin_family = TSa_Family(posix.AF_INET) + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + result = Port(nativesockets.ntohs(name.sin_port)) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's peer address and port number. + ## + ## Similar to POSIX's `getpeername`:idx: + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, cast[cstring](addr result[0][0]), (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + +when useNimNetLite: + + when useWinVersion: + const + INET_ADDRSTRLEN = 16 + INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases + + proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} = + let af_family = sa.sa_family + var nl, v4Slice: cint + var si_addr: ptr InAddr + + if af_family == AF_INET.TSa_Family: + nl = INET_ADDRSTRLEN + si_addr = cast[ptr Sockaddr_in](sa).sin_addr.addr() + elif af_family == AF_INET6.TSa_Family: + nl = INET6_ADDRSTRLEN + let si6_addr = cast[ptr Sockaddr_in6](sa).sin6_addr.addr() + si_addr = cast[ptr InAddr](si6_addr) # let's us reuse logic below + when defined(posix) and not defined(nimdoc) and not defined(zephyr): + if posix.IN6_IS_ADDR_V4MAPPED(si6_addr) != 0: + v4Slice = "::ffff:".len() + else: + when defined(posix) and not defined(nimdoc): + if af_family.cint == nativeAfUnix: + return "unix" + return "" + + result = newString(nl) + let namePtr = result.cstring() + if namePtr == inet_ntop(af_family.cint, si_addr, namePtr, nl): + result.setLen(len(namePtr)) + if v4Slice > 0: result.setSlice(v4Slice.int ..< nl.int) + else: + return "" + + proc sockAddrToStr(sa: var Sockaddr_in | var Sockaddr_in6): string = + result = sockAddrToStr(cast[ptr SockAddr](unsafeAddr(sa))) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + result = sockAddrToStr(sockAddr) + if result.len() == 0: raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) {.noinit.} = + strAddress = getAddrString(sockAddr) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + template sockGetNameOrRaiseError(socket: untyped, name: untyped) = + var namelen = sizeof(socket).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + case domain + of AF_INET: + var name = Sockaddr_in(sin_family: TSa_Family(posix.AF_INET)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name = Sockaddr_in6(sin6_family: TSa_Family(posix.AF_INET6)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin6_port))) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. tags: [ReadIOEffect].} = @@ -610,12 +795,12 @@ proc pruneSocketSet(s: var seq[SocketHandle], fd: var TFdSet) = setLen(s, L) proc selectRead*(readfds: var seq[SocketHandle], timeout = 500): int = - ## When a socket in ``readfds`` is ready to be read from then a non-zero + ## When a socket in `readfds` is ready to be read from then a non-zero ## value will be returned specifying the count of the sockets which can be - ## read from. The sockets which can be read from will also be removed - ## from ``readfds``. + ## read from. The sockets which cannot be read from will also be removed + ## from `readfds`. ## - ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for + ## `timeout` is specified in milliseconds and `-1` can be specified for ## an unlimited time. var tv {.noinit.}: Timeval = timeValFromMilliseconds(timeout) @@ -632,12 +817,12 @@ proc selectRead*(readfds: var seq[SocketHandle], timeout = 500): int = proc selectWrite*(writefds: var seq[SocketHandle], timeout = 500): int {.tags: [ReadIOEffect].} = - ## When a socket in ``writefds`` is ready to be written to then a non-zero + ## When a socket in `writefds` is ready to be written to then a non-zero ## value will be returned specifying the count of the sockets which can be - ## written to. The sockets which can be written to will also be removed - ## from ``writefds``. + ## written to. The sockets which cannot be written to will also be removed + ## from `writefds`. ## - ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for + ## `timeout` is specified in milliseconds and `-1` can be specified for ## an unlimited time. var tv {.noinit.}: Timeval = timeValFromMilliseconds(timeout) @@ -652,19 +837,34 @@ proc selectWrite*(writefds: var seq[SocketHandle], pruneSocketSet(writefds, (wr)) -proc accept*(fd: SocketHandle): (SocketHandle, string) = +proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (SocketHandle, string) = ## Accepts a new client connection. ## + ## `inheritable` decides if the resulting SocketHandle can be inherited by + ## child processes. + ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr_in + var sockAddress: SockAddr var addrLen = sizeof(sockAddress).SockLen - var sock = accept(fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) + var sock = + when (defined(linux) or defined(bsd)) and not defined(nimdoc): + accept4(fd, addr(sockAddress), addr(addrLen), + if inheritable: 0 else: SOCK_CLOEXEC) + else: + accept(fd, addr(sockAddress), addr(addrLen)) + when declared(setInheritable) and not (defined(linux) or defined(bsd)): + if not setInheritable(sock, inheritable): + close sock + sock = osInvalidSocket if sock == osInvalidSocket: return (osInvalidSocket, "") else: - return (sock, $inet_ntoa(sockAddress.sin_addr)) + when useNimNetLite: + var name = sockAddrToStr(addr sockAddress) + return (sock, name) + else: + return (sock, $inet_ntoa(cast[Sockaddr_in](sockAddress).sin_addr)) -when defined(Windows): +when defined(windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 0d4440242..24c94b651 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -9,18 +9,31 @@ ## This module implements a high-level cross-platform sockets interface. ## The procedures implemented in this module are primarily for blocking sockets. -## For asynchronous non-blocking sockets use the ``asyncnet`` module together -## with the ``asyncdispatch`` module. +## For asynchronous non-blocking sockets use the `asyncnet` module together +## with the `asyncdispatch` module. ## ## The first thing you will always need to do in order to start using sockets, -## is to create a new instance of the ``Socket`` type using the ``newSocket`` +## is to create a new instance of the `Socket` type using the `newSocket` ## procedure. ## ## SSL ## ==== ## ## In order to use the SSL procedures defined in this module, you will need to -## compile your application with the ``-d:ssl`` flag. +## compile your application with the `-d:ssl` flag. See the +## `newContext<net.html#newContext%2Cstring%2Cstring%2Cstring%2Cstring>`_ +## procedure for additional details. +## +## +## SSL on Windows +## ============== +## +## On Windows the SSL library checks for valid certificates. +## It uses the `cacert.pem` file for this purpose which was extracted +## from `https://curl.se/ca/cacert.pem`. Besides +## the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you +## also need to ship `cacert.pem` with your `.exe` file. +## ## ## Examples ## ======== @@ -28,62 +41,87 @@ ## Connecting to a server ## ---------------------- ## -## After you create a socket with the ``newSocket`` procedure, you can easily +## After you create a socket with the `newSocket` procedure, you can easily ## connect it to a server running at a known hostname (or IP address) and port. ## To do so over TCP, use the example below. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.connect("google.com", Port(80)) -## + +runnableExamples("-r:off"): + let socket = newSocket() + socket.connect("google.com", Port(80)) + +## For SSL, use the following example: + +runnableExamples("-r:off -d:ssl"): + let socket = newSocket() + let ctx = newContext() + wrapSocket(ctx, socket) + socket.connect("google.com", Port(443)) + ## UDP is a connectionless protocol, so UDP sockets don't have to explicitly -## call the ``connect`` procedure. They can simply start sending data -## immediately. -## -## .. code-block:: Nim -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.sendTo("192.168.0.1", Port(27960), "status\n") -## +## call the `connect <net.html#connect%2CSocket%2Cstring>`_ procedure. They can +## simply start sending data immediately. + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + socket.sendTo("192.168.0.1", Port(27960), "status\n") + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + let ip = parseIpAddress("192.168.0.1") + doAssert socket.sendTo(ip, Port(27960), "status\c\l") == 8 + ## Creating a server ## ----------------- ## -## After you create a socket with the ``newSocket`` procedure, you can create a -## TCP server by calling the ``bindAddr`` and ``listen`` procedures. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.bindAddr(Port(1234)) -## socket.listen() -## -## You can then begin accepting connections using the ``accept`` procedure. -## -## .. code-block:: Nim -## var client: Socket -## var address = "" -## while true: -## socket.acceptAddr(client, address) -## echo("Client connected from: ", address) - -{.deadCodeElim: on.} # dce option deprecated -import nativesockets, os, strutils, parseutils, times, sets, options, - std/monotimes +## After you create a socket with the `newSocket` procedure, you can create a +## TCP server by calling the `bindAddr` and `listen` procedures. + +runnableExamples("-r:off"): + let socket = newSocket() + socket.bindAddr(Port(1234)) + socket.listen() + + # You can then begin accepting connections using the `accept` procedure. + var client: Socket + var address = "" + while true: + socket.acceptAddr(client, address) + echo "Client connected from: ", address + +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + +import std/nativesockets +import std/[os, strutils, times, sets, options, monotimes] +import std/ssl_config export nativesockets.Port, nativesockets.`$`, nativesockets.`==` -export Domain, SockType, Protocol +export Domain, SockType, Protocol, IPPROTO_NONE -const useWinVersion = defined(Windows) or defined(nimdoc) +const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) const defineSsl = defined(ssl) or defined(nimdoc) +when useWinVersion: + from std/winlean import WSAESHUTDOWN + when defineSsl: - import openssl + import std/openssl + when not defined(nimDisableCertificateValidation): + from std/ssl_certs import scanSSLCertificates # Note: The enumerations are mapped to Window's constants. when defineSsl: type - SslError* = object of Exception + Certificate* = string ## DER encoded certificate + + SslError* = object of CatchableError SslCVerifyMode* = enum - CVerifyNone, CVerifyPeer + CVerifyNone, CVerifyPeer, CVerifyPeerUseEnvVars SslProtVersion* = enum protSSLv2, protSSLv3, protTLSv1, protSSLv23 @@ -109,7 +147,7 @@ when defineSsl: else: type - SslContext* = void # TODO: Workaround #4797. + SslContext* = ref object # TODO: Workaround #4797. const BufferSize*: int = 4000 ## size of a buffered socket's buffer @@ -129,6 +167,7 @@ type sslNoHandshake: bool # True if needs handshake. sslHasPeekChar: bool sslPeekChar: char + sslNoShutdown: bool # True if shutdown shouldn't be done. lastError: OSErrorCode ## stores the last error on this socket domain: Domain sockType: SockType @@ -143,7 +182,7 @@ type ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone - TimeoutError* = object of Exception + TimeoutError* = object of CatchableError SocketFlag* {.pure.} = enum Peek, @@ -168,20 +207,51 @@ type when defined(nimHasStyleChecks): {.pop.} + +when defined(posix) and not defined(lwip): + from std/posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds + + template monitorPollEvent(x: var SocketHandle, y: cint, timeout: int): int = + var tpollfd: TPollfd + tpollfd.fd = cast[cint](x) + tpollfd.events = y + posix.poll(addr(tpollfd), Tnfds(1), timeout) + +proc timeoutRead(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectRead(fds, timeout) + else: + monitorPollEvent(fd, POLLIN or POLLPRI, timeout) + +proc timeoutWrite(fd: var SocketHandle, timeout = 500): int = + when defined(windows) or defined(lwip): + var fds = @[fd] + selectWrite(fds, timeout) + else: + monitorPollEvent(fd, POLLOUT or POLLWRBAND, timeout) + proc socketError*(socket: Socket, err: int = -1, async = false, - lastError = (-1).OSErrorCode): void {.gcsafe.} + lastError = (-1).OSErrorCode, + flags: set[SocketFlag] = {}) {.gcsafe.} proc isDisconnectionError*(flags: set[SocketFlag], lastError: OSErrorCode): bool = - ## Determines whether ``lastError`` is a disconnection error. Only does this - ## if flags contains ``SafeDisconn``. + ## Determines whether `lastError` is a disconnection error. Only does this + ## if flags contains `SafeDisconn`. when useWinVersion: SocketFlag.SafeDisconn in flags and - lastError.int32 in {WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, - WSAEDISCON, ERROR_NETNAME_DELETED} + (lastError.int32 == WSAECONNRESET or + lastError.int32 == WSAECONNABORTED or + lastError.int32 == WSAENETRESET or + lastError.int32 == WSAEDISCON or + lastError.int32 == WSAESHUTDOWN or + lastError.int32 == ERROR_NETNAME_DELETED) else: SocketFlag.SafeDisconn in flags and - lastError.int32 in {ECONNRESET, EPIPE, ENETRESET} + (lastError.int32 == ECONNRESET or + lastError.int32 == EPIPE or + lastError.int32 == ENETRESET) proc toOSFlags*(socketFlags: set[SocketFlag]): cint = ## Converts the flags into the underlying OS representation. @@ -209,22 +279,32 @@ proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET, when defined(macosx) and not defined(nimdoc): setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1) -proc newSocket*(domain, sockType, protocol: cint, buffered = true): owned(Socket) = +proc newSocket*(domain, sockType, protocol: cint, buffered = true, + inheritable = defined(nimInheritHandles)): owned(Socket) = ## Creates a new socket. ## + ## The SocketHandle associated with the resulting Socket will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. + ## ## If an error occurs OSError will be raised. - let fd = createNativeSocket(domain, sockType, protocol) + let fd = createNativeSocket(domain, sockType, protocol, inheritable) if fd == osInvalidSocket: raiseOSError(osLastError()) result = newSocket(fd, domain.Domain, sockType.SockType, protocol.Protocol, buffered) proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP, buffered = true): owned(Socket) = + protocol: Protocol = IPPROTO_TCP, buffered = true, + inheritable = defined(nimInheritHandles)): owned(Socket) = ## Creates a new socket. ## + ## The SocketHandle associated with the resulting Socket will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. + ## ## If an error occurs OSError will be raised. - let fd = createNativeSocket(domain, sockType, protocol) + let fd = createNativeSocket(domain, sockType, protocol, inheritable) if fd == osInvalidSocket: raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) @@ -236,14 +316,20 @@ proc parseIPv4Address(addressStr: string): IpAddress = byteCount = 0 currentByte: uint16 = 0 separatorValid = false + leadingZero = false result = IpAddress(family: IpAddressFamily.IPv4) for i in 0 .. high(addressStr): if addressStr[i] in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers are not allowed") currentByte = currentByte * 10 + cast[uint16](ord(addressStr[i]) - ord('0')) - if currentByte > 255'u16: + if currentByte == 0'u16: + leadingZero = true + elif currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -255,6 +341,7 @@ proc parseIPv4Address(addressStr: string): IpAddress = currentByte = 0 byteCount.inc separatorValid = false + leadingZero = false else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -343,10 +430,16 @@ proc parseIPv6Address(addressStr: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) groupCount.inc() else: # Must parse IPv4 address + var leadingZero = false for i, c in addressStr[v4StartPos..high(addressStr)]: if c in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers not allowed") currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: + if currentShort == 0'u32: + leadingZero = true + elif currentShort > 255'u32: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -357,6 +450,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = currentShort = 0 byteCount.inc() separatorValid = false + leadingZero = false else: # Invalid character raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -386,7 +480,12 @@ proc parseIPv6Address(addressStr: string): IpAddress = proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address - ## Raises ValueError on error + ## + ## Raises ValueError on error. + ## + ## For IPv4 addresses, only the strict form as + ## defined in RFC 6943 is considered valid, see + ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. if addressStr.len == 0: raise newException(ValueError, "IP Address string is empty") if addressStr.contains(':'): @@ -411,14 +510,14 @@ proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, of IpAddressFamily.IPv4: sl = sizeof(Sockaddr_in).SockLen let s = cast[ptr Sockaddr_in](addr sa) - s.sin_family = type(s.sin_family)(toInt(AF_INET)) + s.sin_family = typeof(s.sin_family)(toInt(AF_INET)) s.sin_port = port copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr)) of IpAddressFamily.IPv6: sl = sizeof(Sockaddr_in6).SockLen let s = cast[ptr Sockaddr_in6](addr sa) - s.sin6_family = type(s.sin6_family)(toInt(AF_INET6)) + s.sin6_family = typeof(s.sin6_family)(toInt(AF_INET6)) s.sin6_port = port copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr)) @@ -444,25 +543,30 @@ proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: SockLen, proc fromSockAddr*(sa: Sockaddr_storage | SockAddr | Sockaddr_in | Sockaddr_in6, sl: SockLen, address: var IpAddress, port: var Port) {.inline.} = ## Converts `SockAddr` and `SockLen` to `IpAddress` and `Port`. Raises - ## `ObjectConversionError` in case of invalid `sa` and `sl` arguments. + ## `ObjectConversionDefect` in case of invalid `sa` and `sl` arguments. fromSockAddrAux(cast[ptr Sockaddr_storage](unsafeAddr sa), sl, address, port) when defineSsl: - CRYPTO_malloc_init() - doAssert SslLibraryInit() == 1 - SSL_load_error_strings() - ERR_load_BIO_strings() - OpenSSL_add_all_algorithms() - - proc raiseSSLError*(s = "") = + # OpenSSL >= 1.1.0 does not need explicit init. + when not useOpenssl3: + CRYPTO_malloc_init() + doAssert SslLibraryInit() == 1 + SSL_load_error_strings() + ERR_load_BIO_strings() + OpenSSL_add_all_algorithms() + + proc sslHandle*(self: Socket): SslPtr = + ## Retrieve the ssl pointer of `socket`. + ## Useful for interfacing with `openssl`. + self.sslHandle + + proc raiseSSLError*(s = "") {.raises: [SslError].}= ## Raises a new SSL error. if s != "": raise newException(SslError, s) let err = ERR_peek_last_error() if err == 0: raise newException(SslError, "No error reported.") - if err == -1: - raiseOSError(osLastError()) var errStr = $ERR_error_string(err, nil) case err of 336032814, 336032784: @@ -475,7 +579,7 @@ when defineSsl: proc getExtraData*(ctx: SslContext, index: int): RootRef = ## Retrieves arbitrary data stored inside SslContext. if index notin ctx.referencedData: - raise newException(IndexError, "No data with that index.") + raise newException(IndexDefect, "No data with that index.") let res = ctx.context.SSL_CTX_get_ex_data(index.cint) if cast[int](res) == 0: raiseSSLError() @@ -496,10 +600,10 @@ when defineSsl: # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html proc loadCertificates(ctx: SslCtx, certFile, keyFile: string) = - if certFile != "" and not existsFile(certFile): + if certFile != "" and not fileExists(certFile): raise newException(system.IOError, "Certificate file could not be found: " & certFile) - if keyFile != "" and not existsFile(keyFile): + if keyFile != "" and not fileExists(keyFile): raise newException(system.IOError, "Key file could not be found: " & keyFile) if certFile != "": @@ -517,51 +621,115 @@ when defineSsl: raiseSSLError("Verification of private key file failed.") proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, - certFile = "", keyFile = "", cipherList = "ALL"): SslContext = + certFile = "", keyFile = "", cipherList = CiphersIntermediate, + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## - ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 - ## are available with the addition of ``protSSLv23`` which allows for - ## compatibility with all of them. + ## Protocol version is currently ignored by default and TLS is used. + ## With `-d:openssl10`, only SSLv23 and TLSv1 may be used. + ## + ## There are three options for verify mode: + ## `CVerifyNone`: certificates are not verified; + ## `CVerifyPeer`: certificates are verified; + ## `CVerifyPeerUseEnvVars`: certificates are verified and the optional + ## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to + ## locate certificates + ## + ## The `nimDisableCertificateValidation` define overrides verifyMode and + ## disables certificate verification globally! + ## + ## CA certificates will be loaded, in the following order, from: ## - ## There are currently only two options for verify mode; - ## one is ``CVerifyNone`` and with it certificates will not be verified - ## the other is ``CVerifyPeer`` and certificates will be verified for - ## it, ``CVerifyPeer`` is the safest choice. + ## - caFile, caDir, parameters, if set + ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`, + ## the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used + ## - a set of files and directories from the `ssl_certs <ssl_certs.html>`_ file. ## ## The last two parameters specify the certificate file path and the key file ## path, a server socket will most likely not work without these. ## ## Certificates can be generated using the following command: - ## - ``openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout mykey.pem -out mycert.pem`` + ## - `openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout mykey.pem -out mycert.pem` ## or using ECDSA: - ## - ``openssl ecparam -out mykey.pem -name secp256k1 -genkey`` - ## - ``openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem`` - var newCTX: SslCtx - case protVersion - of protSSLv23: - newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. - of protSSLv2: - raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") - of protSSLv3: - raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") - of protTLSv1: - newCTX = SSL_CTX_new(TLSv1_method()) + ## - `openssl ecparam -out mykey.pem -name secp256k1 -genkey` + ## - `openssl req -new -key mykey.pem -x509 -nodes -days 365 -out mycert.pem` + var mtd: PSSL_METHOD + when defined(openssl10): + case protVersion + of protSSLv23: + mtd = SSLv23_method() + of protSSLv2: + raiseSSLError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") + of protSSLv3: + raiseSSLError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") + of protTLSv1: + mtd = TLSv1_method() + else: + mtd = TLS_method() + if mtd == nil: + raiseSSLError("Failed to create TLS context") + var newCTX = SSL_CTX_new(mtd) + if newCTX == nil: + raiseSSLError("Failed to create TLS context") if newCTX.SSL_CTX_set_cipher_list(cipherList) != 1: raiseSSLError() - case verifyMode - of CVerifyPeer: - newCTX.SSL_CTX_set_verify(SSL_VERIFY_PEER, nil) - of CVerifyNone: + when not defined(openssl10) and not defined(libressl): + let sslVersion = getOpenSSLVersion() + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: + # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via + # this API. + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: + raiseSSLError() + # Automatically the best ECDH curve for client exchange. Without this, ECDH + # ciphers will be ignored by the server. + # + # From OpenSSL >= 1.1.0, this setting is set by default and can't be + # overridden. + if newCTX.SSL_CTX_set_ecdh_auto(1) != 1: + raiseSSLError() + + when defined(nimDisableCertificateValidation): newCTX.SSL_CTX_set_verify(SSL_VERIFY_NONE, nil) + else: + case verifyMode + of CVerifyPeer, CVerifyPeerUseEnvVars: + newCTX.SSL_CTX_set_verify(SSL_VERIFY_PEER, nil) + of CVerifyNone: + newCTX.SSL_CTX_set_verify(SSL_VERIFY_NONE, nil) + if newCTX == nil: raiseSSLError() discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) newCTX.loadCertificates(certFile, keyFile) - result = SslContext(context: newCTX, referencedData: initSet[int](), + const VerifySuccess = 1 # SSL_CTX_load_verify_locations returns 1 on success. + + when not defined(nimDisableCertificateValidation): + if verifyMode != CVerifyNone: + # Use the caDir and caFile parameters if set + if caDir != "" or caFile != "": + if newCTX.SSL_CTX_load_verify_locations(if caFile == "": nil else: caFile.cstring, if caDir == "": nil else: caDir.cstring) != VerifySuccess: + raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).") + + else: + # Scan for certs in known locations. For CVerifyPeerUseEnvVars also scan + # the SSL_CERT_FILE and SSL_CERT_DIR env vars + var found = false + let useEnvVars = (if verifyMode == CVerifyPeerUseEnvVars: true else: false) + for fn in scanSSLCertificates(useEnvVars = useEnvVars): + if fn.extractFilename == "": + if newCTX.SSL_CTX_load_verify_locations(nil, cstring(fn.normalizePathEnd(false))) == VerifySuccess: + found = true + break + elif newCTX.SSL_CTX_load_verify_locations(cstring(fn), nil) == VerifySuccess: + found = true + break + if not found: + raise newException(IOError, "No SSL/TLS CA certificates found.") + + result = SslContext(context: newCTX, referencedData: initHashSet[int](), extraInternal: new(SslContextExtraInternal)) proc getExtraInternal(ctx: SslContext): SslContextExtraInternal = @@ -574,7 +742,7 @@ when defineSsl: # That means we can assume that the next internal index is the length of # extra data indexes. for i in ctx.referencedData: - GC_unref(getExtraData(ctx, i).RootRef) + GC_unref(getExtraData(ctx, i)) ctx.context.SSL_CTX_free() proc `pskIdentityHint=`*(ctx: SslContext, hint: string) = @@ -588,17 +756,16 @@ when defineSsl: return ctx.getExtraInternal().clientGetPskFunc proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; - max_identity_len: cuint; psk: ptr cuchar; + max_identity_len: cuint; psk: ptr uint8; max_psk_len: cuint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let hintString = if hint == nil: "" else: $hint let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString) - if psk.len.cuint > max_psk_len: + if pskString.len.cuint > max_psk_len: return 0 if identityString.len.cuint >= max_identity_len: return 0 - - copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte + copyMem(identity, identityString.cstring, identityString.len + 1) # with the last zero byte copyMem(psk, pskString.cstring, pskString.len) return pskString.len.cuint @@ -615,11 +782,11 @@ when defineSsl: proc serverGetPskFunc*(ctx: SslContext): SslServerGetPskFunc = return ctx.getExtraInternal().serverGetPskFunc - proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; + proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.} = let ctx = SslContext(context: ssl.SSL_get_SSL_CTX) let pskString = (ctx.serverGetPskFunc)($identity) - if psk.len.cint > max_psk_len: + if pskString.len.cint > max_psk_len: return 0 copyMem(psk, pskString.cstring, pskString.len) @@ -640,11 +807,12 @@ when defineSsl: proc wrapSocket*(ctx: SslContext, socket: Socket) = ## Wraps a socket in an SSL context. This function effectively turns - ## ``socket`` into an SSL socket. + ## `socket` into an SSL socket. ## ## This must be called on an unconnected socket; an SSL session will ## be started when the socket is connected. ## + ## FIXME: ## **Disclaimer**: This code is not well tested, may be very unsafe and ## prone to security vulnerabilities. @@ -654,23 +822,48 @@ when defineSsl: socket.sslHandle = SSL_new(socket.sslContext.context) socket.sslNoHandshake = false socket.sslHasPeekChar = false + socket.sslNoShutdown = false if socket.sslHandle == nil: raiseSSLError() if SSL_set_fd(socket.sslHandle, socket.fd) != 1: raiseSSLError() + proc checkCertName(socket: Socket, hostname: string) {.raises: [SslError], tags:[RootEffect].} = + ## Check if the certificate Subject Alternative Name (SAN) or Subject CommonName (CN) matches hostname. + ## Wildcards match only in the left-most label. + ## When name starts with a dot it will be matched by a certificate valid for any subdomain + when not defined(nimDisableCertificateValidation) and not defined(windows): + assert socket.isSsl + try: + let certificate = socket.sslHandle.SSL_get_peer_certificate() + if certificate.isNil: + raiseSSLError("No SSL certificate found.") + + const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint + # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html + let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint, + X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil) + # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html + X509_free(certificate) + if match != 1: + raiseSSLError("SSL Certificate check failed.") + + except LibraryError: + raiseSSLError("SSL import failed") + proc wrapConnectedSocket*(ctx: SslContext, socket: Socket, handshake: SslHandshakeType, hostname: string = "") = ## Wraps a connected socket in an SSL context. This function effectively - ## turns ``socket`` into an SSL socket. - ## ``hostname`` should be specified so that the client knows which hostname + ## turns `socket` into an SSL socket. + ## `hostname` should be specified so that the client knows which hostname ## the server certificate should be validated against. ## ## This should be called on a connected socket, and will perform ## an SSL handshake immediately. ## + ## FIXME: ## **Disclaimer**: This code is not well tested, may be very unsafe and ## prone to security vulnerabilities. wrapSocket(ctx, socket) @@ -680,14 +873,66 @@ when defineSsl: # Discard result in case OpenSSL version doesn't support SNI, or we're # not using TLSv1+ discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) + ErrClearError() let ret = SSL_connect(socket.sslHandle) socketError(socket, ret) + when not defined(nimDisableCertificateValidation) and not defined(windows): + # FIXME: this should be skipped on CVerifyNone + if hostname.len > 0 and not isIpAddress(hostname): + socket.checkCertName(hostname) of handshakeAsServer: + ErrClearError() let ret = SSL_accept(socket.sslHandle) socketError(socket, ret) + proc getPeerCertificates*(sslHandle: SslPtr): seq[Certificate] {.since: (1, 1).} = + ## Returns the certificate chain received by the peer we are connected to + ## through the OpenSSL connection represented by `sslHandle`. + ## The handshake must have been completed and the certificate chain must + ## have been verified successfully or else an empty sequence is returned. + ## The chain is ordered from leaf certificate to root certificate. + result = newSeq[Certificate]() + if SSL_get_verify_result(sslHandle) != X509_V_OK: + return + let stack = SSL_get0_verified_chain(sslHandle) + if stack == nil: + return + let length = OPENSSL_sk_num(stack) + if length == 0: + return + for i in 0 .. length - 1: + let x509 = cast[PX509](OPENSSL_sk_value(stack, i)) + result.add(i2d_X509(x509)) + + proc getPeerCertificates*(socket: Socket): seq[Certificate] {.since: (1, 1).} = + ## Returns the certificate chain received by the peer we are connected to + ## through the given socket. + ## The handshake must have been completed and the certificate chain must + ## have been verified successfully or else an empty sequence is returned. + ## The chain is ordered from leaf certificate to root certificate. + if not socket.isSsl: + result = newSeq[Certificate]() + else: + result = getPeerCertificates(socket.sslHandle) + + proc `sessionIdContext=`*(ctx: SslContext, sidCtx: string) = + ## Sets the session id context in which a session can be reused. + ## Used for permitting clients to reuse a session id instead of + ## doing a new handshake. + ## + ## TLS clients might attempt to resume a session using the session id context, + ## thus it must be set if verifyMode is set to CVerifyPeer or CVerifyPeerUseEnvVars, + ## otherwise the connection will fail and SslError will be raised if resumption occurs. + ## + ## - Only useful if set server-side. + ## - Should be unique per-application to prevent clients from malfunctioning. + ## - sidCtx must be at most 32 characters in length. + if sidCtx.len > 32: + raiseSSLError("sessionIdContext must be shorter than 32 characters") + SSL_CTX_set_session_id_context(ctx.context, sidCtx, sidCtx.len) + proc getSocketError*(socket: Socket): OSErrorCode = - ## Checks ``osLastError`` for a valid error. If it has been reset it uses + ## Checks `osLastError` for a valid error. If it has been reset it uses ## the last error stored in the socket object. result = osLastError() if result == 0.OSErrorCode: @@ -696,14 +941,18 @@ proc getSocketError*(socket: Socket): OSErrorCode = raiseOSError(result, "No valid socket error code available") proc socketError*(socket: Socket, err: int = -1, async = false, - lastError = (-1).OSErrorCode) = - ## Raises an OSError based on the error code returned by ``SSL_get_error`` - ## (for SSL sockets) and ``osLastError`` otherwise. + lastError = (-1).OSErrorCode, + flags: set[SocketFlag] = {}) = + ## Raises an OSError based on the error code returned by `SSL_get_error` + ## (for SSL sockets) and `osLastError` otherwise. ## - ## If ``async`` is ``true`` no error will be thrown in the case when the + ## If `async` is `true` no error will be thrown in the case when the ## error was caused by no data being available to be read. ## - ## If ``err`` is not lower than 0 no exception will be raised. + ## If `err` is not lower than 0 no exception will be raised. + ## + ## If `flags` contains `SafeDisconn`, no exception will be raised + ## when the error was caused by a peer disconnection. when defineSsl: if socket.isSsl: if err <= 0: @@ -722,37 +971,43 @@ proc socketError*(socket: Socket, err: int = -1, async = false, of SSL_ERROR_WANT_X509_LOOKUP: raiseSSLError("Function for x509 lookup has been called.") of SSL_ERROR_SYSCALL: - var errStr = "IO error has occurred " - let sslErr = ERR_peek_last_error() - if sslErr == 0 and err == 0: - errStr.add "because an EOF was observed that violates the protocol" - elif sslErr == 0 and err == -1: - errStr.add "in the BIO layer" - else: - let errStr = $ERR_error_string(sslErr, nil) - raiseSSLError(errStr & ": " & errStr) + # SSL shutdown must not be done if a fatal error occurred. + socket.sslNoShutdown = true let osErr = osLastError() - raiseOSError(osErr, errStr) + if not flags.isDisconnectionError(osErr): + var errStr = "IO error has occurred " + let sslErr = ERR_peek_last_error() + if sslErr == 0 and err == 0: + errStr.add "because an EOF was observed that violates the protocol" + elif sslErr == 0 and err == -1: + errStr.add "in the BIO layer" + else: + let errStr = $ERR_error_string(sslErr, nil) + raiseSSLError(errStr & ": " & errStr) + raiseOSError(osErr, errStr) of SSL_ERROR_SSL: + # SSL shutdown must not be done if a fatal error occurred. + socket.sslNoShutdown = true raiseSSLError() else: raiseSSLError("Unknown Error") if err == -1 and not (when defineSsl: socket.isSsl else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError - if async: - when useWinVersion: - if lastE.int32 == WSAEWOULDBLOCK: - return - else: raiseOSError(lastE) - else: - if lastE.int32 == EAGAIN or lastE.int32 == EWOULDBLOCK: - return - else: raiseOSError(lastE) - else: raiseOSError(lastE) + if not flags.isDisconnectionError(lastE): + if async: + when useWinVersion: + if lastE.int32 == WSAEWOULDBLOCK: + return + else: raiseOSError(lastE) + else: + if lastE.int32 == EAGAIN or lastE.int32 == EWOULDBLOCK: + return + else: raiseOSError(lastE) + else: raiseOSError(lastE) proc listen*(socket: Socket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the + ## Marks `socket` as accepting connections. + ## `Backlog` specifies the maximum length of the ## queue of pending connections. ## ## Raises an OSError error upon failure. @@ -761,9 +1016,9 @@ proc listen*(socket: Socket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = proc bindAddr*(socket: Socket, port = Port(0), address = "") {. tags: [ReadIOEffect].} = - ## Binds ``address``:``port`` to the socket. + ## Binds `address`:`port` to the socket. ## - ## If ``address`` is "" then ADDR_ANY will be bound. + ## If `address` is "" then ADDR_ANY will be bound. var realaddr = address if realaddr == "": case socket.domain @@ -775,34 +1030,41 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) - raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) + var address2: string + address2.addQuoted address + raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) + freeAddrInfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, - flags = {SocketFlag.SafeDisconn}) {. - tags: [ReadIOEffect], gcsafe, locks: 0.} = + flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)) {. + tags: [ReadIOEffect], gcsafe.} = ## Blocks until a connection is being made from a client. When a connection - ## is made sets ``client`` to the client socket and ``address`` to the address + ## is made sets `client` to the client socket and `address` to the address ## of the connecting client. ## This function will raise OSError if an error occurs. ## ## The resulting client will inherit any properties of the server socket. For ## example: whether the socket is buffered or not. ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## The SocketHandle associated with the resulting client will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. + ## + ## The `accept` call may result in an error if the connecting socket + ## disconnects during the duration of the `accept`. If the `SafeDisconn` ## flag is specified then this error will not be raised and instead ## accept will be called again. if client.isNil: new(client) - let ret = accept(server.fd) + let ret = accept(server.fd, inheritable) let sock = ret[0] if sock == osInvalidSocket: let err = osLastError() if flags.isDisconnectionError(err): - acceptAddr(server, client, address, flags) + acceptAddr(server, client, address, flags, inheritable) raiseOSError(err) else: address = ret[1] @@ -816,6 +1078,7 @@ proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, # We must wrap the client sock in a ssl context. server.sslContext.wrapSocket(client) + ErrClearError() let ret = SSL_accept(client.sslHandle) socketError(client, ret, false) @@ -826,16 +1089,16 @@ when false: #defineSsl: ## This procedure should only be used for non-blocking **SSL** sockets. ## It will immediately return with one of the following values: ## - ## ``AcceptSuccess`` will be returned when a client has been successfully + ## `AcceptSuccess` will be returned when a client has been successfully ## accepted and the handshake has been successfully performed between - ## ``server`` and the newly connected client. + ## `server` and the newly connected client. ## - ## ``AcceptNoHandshake`` will be returned when a client has been accepted + ## `AcceptNoHandshake` will be returned when a client has been accepted ## but no handshake could be performed. This can happen when the client ## connects but does not yet initiate a handshake. In this case - ## ``acceptAddrSSL`` should be called again with the same parameters. + ## `acceptAddrSSL` should be called again with the same parameters. ## - ## ``AcceptNoClient`` will be returned when no client is currently attempting + ## `AcceptNoClient` will be returned when no client is currently attempting ## to connect. template doHandshake(): untyped = when defineSsl: @@ -845,6 +1108,7 @@ when false: #defineSsl: if not client.isSsl or client.sslHandle == nil: server.sslContext.wrapSocket(client) + ErrClearError() let ret = SSL_accept(client.sslHandle) while ret <= 0: let err = SSL_get_error(client.sslHandle, ret) @@ -872,32 +1136,128 @@ when false: #defineSsl: doHandshake() proc accept*(server: Socket, client: var owned(Socket), - flags = {SocketFlag.SafeDisconn}) {.tags: [ReadIOEffect].} = - ## Equivalent to ``acceptAddr`` but doesn't return the address, only the + flags = {SocketFlag.SafeDisconn}, + inheritable = defined(nimInheritHandles)) + {.tags: [ReadIOEffect].} = + ## Equivalent to `acceptAddr` but doesn't return the address, only the ## socket. ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## The SocketHandle associated with the resulting client will not be + ## inheritable by child processes by default. This can be changed via + ## the `inheritable` parameter. + ## + ## The `accept` call may result in an error if the connecting socket + ## disconnects during the duration of the `accept`. If the `SafeDisconn` ## flag is specified then this error will not be raised and instead ## accept will be called again. var addrDummy = "" acceptAddr(server, client, addrDummy, flags) -proc close*(socket: Socket) = +when defined(posix) and not defined(lwip): + from std/posix import Sigset, sigwait, sigismember, sigemptyset, sigaddset, + sigprocmask, pthread_sigmask, SIGPIPE, SIG_BLOCK, SIG_UNBLOCK + +template blockSigpipe(body: untyped): untyped = + ## Temporary block SIGPIPE within the provided code block. If SIGPIPE is + ## raised for the duration of the code block, it will be queued and will be + ## raised once the block ends. + ## + ## Within the block a `selectSigpipe()` template is provided which can be + ## used to remove SIGPIPE from the queue. Note that if SIGPIPE is **not** + ## raised at the time of call, it will block until SIGPIPE is raised. + ## + ## If SIGPIPE has already been blocked at the time of execution, the + ## signal mask is left as-is and `selectSigpipe()` will become a no-op. + ## + ## For convenience, this template is also available for non-POSIX system, + ## where `body` will be executed as-is. + when not defined(posix) or defined(lwip): + body + else: + template sigmask(how: cint, set, oset: var Sigset): untyped {.gensym.} = + ## Alias for pthread_sigmask or sigprocmask depending on the status + ## of --threads + when compileOption("threads"): + pthread_sigmask(how, set, oset) + else: + sigprocmask(how, set, oset) + + var oldSet, watchSet: Sigset + if sigemptyset(oldSet) == -1: + raiseOSError(osLastError()) + if sigemptyset(watchSet) == -1: + raiseOSError(osLastError()) + + if sigaddset(watchSet, SIGPIPE) == -1: + raiseOSError(osLastError(), "Couldn't add SIGPIPE to Sigset") + + if sigmask(SIG_BLOCK, watchSet, oldSet) == -1: + raiseOSError(osLastError(), "Couldn't block SIGPIPE") + + let alreadyBlocked = sigismember(oldSet, SIGPIPE) == 1 + + template selectSigpipe(): untyped {.used.} = + if not alreadyBlocked: + var signal: cint + let err = sigwait(watchSet, signal) + if err != 0: + raiseOSError(err.OSErrorCode, "Couldn't select SIGPIPE") + assert signal == SIGPIPE + + try: + body + finally: + if not alreadyBlocked: + if sigmask(SIG_UNBLOCK, watchSet, oldSet) == -1: + raiseOSError(osLastError(), "Couldn't unblock SIGPIPE") + +proc close*(socket: Socket, flags = {SocketFlag.SafeDisconn}) = ## Closes a socket. + ## + ## If `socket` is an SSL/TLS socket, this proc will also send a closure + ## notification to the peer. If `SafeDisconn` is in `flags`, failure to do so + ## due to disconnections will be ignored. This is generally safe in + ## practice. See + ## `here <https://security.stackexchange.com/a/82044>`_ for more details. try: when defineSsl: if socket.isSsl and socket.sslHandle != nil: - ErrClearError() - # As we are closing the underlying socket immediately afterwards, - # it is valid, under the TLS standard, to perform a unidirectional - # shutdown i.e not wait for the peers "close notify" alert with a second - # call to SSL_shutdown - let res = SSL_shutdown(socket.sslHandle) - if res == 0: - discard - elif res != 1: - socketError(socket, res) + # Don't call SSL_shutdown if the connection has not been fully + # established, see: + # https://github.com/openssl/openssl/issues/710#issuecomment-253897666 + if not socket.sslNoShutdown and SSL_in_init(socket.sslHandle) == 0: + # As we are closing the underlying socket immediately afterwards, + # it is valid, under the TLS standard, to perform a unidirectional + # shutdown i.e not wait for the peers "close notify" alert with a second + # call to SSL_shutdown + blockSigpipe: + ErrClearError() + let res = SSL_shutdown(socket.sslHandle) + if res == 0: + discard + elif res != 1: + let + err = osLastError() + sslError = SSL_get_error(socket.sslHandle, res) + + # If a close notification is received, failures outside of the + # protocol will be returned as SSL_ERROR_ZERO_RETURN instead + # of SSL_ERROR_SYSCALL. This fact is deduced by digging into + # SSL_get_error() source code. + if sslError == SSL_ERROR_ZERO_RETURN or + sslError == SSL_ERROR_SYSCALL: + when defined(posix) and not defined(macosx) and + not defined(nimdoc): + if err == EPIPE.OSErrorCode: + # Clear the SIGPIPE that's been raised due to + # the disconnection. + selectSigpipe() + else: + discard + if not flags.isDisconnectionError(err): + socketError(socket, res, lastError = err, flags = flags) + else: + socketError(socket, res, lastError = err, flags = flags) finally: when defineSsl: if socket.isSsl and socket.sslHandle != nil: @@ -908,12 +1268,12 @@ proc close*(socket: Socket) = socket.fd = osInvalidSocket when defined(posix): - from posix import TCP_NODELAY + from std/posix import TCP_NODELAY else: - from winlean import TCP_NODELAY + from std/winlean import TCP_NODELAY proc toCInt*(opt: SOBool): cint = - ## Converts a ``SOBool`` into its Socket Option cint representation. + ## Converts a `SOBool` into its Socket Option cint representation. case opt of OptAcceptConn: SO_ACCEPTCONN of OptBroadcast: SO_BROADCAST @@ -927,7 +1287,7 @@ proc toCInt*(opt: SOBool): cint = proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = - ## Retrieves option ``opt`` as a boolean value. + ## Retrieves option `opt` as a boolean value. var res = getSockOptInt(socket.fd, cint(level), toCInt(opt)) result = res != 0 @@ -937,32 +1297,31 @@ proc getLocalAddr*(socket: Socket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: Socket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: Socket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = - ## Sets option ``opt`` to a boolean value specified by ``value``. - ## - ## .. code-block:: Nim - ## var socket = newSocket() - ## socket.setSockOpt(OptReusePort, true) - ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) - ## + ## Sets option `opt` to a boolean value specified by `value`. + runnableExamples("-r:off"): + let socket = newSocket() + socket.setSockOpt(OptReusePort, true) + socket.setSockOpt(OptNoDelay, true, level = IPPROTO_TCP.cint) var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(posix) or defined(nimdoc): +when defined(nimdoc) or (defined(posix) and not useNimNetLite): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.connect(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) proc bindUnix*(socket: Socket, path: string) = @@ -971,15 +1330,15 @@ when defined(posix) or defined(nimdoc): when not defined(nimdoc): var socketAddr = makeUnixAddr(path) if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr), - (sizeof(socketAddr.sun_family) + path.len).SockLen) != 0'i32: + (offsetOf(socketAddr, sun_path) + path.len + 1).SockLen) != 0'i32: raiseOSError(osLastError()) -when defined(ssl): +when defineSsl: proc gotHandshake*(socket: Socket): bool = - ## Determines whether a handshake has occurred between a client (``socket``) - ## and the server that ``socket`` is connected to. + ## Determines whether a handshake has occurred between a client (`socket`) + ## and the server that `socket` is connected to. ## - ## Throws SslError if ``socket`` is not an SSL socket. + ## Throws SslError if `socket` is not an SSL socket. if socket.isSsl: return not socket.sslNoHandshake else: @@ -995,14 +1354,6 @@ proc hasDataBuffered*(s: Socket): bool = if s.isSsl and not result: result = s.sslHasPeekChar -proc select(readfd: Socket, timeout = 500): int = - ## Used for socket operation timeouts. - if readfd.hasDataBuffered: - return 1 - - var fds = @[readfd.fd] - result = selectRead(fds, timeout) - proc isClosed(socket: Socket): bool = socket.fd == osInvalidSocket @@ -1015,6 +1366,7 @@ proc uniRecv(socket: Socket, buffer: pointer, size, flags: cint): int = assert(not socket.isClosed, "Cannot `recv` on a closed socket") when defineSsl: if socket.isSsl: + ErrClearError() return SSL_read(socket.sslHandle, buffer, size) return recv(socket.fd, buffer, size, flags) @@ -1046,7 +1398,7 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ ## Receives data from a socket. ## ## **Note**: This is a low-level function, you may be interested in the higher - ## level versions of this function which are also named ``recv``. + ## level versions of this function which are also named `recv`. if size == 0: return if socket.isBuffered: if socket.bufLen == 0: @@ -1089,11 +1441,11 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ proc waitFor(socket: Socket, waited: var Duration, timeout, size: int, funcName: string): int {.tags: [TimeEffect].} = ## determines the amount of characters that can be read. Result will never - ## be larger than ``size``. For unbuffered sockets this will be ``1``. - ## For buffered sockets it can be as big as ``BufferSize``. + ## be larger than `size`. For unbuffered sockets this will be `1`. + ## For buffered sockets it can be as big as `BufferSize`. ## ## If this function does not determine that there is data on the socket - ## within ``timeout`` ms, a TimeoutError error will be raised. + ## within `timeout` ms, a TimeoutError error will be raised. result = 1 if size <= 0: assert false if timeout == -1: return size @@ -1115,7 +1467,9 @@ proc waitFor(socket: Socket, waited: var Duration, timeout, size: int, return min(sslPending, size) var startTime = getMonoTime() - let selRet = select(socket, (timeout - waited.inMilliseconds).int) + let selRet = if socket.hasDataBuffered: 1 + else: + timeoutRead(socket.fd, (timeout - waited.inMilliseconds).int) if selRet < 0: raiseOSError(osLastError()) if selRet != 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") @@ -1123,7 +1477,7 @@ proc waitFor(socket: Socket, waited: var Duration, timeout, size: int, proc recv*(socket: Socket, data: pointer, size: int, timeout: int): int {. tags: [ReadIOEffect, TimeEffect].} = - ## overload with a ``timeout`` parameter in milliseconds. + ## overload with a `timeout` parameter in milliseconds. var waited: Duration # duration already waited var read = 0 @@ -1141,12 +1495,12 @@ proc recv*(socket: Socket, data: pointer, size: int, timeout: int): int {. proc recv*(socket: Socket, data: var string, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): int = - ## Higher-level version of ``recv``. + ## Higher-level version of `recv`. ## - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. + ## Reads **up to** `size` bytes from `socket` into `data`. ## ## For buffered sockets this function will attempt to read all the requested - ## data. It will read this data in ``BufferSize`` chunks. + ## data. It will read this data in `BufferSize` chunks. ## ## For unbuffered sockets this function makes no effort to read ## all the data requested. It will return as much data as the operating system @@ -1160,9 +1514,7 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, ## A timeout may be specified in milliseconds, if enough data is not received ## within the time specified a TimeoutError exception will be raised. ## - ## **Note**: ``data`` must be initialised. - ## - ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + ## .. warning:: Only the `SafeDisconn` flag is currently supported. data.setLen(size) result = if timeout == -1: @@ -1172,24 +1524,24 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, if result < 0: data.setLen(0) let lastError = getSocketError(socket) - if flags.isDisconnectionError(lastError): return - socket.socketError(result, lastError = lastError) - data.setLen(result) + socket.socketError(result, lastError = lastError, flags = flags) + else: + data.setLen(result) proc recv*(socket: Socket, size: int, timeout = -1, flags = {SocketFlag.SafeDisconn}): string {.inline.} = - ## Higher-level version of ``recv`` which returns a string. + ## Higher-level version of `recv` which returns a string. ## - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. + ## Reads **up to** `size` bytes from `socket` into the result. ## ## For buffered sockets this function will attempt to read all the requested - ## data. It will read this data in ``BufferSize`` chunks. + ## data. It will read this data in `BufferSize` chunks. ## ## For unbuffered sockets this function makes no effort to read ## all the data requested. It will return as much data as the operating system ## gives it. ## - ## When ``""`` is returned the socket's connection has been closed. + ## When `""` is returned the socket's connection has been closed. ## ## This function will throw an OSError exception when an error occurs. ## @@ -1197,7 +1549,7 @@ proc recv*(socket: Socket, size: int, timeout = -1, ## within the time specified a TimeoutError exception will be raised. ## ## - ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + ## .. warning:: Only the `SafeDisconn` flag is currently supported. result = newString(size) discard recv(socket, result, size, timeout, flags) @@ -1221,45 +1573,47 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = return result = recv(socket.fd, addr(c), 1, MSG_PEEK) -proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, +proc readLine*(socket: Socket, line: var string, timeout = -1, flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {. tags: [ReadIOEffect, TimeEffect].} = - ## Reads a line of data from ``socket``. + ## Reads a line of data from `socket`. ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## If a full line is read `\r\L` is not + ## added to `line`, however if solely `\r\L` is read then `line` ## will be set to it. ## - ## If the socket is disconnected, ``line`` will be set to ``""``. + ## If the socket is disconnected, `line` will be set to `""`. ## ## An OSError exception will be raised in the case of a socket error. ## ## A timeout can be specified in milliseconds, if data is not received within ## the specified time a TimeoutError exception will be raised. ## - ## The ``maxLength`` parameter determines the maximum amount of characters + ## The `maxLength` parameter determines the maximum amount of characters ## that can be read. The result is truncated after that. ## - ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + ## .. warning:: Only the `SafeDisconn` flag is currently supported. template addNLIfEmpty() = if line.len == 0: - line.string.add("\c\L") + line.add("\c\L") template raiseSockError() {.dirty.} = let lastError = getSocketError(socket) - if flags.isDisconnectionError(lastError): setLen(line.string, 0); return - socket.socketError(n, lastError = lastError) + if flags.isDisconnectionError(lastError): + setLen(line, 0) + socket.socketError(n, lastError = lastError, flags = flags) + return var waited: Duration - setLen(line.string, 0) + setLen(line, 0) while true: var c: char discard waitFor(socket, waited, timeout, 1, "readLine") var n = recv(socket, addr(c), 1) if n < 0: raiseSockError() - elif n == 0: setLen(line.string, 0); return + elif n == 0: setLen(line, 0); return if c == '\r': discard waitFor(socket, waited, timeout, 1, "readLine") n = peekChar(socket, c) @@ -1271,65 +1625,84 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, elif c == '\L': addNLIfEmpty() return - add(line.string, c) + add(line, c) # Verify that this isn't a DOS attack: #3847. - if line.string.len > maxLength: break + if line.len > maxLength: break proc recvLine*(socket: Socket, timeout = -1, flags = {SocketFlag.SafeDisconn}, - maxLength = MaxLineLength): TaintedString = - ## Reads a line of data from ``socket``. + maxLength = MaxLineLength): string = + ## Reads a line of data from `socket`. ## - ## If a full line is read ``\r\L`` is not - ## added to the result, however if solely ``\r\L`` is read then the result + ## If a full line is read `\r\L` is not + ## added to the result, however if solely `\r\L` is read then the result ## will be set to it. ## - ## If the socket is disconnected, the result will be set to ``""``. + ## If the socket is disconnected, the result will be set to `""`. ## ## An OSError exception will be raised in the case of a socket error. ## ## A timeout can be specified in milliseconds, if data is not received within ## the specified time a TimeoutError exception will be raised. ## - ## The ``maxLength`` parameter determines the maximum amount of characters + ## The `maxLength` parameter determines the maximum amount of characters ## that can be read. The result is truncated after that. ## - ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. - result = "".TaintedString + ## .. warning:: Only the `SafeDisconn` flag is currently supported. + result = "" readLine(socket, result, timeout, flags, maxLength) -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. +proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: int, + address: var T, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = - ## Receives data from ``socket``. This function should normally be used with - ## connection-less sockets (UDP sockets). + ## Receives data from `socket`. This function should normally be used with + ## connection-less sockets (UDP sockets). The source address of the data + ## packet is stored in the `address` argument as either a string or an IpAddress. ## ## If an error occurs an OSError exception will be raised. Otherwise the return ## value will be the length of data received. ## - ## **Warning:** This function does not yet have a buffered implementation, - ## so when ``socket`` is buffered the non-buffered implementation will be - ## used. Therefore if ``socket`` contains something in its buffer this - ## function will make no effort to return it. + ## .. warning:: This function does not yet have a buffered implementation, + ## so when `socket` is buffered the non-buffered implementation will be + ## used. Therefore if `socket` contains something in its buffer this + ## function will make no effort to return it. + template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = + var addrLen = SockLen(sizeof(sockAddress)) + result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, + cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + + if result != -1: + data.setLen(result) + + when typeof(address) is string: + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) + when domain == AF_INET6: + port = ntohs(sockAddress.sin6_port).Port + else: + port = ntohs(sockAddress.sin_port).Port + else: + data.setLen(result) + sockAddress.fromSockAddr(addrLen, address, port) + else: + raiseOSError(osLastError()) assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - if result != -1: - data.setLen(result) - address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) - port = ntohs(sockAddress.sin_port).Port + case socket.domain + of AF_INET6: + var sockAddress: Sockaddr_in6 + adaptRecvFromToDomain(sockAddress, AF_INET6) + of AF_INET: + var sockAddress: Sockaddr_in + adaptRecvFromToDomain(sockAddress, AF_INET) else: - raiseOSError(osLastError()) + raise newException(ValueError, "Unknown socket address family") proc skip*(socket: Socket, size: int, timeout = -1) = - ## Skips ``size`` amount of bytes. + ## Skips `size` amount of bytes. ## ## An optional timeout can be specified in milliseconds, if skipping the ## bytes takes longer than specified a TimeoutError exception will be raised. @@ -1347,11 +1720,12 @@ proc send*(socket: Socket, data: pointer, size: int): int {. tags: [WriteIOEffect].} = ## Sends data to a socket. ## - ## **Note**: This is a low-level version of ``send``. You likely should use + ## **Note**: This is a low-level version of `send`. You likely should use ## the version below. assert(not socket.isClosed, "Cannot `send` on a closed socket") when defineSsl: if socket.isSsl: + ErrClearError() return SSL_write(socket.sslHandle, cast[cstring](data), size) when useWinVersion or defined(macosx): @@ -1362,32 +1736,53 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - if flags.isDisconnectionError(lastError): return - socketError(socket, lastError = lastError) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) + + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) template `&=`*(socket: Socket; data: typed) = ## an alias for 'send'. send(socket, data) proc trySend*(socket: Socket, data: string): bool {.tags: [WriteIOEffect].} = - ## Safe alternative to ``send``. Does not raise an OSError when an error occurs, - ## and instead returns ``false`` on failure. + ## Safe alternative to `send`. Does not raise an OSError when an error occurs, + ## and instead returns `false` on failure. result = send(socket, cstring(data), data.len) == data.len proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, size: int, af: Domain = AF_INET, flags = 0'i32) {. tags: [WriteIOEffect].} = - ## This proc sends ``data`` to the specified ``address``, + ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. + ## this function will try each IP of that hostname. This function + ## should normally be used with connection-less sockets (UDP sockets). ## ## If an error occurs an OSError exception will be raised. ## @@ -1411,25 +1806,51 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, it = it.ai_next let osError = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(osError) proc sendTo*(socket: Socket, address: string, port: Port, data: string) {.tags: [WriteIOEffect].} = - ## This proc sends ``data`` to the specified ``address``, + ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. ## + ## Generally for use with connection-less (UDP) sockets. + ## ## If an error occurs an OSError exception will be raised. ## - ## This is the high-level version of the above ``sendTo`` function. + ## This is the high-level version of the above `sendTo` function. socket.sendTo(address, port, cstring(data), data.len, socket.domain) +proc sendTo*(socket: Socket, address: IpAddress, port: Port, + data: string, flags = 0'i32): int {. + discardable, tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `IpAddress` and returns + ## the number of bytes written. + ## + ## Generally for use with connection-less (UDP) sockets. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This is the high-level version of the above `sendTo` function. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + + var sa: Sockaddr_storage + var sl: SockLen + toSockAddr(address, port, sa, sl) + result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, + cast[ptr SockAddr](addr sa), sl) + + if result == -1'i32: + let osError = osLastError() + raiseOSError(osError) + proc isSsl*(socket: Socket): bool = - ## Determines whether ``socket`` is a SSL socket. + ## Determines whether `socket` is a SSL socket. when defineSsl: result = socket.isSsl else: @@ -1438,6 +1859,16 @@ proc isSsl*(socket: Socket): bool = proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor +when defined(zephyr) or defined(nimNetSocketExtras): # Remove in future + proc getDomain*(socket: Socket): Domain = return socket.domain + ## Returns the socket's domain + + proc getType*(socket: Socket): SockType = return socket.sockType + ## Returns the socket's type + + proc getProtocol*(socket: Socket): Protocol = return socket.protocol + ## Returns the socket's protocol + when defined(nimHasStyleChecks): {.push styleChecks: off.} @@ -1489,14 +1920,18 @@ proc `==`*(lhs, rhs: IpAddress): bool = proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation - result = "" case address.family of IpAddressFamily.IPv4: - for i in 0 .. 3: - if i != 0: - result.add('.') - result.add($address.address_v4[i]) + result = newStringOfCap(15) + result.addInt address.address_v4[0] + result.add '.' + result.addInt address.address_v4[1] + result.add '.' + result.addInt address.address_v4[2] + result.add '.' + result.addInt address.address_v4[3] of IpAddressFamily.IPv6: + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 @@ -1545,14 +1980,18 @@ proc `$`*(address: IpAddress): string = result.add(chr(uint16(ord('a'))+val-0xA)) afterLeadingZeros = true mask = mask shr 4 + + if not afterLeadingZeros: + result.add '0' + printedLastGroup = true proc dial*(address: string, port: Port, protocol = IPPROTO_TCP, buffered = true): owned(Socket) {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Establishes connection to the specified ``address``:``port`` pair via the + ## Establishes connection to the specified `address`:`port` pair via the ## specified protocol. The procedure iterates through possible - ## resolutions of the ``address`` until it succeeds, meaning that it + ## resolutions of the `address` until it succeeds, meaning that it ## seamlessly works with both IPv4 and IPv6. ## Returns Socket ready to send or receive data. let sockType = protocol.toSockType() @@ -1586,7 +2025,7 @@ proc dial*(address: string, port: Port, # network system problem (e.g. not enough FDs), and not an unreachable # address. let err = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds() raiseOSError(err) fdPerDomain[ord(domain)] = lastFd @@ -1595,24 +2034,26 @@ proc dial*(address: string, port: Port, break lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds(ord(domain)) if success: - result = newSocket(lastFd, domain, sockType, protocol) + result = newSocket(lastFd, domain, sockType, protocol, buffered) elif lastError != 0.OSErrorCode: + lastFd.close() raiseOSError(lastError) else: + lastFd.close() raise newException(IOError, "Couldn't resolve address: " & address) proc connect*(socket: Socket, address: string, - port = Port(0)) {.tags: [ReadIOEffect].} = - ## Connects socket to ``address``:``port``. ``Address`` can be an IP address or a - ## host name. If ``address`` is a host name, this function will try each IP - ## of that host name. ``htons`` is already performed on ``port`` so you must + port = Port(0)) {.tags: [ReadIOEffect, RootEffect].} = + ## Connects socket to `address`:`port`. `Address` can be an IP address or a + ## host name. If `address` is a host name, this function will try each IP + ## of that host name. `htons` is already performed on `port` so you must ## not do it. ## - ## If ``socket`` is an SSL socket a handshake will be automatically performed. + ## If `socket` is an SSL socket a handshake will be automatically performed. var aiList = getAddrInfo(address, port, socket.domain) # try all possibilities: var success = false @@ -1625,7 +2066,7 @@ proc connect*(socket: Socket, address: string, else: lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) when defineSsl: @@ -1636,18 +2077,22 @@ proc connect*(socket: Socket, address: string, # not using TLSv1+ discard SSL_set_tlsext_host_name(socket.sslHandle, address) + ErrClearError() let ret = SSL_connect(socket.sslHandle) socketError(socket, ret) + when not defined(nimDisableCertificateValidation) and not defined(windows): + if not isIpAddress(address): + socket.checkCertName(address) proc connectAsync(socket: Socket, name: string, port = Port(0), af: Domain = AF_INET) {.tags: [ReadIOEffect].} = - ## A variant of ``connect`` for non-blocking sockets. + ## A variant of `connect` for non-blocking sockets. ## ## This procedure will immediately return, it will not block until a connection ## is made. It is up to the caller to make sure the connection has been established - ## by checking (using ``select``) whether the socket is writeable. + ## by checking (using `select`) whether the socket is writeable. ## - ## **Note**: For SSL sockets, the ``handshake`` procedure must be called + ## **Note**: For SSL sockets, the `handshake` procedure must be called ## whenever the socket successfully connects to a server. var aiList = getAddrInfo(name, port, af) # try all possibilities: @@ -1673,20 +2118,19 @@ proc connectAsync(socket: Socket, name: string, port = Port(0), it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) proc connect*(socket: Socket, address: string, port = Port(0), - timeout: int) {.tags: [ReadIOEffect, WriteIOEffect].} = - ## Connects to server as specified by ``address`` on port specified by ``port``. + timeout: int) {.tags: [ReadIOEffect, WriteIOEffect, RootEffect].} = + ## Connects to server as specified by `address` on port specified by `port`. ## - ## The ``timeout`` parameter specifies the time in milliseconds to allow for + ## The `timeout` parameter specifies the time in milliseconds to allow for ## the connection to the server to be made. socket.fd.setBlocking(false) socket.connectAsync(address, port, socket.domain) - var s = @[socket.fd] - if selectWrite(s, timeout) != 1: + if timeoutWrite(socket.fd, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR) @@ -1695,7 +2139,18 @@ proc connect*(socket: Socket, address: string, port = Port(0), when defineSsl and not defined(nimdoc): if socket.isSsl: socket.fd.setBlocking(true) - doAssert socket.gotHandshake() + # RFC3546 for SNI specifies that IP addresses are not allowed. + if not isIpAddress(address): + # Discard result in case OpenSSL version doesn't support SNI, or we're + # not using TLSv1+ + discard SSL_set_tlsext_host_name(socket.sslHandle, address) + + ErrClearError() + let ret = SSL_connect(socket.sslHandle) + socketError(socket, ret) + when not defined(nimDisableCertificateValidation): + if not isIpAddress(address): + socket.checkCertName(address) socket.fd.setBlocking(true) proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = @@ -1706,14 +2161,15 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = ## ## Supports IPv4 and v6. ## Raises OSError if external networking is not set up. - ## - ## .. code-block:: Nim - ## echo $getPrimaryIPAddr() # "192.168.1.2" - + runnableExamples("-r:off"): + echo getPrimaryIPAddr() # "192.168.1.2" let socket = if dest.family == IpAddressFamily.IPv4: newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) else: newSocket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP) - socket.connect($dest, 80.Port) - socket.getLocalAddr()[0].parseIpAddress() + try: + socket.connect($dest, 80.Port) + result = socket.getLocalAddr()[0].parseIpAddress() + finally: + socket.close() diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index fad8b66a1..bf8367d1d 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -8,19 +8,22 @@ # ## Profiling support for Nim. This is an embedded profiler that requires -## ``--profiler:on``. You only need to import this module to get a profiling -## report at program exit. +## `--profiler:on`. You only need to import this module to get a profiling +## report at program exit. See `Embedded Stack Trace Profiler <estp.html>`_ +## for usage. when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} -when defined(nimHasUsed): - {.used.} +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} -import hashes, algorithm, strutils, tables, sets +import std/[hashes, algorithm, strutils, tables, sets] + +when defined(nimPreviewSlimSystem): + import std/[syncio, sysatomics] when not defined(memProfiler): include "system/timers" @@ -66,7 +69,7 @@ when not defined(memProfiler): else: interval = intervalInUs * 1000 - tickCountCorrection when withThreads: - import locks + import std/locks var profilingLock: Lock @@ -121,13 +124,13 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: gTicker = SamplingInterval result = true dec gTicker - proc hook(st: StackTrace, size: int) {.nimcall, locks: 0.} = + proc hook(st: StackTrace, size: int) {.nimcall.} = when defined(ignoreAllocationSize): hookAux(st, 1) else: @@ -139,7 +142,7 @@ else: gTicker: int # we use an additional counter to # avoid calling 'getTicks' too frequently - proc requestedHook(): bool {.nimcall, locks: 0.} = + proc requestedHook(): bool {.nimcall.} = if interval == 0: result = true elif gTicker == 0: gTicker = 500 @@ -148,7 +151,7 @@ else: else: dec gTicker - proc hook(st: StackTrace) {.nimcall, locks: 0.} = + proc hook(st: StackTrace) {.nimcall.} = #echo "profiling! ", interval if interval == 0: hookAux(st, 1) @@ -181,7 +184,7 @@ proc writeProfile() {.noconv.} = var perProc = initCountTable[string]() for i in 0..entries-1: - var dups = initSet[string]() + var dups = initHashSet[string]() for ii in 0..high(StackTrace.lines): let procname = profileData[i].st[ii] if isNil(procname): break @@ -222,8 +225,9 @@ proc enableProfiling*() = system.profilingRequestedHook = requestedHook when declared(system.StackTrace): + import std/exitprocs system.profilingRequestedHook = requestedHook system.profilerHook = hook - addQuitProc(writeProfile) + addExitProc(writeProfile) {.pop.} diff --git a/lib/pure/nimtracker.nim b/lib/pure/nimtracker.nim deleted file mode 100644 index a66dfc2ea..000000000 --- a/lib/pure/nimtracker.nim +++ /dev/null @@ -1,88 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Memory tracking support for Nim. - -when not defined(memTracker) and not isMainModule: - {.error: "Memory tracking support is turned off!".} - -{.push memtracker: off.} -# we import the low level wrapper and are careful not to use Nim's -# memory manager for anything here. -import sqlite3 - -var - dbHandle: PSqlite3 - insertStmt {.threadvar.}: Pstmt - -const insertQuery = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - -template sbind(x: int; value) = - when value is cstring: - let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT) - if ret != SQLITE_OK: - quit "could not bind value" - else: - let ret = insertStmt.bindInt64(x, value) - if ret != SQLITE_OK: - quit "could not bind value" - -when defined(memTracker): - proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [], gcsafe.} = - if insertStmt.isNil: - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) != SQLITE_OK: - quit "could not bind query to insertStmt " & $sqlite3.errmsg(dbHandle) - for i in 0..log.count-1: - var success = false - let e = log.data[i] - discard sqlite3.reset(insertStmt) - discard clearBindings(insertStmt) - sbind 1, e.op - sbind(2, cast[int](e.address)) - sbind 3, e.size - sbind 4, e.file - sbind 5, e.line - if step(insertStmt) == SQLITE_DONE: - success = true - if not success: - quit "could not write to database! " & $sqlite3.errmsg(dbHandle) - -proc execQuery(q: string) = - var s: Pstmt - if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK: - discard step(s) - if finalize(s) != SQLITE_OK: - quit "could not finalize " & $sqlite3.errmsg(dbHandle) - else: - quit "could not prepare statement " & $sqlite3.errmsg(dbHandle) - -proc setupDb() = - execQuery """create table if not exists tracking( - id integer primary key, - op varchar not null, - address integer not null, - size integer not null, - file varchar not null, - line integer not null - )""" - execQuery "delete from tracking" - -if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK: - setupDb() - const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" - if prepare_v2(dbHandle, insertQuery, - insertQuery.len, insertStmt, nil) == SQLITE_OK: - when defined(memTracker): - setTrackLogger logEntries - else: - quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle) -else: - quit "could not setup sqlite " & $sqlite3.errmsg(dbHandle) -{.pop.} diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index d3a7c4fb6..4d6ceefd7 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -9,92 +9,91 @@ ## Nim OID support. An OID is a global ID that consists of a timestamp, ## a unique counter and a random value. This combination should suffice to -## produce a globally distributed unique ID. This implementation was extracted -## from the Mongodb interface and it thus binary compatible with a Mongo OID. +## produce a globally distributed unique ID. ## -## This implementation calls ``math.randomize()`` for the first call of -## ``genOid``. +## This implementation calls `initRand()` for the first call of +## `genOid`. -import hashes, times, endians +import std/[hashes, times, endians, random] +from std/private/decode_helpers import handleHexChar + +when defined(nimPreviewSlimSystem): + import std/sysatomics type - Oid* = object ## an OID - time: int32 ## - fuzz: int32 ## - count: int32 ## - -proc `==`*(oid1: Oid, oid2: Oid): bool = - ## Compare two Mongo Object IDs for equality - return (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and + Oid* = object ## An OID. + time: int64 + fuzz: int32 + count: int32 + +proc `==`*(oid1: Oid, oid2: Oid): bool {.inline.} = + ## Compares two OIDs for equality. + result = (oid1.time == oid2.time) and (oid1.fuzz == oid2.fuzz) and (oid1.count == oid2.count) proc hash*(oid: Oid): Hash = - ## Generate hash of Oid for use in hashtables + ## Generates the hash of an OID for use in hashtables. var h: Hash = 0 h = h !& hash(oid.time) h = h !& hash(oid.fuzz) h = h !& hash(oid.count) result = !$h -proc hexbyte*(hex: char): int = - case hex - of '0'..'9': result = (ord(hex) - ord('0')) - of 'a'..'f': result = (ord(hex) - ord('a') + 10) - of 'A'..'F': result = (ord(hex) - ord('A') + 10) - else: discard +proc hexbyte*(hex: char): int {.inline.} = + result = handleHexChar(hex) proc parseOid*(str: cstring): Oid = - ## parses an OID. - var bytes = cast[cstring](addr(result.time)) + ## Parses an OID. + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) inc(i) -proc oidToString*(oid: Oid, str: cstring) = +proc `$`*(oid: Oid): string = + ## Converts an OID to a string. const hex = "0123456789abcdef" - # work around a compiler bug: - var str = str + + result.setLen 24 + var o = oid - var bytes = cast[cstring](addr(o)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 while i < 12: let b = bytes[i].ord - str[2 * i] = hex[(b and 0xF0) shr 4] - str[2 * i + 1] = hex[b and 0xF] + result[2 * i] = hex[(b and 0xF0) shr 4] + result[2 * i + 1] = hex[b and 0xF] inc(i) - str[24] = '\0' -proc `$`*(oid: Oid): string = - result = newString(24) - oidToString(oid, result) +let + t = getTime().toUnix -proc rand(): cint {.importc: "rand", header: "<stdlib.h>", nodecl.} -proc srand(seed: cint) {.importc: "srand", header: "<stdlib.h>", nodecl.} +var + seed = initRand(t) + incr: int = seed.rand(int.high) -var t = getTime().toUnix.int32 -srand(t) +let fuzz = cast[int32](seed.rand(high(int))) -var - incr: int = rand() - fuzz: int32 = rand() -proc genOid*(): Oid = - ## generates a new OID. - t = getTime().toUnix.int32 - var i = int32(atomicInc(incr)) +template genOid(result: var Oid, incr: var int, fuzz: int32) = + var time = getTime().toUnix + var i = cast[int32](atomicInc(incr)) - bigEndian32(addr result.time, addr(t)) + bigEndian64(addr result.time, addr(time)) result.fuzz = fuzz bigEndian32(addr result.count, addr(i)) +proc genOid*(): Oid = + ## Generates a new OID. + runnableExamples: + doAssert ($genOid()).len == 24 + runnableExamples("-r:off"): + echo $genOid() # for example, "5fc7f546ddbbc84800006aaf" + genOid(result, incr, fuzz) + proc generatedTime*(oid: Oid): Time = - ## returns the generated timestamp of the OID. - var tmp: int32 + ## Returns the generated timestamp of the OID. + var tmp: int64 var dummy = oid.time - bigEndian32(addr(tmp), addr(dummy)) + bigEndian64(addr(tmp), addr(dummy)) result = fromUnix(tmp) - -when not defined(testing) and isMainModule: - let xo = genOid() - echo xo.generatedTime diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 28c4e4984..b34ff72c0 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -7,143 +7,166 @@ # distribution, for details about the copyright. # -## This module implements types which encapsulate an optional value. -## -## A value of type `Option[T]` either contains a value `x` (represented as -## `some(x)`) or is empty (`none(T)`). -## -## This can be useful when you have a value that can be present or not. The -## absence of a value is often represented by `nil`, but it is not always -## available, nor is it always a good solution. -## -## -## Basic usage -## =========== -## -## Let's start with an example: a procedure that finds the index of a character -## in a string. -## -## .. code-block:: nim -## -## import options -## -## proc find(haystack: string, needle: char): Option[int] = -## for i, c in haystack: -## if c == needle: -## return some(i) -## return none(int) # This line is actually optional, -## # because the default is empty -## -## .. code-block:: nim -## -## let found = "abc".find('c') -## assert found.isSome and found.get() == 2 -## -## The `get` operation demonstrated above returns the underlying value, or -## raises `UnpackError` if there is no value. Note that `UnpackError` -## inherits from `system.Defect`, and should therefore never be caught. -## Instead, rely on checking if the option contains a value with -## `isSome <#isSome,Option[T]>`_ and `isNone <#isNone,Option[T]>`_ procs. -## -## How to deal with an absence of a value: -## -## .. code-block:: nim -## -## let result = "team".find('i') -## -## # Nothing was found, so the result is `none`. -## assert(result == none(int)) -## # It has no value: -## assert(result.isNone) - -import typetraits +##[ +This module implements types which encapsulate an optional value. -type - SomePointer = ref | ptr | pointer +A value of type `Option[T]` either contains a value `x` (represented as +`some(x)`) or is empty (`none(T)`). + +This can be useful when you have a value that can be present or not. The +absence of a value is often represented by `nil`, but that is not always +available, nor is it always a good solution. + + +Basic usage +=========== + +Let's start with an example: a procedure that finds the index of a character +in a string. +]## + +runnableExamples: + proc find(haystack: string, needle: char): Option[int] = + for i, c in haystack: + if c == needle: + return some(i) + return none(int) # This line is actually optional, + # because the default is empty + + let found = "abc".find('c') + assert found.isSome and found.get() == 2 + +##[ +The `get` operation demonstrated above returns the underlying value, or +raises `UnpackDefect` if there is no value. Note that `UnpackDefect` +inherits from `system.Defect` and should therefore never be caught. +Instead, rely on checking if the option contains a value with the +`isSome <#isSome,Option[T]>`_ and `isNone <#isNone,Option[T]>`_ procs. + + +Pattern matching +================ + +.. note:: This requires the [fusion](https://github.com/nim-lang/fusion) package. + +[fusion/matching](https://nim-lang.github.io/fusion/src/fusion/matching.html) +supports pattern matching on `Option`s, with the `Some(<pattern>)` and +`None()` patterns. + + ```nim + {.experimental: "caseStmtMacros".} + + import fusion/matching + + case some(42) + of Some(@a): + assert a == 42 + of None(): + assert false + + assertMatch(some(some(none(int))), Some(Some(None()))) + ``` +]## +# xxx pending https://github.com/timotheecour/Nim/issues/376 use `runnableExamples` and `whichModule` + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +import std/typetraits + +when defined(nimPreviewSlimSystem): + import std/assertions + + +when (NimMajor, NimMinor) >= (1, 1): + type + SomePointer = ref | ptr | pointer | proc | iterator {.closure.} +else: + type + SomePointer = ref | ptr | pointer type Option*[T] = object - ## An optional type that stores its value and state separately in a boolean. + ## An optional type that may or may not contain a value of type `T`. + ## When `T` is a a pointer type (`ptr`, `pointer`, `ref`, `proc` or `iterator {.closure.}`), + ## `none(T)` is represented as `nil`. when T is SomePointer: val: T else: val: T has: bool - UnpackError* = object of Defect + UnpackDefect* = object of Defect + UnpackError* {.deprecated: "See corresponding Defect".} = UnpackDefect - -proc option*[T](val: T): Option[T] = - ## Can be used to convert a pointer type (`ptr` or `ref`) to an option type. - ## It converts `nil` to `None`. +proc option*[T](val: sink T): Option[T] {.inline.} = + ## Can be used to convert a pointer type (`ptr`, `pointer`, `ref` or `proc`) to an option type. + ## It converts `nil` to `none(T)`. When `T` is no pointer type, this is equivalent to `some(val)`. ## - ## See also: - ## * `some <#some,T>`_ - ## * `none <#none,typedesc>`_ + ## **See also:** + ## * `some proc <#some,T>`_ + ## * `none proc <#none,typedesc>`_ runnableExamples: type Foo = ref object a: int b: string - var c: Foo - assert c.isNil - var d = option(c) - assert d.isNone - result.val = val - when T isnot SomePointer: - result.has = true + assert option[Foo](nil).isNone + assert option(42).isSome + + when T is SomePointer: + result = Option[T](val: val) + else: + result = Option[T](has: true, val: val) -proc some*[T](val: T): Option[T] = +proc some*[T](val: sink T): Option[T] {.inline.} = ## Returns an `Option` that has the value `val`. ## - ## See also: - ## * `option <#option,T>`_ - ## * `none <#none,typedesc>`_ - ## * `isSome <#isSome,Option[T]>`_ + ## **See also:** + ## * `option proc <#option,T>`_ + ## * `none proc <#none,typedesc>`_ + ## * `isSome proc <#isSome,Option[T]>`_ runnableExamples: - var - a = some("abc") - b = some(42) - assert $type(a) == "Option[system.string]" - assert b.isSome + let a = some("abc") + + assert a.isSome assert a.get == "abc" - assert $b == "Some(42)" when T is SomePointer: - assert(not val.isNil) - result.val = val + assert not val.isNil + result = Option[T](val: val) else: - result.has = true - result.val = val + result = Option[T](has: true, val: val) -proc none*(T: typedesc): Option[T] = +proc none*(T: typedesc): Option[T] {.inline.} = ## Returns an `Option` for this type that has no value. ## - ## See also: - ## * `option <#option,T>`_ - ## * `some <#some,T>`_ - ## * `isNone <#isNone,Option[T]>`_ + ## **See also:** + ## * `option proc <#option,T>`_ + ## * `some proc <#some,T>`_ + ## * `isNone proc <#isNone,Option[T]>`_ runnableExamples: - var a = none(int) - assert a.isNone - assert $type(a) == "Option[system.int]" + assert none(int).isNone # the default is the none type - discard + result = Option[T]() -proc none*[T]: Option[T] = - ## Alias for `none(T) proc <#none,typedesc>`_. +proc none*[T]: Option[T] {.inline.} = + ## Alias for `none(T) <#none,typedesc>`_. none(T) proc isSome*[T](self: Option[T]): bool {.inline.} = ## Checks if an `Option` contains a value. + ## + ## **See also:** + ## * `isNone proc <#isNone,Option[T]>`_ + ## * `some proc <#some,T>`_ runnableExamples: - var - a = some(42) - b = none(string) - assert a.isSome - assert not b.isSome + assert some(42).isSome + assert not none(string).isSome when T is SomePointer: not self.val.isNil @@ -152,136 +175,124 @@ proc isSome*[T](self: Option[T]): bool {.inline.} = proc isNone*[T](self: Option[T]): bool {.inline.} = ## Checks if an `Option` is empty. + ## + ## **See also:** + ## * `isSome proc <#isSome,Option[T]>`_ + ## * `none proc <#none,typedesc>`_ runnableExamples: - var - a = some(42) - b = none(string) - assert not a.isNone - assert b.isNone + assert not some(42).isNone + assert none(string).isNone + when T is SomePointer: self.val.isNil else: not self.has -proc get*[T](self: Option[T]): T = - ## Returns contents of an `Option`. If it is `None`, then an exception is - ## thrown. +proc get*[T](self: Option[T]): lent T {.inline.} = + ## Returns the content of an `Option`. If it has no value, + ## an `UnpackDefect` exception is raised. ## - ## See also: - ## * `get proc <#get,Option[T],T>`_ with the default return value + ## **See also:** + ## * `get proc <#get,Option[T],T>`_ with a default return value runnableExamples: - let - a = some(42) - b = none(string) - assert a.get == 42 - doAssertRaises(UnpackError): - echo b.get + assert some(42).get == 42 + doAssertRaises(UnpackDefect): + echo none(string).get if self.isNone: - raise newException(UnpackError, "Can't obtain a value from a `none`") - self.val + raise newException(UnpackDefect, "Can't obtain a value from a `none`") + result = self.val -proc get*[T](self: Option[T], otherwise: T): T = - ## Returns the contents of the `Option` or an `otherwise` value if - ## the `Option` is `None`. +proc get*[T](self: Option[T], otherwise: T): T {.inline.} = + ## Returns the content of the `Option` or `otherwise` if + ## the `Option` has no value. runnableExamples: - var - a = some(42) - b = none(int) - assert a.get(9999) == 42 - assert b.get(9999) == 9999 + assert some(42).get(9999) == 42 + assert none(int).get(9999) == 9999 if self.isSome: self.val else: otherwise -proc get*[T](self: var Option[T]): var T = - ## Returns contents of the `var Option`. If it is `None`, then an exception - ## is thrown. +proc get*[T](self: var Option[T]): var T {.inline.} = + ## Returns the content of the `var Option` mutably. If it has no value, + ## an `UnpackDefect` exception is raised. runnableExamples: - let + var a = some(42) b = none(string) - assert a.get == 42 - doAssertRaises(UnpackError): + inc(a.get) + assert a.get == 43 + doAssertRaises(UnpackDefect): echo b.get if self.isNone: - raise newException(UnpackError, "Can't obtain a value from a `none`") + raise newException(UnpackDefect, "Can't obtain a value from a `none`") return self.val -proc map*[T](self: Option[T], callback: proc (input: T)) = +proc map*[T](self: Option[T], callback: proc (input: T)) {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option`, if it has one. ## - ## See also: + ## **See also:** ## * `map proc <#map,Option[T],proc(T)_2>`_ for a version with a callback ## which returns a value - ## * `filter proc <#filter,Option[T],proc(T)>`_ runnableExamples: var d = 0 proc saveDouble(x: int) = - d = 2*x + d = 2 * x - let - a = some(42) - b = none(int) - - b.map(saveDouble) + none(int).map(saveDouble) assert d == 0 - a.map(saveDouble) + some(42).map(saveDouble) assert d == 84 if self.isSome: callback(self.val) -proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] = +proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline, effectsOf: callback.} = ## Applies a `callback` function to the value of the `Option` and returns an ## `Option` containing the new value. ## - ## If the `Option` is `None`, `None` of the return type of the `callback` - ## will be returned. + ## If the `Option` has no value, `none(R)` will be returned. ## - ## See also: - ## * `flatMap proc <#flatMap,Option[A],proc(A)>`_ for a version with a - ## callback which returns an `Option` - ## * `filter proc <#filter,Option[T],proc(T)>`_ + ## **See also:** + ## * `map proc <#map,Option[T],proc(T)>`_ + ## * `flatMap proc <#flatMap,Option[T],proc(T)>`_ for a version with a + ## callback that returns an `Option` runnableExamples: - var - a = some(42) - b = none(int) - proc isEven(x: int): bool = x mod 2 == 0 - assert $(a.map(isEven)) == "Some(true)" - assert $(b.map(isEven)) == "None[bool]" + assert some(42).map(isEven) == some(true) + assert none(int).map(isEven) == none(bool) if self.isSome: some[R](callback(self.val)) else: none(R) -proc flatten*[A](self: Option[Option[A]]): Option[A] = +proc flatten*[T](self: Option[Option[T]]): Option[T] {.inline.} = ## Remove one level of structure in a nested `Option`. + ## + ## **See also:** + ## * `flatMap proc <#flatMap,Option[T],proc(T)>`_ runnableExamples: - let a = some(some(42)) - assert $flatten(a) == "Some(42)" + assert flatten(some(some(42))) == some(42) + assert flatten(none(Option[int])) == none(int) if self.isSome: self.val else: - none(A) + none(T) -proc flatMap*[A, B](self: Option[A], - callback: proc (input: A): Option[B]): Option[B] = - ## Applies a `callback` function to the value of the `Option` and returns an - ## `Option` containing the new value. +proc flatMap*[T, R](self: Option[T], + callback: proc (input: T): Option[R]): Option[R] {.inline, effectsOf: callback.} = + ## Applies a `callback` function to the value of the `Option` and returns the new value. ## - ## If the `Option` is `None`, `None` of the return type of the `callback` - ## will be returned. + ## If the `Option` has no value, `none(R)` will be returned. ## - ## Similar to `map`, with the difference that the `callback` returns an + ## This is similar to `map`, with the difference that the `callback` returns an ## `Option`, not a raw value. This allows multiple procs with a ## signature of `A -> Option[B]` to be chained together. ## @@ -291,47 +302,40 @@ proc flatMap*[A, B](self: Option[A], runnableExamples: proc doublePositives(x: int): Option[int] = if x > 0: - return some(2*x) + some(2 * x) else: - return none(int) - let - a = some(42) - b = none(int) - c = some(-11) - assert a.flatMap(doublePositives) == some(84) - assert b.flatMap(doublePositives) == none(int) - assert c.flatMap(doublePositives) == none(int) + none(int) + + assert some(42).flatMap(doublePositives) == some(84) + assert none(int).flatMap(doublePositives) == none(int) + assert some(-11).flatMap(doublePositives) == none(int) map(self, callback).flatten() -proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] = +proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline, effectsOf: callback.} = ## Applies a `callback` to the value of the `Option`. ## - ## If the `callback` returns `true`, the option is returned as `Some`. - ## If it returns `false`, it is returned as `None`. + ## If the `callback` returns `true`, the option is returned as `some`. + ## If it returns `false`, it is returned as `none`. ## - ## See also: - ## * `map proc <#map,Option[T],proc(T)_2>`_ + ## **See also:** ## * `flatMap proc <#flatMap,Option[A],proc(A)>`_ runnableExamples: proc isEven(x: int): bool = x mod 2 == 0 - let - a = some(42) - b = none(int) - c = some(-11) - assert a.filter(isEven) == some(42) - assert b.filter(isEven) == none(int) - assert c.filter(isEven) == none(int) + + assert some(42).filter(isEven) == some(42) + assert none(int).filter(isEven) == none(int) + assert some(-11).filter(isEven) == none(int) if self.isSome and not callback(self.val): none(T) else: self -proc `==`*(a, b: Option): bool = - ## Returns `true` if both `Option`s are `None`, - ## or if they are both `Some` and have equal values. +proc `==`*[T](a, b: Option[T]): bool {.inline.} = + ## Returns `true` if both `Option`s are `none`, + ## or if they are both `some` and have equal values. runnableExamples: let a = some(42) @@ -343,165 +347,35 @@ proc `==`*(a, b: Option): bool = assert b == d assert not (a == b) - (a.isSome and b.isSome and a.val == b.val) or (not a.isSome and not b.isSome) + when T is SomePointer: + a.val == b.val + else: + (a.isSome and b.isSome and a.val == b.val) or (a.isNone and b.isNone) proc `$`*[T](self: Option[T]): string = ## Get the string representation of the `Option`. - ## - ## If the `Option` has a value, the result will be `Some(x)` where `x` - ## is the string representation of the contained value. - ## If the `Option` does not have a value, the result will be `None[T]` - ## where `T` is the name of the type contained in the `Option`. + runnableExamples: + assert $some(42) == "some(42)" + assert $none(int) == "none(int)" + if self.isSome: - result = "Some(" + when defined(nimLagacyOptionsDollar): + result = "Some(" + else: + result = "some(" result.addQuoted self.val result.add ")" else: - result = "None[" & name(T) & "]" + when defined(nimLagacyOptionsDollar): + result = "None[" & name(T) & "]" + else: + result = "none(" & name(T) & ")" -proc unsafeGet*[T](self: Option[T]): T = - ## Returns the value of a `some`. Behavior is undefined for `none`. +proc unsafeGet*[T](self: Option[T]): lent T {.inline.}= + ## Returns the value of a `some`. The behavior is undefined for `none`. ## - ## **Note:** Use it only when you are **absolutely sure** the value is present - ## (e.g. after checking `isSome <#isSome,Option[T]>`_). - ## Generally, using `get proc <#get,Option[T]>`_ is preferred. + ## **Note:** Use this only when you are **absolutely sure** the value is present + ## (e.g. after checking with `isSome <#isSome,Option[T]>`_). + ## Generally, using the `get proc <#get,Option[T]>`_ is preferred. assert self.isSome - self.val - - -when isMainModule: - import unittest, sequtils - - # RefPerson is used to test that overloaded `==` operator is not called by - # options. It is defined here in the global scope, because otherwise the test - # will not even consider the `==` operator. Different bug? - type RefPerson = ref object - name: string - - proc `==`(a, b: RefPerson): bool = - assert(not a.isNil and not b.isNil) - a.name == b.name - - suite "options": - # work around a bug in unittest - let intNone = none(int) - let stringNone = none(string) - - test "example": - proc find(haystack: string, needle: char): Option[int] = - for i, c in haystack: - if c == needle: - return some i - - check("abc".find('c').get() == 2) - - let result = "team".find('i') - - check result == intNone - check result.isNone - - test "some": - check some(6).get() == 6 - check some("a").unsafeGet() == "a" - check some(6).isSome - check some("a").isSome - - test "none": - expect UnpackError: - discard none(int).get() - check(none(int).isNone) - check(not none(string).isSome) - - test "equality": - check some("a") == some("a") - check some(7) != some(6) - check some("a") != stringNone - check intNone == intNone - - when compiles(some("a") == some(5)): - check false - when compiles(none(string) == none(int)): - check false - - test "get with a default value": - check(some("Correct").get("Wrong") == "Correct") - check(stringNone.get("Correct") == "Correct") - - test "$": - check($(some("Correct")) == "Some(\"Correct\")") - check($(stringNone) == "None[string]") - - test "map with a void result": - var procRan = 0 - some(123).map(proc (v: int) = procRan = v) - check procRan == 123 - intNone.map(proc (v: int) = check false) - - test "map": - check(some(123).map(proc (v: int): int = v * 2) == some(246)) - check(intNone.map(proc (v: int): int = v * 2).isNone) - - test "filter": - check(some(123).filter(proc (v: int): bool = v == 123) == some(123)) - check(some(456).filter(proc (v: int): bool = v == 123).isNone) - check(intNone.filter(proc (v: int): bool = check false).isNone) - - test "flatMap": - proc addOneIfNotZero(v: int): Option[int] = - if v != 0: - result = some(v + 1) - else: - result = none(int) - - check(some(1).flatMap(addOneIfNotZero) == some(2)) - check(some(0).flatMap(addOneIfNotZero) == none(int)) - check(some(1).flatMap(addOneIfNotZero).flatMap(addOneIfNotZero) == some(3)) - - proc maybeToString(v: int): Option[string] = - if v != 0: - result = some($v) - else: - result = none(string) - - check(some(1).flatMap(maybeToString) == some("1")) - - proc maybeExclaim(v: string): Option[string] = - if v != "": - result = some v & "!" - else: - result = none(string) - - check(some(1).flatMap(maybeToString).flatMap(maybeExclaim) == some("1!")) - check(some(0).flatMap(maybeToString).flatMap(maybeExclaim) == none(string)) - - test "SomePointer": - var intref: ref int - check(option(intref).isNone) - intref.new - check(option(intref).isSome) - - let tmp = option(intref) - check(sizeof(tmp) == sizeof(ptr int)) - - test "none[T]": - check(none[int]().isNone) - check(none(int) == none[int]()) - - test "$ on typed with .name": - type Named = object - name: string - - let nobody = none(Named) - check($nobody == "None[Named]") - - test "$ on type with name()": - type Person = object - myname: string - - let noperson = none(Person) - check($noperson == "None[Person]") - - test "Ref type with overloaded `==`": - let p = some(RefPerson.new()) - check p.isSome - + result = self.val diff --git a/lib/pure/os.nim b/lib/pure/os.nim index abb1b4a19..78ebb1c88 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -8,916 +8,121 @@ # ## This module contains basic operating system facilities like -## retrieving environment variables, reading command line arguments, -## working with directories, running shell commands, etc. -## -## .. code-block:: -## import os -## -## let myFile = "/path/to/my/file.nim" -## -## let pathSplit = splitPath(myFile) -## assert pathSplit.head == "/path/to/my" -## assert pathSplit.tail == "file.nim" -## -## assert parentDir(myFile) == "/path/to/my" -## -## let fileSplit = splitFile(myFile) -## assert fileSplit.dir == "/path/to/my" -## assert fileSplit.name == "file" -## assert fileSplit.ext == ".nim" -## -## assert myFile.changeFileExt("c") == "/path/to/my/file.c" - -## -## +## retrieving environment variables, working with directories, +## running shell commands, etc. + +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim + +runnableExamples: + let myFile = "/path/to/my/file.nim" + assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim") + when defined(posix): + assert parentDir(myFile) == "/path/to/my" + assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim") + assert myFile.changeFileExt("c") == "/path/to/my/file.c" + ## **See also:** +## * `paths <paths.html>`_ and `files <files.html>`_ modules for high-level file manipulation ## * `osproc module <osproc.html>`_ for process communication beyond -## `execShellCmd proc <#execShellCmd,string>`_ -## * `parseopt module <parseopt.html>`_ for command-line parser beyond -## `parseCmdLine proc <#parseCmdLine,string>`_ +## `execShellCmd proc`_ ## * `uri module <uri.html>`_ ## * `distros module <distros.html>`_ ## * `dynlib module <dynlib.html>`_ ## * `streams module <streams.html>`_ +import std/private/ospaths2 +export ospaths2 + +import std/private/osfiles +export osfiles + +import std/private/osdirs +export osdirs + +import std/private/ossymlinks +export ossymlinks + +import std/private/osappdirs +export osappdirs + +import std/private/oscommon -include "system/inclrtl" +include system/inclrtl +import std/private/since -import - strutils, pathnorm +import std/cmdline +export cmdline + +import std/[strutils, pathnorm] + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] const weirdTarget = defined(nimscript) or defined(js) +since (1, 1): + const + invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ + ## Characters that may produce invalid filenames across Linux, Windows and Mac. + ## You can check if your filename contains any of these chars and strip them for safety. + ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. + invalidFilenames* = [ + "CON", "PRN", "AUX", "NUL", + "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"] ## \ + ## Filenames that may be invalid across Linux, Windows, Mac, etc. + ## You can check if your filename match these and rename it for safety + ## (Currently all invalid filenames are from Windows only). + when weirdTarget: discard elif defined(windows): - import winlean, times + import std/[winlean, times] elif defined(posix): - import posix, times + import std/[posix, times] proc toTime(ts: Timespec): times.Time {.inline.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) else: {.error: "OS module not ported to your operating system!".} -when weirdTarget and defined(nimErrorProcCanHaveBody): - {.pragma: noNimScript, error: "this proc is not available on the NimScript target".} +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} else: - {.pragma: noNimScript.} - -type - ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read - ## from an environment variable. - WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write - ## to an environment variable. - - ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read - ## operation from the directory - ## structure. - WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write - ## operation to - ## the directory structure. - - OSErrorCode* = distinct int32 ## Specifies an OS Error Code. - -include "includes/osseps" - -proc normalizePathEnd(path: var string, trailingSep = false) = - ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on - ## ``trailingSep``, and taking care of edge cases: it preservers whether - ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, - ## not `AltSep`. Trailing `/.` are compressed, see examples. - if path.len == 0: return - var i = path.len - while i >= 1: - if path[i-1] in {DirSep, AltSep}: dec(i) - elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) - else: break - if trailingSep: - # foo// => foo - path.setLen(i) - # foo => foo/ - path.add DirSep - elif i > 0: - # foo// => foo - path.setLen(i) - else: - # // => / (empty case was already taken care of) - path = $DirSep - -proc normalizePathEnd(path: string, trailingSep = false): string = - ## outplace overload - runnableExamples: - when defined(posix): - assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" - assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" - assert normalizePathEnd(".//./.", trailingSep = false) == "." - assert normalizePathEnd("", trailingSep = true) == "" # not / ! - assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! - result = path - result.normalizePathEnd(trailingSep) - -since((1, 1)): - export normalizePathEnd - -proc joinPath*(head, tail: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Joins two directory names to one. - ## - ## If `head` is the empty string, `tail` is returned. If `tail` is the empty - ## string, `head` is returned with a trailing path separator. If `tail` starts - ## with a path separator it will be removed when concatenated to `head`. - ## Path separators will be normalized. - ## - ## See also: - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ - ## * `uri./ proc <uri.html#/,Uri,string>`_ - runnableExamples: - when defined(posix): - assert joinPath("usr", "lib") == "usr/lib" - assert joinPath("usr", "") == "usr/" - assert joinPath("", "lib") == "lib" - assert joinPath("", "/lib") == "/lib" - assert joinPath("usr/", "/lib") == "usr/lib" - assert joinPath("usr/lib", "../bin") == "usr/bin" - - result = newStringOfCap(head.len + tail.len) - var state = 0 - addNormalizePath(head, result, state, DirSep) - if tail.len == 0: - result.add DirSep - else: - addNormalizePath(tail, result, state, DirSep) - when false: - if len(head) == 0: - result = tail - elif head[len(head)-1] in {DirSep, AltSep}: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & substr(tail, 1) - else: - result = head & tail - else: - if tail.len > 0 and tail[0] in {DirSep, AltSep}: - result = head & tail - else: - result = head & DirSep & tail - -proc joinPath*(parts: varargs[string]): string {.noSideEffect, - rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_, - ## but works with any number of directory parts. - ## - ## You need to pass at least one element or the proc - ## will assert in debug builds and crash on release builds. - ## - ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - runnableExamples: - when defined(posix): - assert joinPath("a") == "a" - assert joinPath("a", "b", "c") == "a/b/c" - assert joinPath("usr/lib", "../../var", "log") == "var/log" - - var estimatedLen = 0 - for p in parts: estimatedLen += p.len - result = newStringOfCap(estimatedLen) - var state = 0 - for i in 0..high(parts): - addNormalizePath(parts[i], result, state, DirSep) - -proc `/`*(head, tail: string): string {.noSideEffect.} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_. - ## - ## See also: - ## * `/../ proc <#/../,string,string>`_ - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ - ## * `uri./ proc <uri.html#/,Uri,string>`_ - runnableExamples: - when defined(posix): - assert "usr" / "" == "usr/" - assert "" / "lib" == "lib" - assert "" / "/lib" == "/lib" - assert "usr/" / "/lib" == "usr/lib" - assert "usr" / "lib" / "../bin" == "usr/bin" - - return joinPath(head, tail) - -proc splitPath*(path: string): tuple[head, tail: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a directory into `(head, tail)` tuple, so that - ## ``head / tail == path`` (except for edge cases like "/usr"). - ## - ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `relativePath proc <#relativePath,string,string>`_ - runnableExamples: - assert splitPath("usr/local/bin") == ("usr/local", "bin") - assert splitPath("usr/local/bin/") == ("usr/local/bin", "") - assert splitPath("/bin/") == ("/bin", "") - when (NimMajor, NimMinor) <= (1, 0): - assert splitPath("/bin") == ("", "bin") - else: - assert splitPath("/bin") == ("/", "bin") - assert splitPath("bin") == ("", "bin") - assert splitPath("") == ("", "") - - var sepPos = -1 - for i in countdown(len(path)-1, 0): - if path[i] in {DirSep, AltSep}: - sepPos = i - break - if sepPos >= 0: - result.head = substr(path, 0, - when (NimMajor, NimMinor) <= (1, 0): - sepPos-1 - else: - if likely(sepPos >= 1): sepPos-1 else: 0 - ) - result.tail = substr(path, sepPos+1) - else: - result.head = "" - result.tail = path + {.pragma: noWeirdTarget.} -proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = - ## Checks whether a given `path` is absolute. - ## - ## On Windows, network paths are considered absolute too. - runnableExamples: - assert not "".isAbsolute - assert not ".".isAbsolute - when defined(posix): - assert "/".isAbsolute - assert not "a/".isAbsolute - assert "/a/".isAbsolute - - if len(path) == 0: return false - - when doslikeFileSystem: - var len = len(path) - result = (path[0] in {'/', '\\'}) or - (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') - elif defined(macos): - # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path - result = path[0] != ':' - elif defined(RISCOS): - result = path[0] == '$' - elif defined(posix): - result = path[0] == '/' - -when FileSystemCaseSensitive: - template `!=?`(a, b: char): bool = a != b +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} else: - template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) - -when doslikeFileSystem: - proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: []} = - ## An absolute path from the root of the current drive (e.g. "\foo") - path.len > 0 and - (path[0] == AltSep or - (path[0] == DirSep and - (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) - - proc isUNCPrefix(path: string): bool {.noSideEffect, raises: []} = - path[0] == DirSep and path[1] == DirSep - - proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: []} = - ## Return true if path1 and path2 have a same root. - ## - ## Detail of windows path formats: - ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats - - assert(isAbsolute(path1)) - assert(isAbsolute(path2)) - - let - len1 = path1.len - len2 = path2.len - assert(len1 != 0 and len2 != 0) - - if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): - return true - elif len1 == 1 or len2 == 1: - return false - else: - if path1[1] == ':' and path2[1] == ':': - return path1[0].toLowerAscii() == path2[0].toLowerAscii() - else: - var - p1, p2: PathIter - pp1 = next(p1, path1) - pp2 = next(p2, path2) - if pp1[1] - pp1[0] == 1 and pp2[1] - pp2[0] == 1 and - isUNCPrefix(path1) and isUNCPrefix(path2): - #UNC - var h = 0 - while p1.hasNext(path1) and p2.hasNext(path2) and h < 2: - pp1 = next(p1, path1) - pp2 = next(p2, path2) - let diff = pp1[1] - pp1[0] - if diff != pp2[1] - pp2[0]: - return false - for i in 0..diff: - if path1[i + pp1[0]] !=? path2[i + pp2[0]]: - return false - inc h - return h == 2 - else: - return false - -proc relativePath*(path, base: string; sep = DirSep): string {. - noSideEffect, rtl, extern: "nos$1", raises: [].} = - ## Converts `path` to a path relative to `base`. - ## - ## The `sep` (default: `DirSep <#DirSep>`_) is used for the path normalizations, - ## this can be useful to ensure the relative path only contains `'/'` - ## so that it can be used for URL constructions. - ## - ## On windows, if a root of `path` and a root of `base` are different, - ## returns `path` as is because it is impossible to make a relative path. - ## That means an absolute path can be returned. - ## - ## See also: - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - runnableExamples: - assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" - assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" - assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" - assert relativePath("", "/users/moo", '/') == "" - assert relativePath("foo", ".", '/') == "foo" - - if path.len == 0: return "" - let base = if base == ".": "" else: base - - when doslikeFileSystem: - if isAbsolute(path) and isAbsolute(base): - if not sameRoot(path, base): - return path - - var f, b: PathIter - var ff = (0, -1) - var bb = (0, -1) # (int, int) - result = newStringOfCap(path.len) - # skip the common prefix: - while f.hasNext(path) and b.hasNext(base): - ff = next(f, path) - bb = next(b, base) - let diff = ff[1] - ff[0] - if diff != bb[1] - bb[0]: break - var same = true - for i in 0..diff: - if path[i + ff[0]] !=? base[i + bb[0]]: - same = false - break - if not same: break - ff = (0, -1) - bb = (0, -1) - # for i in 0..diff: - # result.add base[i + bb[0]] - - # /foo/bar/xxx/ -- base - # /foo/bar/baz -- path path - # ../baz - # every directory that is in 'base', needs to add '..' - while true: - if bb[1] >= bb[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - result.add ".." - if not b.hasNext(base): break - bb = b.next(base) - - # add the rest of 'path': - while true: - if ff[1] >= ff[0]: - if result.len > 0 and result[^1] != sep: - result.add sep - for i in 0..ff[1] - ff[0]: - result.add path[i + ff[0]] - if not f.hasNext(path): break - ff = f.next(path) - -proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = - ## Returns true if `path` is relative to `base`. - runnableExamples: - doAssert isRelativeTo("./foo//bar", "foo") - doAssert isRelativeTo("foo/bar", ".") - doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") - doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") - let path = path.normalizePath - let base = base.normalizePath - let ret = relativePath(path, base) - result = path.len > 0 and not ret.startsWith ".." - -proc parentDirPos(path: string): int = - var q = 1 - if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 - for i in countdown(len(path)-q, 0): - if path[i] in {DirSep, AltSep}: return i - result = -1 - -proc parentDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the parent directory of `path`. - ## - ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end - ## in a dir separator, but also takes care of path normalizations. - ## The remainder can be obtained with `lastPathPart(path) proc - ## <#lastPathPart,string>`_. - ## - ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - ## * `parentDirs iterator <#parentDirs.i,string>`_ - runnableExamples: - assert parentDir("") == "" - when defined(posix): - assert parentDir("/usr/local/bin") == "/usr/local" - assert parentDir("foo/bar//") == "foo" - assert parentDir("//foo//bar//.") == "/foo" - assert parentDir("./foo") == "." - assert parentDir("/./foo//./") == "/" - assert parentDir("a//./") == "." - assert parentDir("a/b/c/..") == "a" - result = pathnorm.normalizePath(path) - var sepPos = parentDirPos(result) - if sepPos >= 0: - result = substr(result, 0, sepPos) - normalizePathEnd(result) - elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: - # `.` => `..` and .. => `../..`(etc) would be a sensible alternative - # `/` => `/` (as done with splitFile) would be a sensible alternative - result = "" - else: - result = "." - -proc tailDir*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Returns the tail part of `path`. - ## - ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - runnableExamples: - assert tailDir("/bin") == "bin" - assert tailDir("bin") == "" - assert tailDir("bin/") == "" - assert tailDir("/usr/local/bin") == "usr/local/bin" - assert tailDir("//usr//local//bin//") == "usr//local//bin//" - assert tailDir("./usr/local/bin") == "usr/local/bin" - assert tailDir("usr/local/bin") == "local/bin" - - var i = 0 - while i < len(path): - if path[i] in {DirSep, AltSep}: - while i < len(path) and path[i] in {DirSep, AltSep}: inc i - return substr(path, i) - inc i - result = "" - -proc isRootDir*(path: string): bool {. - noSideEffect, rtl, extern: "nos$1".} = - ## Checks whether a given `path` is a root directory. - runnableExamples: - assert isRootDir("") - assert isRootDir(".") - assert isRootDir("/") - assert isRootDir("a") - assert not isRootDir("/a") - assert not isRootDir("a/b/c") - - result = parentDirPos(path) < 0 - -iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = - ## Walks over all parent directories of a given `path`. - ## - ## If `fromRoot` is true (default: false), the traversal will start from - ## the file system root directory. - ## If `inclusive` is true (default), the original argument will be included - ## in the traversal. - ## - ## Relative paths won't be expanded by this iterator. Instead, it will traverse - ## only the directories appearing in the relative path. - ## - ## See also: - ## * `parentDir proc <#parentDir,string>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## let g = "a/b/c" - ## - ## for p in g.parentDirs: - ## echo p - ## # a/b/c - ## # a/b - ## # a - ## - ## for p in g.parentDirs(fromRoot=true): - ## echo p - ## # a/ - ## # a/b/ - ## # a/b/c - ## - ## for p in g.parentDirs(inclusive=false): - ## echo p - ## # a/b - ## # a - - if not fromRoot: - var current = path - if inclusive: yield path - while true: - if current.isRootDir: break - current = current.parentDir - yield current - else: - for i in countup(0, path.len - 2): # ignore the last / - # deal with non-normalized paths such as /foo//bar//baz - if path[i] in {DirSep, AltSep} and - (i == 0 or path[i-1] notin {DirSep, AltSep}): - yield path.substr(0, i) - - if inclusive: yield path - -proc `/../`*(head, tail: string): string {.noSideEffect.} = - ## The same as ``parentDir(head) / tail``, unless there is no parent - ## directory. Then ``head / tail`` is performed instead. - ## - ## See also: - ## * `/ proc <#/,string,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - runnableExamples: - when defined(posix): - assert "a/b/c" /../ "d/e" == "a/b/d/e" - assert "a" /../ "d/e" == "a/d/e" - - let sepPos = parentDirPos(head) - if sepPos >= 0: - result = substr(head, 0, sepPos-1) / tail - else: - result = head / tail - -proc normExt(ext: string): string = - if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here - else: result = ExtSep & ext - -proc searchExtPos*(path: string): int = - ## Returns index of the `'.'` char in `path` if it signifies the beginning - ## of extension. Returns -1 otherwise. - ## - ## See also: - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert searchExtPos("a/b/c") == -1 - assert searchExtPos("c.nim") == 1 - assert searchExtPos("a/b/c.nim") == 5 - assert searchExtPos("a.b.c.nim") == 5 - - # BUGFIX: do not search until 0! .DS_Store is no file extension! - result = -1 - for i in countdown(len(path)-1, 1): - if path[i] == ExtSep: - result = i - break - elif path[i] in {DirSep, AltSep}: - break # do not skip over path - -proc splitFile*(path: string): tuple[dir, name, ext: string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a filename into `(dir, name, extension)` tuple. - ## - ## `dir` does not end in `DirSep <#DirSep>`_ unless it's `/`. - ## `extension` includes the leading dot. - ## - ## If `path` has no extension, `ext` is the empty string. - ## If `path` has no directory component, `dir` is the empty string. - ## If `path` has no filename component, `name` and `ext` are empty strings. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - var (dir, name, ext) = splitFile("usr/local/nimc.html") - assert dir == "usr/local" - assert name == "nimc" - assert ext == ".html" - (dir, name, ext) = splitFile("/usr/local/os") - assert dir == "/usr/local" - assert name == "os" - assert ext == "" - (dir, name, ext) = splitFile("/usr/local/") - assert dir == "/usr/local" - assert name == "" - assert ext == "" - (dir, name, ext) = splitFile("/tmp.txt") - assert dir == "/" - assert name == "tmp" - assert ext == ".txt" - - var namePos = 0 - var dotPos = 0 - for i in countdown(len(path) - 1, 0): - if path[i] in {DirSep, AltSep} or i == 0: - if path[i] in {DirSep, AltSep}: - result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) - namePos = i + 1 - if dotPos > i: - result.name = substr(path, namePos, dotPos - 1) - result.ext = substr(path, dotPos) - else: - result.name = substr(path, namePos) - break - elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and - path[i - 1] notin {DirSep, AltSep} and - path[i + 1] != ExtSep and dotPos == 0: - dotPos = i - -proc extractFilename*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Extracts the filename of a given `path`. - ## - ## This is the same as ``name & ext`` from `splitFile(path) proc - ## <#splitFile,string>`_. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert extractFilename("foo/bar/") == "" - assert extractFilename("foo/bar") == "bar" - assert extractFilename("foo/bar.baz") == "bar.baz" + {.pragma: noNimJs.} - if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: - result = "" - else: - result = splitPath(path).tail - -proc lastPathPart*(path: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Like `extractFilename proc <#extractFilename,string>`_, but ignores - ## trailing dir separator; aka: `baseName`:idx: in some other languages. - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert lastPathPart("foo/bar/") == "bar" - assert lastPathPart("foo/bar") == "bar" - let path = path.normalizePathEnd(trailingSep = false) - result = extractFilename(path) +import std/oserrors +export oserrors +import std/envvars +export envvars -proc changeFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Changes the file extension to `ext`. - ## - ## If the `filename` has no extension, `ext` will be added. - ## If `ext` == "" then any extension is removed. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. (Although I know - ## of none such beast.) - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ - runnableExamples: - assert changeFileExt("foo.bar", "baz") == "foo.baz" - assert changeFileExt("foo.bar", "") == "foo" - assert changeFileExt("foo", "baz") == "foo.baz" - - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = substr(filename, 0, extPos-1) & normExt(ext) - -proc addFileExt*(filename, ext: string): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Adds the file extension `ext` to `filename`, unless - ## `filename` already has an extension. - ## - ## `Ext` should be given without the leading `'.'`, because some - ## filesystems may use a different character. - ## (Although I know of none such beast.) - ## - ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - runnableExamples: - assert addFileExt("foo.bar", "baz") == "foo.bar" - assert addFileExt("foo.bar", "") == "foo.bar" - assert addFileExt("foo", "baz") == "foo.baz" +import std/private/osseps +export osseps - var extPos = searchExtPos(filename) - if extPos < 0: result = filename & normExt(ext) - else: result = filename -proc cmpPaths*(pathA, pathB: string): int {. - noSideEffect, rtl, extern: "nos$1".} = - ## Compares two paths. - ## - ## On a case-sensitive filesystem this is done - ## case-sensitively otherwise case-insensitively. Returns: - ## - ## | 0 iff pathA == pathB - ## | < 0 iff pathA < pathB - ## | > 0 iff pathA > pathB - runnableExamples: - when defined(macosx): - assert cmpPaths("foo", "Foo") == 0 - elif defined(posix): - assert cmpPaths("foo", "Foo") > 0 - - let a = normalizePath(pathA) - let b = normalizePath(pathB) - if FileSystemCaseSensitive: - result = cmp(a, b) - else: - when defined(nimscript): - result = cmpic(a, b) - elif defined(nimdoc): discard - else: - result = cmpIgnoreCase(a, b) - -proc unixToNativePath*(path: string, drive=""): string {. - noSideEffect, rtl, extern: "nos$1".} = - ## Converts an UNIX-like path to a native one. - ## - ## On an UNIX system this does nothing. Else it converts - ## `'/'`, `'.'`, `'..'` to the appropriate things. - ## - ## On systems with a concept of "drives", `drive` is used to determine - ## which drive label to use during absolute path conversion. - ## `drive` defaults to the drive of the current working directory, and is - ## ignored on systems that do not have a concept of "drives". - when defined(unix): - result = path - else: - if path.len == 0: return "" - - var start: int - if path[0] == '/': - # an absolute path - when doslikeFileSystem: - if drive != "": - result = drive & ":" & DirSep - else: - result = $DirSep - elif defined(macos): - result = "" # must not start with ':' - else: - result = $DirSep - start = 1 - elif path[0] == '.' and (path.len == 1 or path[1] == '/'): - # current directory - result = $CurDir - start = when doslikeFileSystem: 1 else: 2 - else: - result = "" - start = 0 - - var i = start - while i < len(path): # ../../../ --> :::: - if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': - # parent directory - when defined(macos): - if result[high(result)] == ':': - add result, ':' - else: - add result, ParDir - else: - add result, ParDir & DirSep - inc(i, 3) - elif path[i] == '/': - add result, DirSep - inc(i) - else: - add result, path[i] - inc(i) - -include "includes/oserr" -when not defined(nimscript): - include "includes/osenv" - -proc getHomeDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the home directory of the current user. - ## - ## This proc is wrapped by the `expandTilde proc <#expandTilde,string>`_ - ## for the convenience of processing paths coming from user configuration files. - ## - ## See also: - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - runnableExamples: - assert getHomeDir() == expandTilde("~") - - when defined(windows): return string(getEnv("USERPROFILE")) & "\\" - else: return string(getEnv("HOME")) & "/" - -proc getConfigDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the config directory of the current user for applications. - ## - ## On non-Windows OSs, this proc conforms to the XDG Base Directory - ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment - ## variable if it is set, otherwise it returns the default configuration - ## directory ("~/.config/"). - ## - ## An OS-dependent trailing slash is always present at the end of the - ## returned string: `\\` on Windows and `/` on all other OSs. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - when defined(windows): - result = getEnv("APPDATA").string - else: - result = getEnv("XDG_CONFIG_HOME", getEnv("HOME").string / ".config").string - result.normalizePathEnd(trailingSep = true) - -proc getTempDir*(): string {.rtl, extern: "nos$1", - tags: [ReadEnvEffect, ReadIOEffect].} = - ## Returns the temporary directory of the current user for applications to - ## save temporary files in. - ## - ## **Please do not use this**: On Android, it currently - ## returns ``getHomeDir()``, and on other Unix based systems it can cause - ## security problems too. That said, you can override this implementation - ## by adding ``-d:tempDir=mytempname`` to your compiler invocation. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - const tempDirDefault = "/tmp" - result = tempDirDefault - when defined(tempDir): - const tempDir {.strdefine.}: string = tempDirDefault - result = tempDir - elif defined(windows): result = string(getEnv("TEMP")) - elif defined(android): result = getHomeDir() - else: - if existsEnv("TMPDIR"): result = string(getEnv("TMPDIR")) - normalizePathEnd(result, trailingSep=true) proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing - ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## ``~`` with `getHomeDir()`_ (otherwise returns ``path`` unmodified). ## - ## Windows: this is still supported despite Windows platform not having this + ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" @@ -933,15 +138,12 @@ proc expandTilde*(path: string): string {. # TODO: handle `~bob` and `~bob/` which means home of bob result = path -# TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand -# belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote `s`, so it can be safely passed to Windows API. ## ## Based on Python's `subprocess.list2cmdline`. ## See `this link <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_ ## for more details. - let needQuote = {' ', '\t'} in s or s.len == 0 result = "" var backslashBuff = "" @@ -952,8 +154,8 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" if c == '\\': backslashBuff.add(c) elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) + for i in 0..<backslashBuff.len*2: + result.add('\\') backslashBuff.setLen(0) result.add("\\\"") else: @@ -962,35 +164,34 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" backslashBuff.setLen(0) result.add(c) + if backslashBuff.len > 0: + result.add(backslashBuff) if needQuote: + result.add(backslashBuff) result.add("\"") + proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's `pipes.quote`. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s + result = "''" + elif s.allCharsInSet(safeUnixChars): + result = s else: - return "'" & s.replace("'", "'\"'\"'") & "'" + result = "'" & s.replace("'", "'\"'\"'") & "'" when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. ## - ## When on Windows, it calls `quoteShellWindows proc - ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc - ## <#quoteShellPosix,string>`_. + ## When on Windows, it calls `quoteShellWindows proc`_. + ## Otherwise, calls `quoteShellPosix proc`_. when defined(windows): - return quoteShellWindows(s) + result = quoteShellWindows(s) else: - return quoteShellPosix(s) + result = quoteShellPosix(s) proc quoteShellCommand*(args: openArray[string]): string = ## Concatenates and quotes shell arguments `args`. @@ -1001,128 +202,19 @@ when defined(windows) or defined(posix) or defined(nintendoswitch): assert quoteShellCommand(["aaa", "", "c d"]) == "aaa \"\" \"c d\"" # can't use `map` pending https://github.com/nim-lang/Nim/issues/8303 + result = "" for i in 0..<args.len: if i > 0: result.add " " result.add quoteShell(args[i]) when not weirdTarget: - proc c_rename(oldname, newname: cstring): cint {. - importc: "rename", header: "<stdio.h>".} proc c_system(cmd: cstring): cint {. importc: "system", header: "<stdlib.h>".} - proc c_strlen(a: cstring): cint {. - importc: "strlen", header: "<string.h>", noSideEffect.} - proc c_free(p: pointer) {. - importc: "free", header: "<stdlib.h>".} - - -when defined(windows) and not weirdTarget: - when useWinUnicode: - template wrapUnary(varname, winApiProc, arg: untyped) = - var varname = winApiProc(newWideCString(arg)) - - template wrapBinary(varname, winApiProc, arg, arg2: untyped) = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile(a, b: untyped): untyped = findNextFileW(a, b) - template getCommandLine(): untyped = getCommandLineW() - - template getFilename(f: untyped): untyped = - $cast[WideCString](addr(f.cFileName[0])) - else: - template findFirstFile(a, b: untyped): untyped = findFirstFileA(a, b) - template findNextFile(a, b: untyped): untyped = findNextFileA(a, b) - template getCommandLine(): untyped = getCommandLineA() - - template getFilename(f: untyped): untyped = $f.cFileName - - proc skipFindData(f: WIN32_FIND_DATA): bool {.inline.} = - # Note - takes advantage of null delimiter in the cstring - const dot = ord('.') - result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or - f.cFileName[1].int == dot and f.cFileName[2].int == 0) -proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimScript.} = - ## Returns true if `filename` exists and is a regular file or symlink. - ## - ## Directories, device files, named pipes and sockets return false. - ## - ## See also: - ## * `existsDir proc <#existsDir,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) - else: - var a = getFileAttributesA(filename) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 - else: - var res: Stat - return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) - -proc existsDir*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], - noNimScript.} = - ## Returns true iff the directory `dir` exists. If `dir` is a file, false - ## is returned. Follows symlinks. - ## - ## See also: - ## * `existsFile proc <#existsFile,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) - else: - var a = getFileAttributesA(dir) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - else: - var res: Stat - return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) - -proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], - noNimScript.} = - ## Returns true iff the symlink `link` exists. Will return true - ## regardless of whether the link points to a directory or file. - ## - ## See also: - ## * `existsFile proc <#existsFile,string>`_ - ## * `existsDir proc <#existsDir,string>`_ - when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) - else: - var a = getFileAttributesA(link) - if a != -1'i32: - result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 - else: - var res: Stat - return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) - -proc fileExists*(filename: string): bool {.inline, noNimScript.} = - ## Alias for `existsFile proc <#existsFile,string>`_. - ## - ## See also: - ## * `existsDir proc <#existsDir,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - existsFile(filename) - -proc dirExists*(dir: string): bool {.inline, noNimScript.} = - ## Alias for `existsDir proc <#existsDir,string>`_. - ## - ## See also: - ## * `existsFile proc <#existsFile,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - existsDir(dir) + when not defined(windows): + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} -when not defined(windows) and not weirdTarget: - proc checkSymlink(path: string): bool = - var rawInfo: Stat - if lstat(path, rawInfo) < 0'i32: result = false - else: result = S_ISLNK(rawInfo.st_mode) const ExeExts* = ## Platform specific file extension for executables. @@ -1131,12 +223,12 @@ const proc findExe*(exe: string, followSymlinks: bool = true; extensions: openArray[string]=ExeExts): string {. - tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect], noNimScript.} = + tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect], noNimJs.} = ## Searches for `exe` in the current working directory and then ## in directories listed in the ``PATH`` environment variable. ## ## Returns `""` if the `exe` cannot be found. `exe` - ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. + ## is added the `ExeExts`_ file extensions if it has none. ## ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired @@ -1146,12 +238,12 @@ proc findExe*(exe: string, followSymlinks: bool = true; template checkCurrentDir() = for ext in extensions: result = addFileExt(exe, ext) - if existsFile(result): return + if fileExists(result): return when defined(posix): if '/' in exe: checkCurrentDir() else: checkCurrentDir() - let path = string(getEnv("PATH")) + let path = getEnv("PATH") for candidate in split(path, PathSep): if candidate.len == 0: continue when defined(windows): @@ -1162,17 +254,17 @@ proc findExe*(exe: string, followSymlinks: bool = true; var x = expandTilde(candidate) / exe for ext in extensions: var x = addFileExt(x, ext) - if existsFile(x): - when not defined(windows): + if fileExists(x): + when not (defined(windows) or defined(nintendoswitch)): while followSymlinks: # doubles as if here - if x.checkSymlink: - var r = newString(256) - var len = readlink(x, r, 256) + if x.symlinkExists: + var r = newString(maxSymlinkLen) + var len = readlink(x.cstring, r.cstring, maxSymlinkLen) if len < 0: - raiseOSError(osLastError()) - if len > 256: + raiseOSError(osLastError(), exe) + if len > maxSymlinkLen: r = newString(len+1) - len = readlink(x, r, len) + len = readlink(x.cstring, r.cstring, len) setLen(r, len) if isAbsolute(r): x = r @@ -1187,43 +279,43 @@ when weirdTarget: const times = "fake const" template Time(x: untyped): untyped = string -proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScript.} = +proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns the `file`'s last modification time. ## ## See also: - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat - if stat(file, res) < 0'i32: raiseOSError(osLastError()) + if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) result = res.st_mtim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) - if h == -1'i32: raiseOSError(osLastError()) + if h == -1'i32: raiseOSError(osLastError(), file) result = fromWinTime(rdFileTime(f.ftLastWriteTime)) findClose(h) -proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScript.} = +proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns the `file`'s last read or write access time. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat - if stat(file, res) < 0'i32: raiseOSError(osLastError()) + if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) result = res.st_atim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) - if h == -1'i32: raiseOSError(osLastError()) + if h == -1'i32: raiseOSError(osLastError(), file) result = fromWinTime(rdFileTime(f.ftLastAccessTime)) findClose(h) -proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScript.} = +proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns the `file`'s creation time. ## ## **Note:** Under POSIX OS's, the returned time may actually be the time at @@ -1231,28 +323,28 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noNimScr ## `here <https://github.com/nim-lang/Nim/issues/1058>`_ for details. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat - if stat(file, res) < 0'i32: raiseOSError(osLastError()) + if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) result = res.st_ctim.toTime else: var f: WIN32_FIND_DATA var h = findFirstFile(file, f) - if h == -1'i32: raiseOSError(osLastError()) + if h == -1'i32: raiseOSError(osLastError(), file) result = fromWinTime(rdFileTime(f.ftCreationTime)) findClose(h) -proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noNimScript.} = +proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = ## Returns true if the file `a` is newer than file `b`, i.e. if `a`'s ## modification time is later than `b`'s. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ when defined(posix): # If we don't have access to nanosecond resolution, use '>=' when not StatHasNanoseconds: @@ -1262,529 +354,38 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noNimScript.} = else: result = getLastModificationTime(a) > getLastModificationTime(b) -proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [], noNimScript.} = - ## Returns the `current working directory`:idx: i.e. where the built - ## binary is run. - ## - ## So the path returned by this proc is determined at run time. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ - ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ - ## * `getProjectPath proc <macros.html#getProjectPath>`_ - when defined(windows): - var bufsize = MAX_PATH.int32 - when useWinUnicode: - var res = newWideCString("", bufsize) - while true: - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - result = newString(bufsize) - while true: - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break - else: - var bufsize = 1024 # should be enough - result = newString(bufsize) - while true: - if getcwd(result, bufsize) != nil: - setLen(result, c_strlen(result)) - break - else: - let err = osLastError() - if err.int32 == ERANGE: - bufsize = bufsize shl 1 - doAssert(bufsize >= 0) - result = newString(bufsize) - else: - raiseOSError(osLastError()) - -proc setCurrentDir*(newDir: string) {.inline, tags: [], noNimScript.} = - ## Sets the `current working directory`:idx:; `OSError` - ## is raised if `newDir` cannot been set. - ## - ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - when defined(Windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError()) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError()) - else: - if chdir(newDir) != 0'i32: raiseOSError(osLastError()) - -when not weirdTarget: - proc absolutePath*(path: string, root = getCurrentDir()): string {.noNimScript.} = - ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; - ## default: current directory). - ## If `path` is absolute, return it, ignoring `root`. - ## - ## See also: - ## * `normalizedPath proc <#normalizedPath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ - runnableExamples: - assert absolutePath("a") == getCurrentDir() / "a" - - if isAbsolute(path): path - else: - if not root.isAbsolute: - raise newException(ValueError, "The specified root is not absolute: " & root) - joinPath(root, path) - -proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [], noNimScript.} = - ## Normalize a path. - ## - ## Consecutive directory separators are collapsed, including an initial double slash. - ## - ## On relative paths, double dot (`..`) sequences are collapsed if possible. - ## On absolute paths they are always collapsed. - ## - ## Warning: URL-encoded and Unicode attempts at directory traversal are not detected. - ## Triple dot is not handled. - ## - ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizedPath proc <#normalizedPath,string>`_ for a version which returns - ## a new string - runnableExamples: - when defined(posix): - var a = "a///b//..//c///d" - a.normalizePath() - assert a == "a/c/d" - - path = pathnorm.normalizePath(path) - when false: - let isAbs = isAbsolute(path) - var stack: seq[string] = @[] - for p in split(path, {DirSep}): - case p - of "", ".": - continue - of "..": - if stack.len == 0: - if isAbs: - discard # collapse all double dots on absoluta paths - else: - stack.add(p) - elif stack[^1] == "..": - stack.add(p) - else: - discard stack.pop() - else: - stack.add(p) - - if isAbs: - path = DirSep & join(stack, $DirSep) - elif stack.len > 0: - path = join(stack, $DirSep) - else: - path = "." - -proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [], noNimScript.} = - ## Returns a normalized path for the current OS. - ## - ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version - runnableExamples: - when defined(posix): - assert normalizedPath("a///b//..//c///d") == "a/c/d" - result = pathnorm.normalizePath(path) - -when defined(Windows) and not weirdTarget: - proc openHandle(path: string, followSymlink=true, writeAccess=false): Handle = - var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL - if not followSymlink: - flags = flags or FILE_FLAG_OPEN_REPARSE_POINT - let access = if writeAccess: GENERIC_WRITE else: 0'i32 - - when useWinUnicode: - result = createFileW( - newWideCString(path), access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - -proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimScript.} = - ## Returns true if both pathname arguments refer to the same physical - ## file or directory. - ## - ## Raises `OSError` if any of the files does not - ## exist or information about it can not be obtained. - ## - ## This proc will return true if given two alternative hard-linked or - ## sym-linked paths to the same file or directory. - ## - ## See also: - ## * `sameFileContent proc <#sameFileContent,string,string>`_ - when defined(Windows): - var success = true - var f1 = openHandle(path1) - var f2 = openHandle(path2) - - var lastErr: OSErrorCode - if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: - var fi1, fi2: BY_HANDLE_FILE_INFORMATION - - if getFileInformationByHandle(f1, addr(fi1)) != 0 and - getFileInformationByHandle(f2, addr(fi2)) != 0: - result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and - fi1.nFileIndexHigh == fi2.nFileIndexHigh and - fi1.nFileIndexLow == fi2.nFileIndexLow - else: - lastErr = osLastError() - success = false - else: - lastErr = osLastError() - success = false - - discard closeHandle(f1) - discard closeHandle(f2) - - if not success: raiseOSError(lastErr) - else: - var a, b: Stat - if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: - raiseOSError(osLastError()) - else: - result = a.st_dev == b.st_dev and a.st_ino == b.st_ino - -proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", - tags: [ReadIOEffect], noNimScript.} = - ## Returns true if both pathname arguments refer to files with identical - ## binary content. - ## - ## See also: - ## * `sameFile proc <#sameFile,string,string>`_ - const - bufSize = 8192 # 8K buffer - var - a, b: File - if not open(a, path1): return false - if not open(b, path2): - close(a) - return false - var bufA = alloc(bufSize) - var bufB = alloc(bufSize) - while true: - var readA = readBuffer(a, bufA, bufSize) - var readB = readBuffer(b, bufB, bufSize) - if readA != readB: - result = false - break - if readA == 0: - result = true - break - result = equalMem(bufA, bufB, readA) - if not result: break - if readA != bufSize: break # end of file - dealloc(bufA) - dealloc(bufB) - close(a) - close(b) -type - FilePermission* = enum ## File access permission, modelled after UNIX. - ## - ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FileInfo object <#FileInfo>`_ - fpUserExec, ## execute access for the file owner - fpUserWrite, ## write access for the file owner - fpUserRead, ## read access for the file owner - fpGroupExec, ## execute access for the group - fpGroupWrite, ## write access for the group - fpGroupRead, ## read access for the group - fpOthersExec, ## execute access for others - fpOthersWrite, ## write access for others - fpOthersRead ## read access for others - -proc getFilePermissions*(filename: string): set[FilePermission] {. - rtl, extern: "nos$1", tags: [ReadDirEffect], noNimScript.} = - ## Retrieves file permissions for `filename`. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is checked, every other - ## permission is available in any case. - ## - ## See also: - ## * `setFilePermissions proc <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FilePermission enum <#FilePermission>`_ - when defined(posix): - var a: Stat - if stat(filename, a) < 0'i32: raiseOSError(osLastError()) - result = {} - if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) - if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) - if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) - - if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) - if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) - if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) - - if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) - if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) - if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError()) - if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: - result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, - fpOthersExec, fpOthersRead} - else: - result = {fpUserExec..fpOthersRead} - -proc setFilePermissions*(filename: string, permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [WriteDirEffect], noNimScript.} = - ## Sets the file permissions for `filename`. - ## - ## `OSError` is raised in case of an error. - ## On Windows, only the ``readonly`` flag is changed, depending on - ## ``fpUserWrite`` permission. - ## - ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `FilePermission enum <#FilePermission>`_ - when defined(posix): - var p = 0.Mode - if fpUserRead in permissions: p = p or S_IRUSR.Mode - if fpUserWrite in permissions: p = p or S_IWUSR.Mode - if fpUserExec in permissions: p = p or S_IXUSR.Mode - - if fpGroupRead in permissions: p = p or S_IRGRP.Mode - if fpGroupWrite in permissions: p = p or S_IWGRP.Mode - if fpGroupExec in permissions: p = p or S_IXGRP.Mode - - if fpOthersRead in permissions: p = p or S_IROTH.Mode - if fpOthersWrite in permissions: p = p or S_IWOTH.Mode - if fpOthersExec in permissions: p = p or S_IXOTH.Mode - - if chmod(filename, cast[Mode](p)) != 0: raiseOSError(osLastError()) - else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) - if res == -1'i32: raiseOSError(osLastError()) - if fpUserWrite in permissions: - res = res and not FILE_ATTRIBUTE_READONLY - else: - res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) - if res2 == - 1'i32: raiseOSError(osLastError()) - -proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = - ## Copies a file from `source` to `dest`. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will - ## copy the source file's attributes into dest. - ## - ## On other platforms you need - ## to use `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ procs - ## to copy them by hand (or use the convenience `copyFileWithPermissions - ## proc <#copyFileWithPermissions,string,string>`_), - ## otherwise `dest` will inherit the default permissions of a newly - ## created file for the user. - ## - ## If `dest` already exists, the file attributes - ## will be preserved and the content overwritten. - ## - ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - - when defined(Windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: raiseOSError(osLastError()) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError()) - else: - # generic version of copyFile which works for any platform: - const bufSize = 8000 # better for memory manager - var d, s: File - if not open(s, source): raiseOSError(osLastError()) - if not open(d, dest, fmWrite): - close(s) - raiseOSError(osLastError()) - var buf = alloc(bufSize) - while true: - var bytesread = readBuffer(s, buf, bufSize) - if bytesread > 0: - var byteswritten = writeBuffer(d, buf, bytesread) - if bytesread != byteswritten: - dealloc(buf) - close(s) - close(d) - raiseOSError(osLastError()) - if bytesread != bufSize: break - dealloc(buf) - close(s) - flushFile(d) - close(d) - -when not declared(ENOENT) and not defined(Windows): - when NoFakeVars: - when not defined(haiku): - const ENOENT = cint(2) # 2 on most systems including Solaris - else: - const ENOENT = cint(-2147459069) - else: - var ENOENT {.importc, header: "<errno.h>".}: cint - -when defined(Windows) and not weirdTarget: - when useWinUnicode: - template deleteFile(file: untyped): untyped = deleteFileW(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesW(file, attrs) - else: - template deleteFile(file: untyped): untyped = deleteFileA(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesA(file, attrs) +proc isAdmin*: bool {.noWeirdTarget.} = + ## Returns whether the caller's process is a member of the Administrators local + ## group (on Windows) or a root (on POSIX), via `geteuid() == 0`. + when defined(windows): + # Rewrite of the example from Microsoft Docs: + # https://docs.microsoft.com/en-us/windows/win32/api/securitybaseapi/nf-securitybaseapi-checktokenmembership#examples + # and corresponding PostgreSQL function: + # https://doxygen.postgresql.org/win32security_8c.html#ae6b61e106fa5d6c5d077a9d14ee80569 + var ntAuthority = SID_IDENTIFIER_AUTHORITY(value: SECURITY_NT_AUTHORITY) + var administratorsGroup: PSID + if not isSuccess(allocateAndInitializeSid(addr ntAuthority, + BYTE(2), + SECURITY_BUILTIN_DOMAIN_RID, + DOMAIN_ALIAS_RID_ADMINS, + 0, 0, 0, 0, 0, 0, + addr administratorsGroup)): + raiseOSError(osLastError(), "could not get SID for Administrators group") -proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noNimScript.} = - ## Removes the `file`. - ## - ## If this fails, returns `false`. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - result = true - when defined(Windows): - when useWinUnicode: - let f = newWideCString(file) - else: - let f = file - if deleteFile(f) == 0: - result = false - let err = getLastError() - if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: - result = true - elif err == ERROR_ACCESS_DENIED and - setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and - deleteFile(f) != 0: - result = true - else: - if unlink(file) != 0'i32 and errno != ENOENT: - result = false + try: + var b: WINBOOL + if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): + raiseOSError(osLastError(), "could not check access token membership") -proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noNimScript.} = - ## Removes the `file`. - ## - ## If this fails, `OSError` is raised. This does not fail - ## if the file never existed in the first place. - ## - ## On Windows, ignores the read-only attribute. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - if not tryRemoveFile(file): - when defined(Windows): - raiseOSError(osLastError()) - else: - raiseOSError(osLastError(), $strerror(errno)) + result = isSuccess(b) + finally: + if freeSid(administratorsGroup) != nil: + raiseOSError(osLastError(), "failed to free SID for Administrators group") -proc tryMoveFSObject(source, dest: string): bool {.noNimScript.} = - ## Moves a file or directory from `source` to `dest`. - ## - ## Returns false in case of `EXDEV` error. - ## In case of other errors `OSError` is raised. - ## Returns true in case of success. - when defined(Windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED) == 0'i32: raiseOSError(osLastError()) - else: - if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED) == 0'i32: raiseOSError(osLastError()) else: - if c_rename(source, dest) != 0'i32: - let err = osLastError() - if err == EXDEV.OSErrorCode: - return false - else: - raiseOSError(err, $strerror(errno)) - return true + result = geteuid() == 0 -proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = - ## Moves a file from `source` to `dest`. - ## - ## If this fails, `OSError` is raised. - ## If `dest` already exists, it will be overwritten. - ## - ## Can be used to `rename files`:idx:. - ## - ## See also: - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyFile(source, dest) - try: - removeFile(source) - except: - discard tryRemoveFile(dest) - raise proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. @@ -1798,7 +399,7 @@ proc exitStatusLikeShell*(status: cint): cint = status proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", - tags: [ExecIOEffect], noNimScript.} = + tags: [ExecIOEffect], noWeirdTarget.} = ## Executes a `shell command`:idx:. ## ## Command has the form 'program args' where args are the command @@ -1810,925 +411,96 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## <osproc.html#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## discard execShellCmd("ls -la") + ## ``` result = exitStatusLikeShell(c_system(command)) -# Templates for filtering directories and files -when defined(windows) and not weirdTarget: - template isDir(f: WIN32_FIND_DATA): bool = - (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 - template isFile(f: WIN32_FIND_DATA): bool = - not isDir(f) -else: - template isDir(f: string): bool {.dirty.} = - dirExists(f) - template isFile(f: string): bool {.dirty.} = - fileExists(f) - -template defaultWalkFilter(item): bool = - ## Walk filter used to return true on both - ## files and directories - true - -template walkCommon(pattern: string, filter) = - ## Common code for getting the files and directories with the - ## specified `pattern` - when defined(windows): - var - f: WIN32_FIND_DATA - res: int - res = findFirstFile(pattern, f) - if res != -1: - defer: findClose(res) - let dotPos = searchExtPos(pattern) - while true: - if not skipFindData(f) and filter(f): - # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check - # that the file extensions have the same length ... - let ff = getFilename(f) - let idx = ff.len - pattern.len + dotPos - if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or - (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): - yield splitFile(pattern).dir / extractFilename(ff) - if findNextFile(res, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: # here we use glob - var - f: Glob - res: int - f.gl_offs = 0 - f.gl_pathc = 0 - f.gl_pathv = nil - res = glob(pattern, 0, nil, addr(f)) - defer: globfree(addr(f)) - if res == 0: - for i in 0.. f.gl_pathc - 1: - assert(f.gl_pathv[i] != nil) - let path = $f.gl_pathv[i] - if filter(path): - yield path - -iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noNimScript.} = - ## Iterate over all the files and directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - walkCommon(pattern, defaultWalkFilter) - -iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noNimScript.} = - ## Iterate over all the files that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - walkCommon(pattern, isFile) - -iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noNimScript.} = - ## Iterate over all the directories that match the `pattern`. - ## - ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` - ## notation is supported. - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - walkCommon(pattern, isDir) - proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", - tags: [ReadDirEffect], noNimScript.} = + tags: [ReadDirEffect], noWeirdTarget.} = ## Returns the full (`absolute`:idx:) path of an existing file `filename`. ## ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var unused: WideCString = nil + var res = newWideCString(bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break # getFullPathName doesn't do case corrections, so we have to use this convoluted # way of retrieving the true filename for x in walkFiles(result): result = x - if not existsFile(result) and not existsDir(result): + if not fileExists(result) and not dirExists(result): + # consider using: `raiseOSError(osLastError(), result)` raise newException(OSError, "file '" & result & "' does not exist") else: # according to Posix we don't need to allocate space for result pathname. # But we need to free return value with free(3). var r = realpath(filename, nil) if r.isNil: - raiseOSError(osLastError()) + raiseOSError(osLastError(), filename) else: result = $r c_free(cast[pointer](r)) -type - PathComponent* = enum ## Enumeration specifying a path component. - ## - ## See also: - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - ## * `FileInfo object <#FileInfo>`_ - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory - proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## This is `getAppFilename() <#getAppFilename>`_ at compile time. + ## Returns the path of the currently running Nim compiler or nimble executable. ## ## Can be used to retrieve the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary ## inside a nimble program (likewise with other binaries built from ## compiler API). -when defined(posix) and not weirdTarget: - proc getSymlinkFileKind(path: string): PathComponent = - # Helper function. - var s: Stat - assert(path != "") - if stat(path, s) == 0'i32 and S_ISDIR(s.st_mode): - result = pcLinkToDir - else: - result = pcLinkToFile - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent, path: string]] = - discard - -iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. - tags: [ReadDirEffect].} = - ## Walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item are returned. - ## - ## Walking is not recursive. If ``relative`` is true (default: false) - ## the resulting path is shortened to be relative to ``dir``. - ## Example: This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: - ## - ## .. code-block:: Nim - ## for kind, path in walkDir("dirA"): - ## echo(path) - ## - ## produce this output (but not necessarily in this order!):: - ## dirA/dirB - ## dirA/dirC - ## dirA/fileA1.txt - ## dirA/fileA2.txt - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - - when nimvm: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - else: - when weirdTarget: - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - elif defined(windows): - var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h != -1: - defer: findClose(h) - while true: - var k = pcFile - if not skipFindData(f): - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - yield (k, xx) - if findNextFile(h, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) - else: - var d = opendir(dir) - if d != nil: - defer: discard closedir(d) - while true: - var x = readdir(d) - if x == nil: break - when defined(nimNoArrayToCstringConversion): - var y = $cstring(addr x.d_name) - else: - var y = $x.d_name.cstring - if y != "." and y != "..": - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: k = pcDir - if x.d_type == DT_LNK: - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - yield (k, y) - continue - - if lstat(path, s) < 0'i32: break - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - yield (k, y) - -iterator walkDirRec*(dir: string, - yieldFilter = {pcFile}, followFilter = {pcDir}, - relative = false): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file - ## or directory in `dir`. - ## - ## If ``relative`` is true (default: false) the resulting path is - ## shortened to be relative to ``dir``, otherwise the full path is returned. - ## - ## **Warning**: - ## Modifying the directory structure while the iterator - ## is traversing may result in undefined behavior! - ## - ## Walking is recursive. `followFilter` controls the behaviour of the iterator: - ## - ## --------------------- --------------------------------------------- - ## yieldFilter meaning - ## --------------------- --------------------------------------------- - ## ``pcFile`` yield real files (default) - ## ``pcLinkToFile`` yield symbolic links to files - ## ``pcDir`` yield real directories - ## ``pcLinkToDir`` yield symbolic links to directories - ## --------------------- --------------------------------------------- - ## - ## --------------------- --------------------------------------------- - ## followFilter meaning - ## --------------------- --------------------------------------------- - ## ``pcDir`` follow real directories (default) - ## ``pcLinkToDir`` follow symbolic links to directories - ## --------------------- --------------------------------------------- - ## - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - - var stack = @[""] - while stack.len > 0: - let d = stack.pop() - for k, p in walkDir(dir / d, relative = true): - let rel = d / p - if k in {pcDir, pcLinkToDir} and k in followFilter: - stack.add rel - if k in yieldFilter: - yield if relative: rel else: dir / rel - -proc rawRemoveDir(dir: string) {.noNimScript.} = - when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) - let lastError = osLastError() - if res == 0'i32 and lastError.int32 != 3'i32 and - lastError.int32 != 18'i32 and lastError.int32 != 2'i32: - raiseOSError(lastError) - else: - if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError()) - -proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ - WriteDirEffect, ReadDirEffect], benign, noNimScript.} = - ## Removes the directory `dir` including all subdirectories and files - ## in `dir` (recursively). - ## - ## If this fails, `OSError` is raised. This does not fail if the directory never - ## existed in the first place. - ## - ## See also: - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - for kind, path in walkDir(dir): - case kind - of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) - of pcDir: removeDir(path) - rawRemoveDir(dir) - -proc rawCreateDir(dir: string): bool {.noNimScript.} = - # Try to create one directory (not the whole path). - # returns `true` for success, `false` if the path has previously existed - # - # This is a thin wrapper over mkDir (or alternatives on other systems), - # so in case of a pre-existing path we don't check that it is a directory. - when defined(solaris): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno in {EEXIST, ENOSYS}: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(haiku): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST or errno == EROFS: - result = false - else: - raiseOSError(osLastError(), dir) - elif defined(posix): - let res = mkdir(dir, 0o777) - if res == 0'i32: - result = true - elif errno == EEXIST: - result = false - else: - #echo res - raiseOSError(osLastError(), dir) - else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) - else: - let res = createDirectoryA(dir) - - if res != 0'i32: - result = true - elif getLastError() == 183'i32: - result = false - else: - raiseOSError(osLastError(), dir) - -proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noNimScript.} = - ## Check if a `directory`:idx: `dir` exists, and create it otherwise. - ## - ## Does not create parent directories (fails if parent does not exist). - ## Returns `true` if the directory already exists, and `false` - ## otherwise. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - result = not rawCreateDir(dir) - if result: - # path already exists - need to check that it is indeed a directory - if not existsDir(dir): - raise newException(IOError, "Failed to create '" & dir & "'") - -proc createDir*(dir: string) {.rtl, extern: "nos$1", - tags: [WriteDirEffect, ReadDirEffect], noNimScript.} = - ## Creates the `directory`:idx: `dir`. - ## - ## The directory may contain several subdirectories that do not exist yet. - ## The full path is created. If this fails, `OSError` is raised. - ## - ## It does **not** fail if the directory already exists because for - ## most usages this does not indicate an error. - ## - ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - var omitNext = false - when doslikeFileSystem: - omitNext = isAbsolute(dir) - for i in 1.. dir.len-1: - if dir[i] in {DirSep, AltSep}: - if omitNext: - omitNext = false - else: - discard existsOrCreateDir(substr(dir, 0, i-1)) - - # The loop does not create the dir itself if it doesn't end in separator - if dir.len > 0 and not omitNext and - dir[^1] notin {DirSep, AltSep}: - discard existsOrCreateDir(dir) - -proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect], benign, noNimScript.} = - ## Copies a directory from `source` to `dest`. - ## - ## If this fails, `OSError` is raised. - ## - ## On the Windows platform this proc will copy the attributes from - ## `source` into `dest`. - ## - ## On other platforms created files and directories will inherit the - ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## to preserve attributes recursively on these platforms. - ## - ## See also: - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - createDir(dest) - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - case kind - of pcFile: - copyFile(path, dest / noSource) - of pcDir: - copyDir(path, dest / noSource) - else: discard - -proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noNimScript.} = - ## Moves a directory from `source` to `dest`. - ## - ## If this fails, `OSError` is raised. - ## - ## See also: - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) - -proc createSymlink*(src, dest: string) {.noNimScript.} = - ## Create a symbolic link at `dest` which points to the item specified - ## by `src`. On most operating systems, will fail if a link already exists. - ## - ## **Warning**: - ## Some OS's (such as Microsoft Windows) restrict the creation - ## of symlinks to root users (administrators). - ## - ## See also: - ## * `createHardlink proc <#createHardlink,string,string>`_ - ## * `expandSymlink proc <#expandSymlink,string>`_ - - when defined(Windows): - # 2 is the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE. This allows - # anyone with developer mode on to create a link - let flag = dirExists(src).int32 or 2 - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError()) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError()) - else: - if symlink(src, dest) != 0: - raiseOSError(osLastError()) - -proc createHardlink*(src, dest: string) {.noNimScript.} = +proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified ## by `src`. ## - ## **Warning**: Some OS's restrict the creation of hard links to - ## root users (administrators). + ## .. warning:: Some OS's restrict the creation of hard links to + ## root users (administrators). ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ - when defined(Windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError()) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError()) + ## * `createSymlink proc`_ + when defined(windows): + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: - raiseOSError(osLastError()) - -proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.noNimScript.} = - ## Copies a file from `source` to `dest` preserving file permissions. - ## - ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, - ## `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions<#setFilePermissions,string,set[FilePermission]>`_ - ## procs on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyFile proc - ## <#copyFile,string,string>`_ since that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file itself has - ## been copied, which won't happen atomically and could lead to a race - ## condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - copyFile(source, dest) - when not defined(Windows): - try: - setFilePermissions(dest, getFilePermissions(source)) - except: - if not ignorePermissionErrors: - raise - -proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect], benign, noNimScript.} = - ## Copies a directory from `source` to `dest` preserving file permissions. - ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir - ## <#copyDir,string,string>`_ and `copyFileWithPermissions - ## <#copyFileWithPermissions,string,string>`_ procs - ## on non-Windows platforms. - ## - ## On Windows this proc is just a wrapper for `copyDir proc - ## <#copyDir,string,string>`_ since that proc already copies attributes. - ## - ## On non-Windows systems permissions are copied after the file or directory - ## itself has been copied, which won't happen atomically and could lead to a - ## race condition. If `ignorePermissionErrors` is true (default), errors while - ## reading/setting file attributes will be ignored, otherwise will raise - ## `OSError`. - ## - ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - createDir(dest) - when not defined(Windows): - try: - setFilePermissions(dest, getFilePermissions(source)) - except: - if not ignorePermissionErrors: - raise - for kind, path in walkDir(source): - var noSource = splitPath(path).tail - case kind - of pcFile: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors) - of pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) - else: discard + raiseOSError(osLastError(), $(src, dest)) proc inclFilePermissions*(filename: string, permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noNimScript.} = + rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)+permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)+permissions) proc exclFilePermissions*(filename: string, permissions: set[FilePermission]) {. - rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noNimScript.} = + rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], noWeirdTarget.} = ## A convenience proc for: - ## - ## .. code-block:: nim + ## ```nim ## setFilePermissions(filename, getFilePermissions(filename)-permissions) + ## ``` setFilePermissions(filename, getFilePermissions(filename)-permissions) -proc expandSymlink*(symlinkPath: string): string {.noNimScript.} = - ## Returns a string representing the path to which the symbolic link points. - ## - ## On Windows this is a noop, ``symlinkPath`` is simply returned. - ## - ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ - when defined(windows): - result = symlinkPath - else: - result = newString(256) - var len = readlink(symlinkPath, result, 256) - if len < 0: - raiseOSError(osLastError()) - if len > 256: - result = newString(len+1) - len = readlink(symlinkPath, result, len) - setLen(result, len) - -proc parseCmdLine*(c: string): seq[string] {. - noSideEffect, rtl, extern: "nos$1".} = - ## Splits a `command line`:idx: into several components. - ## - ## **Note**: This proc is only occasionally useful, better use the - ## `parseopt module <parseopt.html>`_. - ## - ## On Windows, it uses the `following parsing rules - ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: - ## - ## * Arguments are delimited by white space, which is either a space or a tab. - ## * The caret character (^) is not recognized as an escape character or - ## delimiter. The character is handled completely by the command-line parser - ## in the operating system before being passed to the argv array in the - ## program. - ## * A string surrounded by double quotation marks ("string") is interpreted - ## as a single argument, regardless of white space contained within. A - ## quoted string can be embedded in an argument. - ## * A double quotation mark preceded by a backslash (\") is interpreted as a - ## literal double quotation mark character ("). - ## * Backslashes are interpreted literally, unless they immediately precede - ## a double quotation mark. - ## * If an even number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is interpreted as a string delimiter. - ## * If an odd number of backslashes is followed by a double quotation mark, - ## one backslash is placed in the argv array for every pair of backslashes, - ## and the double quotation mark is "escaped" by the remaining backslash, - ## causing a literal double quotation mark (") to be placed in argv. - ## - ## On Posix systems, it uses the following parsing rules: - ## Components are separated by whitespace unless the whitespace - ## occurs within ``"`` or ``'`` quotes. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - - result = @[] - var i = 0 - var a = "" - while true: - setLen(a, 0) - # eat all delimiting whitespace - while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) - if i >= c.len: break - when defined(windows): - # parse a single argument according to the above rules: - var inQuote = false - while i < c.len: - case c[i] - of '\\': - var j = i - while j < c.len and c[j] == '\\': inc(j) - if j < c.len and c[j] == '"': - for k in 1..(j-i) div 2: a.add('\\') - if (j-i) mod 2 == 0: - i = j - else: - a.add('"') - i = j+1 - else: - a.add(c[i]) - inc(i) - of '"': - inc(i) - if not inQuote: inQuote = true - elif i < c.len and c[i] == '"': - a.add(c[i]) - inc(i) - else: - inQuote = false - break - of ' ', '\t': - if not inQuote: break - a.add(c[i]) - inc(i) - else: - a.add(c[i]) - inc(i) - else: - case c[i] - of '\'', '\"': - var delim = c[i] - inc(i) # skip ' or " - while i < c.len and c[i] != delim: - add a, c[i] - inc(i) - if i < c.len: inc(i) - else: - while i < c.len and c[i] > ' ': - add(a, c[i]) - inc(i) - add(result, a) - -when defined(nimdoc): - # Common forward declaration docstring block for parameter retrieval procs. - proc paramCount*(): int {.tags: [ReadIOEffect].} = - ## Returns the number of `command line arguments`:idx: given to the - ## application. - ## - ## Unlike `argc`:idx: in C, if your binary was called without parameters this - ## will return zero. - ## You can query each individual parameter with `paramStr proc <#paramStr,int>`_ - ## or retrieve all of them in one go with `commandLineParams proc - ## <#commandLineParams>`_. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramCount): - ## # Use paramCount() here - ## else: - ## # Do something else! - - proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} = - ## Returns the `i`-th `command line argument`:idx: given to the application. - ## - ## `i` should be in the range `1..paramCount()`, the `IndexError` - ## exception will be raised for invalid values. Instead of iterating over - ## `paramCount() <#paramCount>`_ with this proc you can call the - ## convenience `commandLineParams() <#commandLineParams>`_. - ## - ## Similarly to `argv`:idx: in C, - ## it is possible to call ``paramStr(0)`` but this will return OS specific - ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename() <#getAppFilename>`_ instead. - ## - ## **Availability**: When generating a dynamic library (see `--app:lib`) on - ## Posix this proc is not defined. - ## Test for availability using `declared() <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## * `getAppFilename proc <#getAppFilename>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(paramStr): - ## # Use paramStr() here - ## else: - ## # Do something else! - -elif defined(nintendoswitch) or weirdTarget: - proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on Nintendo Switch") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on Nintendo Switch") - -elif defined(windows): - # Since we support GUI applications with Nim, we sometimes generate - # a WinMain entry proc. But a WinMain proc has no access to the parsed - # command line arguments. The way to get them differs. Thus we parse them - # ourselves. This has the additional benefit that the program's behaviour - # is always the same -- independent of the used C compiler. - var - ownArgv {.threadvar.}: seq[string] - ownParsedArgv {.threadvar.}: bool - - proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - result = ownArgv.len-1 - - proc paramStr*(i: int): TaintedString {.rtl, extern: "nos$1", - tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if not ownParsedArgv: - ownArgv = parseCmdLine($getCommandLine()) - ownParsedArgv = true - if i < ownArgv.len and i >= 0: return TaintedString(ownArgv[i]) - raise newException(IndexError, formatErrorIndexBound(i, ownArgv.len-1)) - -elif defined(genode): - proc paramStr*(i: int): TaintedString = - raise newException(OSError, "paramStr is not implemented on Genode") - - proc paramCount*(): int = - raise newException(OSError, "paramCount is not implemented on Genode") - -elif not defined(createNimRtl) and - not(defined(posix) and appType == "lib"): - # On Posix, there is no portable way to get the command line from a DLL. - var - cmdCount {.importc: "cmdCount".}: cint - cmdLine {.importc: "cmdLine".}: cstringArray - - proc paramStr*(i: int): TaintedString {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - if i < cmdCount and i >= 0: return TaintedString($cmdLine[i]) - raise newException(IndexError, formatErrorIndexBound(i, cmdCount-1)) - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - # Docstring in nimdoc block. - result = cmdCount-1 - -when declared(paramCount) or defined(nimdoc): - proc commandLineParams*(): seq[TaintedString] = - ## Convenience proc which returns the command line parameters. - ## - ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename() <#getAppFilename>`_. - ## - ## **Availability**: On Posix there is no portable way to get the command - ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `declared() - ## <system.html#declared,untyped>`_. - ## - ## See also: - ## * `parseopt module <parseopt.html>`_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `getAppFilename proc <#getAppFilename>`_ - ## - ## **Examples:** - ## - ## .. code-block:: nim - ## when declared(commandLineParams): - ## # Use commandLineParams() here - ## else: - ## # Do something else! - result = @[] - for i in 1..paramCount(): - result.add(paramStr(i)) - -when not weirdTarget and (defined(freebsd) or defined(dragonfly)): +when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netbsd)): proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include <sys/types.h> - #include <sys/sysctl.h>"""} + #include <sys/sysctl.h>""".} const CTL_KERN = 1 KERN_PROC = 14 @@ -2736,42 +508,45 @@ when not weirdTarget and (defined(freebsd) or defined(dragonfly)): when defined(freebsd): const KERN_PROC_PATHNAME = 12 + elif defined(netbsd): + const KERN_PROC_ARGS = 48 + const KERN_PROC_PATHNAME = 5 else: const KERN_PROC_PATHNAME = 9 proc getApplFreebsd(): string = - var pathLength = csize_t(MAX_PATH) - result = newString(pathLength) - var req = [CTL_KERN.cint, KERN_PROC.cint, KERN_PROC_PATHNAME.cint, -1.cint] - while true: - let res = sysctl(addr req[0], 4, cast[pointer](addr result[0]), - addr pathLength, nil, 0) - if res < 0: - let err = osLastError() - if err.int32 == ENOMEM: - result = newString(pathLength) - else: - result.setLen(0) # error! - break - else: - # trim the trailing null byte, as the result is a string not a cstring - result.setLen(pathLength-1) - break + var pathLength = csize_t(0) + + when defined(netbsd): + var req = [CTL_KERN.cint, KERN_PROC_ARGS.cint, -1.cint, KERN_PROC_PATHNAME.cint] + else: + var req = [CTL_KERN.cint, KERN_PROC.cint, KERN_PROC_PATHNAME.cint, -1.cint] + + # first call to get the required length + var res = sysctl(addr req[0], 4, nil, addr pathLength, nil, 0) + + if res < 0: + return "" + + result.setLen(pathLength) + res = sysctl(addr req[0], 4, addr result[0], addr pathLength, nil, 0) + + if res < 0: + return "" + + let realLen = len(cstring(result)) + setLen(result, realLen) when not weirdTarget and (defined(linux) or defined(solaris) or defined(bsd) or defined(aix)): proc getApplAux(procPath: string): string = - result = newString(256) - var len = readlink(procPath, result, 256) - if len > 256: + result = newString(maxSymlinkLen) + var len = readlink(procPath, result.cstring, maxSymlinkLen) + if len > maxSymlinkLen: result = newString(len+1) - len = readlink(procPath, result, len) + len = readlink(procPath, result.cstring, len) setLen(result, len) -when defined(openbsd): - proc isExecutable(path: string): bool = - let p = getFilePermissions(path) - result = fpUserExec in p and fpGroupExec in p and fpOthersExec in p - +when not weirdTarget and defined(openbsd): proc getApplOpenBsd(): string = # similar to getApplHeuristic, but checks current working directory when declared(paramStr): @@ -2779,7 +554,7 @@ when defined(openbsd): # POSIX guaranties that this contains the executable # as it has been executed by the calling process - let exePath = string(paramStr(0)) + let exePath = paramStr(0) if len(exePath) == 0: return "" @@ -2795,15 +570,12 @@ when defined(openbsd): break if len(result) > 0: - if isExecutable(result): - return expandFilename(result) - - return "" + return expandFilename(result) # search in path - for p in split(string(getEnv("PATH")), {PathSep}): + for p in split(getEnv("PATH"), {PathSep}): var x = joinPath(p, exePath) - if existsFile(x) and isExecutable(x): + if fileExists(x): return expandFilename(x) else: result = "" @@ -2811,14 +583,14 @@ when defined(openbsd): when not (defined(windows) or defined(macosx) or weirdTarget): proc getApplHeuristic(): string = when declared(paramStr): - result = string(paramStr(0)) + result = paramStr(0) # POSIX guaranties that this contains the executable # as it has been executed by the calling process if len(result) > 0 and result[0] != DirSep: # not an absolute path? # iterate over any path in the $PATH environment variable - for p in split(string(getEnv("PATH")), {PathSep}): + for p in split(getEnv("PATH"), {PathSep}): var x = joinPath(p, result) - if existsFile(x): return x + if fileExists(x): return x else: result = "" @@ -2840,7 +612,7 @@ when defined(haiku): B_FIND_PATH_IMAGE_PATH = 1000 proc find_path(codePointer: pointer, baseDirectory: cint, subPath: cstring, - pathBuffer: cstring, bufferSize: csize): int32 + pathBuffer: cstring, bufferSize: csize_t): int32 {.importc, header: "<FindDirectory.h>".} proc getApplHaiku(): string = @@ -2852,13 +624,15 @@ when defined(haiku): else: result = "" -proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noNimScript.} = +proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget, raises: [].} = ## Returns the filename of the application's executable. ## This proc will resolve symlinks. ## + ## Returns empty string when name is unavailable + ## ## See also: - ## * `getAppDir proc <#getAppDir>`_ - ## * `getCurrentCompilerExe proc <#getCurrentCompilerExe>`_ + ## * `getAppDir proc`_ + ## * `getCurrentCompilerExe proc`_ # Linux: /proc/<pid>/exe # Solaris: @@ -2866,68 +640,62 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noN # /proc/<pid>/path/a.out (complete pathname) when defined(windows): var bufsize = int32(MAX_PATH) - when useWinUnicode: - var buf = newWideCString("", bufsize) - while true: - var L = getModuleFileNameW(0, buf, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - buf = newWideCString("", L) - bufsize = L - else: - result = buf$L - break - else: - result = newString(bufsize) - while true: - var L = getModuleFileNameA(0, result, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var buf = newWideCString(bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString(L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): - var size: cuint32 + var size = cuint32(0) getExecPath1(nil, size) result = newString(int(size)) - if getExecPath2(result, size): + if getExecPath2(result.cstring, size): result = "" # error! if result.len > 0: - result = result.expandFilename + try: + result = result.expandFilename + except OSError: + result = "" else: - when defined(linux) or defined(aix) or defined(netbsd): + when defined(linux) or defined(aix): result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") - elif defined(genode) or defined(nintendoswitch): - raiseOSError(OSErrorCode(-1), "POSIX command line not supported") - elif defined(freebsd) or defined(dragonfly): + elif defined(genode): + result = "" # Not supported + elif defined(freebsd) or defined(dragonfly) or defined(netbsd): result = getApplFreebsd() elif defined(haiku): result = getApplHaiku() elif defined(openbsd): - result = getApplOpenBsd() + result = try: getApplOpenBsd() except OSError: "" + elif defined(nintendoswitch): + result = "" # little heuristic that may work on other POSIX-like systems: if result.len == 0: - result = getApplHeuristic() + result = try: getApplHeuristic() except OSError: "" -proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noNimScript.} = +proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdTarget.} = ## Returns the directory of the application's executable. ## ## See also: - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `getAppFilename proc`_ result = splitFile(getAppFilename()).dir -proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noNimScript.} = +proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = ## Sleeps `milsecs` milliseconds. + ## A negative `milsecs` causes sleep to return immediately. when defined(windows): + if milsecs < 0: + return # fixes #23732 winlean.sleep(int32(milsecs)) else: var a, b: Timespec @@ -2936,23 +704,22 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noNimScrip discard posix.nanosleep(a, b) proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", - tags: [ReadIOEffect], noNimScript.} = + tags: [ReadIOEffect], noWeirdTarget.} = ## Returns the file size of `file` (in bytes). ``OSError`` is ## raised in case of an error. when defined(windows): var a: WIN32_FIND_DATA var resA = findFirstFile(file, a) - if resA == -1: raiseOSError(osLastError()) + if resA == -1: raiseOSError(osLastError(), file) result = rdFileSize(a) findClose(resA) else: - var f: File - if open(f, file): - result = getFileSize(f) - close(f) - else: raiseOSError(osLastError()) + var rawInfo: Stat + if stat(file, rawInfo) < 0'i32: + raiseOSError(osLastError(), file) + rawInfo.st_size -when defined(Windows) or weirdTarget: +when defined(windows) or weirdTarget: type DeviceId* = int32 FileId* = int64 @@ -2966,9 +733,9 @@ type ## Contains information associated with a file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ id*: tuple[device: DeviceId, file: FileId] ## Device and file id. kind*: PathComponent ## Kind of file object - directory, symlink, etc. size*: BiggestInt ## Size of file. @@ -2977,34 +744,47 @@ type lastAccessTime*: times.Time ## Time file was last accessed. lastWriteTime*: times.Time ## Time file was last modified/written to. creationTime*: times.Time ## Time file was created. Not supported on all systems! + blockSize*: int ## Preferred I/O block size for this object. + ## In some filesystems, this may vary from file to file. + isSpecial*: bool ## Is file special? (on Unix some "files" + ## can be special=non-regular like FIFOs, + ## devices); for directories `isSpecial` + ## is always `false`, for symlinks it is + ## the same as for the link's target. template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix - when defined(Windows): - template merge(a, b): untyped = a or (b shl 32) + when defined(windows): + template merge[T](a, b): untyped = + cast[T]( + (uint64(cast[uint32](a))) or + (uint64(cast[uint32](b)) shl 32) + ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber - formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) - formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) + formalInfo.id.file = merge[FileId](rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) + formalInfo.size = merge[BiggestInt](rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) formalInfo.linkCount = rawInfo.nNumberOfLinks formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) + formalInfo.blockSize = 8192 # xxx use Windows API instead of hardcoding # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: formalInfo.permissions = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, fpOthersExec, fpOthersRead} else: - result.permissions = {fpUserExec..fpOthersRead} + formalInfo.permissions = {fpUserExec..fpOthersRead} # Retrieve basic file kind - result.kind = pcFile if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: formalInfo.kind = pcDir + else: + formalInfo.kind = pcFile if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - formalInfo.kind = succ(result.kind) + formalInfo.kind = succ(formalInfo.kind) else: template checkAndIncludeMode(rawMode, formalMode: untyped) = @@ -3016,8 +796,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.lastAccessTime = rawInfo.st_atim.toTime formalInfo.lastWriteTime = rawInfo.st_mtim.toTime formalInfo.creationTime = rawInfo.st_ctim.toTime + formalInfo.blockSize = rawInfo.st_blksize - result.permissions = {} + formalInfo.permissions = {} checkAndIncludeMode(S_IRUSR, fpUserRead) checkAndIncludeMode(S_IWUSR, fpUserWrite) checkAndIncludeMode(S_IXUSR, fpUserExec) @@ -3030,12 +811,14 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = checkAndIncludeMode(S_IWOTH, fpOthersWrite) checkAndIncludeMode(S_IXOTH, fpOthersExec) - formalInfo.kind = pcFile - if S_ISDIR(rawInfo.st_mode): - formalInfo.kind = pcDir - elif S_ISLNK(rawInfo.st_mode): - assert(path != "") # symlinks can't occur for file handles - formalInfo.kind = getSymlinkFileKind(path) + (formalInfo.kind, formalInfo.isSpecial) = + if S_ISDIR(rawInfo.st_mode): + (pcDir, false) + elif S_ISLNK(rawInfo.st_mode): + assert(path != "") # symlinks can't occur for file handles + getSymlinkFileKind(path) + else: + (pcFile, not S_ISREG(rawInfo.st_mode)) when defined(js): when not declared(FileHandle): @@ -3043,7 +826,7 @@ when defined(js): when not declared(File): type File = object -proc getFileInfo*(handle: FileHandle): FileInfo {.noNimScript.} = +proc getFileInfo*(handle: FileHandle): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object represented by the given ## handle. ## @@ -3051,73 +834,107 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noNimScript.} = ## is invalid, `OSError` is raised. ## ## See also: - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ # Done: ID, Kind, Size, Permissions, Link Count - when defined(Windows): + when defined(windows): var rawInfo: BY_HANDLE_FILE_INFORMATION # We have to use the super special '_get_osfhandle' call (wrapped above) # To transform the C file descriptor to a native file handle. var realHandle = get_osfhandle(handle) if getFileInformationByHandle(realHandle, addr rawInfo) == 0: - raiseOSError(osLastError()) + raiseOSError(osLastError(), $handle) rawToFormalFileInfo(rawInfo, "", result) else: var rawInfo: Stat if fstat(handle, rawInfo) < 0'i32: - raiseOSError(osLastError()) + raiseOSError(osLastError(), $handle) rawToFormalFileInfo(rawInfo, "", result) -proc getFileInfo*(file: File): FileInfo {.noNimScript.} = +proc getFileInfo*(file: File): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) -proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noNimScript.} = +proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo object <#FileInfo>`_ will be slightly + ## contained by the returned `FileInfo object`_ will be slightly ## different across platforms, and in some cases, incomplete or inaccurate. ## ## When `followSymlink` is true (default), symlinks are followed and the ## information retrieved is information related to the symlink's target. - ## Otherwise, information on the symlink itself is retrieved. + ## Otherwise, information on the symlink itself is retrieved (however, + ## field `isSpecial` is still determined from the target on Unix). ## ## If the information cannot be retrieved, such as when the path doesn't ## exist, or when permission restrictions prevent the program from retrieving ## file information, `OSError` is raised. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - when defined(Windows): + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + when defined(windows): var handle = openHandle(path, followSymlink) rawInfo: BY_HANDLE_FILE_INFORMATION if handle == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) + raiseOSError(osLastError(), path) if getFileInformationByHandle(handle, addr rawInfo) == 0: - raiseOSError(osLastError()) + raiseOSError(osLastError(), path) rawToFormalFileInfo(rawInfo, path, result) discard closeHandle(handle) else: var rawInfo: Stat if followSymlink: if stat(path, rawInfo) < 0'i32: - raiseOSError(osLastError()) + raiseOSError(osLastError(), path) else: if lstat(path, rawInfo) < 0'i32: - raiseOSError(osLastError()) + raiseOSError(osLastError(), path) rawToFormalFileInfo(rawInfo, path, result) -proc isHidden*(path: string): bool {.noNimScript.} = +proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadIOEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to files with identical + ## binary content. + ## + ## See also: + ## * `sameFile proc`_ + var + a, b: File + if not open(a, path1): return false + if not open(b, path2): + close(a) + return false + let bufSize = getFileInfo(a).blockSize + var bufA = alloc(bufSize) + var bufB = alloc(bufSize) + while true: + var readA = readBuffer(a, bufA, bufSize) + var readB = readBuffer(b, bufB, bufSize) + if readA != readB: + result = false + break + if readA == 0: + result = true + break + result = equalMem(bufA, bufB, readA) + if not result: break + if readA != bufSize: break # end of file + dealloc(bufA) + dealloc(bufB) + close(a) + close(b) + +proc isHidden*(path: string): bool {.noWeirdTarget.} = ## Determines whether ``path`` is hidden or not, using `this ## reference <https://en.wikipedia.org/wiki/Hidden_file_and_hidden_directory>`_. ## @@ -3136,18 +953,15 @@ proc isHidden*(path: string): bool {.noNimScript.} = assert not "".isHidden assert ".foo/".isHidden - when defined(Windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + when defined(windows): + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: let fileName = lastPathPart(path) result = len(fileName) >= 2 and fileName[0] == '.' and fileName != ".." -proc getCurrentProcessId*(): int {.noNimScript.} = +proc getCurrentProcessId*(): int {.noWeirdTarget.} = ## Return current process ID. ## ## See also: @@ -3159,7 +973,7 @@ proc getCurrentProcessId*(): int {.noNimScript.} = else: result = getpid() -proc setLastModificationTime*(file: string, t: times.Time) {.noNimScript.} = +proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = ## Sets the `file`'s last modification time. `OSError` is raised in case of ## an error. when defined(posix): @@ -3167,46 +981,52 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noNimScript.} = let micro = convert(Nanoseconds, Microseconds, t.nanosecond) var timevals = [Timeval(tv_sec: unixt, tv_usec: micro), Timeval(tv_sec: unixt, tv_usec: micro)] # [last access, last modification] - if utimes(file, timevals.addr) != 0: raiseOSError(osLastError()) + if utimes(file, timevals.addr) != 0: raiseOSError(osLastError(), file) else: let h = openHandle(path = file, writeAccess = true) - if h == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) + if h == INVALID_HANDLE_VALUE: raiseOSError(osLastError(), file) var ft = t.toWinTime.toFILETIME let res = setFileTime(h, nil, nil, ft.addr) discard h.closeHandle - if res == 0'i32: raiseOSError(osLastError()) - + if res == 0'i32: raiseOSError(osLastError(), file) -when isMainModule: - assert quoteShellWindows("aaa") == "aaa" - assert quoteShellWindows("aaa\"") == "aaa\\\"" - assert quoteShellWindows("") == "\"\"" - assert quoteShellPosix("aaa") == "aaa" - assert quoteShellPosix("aaa a") == "'aaa a'" - assert quoteShellPosix("") == "''" - assert quoteShellPosix("a'a") == "'a'\"'\"'a'" +func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = + ## Returns `true` if `filename` is valid for crossplatform use. + ## + ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. + ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. + ## + ## See also: + ## + ## * https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception + ## * https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + ## * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx + ## + ## .. warning:: This only checks filenames, not whole paths + ## (because basically you can mount anything as a path on Linux). + runnableExamples: + assert not isValidFilename(" foo") # Leading white space + assert not isValidFilename("foo ") # Trailing white space + assert not isValidFilename("foo.") # Ends with dot + assert not isValidFilename("con.txt") # "CON" is invalid (Windows) + assert not isValidFilename("OwO:UwU") # ":" is invalid (Mac) + assert not isValidFilename("aux.bat") # "AUX" is invalid (Windows) + assert not isValidFilename("") # Empty string + assert not isValidFilename("foo/") # Filename is empty - when defined(posix): - assert quoteShell("") == "''" + result = true + let f = filename.splitFile() + if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or + f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or + find(f.name, invalidFilenameChars) != -1): return false + for invalid in invalidFilenames: + if cmpIgnoreCase(f.name, invalid) == 0: return false - block normalizePathEndTest: - # handle edge cases correctly: shouldn't affect whether path is - # absolute/relative - doAssert "".normalizePathEnd(true) == "" - doAssert "".normalizePathEnd(false) == "" - doAssert "/".normalizePathEnd(true) == $DirSep - doAssert "/".normalizePathEnd(false) == $DirSep - when defined(posix): - doAssert "//".normalizePathEnd(false) == "/" - doAssert "foo.bar//".normalizePathEnd == "foo.bar" - doAssert "bar//".normalizePathEnd(trailingSep = true) == "bar/" - when defined(Windows): - doAssert r"C:\foo\\".normalizePathEnd == r"C:\foo" - doAssert r"C:\foo".normalizePathEnd(trailingSep = true) == r"C:\foo\" - # this one is controversial: we could argue for returning `D:\` instead, - # but this is simplest. - doAssert r"D:\".normalizePathEnd == r"D:" - doAssert r"E:/".normalizePathEnd(trailingSep = true) == r"E:\" - doAssert "/".normalizePathEnd == r"\" +# deprecated declarations +when not weirdTarget: + template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = + fileExists(args) + template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = + dirExists(args) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 9b11d440b..c304ecca6 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -18,17 +18,24 @@ include "system/inclrtl" import - strutils, os, strtabs, streams, cpuinfo + std/[strutils, os, strtabs, streams, cpuinfo, streamwrapper, + private/since] export quoteShell, quoteShellWindows, quoteShellPosix when defined(windows): - import winlean + import std/winlean else: - import posix + import std/posix when defined(linux) and defined(useClone): - import linux + import std/linux + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + when defined(windows): + import std/widestrs + type ProcessOption* = enum ## Options that can be passed to `startProcess proc @@ -65,23 +72,17 @@ type Process* = ref ProcessObj ## Represents an operating system process. -const poDemon* {.deprecated.} = poDaemon ## Nim versions before 0.20 - ## used the wrong spelling ("demon"). - ## Now `ProcessOption` uses the correct spelling ("daemon"), - ## and this is needed just for backward compatibility. - proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - TaintedString {.rtl, extern: "nosp$1", + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. ## - ## **WARNING:** This function uses `poEvalCommand` by default for backward - ## compatibility. - ## Make sure to pass options explicitly. + ## .. warning:: This function uses `poEvalCommand` by default for backwards + ## compatibility. Make sure to pass options explicitly. ## ## See also: ## * `startProcess proc @@ -90,12 +91,12 @@ proc execProcess*(command: string, workingDir: string = "", ## * `execCmd proc <#execCmd,string>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) - ## let outp_shell = execProcess("nim c -r mytestfile.nim") - ## # Note: outp may have an interleave of text from the nim compile - ## # and any output from mytestfile when it runs + ## ```Nim + ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) + ## let outp_shell = execProcess("nim c -r mytestfile.nim") + ## # Note: outp may have an interleave of text from the nim compile + ## # and any output from mytestfile when it runs + ## ``` proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadIOEffect, RootEffect].} @@ -105,21 +106,21 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", ## This operation is also often called `system`:idx:. ## ## See also: - ## * `execCmdEx proc <#execCmdEx,string,set[ProcessOption]>`_ + ## * `execCmdEx proc <#execCmdEx,string,set[ProcessOption],StringTableRef,string,string>`_ ## * `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execProcess proc ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let errC = execCmd("nim c -r mytestfile.nim") + ## ```Nim + ## let errC = execCmd("nim c -r mytestfile.nim") + ## ``` proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): - owned(Process) {.rtl, extern: "nosp$1", + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -151,12 +152,12 @@ proc startProcess*(command: string, workingDir: string = "", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## - ## **WARNING:** If the process has not finished executing, this will forcibly - ## terminate the process. Doing so may result in zombie processes and - ## `pty leaks <http://stackoverflow.com/questions/27021641/how-to-fix-request-failed-on-channel-0>`_. + ## .. warning:: If the process has not finished executing, this will forcibly + ## terminate the process. Doing so may result in zombie processes and + ## `pty leaks <http://stackoverflow.com/questions/27021641/how-to-fix-request-failed-on-channel-0>`_. proc suspend*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Suspends the process `p`. @@ -186,6 +187,7 @@ proc terminate*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `suspend proc <#suspend,Process>`_ ## * `resume proc <#resume,Process>`_ ## * `kill proc <#kill,Process>`_ + ## * `posix_utils.sendSignal(pid: Pid, signal: int) <posix_utils.html#sendSignal,Pid,int>`_ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Kill the process `p`. @@ -197,9 +199,10 @@ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `suspend proc <#suspend,Process>`_ ## * `resume proc <#resume,Process>`_ ## * `terminate proc <#terminate,Process>`_ + ## * `posix_utils.sendSignal(pid: Pid, signal: int) <posix_utils.html#sendSignal,Pid,int>`_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} - ## Returns true iff the process `p` is still running. Returns immediately. +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} + ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = ## Returns `p`'s process ID. @@ -209,16 +212,20 @@ proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} + extern: "nosp$1", raises: [OSError, ValueError], tags: [TimeEffect].} ## Waits for the process to finish and returns `p`'s error code. ## - ## **WARNING**: Be careful when using `waitForExit` for processes created without - ## `poParentStreams` because they may fill output buffers, causing deadlock. + ## .. warning:: Be careful when using `waitForExit` for processes created without + ## `poParentStreams` because they may fill output buffers, causing deadlock. ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. + ## + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. -proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -227,18 +234,22 @@ proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s input stream for writing to. ## - ## **WARNING**: The returned `Stream` should not be closed manually as it - ## is closed when closing the Process ``p``. + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. ## ## See also: ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} ## Returns ``p``'s output stream for reading from. ## - ## **WARNING**: The returned `Stream` should not be closed manually as it - ## is closed when closing the Process ``p``. + ## You cannot perform peek/write/setOption operations to this stream. + ## Use `peekableOutputStream proc <#peekableOutputStream,Process>`_ + ## if you need to peek stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. ## ## See also: ## * `inputStream proc <#inputStream,Process>`_ @@ -247,19 +258,47 @@ proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} proc errorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s error stream for reading from. ## - ## **WARNING**: The returned `Stream` should not be closed manually as it - ## is closed when closing the Process ``p``. + ## You cannot perform peek/write/setOption operations to this stream. + ## Use `peekableErrorStream proc <#peekableErrorStream,Process>`_ + ## if you need to peek stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. ## ## See also: ## * `inputStream proc <#inputStream,Process>`_ ## * `outputStream proc <#outputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc peekableOutputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], since: (1, 3).} + ## Returns ``p``'s output stream for reading from. + ## + ## You can peek returned stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `outputStream proc <#outputStream,Process>`_ + ## * `peekableErrorStream proc <#peekableErrorStream,Process>`_ + +proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], since: (1, 3).} + ## Returns ``p``'s error stream for reading from. + ## + ## You can run peek operation to returned stream. + ## + ## .. warning:: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `errorStream proc <#errorStream,Process>`_ + ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ + +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## - ## **WARNING**: The returned `FileHandle` should not be closed manually as - ## it is closed when closing the Process ``p``. + ## .. warning:: The returned `FileHandle` should not be closed manually as + ## it is closed when closing the Process ``p``. ## ## See also: ## * `outputHandle proc <#outputHandle,Process>`_ @@ -267,11 +306,11 @@ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s output file handle for reading from. ## - ## **WARNING**: The returned `FileHandle` should not be closed manually as - ## it is closed when closing the Process ``p``. + ## .. warning:: The returned `FileHandle` should not be closed manually as + ## it is closed when closing the Process ``p``. ## ## See also: ## * `inputHandle proc <#inputHandle,Process>`_ @@ -279,28 +318,34 @@ proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s error file handle for reading from. ## - ## **WARNING**: The returned `FileHandle` should not be closed manually as - ## it is closed when closing the Process ``p``. + ## .. warning:: The returned `FileHandle` should not be closed manually as + ## it is closed when closing the Process ``p``. ## ## See also: ## * `inputHandle proc <#inputHandle,Process>`_ ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. + ## It is implemented just calling `cpuinfo.countProcessors`. result = cpuinfo.countProcessors() +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", - tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect].} = + raises: [ValueError, OSError, IOError], + tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], + effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## @@ -380,6 +425,8 @@ proc execProcesses*(cmds: openArray[string], raiseOSError(err) if rexit >= 0: + when defined(windows): + let processHandle = q[rexit].fProcessHandle result = max(result, abs(q[rexit].peekExitCode())) if afterRunEvent != nil: afterRunEvent(idxs[rexit], q[rexit]) close(q[rexit]) @@ -394,7 +441,7 @@ proc execProcesses*(cmds: openArray[string], else: when defined(windows): for k in 0..wcount - 1: - if w[k] == q[rexit].fProcessHandle: + if w[k] == processHandle: w[k] = w[wcount - 1] w[wcount - 1] = 0 dec(wcount) @@ -410,37 +457,101 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = + ## Convenience iterator for working with `startProcess` to read data from a + ## background process. + ## + ## See also: + ## * `readLines proc <#readLines,Process>`_ + ## + ## Example: + ## ```Nim + ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} + ## var ps: seq[Process] + ## for prog in ["a", "b"]: # run 2 progs in parallel + ## ps.add startProcess("nim", "", ["r", prog], nil, opts) + ## for p in ps: + ## var i = 0 + ## for line in p.lines: + ## echo line + ## i.inc + ## if i > 100: break + ## p.close + ## ``` + var outp = p.outputStream + var line = newStringOfCap(120) + while outp.readLine(line): + if keepNewLines: + line.add("\n") + yield line + discard waitForExit(p) + +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect, TimeEffect].} = + ## Convenience function for working with `startProcess` to read data from a + ## background process. + ## + ## See also: + ## * `lines iterator <#lines.i,Process>`_ + ## + ## Example: + ## ```Nim + ## const opts = {poUsePath, poDaemon, poStdErrToStdOut} + ## var ps: seq[Process] + ## for prog in ["a", "b"]: # run 2 progs in parallel + ## ps.add startProcess("nim", "", ["r", prog], nil, opts) + ## for p in ps: + ## let (lines, exCode) = p.readLines + ## if exCode != 0: + ## for line in lines: echo line + ## p.close + ## ``` + for line in p.lines: result[0].add(line) + result[1] = p.peekExitCode + when not defined(useNimRtl): proc execProcess(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - TaintedString = + string = var p = startProcess(command, workingDir = workingDir, args = args, env = env, options = options) var outp = outputStream(p) - result = TaintedString"" - var line = newStringOfCap(120).TaintedString + result = "" + var line = newStringOfCap(120) + # consider `p.lines(keepNewLines=true)` to circumvent `running` busy-wait while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): - result.string.add(line.string) - result.string.add("\n") + result.add(line) + result.add("\n") elif not running(p): break close(p) template streamAccess(p) = assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams" -when defined(Windows) and not defined(useNimRtl): +when defined(windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: type FileHandleStream = ref object of StreamObj handle: Handle atTheEnd: bool - proc hsClose(s: Stream) = discard # nothing to do here + proc closeHandleCheck(handle: Handle) {.inline.} = + if handle.closeHandle() == 0: + raiseOSError(osLastError()) + + proc fileClose[T: Handle | FileHandle](h: var T) {.inline.} = + if h > 4: + closeHandleCheck(h) + h = INVALID_HANDLE_VALUE.T + + proc hsClose(s: Stream) = + FileHandleStream(s).handle.fileClose() + proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd proc hsReadData(s: Stream, buffer: pointer, bufLen: int): int = @@ -463,12 +574,8 @@ when defined(Windows) and not defined(useNimRtl): if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) @@ -544,8 +651,8 @@ when defined(Windows) and not defined(useNimRtl): stdin = myDup(pipeIn, 0) stdout = myDup(pipeOut, 0) - discard closeHandle(pipeIn) - discard closeHandle(pipeOut) + closeHandleCheck(pipeIn) + closeHandleCheck(pipeOut) stderr = stdout proc createPipeHandles(rdHandle, wrHandle: var Handle) = @@ -556,9 +663,6 @@ when defined(Windows) and not defined(useNimRtl): if createPipe(rdHandle, wrHandle, sa, 0) == 0'i32: raiseOSError(osLastError()) - proc fileClose(h: Handle) {.inline.} = - if h > 4: discard closeHandle(h) - proc startProcess(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): @@ -614,19 +718,15 @@ when defined(Windows) and not defined(useNimRtl): if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = - if e.str.isNil: newWideCString(cstring(nil)) - else: newWideCString(e.str, e.len) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDaemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, e, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -642,7 +742,7 @@ when defined(Windows) and not defined(useNimRtl): const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess @@ -650,13 +750,31 @@ when defined(Windows) and not defined(useNimRtl): result.id = procInfo.dwProcessId result.exitFlag = false + proc closeThreadAndProcessHandle(p: Process) = + if p.fThreadHandle != 0: + closeHandleCheck(p.fThreadHandle) + p.fThreadHandle = 0 + + if p.fProcessHandle != 0: + closeHandleCheck(p.fProcessHandle) + p.fProcessHandle = 0 + proc close(p: Process) = if poParentStreams notin p.options: - discard closeHandle(p.inHandle) - discard closeHandle(p.outHandle) - discard closeHandle(p.errHandle) - discard closeHandle(p.fThreadHandle) - discard closeHandle(p.fProcessHandle) + if p.inStream == nil: + p.inHandle.fileClose() + else: + # p.inHandle can be already closed via inputStream. + p.inStream.close + + # You may NOT close outputStream and errorStream. + assert p.outStream == nil or FileHandleStream(p.outStream).handle != INVALID_HANDLE_VALUE + assert p.errStream == nil or FileHandleStream(p.errStream).handle != INVALID_HANDLE_VALUE + + if p.outHandle != p.errHandle: + p.errHandle.fileClose() + p.outHandle.fileClose() + p.closeThreadAndProcessHandle() proc suspend(p: Process) = discard suspendThread(p.fThreadHandle) @@ -690,8 +808,7 @@ when defined(Windows) and not defined(useNimRtl): if status != STILL_ACTIVE: p.exitFlag = true p.exitStatus = status - discard closeHandle(p.fThreadHandle) - discard closeHandle(p.fProcessHandle) + p.closeThreadAndProcessHandle() result = status else: result = -1 @@ -707,8 +824,7 @@ when defined(Windows) and not defined(useNimRtl): discard getExitCodeProcess(p.fProcessHandle, status) p.exitFlag = true p.exitStatus = status - discard closeHandle(p.fThreadHandle) - discard closeHandle(p.fProcessHandle) + p.closeThreadAndProcessHandle() result = status proc inputStream(p: Process): Stream = @@ -729,6 +845,18 @@ when defined(Windows) and not defined(useNimRtl): p.errStream = newFileHandleStream(p.errHandle) result = p.errStream + proc peekableOutputStream(p: Process): Stream = + streamAccess(p) + if p.outStream == nil: + p.outStream = newFileHandleStream(p.outHandle).newPipeOutStream + result = p.outStream + + proc peekableErrorStream(p: Process): Stream = + streamAccess(p) + if p.errStream == nil: + p.errStream = newFileHandleStream(p.errHandle).newPipeOutStream + result = p.errStream + proc execCmd(command: string): int = var si: STARTUPINFO @@ -739,13 +867,9 @@ when defined(Windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: @@ -804,7 +928,7 @@ elif not defined(useNimRtl): result = cast[cstringArray](alloc0((counter + 1) * sizeof(cstring))) var i = 0 for key, val in envPairs(): - var x = key.string & "=" & val.string + var x = key & "=" & val result[i] = cast[cstring](alloc(x.len+1)) copyMem(result[i], addr(x[0]), x.len+1) inc(i) @@ -822,14 +946,14 @@ elif not defined(useNimRtl): not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} - {.push stacktrace: off, profiler: off.} - proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} - {.pop.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + {.push stacktrace: off, profiler: off.} + proc startProcessAfterFork(data: ptr StartProcessData) {. + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + {.pop.} proc startProcess(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, @@ -853,7 +977,7 @@ elif not defined(useNimRtl): when not defined(android): "/bin/sh" else: "/system/bin/sh" data.sysCommand = useShPath - sysArgsRaw = @[data.sysCommand, "-c", command] + sysArgsRaw = @[useShPath, "-c", command] assert args.len == 0, "`args` has to be empty when using poEvalCommand." else: data.sysCommand = command @@ -891,7 +1015,7 @@ elif not defined(useNimRtl): # Parent process. Copy process information. if poEchoCmd in options: - when declared(echo): echo(command, " ", join(args, " ")) + echo(command, " ", join(args, " ")) result.id = pid result.exitFlag = false @@ -929,13 +1053,15 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - if poDaemon in data.options: - chck posix_spawnattr_setpgroup(attr, 0'i32) + when not defined(nuttx): + if poDaemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) var flags = POSIX_SPAWN_USEVFORK or POSIX_SPAWN_SETSIGMASK - if poDaemon in data.options: - flags = flags or POSIX_SPAWN_SETPGROUP + when not defined(nuttx): + if poDaemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP chck posix_spawnattr_setflags(attr, flags) if not (poParentStreams in data.options): @@ -955,9 +1081,9 @@ elif not defined(useNimRtl): var pid: Pid if (poUsePath in data.options): - res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawnp(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) else: - res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + res = posix_spawn(pid, data.sysCommand.cstring, fops, attr, data.sysArgs, data.sysEnv) discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) @@ -997,62 +1123,65 @@ elif not defined(useNimRtl): var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): - raiseOSError(osLastError(), - "Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + raiseOSError(OSErrorCode(error), + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid - {.push stacktrace: off, profiler: off.} - proc startProcessFail(data: ptr StartProcessData) = - var error: cint = errno - discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) - exitnow(1) - - when not defined(uClibc) and (not defined(linux) or defined(android)): - var environ {.importc.}: cstringArray - - proc startProcessAfterFork(data: ptr StartProcessData) = - # Warning: no GC here! - # Or anything that touches global structures - all called nim procs - # must be marked with stackTrace:off. Inspect C code after making changes. - if not (poParentStreams in data.options): - discard close(data.pStdin[writeIdx]) - if dup2(data.pStdin[readIdx], readIdx) < 0: - startProcessFail(data) - discard close(data.pStdout[readIdx]) - if dup2(data.pStdout[writeIdx], writeIdx) < 0: - startProcessFail(data) - discard close(data.pStderr[readIdx]) - if (poStdErrToStdOut in data.options): - if dup2(data.pStdout[writeIdx], 2) < 0: + {.push stacktrace: off, profiler: off.} + proc startProcessFail(data: ptr StartProcessData, error: cint = errno) = + discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) + exitnow(1) + + when not defined(uClibc) and (not defined(linux) or defined(android)) and + not defined(haiku): + var environ {.importc.}: cstringArray + + proc startProcessAfterFork(data: ptr StartProcessData) = + # Warning: no GC here! + # Or anything that touches global structures - all called nim procs + # must be marked with stackTrace:off. Inspect C code after making changes. + if not (poParentStreams in data.options): + discard close(data.pStdin[writeIdx]) + if dup2(data.pStdin[readIdx], readIdx) < 0: startProcessFail(data) - else: - if dup2(data.pStderr[writeIdx], 2) < 0: + discard close(data.pStdout[readIdx]) + if dup2(data.pStdout[writeIdx], writeIdx) < 0: startProcessFail(data) + discard close(data.pStderr[readIdx]) + if (poStdErrToStdOut in data.options): + if dup2(data.pStdout[writeIdx], 2) < 0: + startProcessFail(data) + else: + if dup2(data.pStderr[writeIdx], 2) < 0: + startProcessFail(data) - if data.workingDir.len > 0: - if chdir(data.workingDir) < 0: - startProcessFail(data) + if data.workingDir.len > 0: + if chdir(data.workingDir) < 0: + startProcessFail(data) - discard close(data.pErrorPipe[readIdx]) - discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) + discard close(data.pErrorPipe[readIdx]) + discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) - if (poUsePath in data.options): - when defined(uClibc) or defined(linux): - # uClibc environment (OpenWrt included) doesn't have the full execvpe - let exe = findExe(data.sysCommand) - discard execve(exe, data.sysArgs, data.sysEnv) + if (poUsePath in data.options): + when defined(uClibc) or defined(linux) or defined(haiku): + # uClibc environment (OpenWrt included) doesn't have the full execvpe + var exe: string + try: + exe = findExe(data.sysCommand) + except OSError as e: + startProcessFail(data, e.errorCode) + discard execve(exe.cstring, data.sysArgs, data.sysEnv) + else: + # MacOSX doesn't have execvpe, so we need workaround. + # On MacOSX we can arrive here only from fork, so this is safe: + environ = data.sysEnv + discard execvp(data.sysCommand.cstring, data.sysArgs) else: - # MacOSX doesn't have execvpe, so we need workaround. - # On MacOSX we can arrive here only from fork, so this is safe: - environ = data.sysEnv - discard execvp(data.sysCommand, data.sysArgs) - else: - discard execve(data.sysCommand, data.sysArgs, data.sysEnv) + discard execve(data.sysCommand.cstring, data.sysArgs, data.sysEnv) - startProcessFail(data) - {.pop.} + startProcessFail(data) + {.pop.} proc close(p: Process) = if poParentStreams notin p.options: @@ -1105,7 +1234,7 @@ elif not defined(useNimRtl): when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or defined(dragonfly): - import kqueue, times + import std/kqueue proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: @@ -1165,37 +1294,23 @@ elif not defined(useNimRtl): discard posix.close(kqFD) result = exitStatusLikeShell(p.exitStatus) - else: - import times - + elif defined(haiku): const - hasThreadSupport = compileOption("threads") and not defined(nimscript) + B_OBJECT_TYPE_THREAD = 3 + B_EVENT_INVALID = 0x1000 + B_RELATIVE_TIMEOUT = 0x8 - proc waitForExit(p: Process, timeout: int = -1): int = - template adjustTimeout(t, s, e: Timespec) = - var diff: int - var b: Timespec - b.tv_sec = e.tv_sec - b.tv_nsec = e.tv_nsec - e.tv_sec = e.tv_sec - s.tv_sec - if e.tv_nsec >= s.tv_nsec: - e.tv_nsec -= s.tv_nsec - else: - if e.tv_sec == posix.Time(0): - raise newException(ValueError, "System time was modified") - else: - diff = s.tv_nsec - e.tv_nsec - e.tv_nsec = 1_000_000_000 - diff - t.tv_sec = t.tv_sec - e.tv_sec - if t.tv_nsec >= e.tv_nsec: - t.tv_nsec -= e.tv_nsec - else: - t.tv_sec = t.tv_sec - posix.Time(1) - diff = e.tv_nsec - t.tv_nsec - t.tv_nsec = 1_000_000_000 - diff - s.tv_sec = b.tv_sec - s.tv_nsec = b.tv_nsec + type + ObjectWaitInfo {.importc: "object_wait_info", header: "OS.h".} = object + obj {.importc: "object".}: int32 + typ {.importc: "type".}: uint16 + events: uint16 + + proc waitForObjects(infos: ptr ObjectWaitInfo, numInfos: cint, flags: uint32, + timeout: int64): clong + {.importc: "wait_for_objects_etc", header: "OS.h".} + proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: return exitStatusLikeShell(p.exitStatus) @@ -1206,74 +1321,98 @@ elif not defined(useNimRtl): p.exitFlag = true p.exitStatus = status else: - var nmask, omask: Sigset - var sinfo: SigInfo - var stspec, enspec, tmspec: Timespec + var info = ObjectWaitInfo( + obj: p.id, # Haiku's PID is actually the main thread ID. + typ: B_OBJECT_TYPE_THREAD, + events: B_EVENT_INVALID # notify when the thread die. + ) + + while true: + var status: cint = 1 + let count = waitForObjects(addr info, 1, B_RELATIVE_TIMEOUT, timeout) + + if count < 0: + let err = count.cint + if err == ETIMEDOUT: + # timeout expired, so we try to kill the process + if posix.kill(p.id, SIGKILL) == -1: + raiseOSError(osLastError()) + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + break + elif err != EINTR: + raiseOSError(err.OSErrorCode) + elif count > 0: + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + break + else: + raiseAssert "unreachable!" - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, SIGCHLD) + result = exitStatusLikeShell(p.exitStatus) - when hasThreadSupport: - if pthread_sigmask(SIG_BLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) - else: - if sigprocmask(SIG_BLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + else: + import std/times except getTime + import std/monotimes - if timeout >= 1000: - tmspec.tv_sec = posix.Time(timeout div 1_000) - tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 - else: - tmspec.tv_sec = posix.Time(0) - tmspec.tv_nsec = (timeout * 1_000_000) + proc waitForExit(p: Process, timeout: int = -1): int = + if p.exitFlag: + return exitStatusLikeShell(p.exitStatus) - try: - if clock_gettime(CLOCK_REALTIME, stspec) == -1: - raiseOSError(osLastError()) - while true: - let res = sigtimedwait(nmask, sinfo, tmspec) - if res == SIGCHLD: - if sinfo.si_pid == p.id: - var status: cint = 1 - if waitpid(p.id, status, 0) < 0: - raiseOSError(osLastError()) - p.exitFlag = true - p.exitStatus = status - break - else: - # we have SIGCHLD, but not for process we are waiting, - # so we need to adjust timeout value and continue - if clock_gettime(CLOCK_REALTIME, enspec) == -1: - raiseOSError(osLastError()) - adjustTimeout(tmspec, stspec, enspec) - elif res < 0: - let err = osLastError() - if err.cint == EINTR: - # we have received another signal, so we need to - # adjust timeout and continue - if clock_gettime(CLOCK_REALTIME, enspec) == -1: - raiseOSError(osLastError()) - adjustTimeout(tmspec, stspec, enspec) - elif err.cint == EAGAIN: - # timeout expired, so we trying to kill process - if posix.kill(p.id, SIGKILL) == -1: - raiseOSError(osLastError()) - var status: cint = 1 - if waitpid(p.id, status, 0) < 0: - raiseOSError(osLastError()) - p.exitFlag = true - p.exitStatus = status - break - else: - raiseOSError(err) - finally: - when hasThreadSupport: - if pthread_sigmask(SIG_UNBLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + if timeout < 0: + # Backwards compatibility with previous verison to + # handle cases where timeout == -1, but extend + # to handle cases where timeout < 0 + var status: cint + if waitpid(p.id, status, 0) < 0: + raiseOSError(osLastError()) + p.exitFlag = true + p.exitStatus = status + else: + # Max 50ms delay + const maxWait = initDuration(milliseconds = 50) + let wait = initDuration(milliseconds = timeout) + let deadline = getMonoTime() + wait + # starting 50μs delay + var delay = initDuration(microseconds = 50) + + while true: + var status: cint + let pid = waitpid(p.id, status, WNOHANG) + if p.id == pid : + p.exitFlag = true + p.exitStatus = status + break + elif pid.int == -1: + raiseOsError(osLastError()) else: - if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: - raiseOSError(osLastError()) + # Continue waiting if needed + if getMonoTime() >= deadline: + # Previous version of `waitForExit` + # foricibly killed the process. + # We keep this so we don't break programs + # that depend on this behavior + if posix.kill(p.id, SIGKILL) < 0: + raiseOSError(osLastError()) + else: + const max = 1_000_000_000 + let + newWait = getMonoTime() + delay + ticks = newWait.ticks() + ns = ticks mod max + secs = ticks div max + var + waitSpec: TimeSpec + unused: Timespec + waitSpec.tv_sec = posix.Time(secs) + waitSpec.tv_nsec = clong ns + discard posix.clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, waitSpec, unused) + let remaining = deadline - getMonoTime() + delay = min([delay * 2, remaining, maxWait]) result = exitStatusLikeShell(p.exitStatus) @@ -1290,35 +1429,47 @@ elif not defined(useNimRtl): p.exitStatus = status result = exitStatusLikeShell(status) - proc createStream(stream: var owned(Stream), handle: var FileHandle, - fileMode: FileMode) = + proc createStream(handle: var FileHandle, + fileMode: FileMode): owned FileStream = var f: File if not open(f, handle, fileMode): raiseOSError(osLastError()) - stream = newFileStream(f) + return newFileStream(f) proc inputStream(p: Process): Stream = streamAccess(p) if p.inStream == nil: - createStream(p.inStream, p.inHandle, fmWrite) + p.inStream = createStream(p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = streamAccess(p) if p.outStream == nil: - createStream(p.outStream, p.outHandle, fmRead) + p.outStream = createStream(p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = streamAccess(p) if p.errStream == nil: - createStream(p.errStream, p.errHandle, fmRead) + p.errStream = createStream(p.errHandle, fmRead) + return p.errStream + + proc peekableOutputStream(p: Process): Stream = + streamAccess(p) + if p.outStream == nil: + p.outStream = createStream(p.outHandle, fmRead).newPipeOutStream + return p.outStream + + proc peekableErrorStream(p: Process): Stream = + streamAccess(p) + if p.errStream == nil: + p.errStream = createStream(p.errHandle, fmRead).newPipeOutStream return p.errStream proc csystem(cmd: cstring): cint {.nodecl, importc: "system", header: "<stdlib.h>".} proc execCmd(command: string): int = - when defined(linux): + when defined(posix): let tmp = csystem(command) result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: @@ -1368,12 +1519,17 @@ elif not defined(useNimRtl): proc execCmdEx*(command: string, options: set[ProcessOption] = { - poStdErrToStdOut, poUsePath}): tuple[ - output: TaintedString, - exitCode: int] {.tags: + poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, + workingDir = "", input = ""): tuple[ + output: string, + exitCode: int] {.raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = - ## A convenience proc that runs the `command`, grabs all its output and - ## exit code and returns both. + ## A convenience proc that runs the `command`, and returns its `output` and + ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. + ## If `input.len > 0`, it is passed as stdin. + ## + ## Note: this could block if `input.len` is greater than your OS's maximum + ## pipe buffer size. ## ## See also: ## * `execCmd proc <#execCmd,string>`_ @@ -1383,24 +1539,42 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: - ## - ## .. code-block:: Nim - ## let (outp, errC) = execCmdEx("nim c -r mytestfile.nim") - - var p = startProcess(command, options = options + {poEvalCommand}) + ## ```Nim + ## var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") + ## import std/[strutils, strtabs] + ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any + ## doAssert result == ("12", 0) + ## doAssert execCmdEx("ls --nonexistent").exitCode != 0 + ## when defined(posix): + ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) + ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + ## ``` + + when (NimMajor, NimMinor, NimPatch) < (1, 3, 5): + doAssert input.len == 0 + doAssert workingDir.len == 0 + doAssert env == nil + + var p = startProcess(command, options = options + {poEvalCommand}, + workingDir = workingDir, env = env) var outp = outputStream(p) - # There is no way to provide input for the child process - # anymore. Closing it will create EOF on stdin instead of eternal - # blocking. + if input.len > 0: + # There is no way to provide input for the child process + # anymore. Closing it will create EOF on stdin instead of eternal + # blocking. + # Writing in chunks would require a selectors (eg kqueue/epoll) to avoid + # blocking on io. + inputStream(p).write(input) close inputStream(p) - result = (TaintedString"", -1) - var line = newStringOfCap(120).TaintedString + # consider `p.lines(keepNewLines=true)` to avoid exit code test + result = ("", -1) + var line = newStringOfCap(120) while true: if outp.readLine(line): - result[0].string.add(line.string) - result[0].string.add("\n") + result[0].add(line) + result[0].add("\n") else: result[1] = peekExitCode(p) if result[1] != -1: break diff --git a/lib/pure/oswalkdir.nim b/lib/pure/oswalkdir.nim deleted file mode 100644 index 1cf561c22..000000000 --- a/lib/pure/oswalkdir.nim +++ /dev/null @@ -1,36 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Compile-time only version for walkDir if you need it at compile-time -## for JavaScript. - -type - PathComponent* = enum ## Enumeration specifying a path component. - pcFile, ## path refers to a file - pcLinkToFile, ## path refers to a symbolic link to a file - pcDir, ## path refers to a directory - pcLinkToDir ## path refers to a symbolic link to a directory - -proc staticWalkDir(dir: string; relative: bool): seq[ - tuple[kind: PathComponent; path: string]] = - discard - -iterator walkDir*(dir: string; relative = false): tuple[kind: PathComponent; - path: string] = - for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) - -iterator walkDirRec*(dir: string; filter = {pcFile, pcDir}): string = - var stack = @[dir] - while stack.len > 0: - for k, p in walkDir(stack.pop()): - if k in filter: - case k - of pcFile, pcLinkToFile: yield p - of pcDir, pcLinkToDir: stack.add(p) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index c8e936ef1..8a43daf54 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -7,8 +7,8 @@ # distribution, for details about the copyright. # -## The ``parsecfg`` module implements a high performance configuration file -## parser. The configuration file's syntax is similar to the Windows ``.ini`` +## The `parsecfg` module implements a high performance configuration file +## parser. The configuration file's syntax is similar to the Windows `.ini` ## format, but much more powerful, as it is not a line based parser. String ## literals, raw string literals and triple quoted string literals are supported ## as in the Nim programming language. @@ -19,106 +19,173 @@ ## :literal: ## ## Here is an example of how to use the configuration file parser: -## -## .. code-block:: nim -## -## import os, parsecfg, strutils, streams -## -## var f = newFileStream(paramStr(1), fmRead) -## if f != nil: -## var p: CfgParser -## open(p, f, paramStr(1)) -## while true: -## var e = next(p) -## case e.kind -## of cfgEof: break -## of cfgSectionStart: ## a ``[section]`` has been parsed -## echo("new section: " & e.section) -## of cfgKeyValuePair: -## echo("key-value-pair: " & e.key & ": " & e.value) -## of cfgOption: -## echo("command: " & e.key & ": " & e.value) -## of cfgError: -## echo(e.msg) -## close(p) -## else: -## echo("cannot open: " & paramStr(1)) -## -## -## Examples -## ======== -## +runnableExamples("-r:off"): + import std/[strutils, streams] + + let configFile = "example.ini" + var f = newFileStream(configFile, fmRead) + assert f != nil, "cannot open " & configFile + var p: CfgParser + open(p, f, configFile) + while true: + var e = next(p) + case e.kind + of cfgEof: break + of cfgSectionStart: ## a `[section]` has been parsed + echo "new section: " & e.section + of cfgKeyValuePair: + echo "key-value-pair: " & e.key & ": " & e.value + of cfgOption: + echo "command: " & e.key & ": " & e.value + of cfgError: + echo e.msg + close(p) + +##[ ## Configuration file example -## -------------------------- -## -## .. code-block:: nim -## +]## + +## ```none ## charset = "utf-8" ## [Package] ## name = "hello" ## --threads:on ## [Author] -## name = "lihf8515" -## qq = "10214028" -## email = "lihaifeng@wxm.com" -## +## name = "nim-lang" +## website = "nim-lang.org" +## ``` + +##[ ## Creating a configuration file -## ----------------------------- -## .. code-block:: nim -## -## import parsecfg -## var dict=newConfig() -## dict.setSectionKey("","charset","utf-8") -## dict.setSectionKey("Package","name","hello") -## dict.setSectionKey("Package","--threads","on") -## dict.setSectionKey("Author","name","lihf8515") -## dict.setSectionKey("Author","qq","10214028") -## dict.setSectionKey("Author","email","lihaifeng@wxm.com") -## dict.writeConfig("config.ini") -## +]## + +runnableExamples: + var dict = newConfig() + dict.setSectionKey("","charset", "utf-8") + dict.setSectionKey("Package", "name", "hello") + dict.setSectionKey("Package", "--threads", "on") + dict.setSectionKey("Author", "name", "nim-lang") + dict.setSectionKey("Author", "website", "nim-lang.org") + assert $dict == """ +charset=utf-8 +[Package] +name=hello +--threads:on +[Author] +name=nim-lang +website=nim-lang.org +""" + +##[ ## Reading a configuration file -## ---------------------------- -## .. code-block:: nim -## -## import parsecfg -## var dict = loadConfig("config.ini") -## var charset = dict.getSectionValue("","charset") -## var threads = dict.getSectionValue("Package","--threads") -## var pname = dict.getSectionValue("Package","name") -## var name = dict.getSectionValue("Author","name") -## var qq = dict.getSectionValue("Author","qq") -## var email = dict.getSectionValue("Author","email") -## echo pname & "\n" & name & "\n" & qq & "\n" & email -## +]## + +runnableExamples("-r:off"): + let dict = loadConfig("config.ini") + let charset = dict.getSectionValue("","charset") + let threads = dict.getSectionValue("Package","--threads") + let pname = dict.getSectionValue("Package","name") + let name = dict.getSectionValue("Author","name") + let website = dict.getSectionValue("Author","website") + echo pname & "\n" & name & "\n" & website + +##[ ## Modifying a configuration file -## ------------------------------ -## .. code-block:: nim -## -## import parsecfg -## var dict = loadConfig("config.ini") -## dict.setSectionKey("Author","name","lhf") -## dict.writeConfig("config.ini") -## -## Deleting a section key in a configuration file -## ---------------------------------------------- -## .. code-block:: nim -## -## import parsecfg -## var dict = loadConfig("config.ini") -## dict.delSectionKey("Author","email") -## dict.writeConfig("config.ini") +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.setSectionKey("Author", "name", "nim-lang") + dict.writeConfig("config.ini") -import - strutils, lexbase, streams, tables +##[ +## Deleting a section key in a configuration file +]## + +runnableExamples("-r:off"): + var dict = loadConfig("config.ini") + dict.delSectionKey("Author", "website") + dict.writeConfig("config.ini") + +##[ +## Supported INI File structure +]## + +# taken from https://docs.python.org/3/library/configparser.html#supported-ini-file-structure +runnableExamples: + import std/streams + + var dict = loadConfig(newStringStream("""[Simple Values] + key=value + spaces in keys=allowed + spaces in values=allowed as well + spaces around the delimiter = obviously + you can also use : to delimit keys from values + [All Values Are Strings] + values like this: 19990429 + or this: 3.14159265359 + are they treated as numbers : no + integers floats and booleans are held as: strings + can use the API to get converted values directly: true + [No Values] + key_without_value + # empty string value is not allowed = + [ Seletion A ] + space around section name will be ignored + [You can use comments] + # like this + ; or this + # By default only in an empty line. + # Inline comments can be harmful because they prevent users + # from using the delimiting characters as parts of values. + # That being said, this can be customized. + [Sections Can Be Indented] + can_values_be_as_well = True + does_that_mean_anything_special = False + purpose = formatting for readability + # Did I mention we can indent comments, too? + """) + ) + + let section1 = "Simple Values" + assert dict.getSectionValue(section1, "key") == "value" + assert dict.getSectionValue(section1, "spaces in keys") == "allowed" + assert dict.getSectionValue(section1, "spaces in values") == "allowed as well" + assert dict.getSectionValue(section1, "spaces around the delimiter") == "obviously" + assert dict.getSectionValue(section1, "you can also use") == "to delimit keys from values" + + let section2 = "All Values Are Strings" + assert dict.getSectionValue(section2, "values like this") == "19990429" + assert dict.getSectionValue(section2, "or this") == "3.14159265359" + assert dict.getSectionValue(section2, "are they treated as numbers") == "no" + assert dict.getSectionValue(section2, "integers floats and booleans are held as") == "strings" + assert dict.getSectionValue(section2, "can use the API to get converted values directly") == "true" + + let section3 = "Seletion A" + assert dict.getSectionValue(section3, + "space around section name will be ignored", "not an empty value") == "" + + let section4 = "Sections Can Be Indented" + assert dict.getSectionValue(section4, "can_values_be_as_well") == "True" + assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" + assert dict.getSectionValue(section4, "purpose") == "formatting for readability" + +import std/[strutils, lexbase, streams, tables] +import std/private/decode_helpers +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/syncio include "system/inclrtl" + type CfgEventKind* = enum ## enumeration of all events that may occur when parsing cfgEof, ## end of file reached - cfgSectionStart, ## a ``[section]`` has been parsed - cfgKeyValuePair, ## a ``key=value`` pair has been detected - cfgOption, ## a ``--key=value`` command line option + cfgSectionStart, ## a `[section]` has been parsed + cfgKeyValuePair, ## a `key=value` pair has been detected + cfgOption, ## a `--key=value` command line option cfgError ## an error occurred during parsing CfgEvent* = object of RootObj ## describes a parsing event @@ -126,12 +193,12 @@ type of cfgEof: nil of cfgSectionStart: section*: string ## `section` contains the name of the - ## parsed section start (syntax: ``[section]``) + ## parsed section start (syntax: `[section]`) of cfgKeyValuePair, cfgOption: key*, value*: string ## contains the (key, value) pair if an option - ## of the form ``--key: value`` or an ordinary - ## ``key= value`` pair has been parsed. - ## ``value==""`` if it was not specified in the + ## of the form `--key: value` or an ordinary + ## `key= value` pair has been parsed. + ## `value==""` if it was not specified in the ## configuration file. of cfgError: ## the parser encountered an error: `msg` msg*: string ## contains the error message. No exceptions @@ -151,13 +218,13 @@ type # implementation const - SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.', '/', '\\', '-'} + SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', ' ', '\x80'..'\xFF', '.', '/', '\\', '-'} proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe.} proc open*(c: var CfgParser, input: Stream, filename: string, lineOffset = 0) {.rtl, extern: "npc$1".} = - ## initializes the parser with an input stream. `Filename` is only used + ## Initializes the parser with an input stream. `Filename` is only used ## for nice error messages. `lineOffset` can be used to influence the line ## number information in the generated error messages. lexbase.open(c, input) @@ -168,35 +235,21 @@ proc open*(c: var CfgParser, input: Stream, filename: string, rawGetTok(c, c.tok) proc close*(c: var CfgParser) {.rtl, extern: "npc$1".} = - ## closes the parser `c` and its associated input stream. + ## Closes the parser `c` and its associated input stream. lexbase.close(c) proc getColumn*(c: CfgParser): int {.rtl, extern: "npc$1".} = - ## get the current column the parser has arrived at. + ## Gets the current column the parser has arrived at. result = getColNumber(c, c.bufpos) proc getLine*(c: CfgParser): int {.rtl, extern: "npc$1".} = - ## get the current line the parser has arrived at. + ## Gets the current line the parser has arrived at. result = c.lineNumber proc getFilename*(c: CfgParser): string {.rtl, extern: "npc$1".} = - ## get the filename of the file that the parser processes. + ## Gets the filename of the file that the parser processes. result = c.filename -proc handleHexChar(c: var CfgParser, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: - discard - proc handleDecChars(c: var CfgParser, xi: var int) = while c.buf[c.bufpos] in {'0'..'9'}: xi = (xi * 10) + (ord(c.buf[c.bufpos]) - ord('0')) @@ -241,8 +294,10 @@ proc getEscapedChar(c: var CfgParser, tok: var Token) = of 'x', 'X': inc(c.bufpos) var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) add(tok.literal, chr(xi)) of '0'..'9': var xi = 0 @@ -306,6 +361,10 @@ proc getSymbol(c: var CfgParser, tok: var Token) = add(tok.literal, c.buf[pos]) inc(pos) if not (c.buf[pos] in SymChars): break + + while tok.literal.len > 0 and tok.literal[^1] == ' ': + tok.literal.setLen(tok.literal.len - 1) + c.bufpos = pos tok.kind = tkSymbol @@ -367,19 +426,19 @@ proc rawGetTok(c: var CfgParser, tok: var Token) = else: getSymbol(c, tok) proc errorStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} = - ## returns a properly formatted error message containing current line and + ## Returns a properly formatted error message containing current line and ## column information. result = `%`("$1($2, $3) Error: $4", [c.filename, $getLine(c), $getColumn(c), msg]) proc warningStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} = - ## returns a properly formatted warning message containing current line and + ## Returns a properly formatted warning message containing current line and ## column information. result = `%`("$1($2, $3) Warning: $4", [c.filename, $getLine(c), $getColumn(c), msg]) proc ignoreMsg*(c: CfgParser, e: CfgEvent): string {.rtl, extern: "npc$1".} = - ## returns a properly formatted warning message containing that + ## Returns a properly formatted warning message containing that ## an entry is ignored. case e.kind of cfgSectionStart: result = c.warningStr("section ignored: " & e.section) @@ -393,7 +452,7 @@ proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent = if c.tok.kind == tkSymbol: case kind of cfgOption, cfgKeyValuePair: - result = CfgEvent(kind: kind, key: c.tok.literal, value: "") + result = CfgEvent(kind: kind, key: c.tok.literal.move, value: "") else: discard rawGetTok(c, c.tok) if c.tok.kind in {tkEquals, tkColon}: @@ -410,7 +469,7 @@ proc getKeyValPair(c: var CfgParser, kind: CfgEventKind): CfgEvent = rawGetTok(c, c.tok) proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = - ## retrieves the first/next event. This controls the parser. + ## Retrieves the first/next event. This controls the parser. case c.tok.kind of tkEof: result = CfgEvent(kind: cfgEof) @@ -422,7 +481,7 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = of tkBracketLe: rawGetTok(c, c.tok) if c.tok.kind == tkSymbol: - result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal) + result = CfgEvent(kind: cfgSectionStart, section: c.tok.literal.move) else: result = CfgEvent(kind: cfgError, msg: errorStr(c, "symbol expected, but found: " & c.tok.literal)) @@ -439,17 +498,17 @@ proc next*(c: var CfgParser): CfgEvent {.rtl, extern: "npc$1".} = # ---------------- Configuration file related operations ---------------- type - Config* = OrderedTableRef[string, <//>OrderedTableRef[string, string]] + Config* = OrderedTableRef[string, OrderedTableRef[string, string]] proc newConfig*(): Config = - ## Create a new configuration table. + ## Creates a new configuration table. ## Useful when wanting to create a configuration file. - result = newOrderedTable[string, <//>OrderedTableRef[string, string]]() + result = newOrderedTable[string, OrderedTableRef[string, string]]() -proc loadConfig*(stream: Stream, filename: string = "[stream]"): <//>Config = - ## Load the specified configuration from stream into a new Config instance. +proc loadConfig*(stream: Stream, filename: string = "[stream]"): Config = + ## Loads the specified configuration from stream into a new Config instance. ## `filename` parameter is only used for nicer error messages. - var dict = newOrderedTable[string, <//>OrderedTableRef[string, string]]() + var dict = newOrderedTable[string, OrderedTableRef[string, string]]() var curSection = "" ## Current section, ## the default value of the current section is "", ## which means that the current section is a common @@ -479,8 +538,8 @@ proc loadConfig*(stream: Stream, filename: string = "[stream]"): <//>Config = close(p) result = dict -proc loadConfig*(filename: string): <//>Config = - ## Load the specified configuration file into a new Config instance. +proc loadConfig*(filename: string): Config = + ## Loads the specified configuration file into a new Config instance. let file = open(filename, fmRead) let fileStream = newFileStream(file) defer: fileStream.close() @@ -505,9 +564,9 @@ proc replace(s: string): string = result = d proc writeConfig*(dict: Config, stream: Stream) = - ## Writes the contents of the table to the specified stream + ## Writes the contents of the table to the specified stream. ## - ## **Note:** Comment statement will be ignored. + ## .. note:: Comment statement will be ignored. for section, sectionData in dict.pairs(): if section != "": ## Not general section if not allCharsInSet(section, SymChars): ## Non system character @@ -546,7 +605,8 @@ proc writeConfig*(dict: Config, stream: Stream) = proc `$`*(dict: Config): string = ## Writes the contents of the table to string. - ## Note: Comment statement will be ignored. + ## + ## .. note:: Comment statement will be ignored. let stream = newStringStream() defer: stream.close() dict.writeConfig(stream) @@ -554,21 +614,23 @@ proc `$`*(dict: Config): string = proc writeConfig*(dict: Config, filename: string) = ## Writes the contents of the table to the specified configuration file. - ## Note: Comment statement will be ignored. + ## + ## .. note:: Comment statement will be ignored. let file = open(filename, fmWrite) defer: file.close() let fileStream = newFileStream(file) dict.writeConfig(fileStream) -proc getSectionValue*(dict: Config, section, key: string): string = - ## Gets the Key value of the specified Section, returns an empty string if the key does not exist. +proc getSectionValue*(dict: Config, section, key: string, defaultVal = ""): string = + ## Gets the key value of the specified Section. + ## Returns the specified default value if the specified key does not exist. if dict.hasKey(section): if dict[section].hasKey(key): result = dict[section][key] else: - result = "" + result = defaultVal else: - result = "" + result = defaultVal proc setSectionKey*(dict: var Config, section, key, value: string) = ## Sets the Key value of the specified Section. @@ -583,10 +645,15 @@ proc delSection*(dict: var Config, section: string) = dict.del(section) proc delSectionKey*(dict: var Config, section, key: string) = - ## Delete the key of the specified section. + ## Deletes the key of the specified section. if dict.hasKey(section): if dict[section].hasKey(key): if dict[section].len == 1: dict.del(section) else: dict[section].del(key) + +iterator sections*(dict: Config): lent string {.since: (1, 5).} = + ## Iterates through the sections in the `dict`. + for section in dict.keys: + yield section diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 44703e3f8..c7bf0c9c1 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -13,10 +13,10 @@ ## Basic usage ## =========== ## -## .. code-block:: nim -## import parsecsv -## from os import paramStr -## from streams import newFileStream +## ```nim +## import std/parsecsv +## from std/os import paramStr +## from std/streams import newFileStream ## ## var s = newFileStream(paramStr(1), fmRead) ## if s == nil: @@ -29,12 +29,13 @@ ## for val in items(x.row): ## echo "##", val, "##" ## close(x) +## ``` ## ## For CSV files with a header row, the header can be read and then used as a ## reference for item access with `rowEntry <#rowEntry,CsvParser,string>`_: ## -## .. code-block:: nim -## import parsecsv +## ```nim +## import std/parsecsv ## ## # Prepare a file ## let content = """One,Two,Three,Four @@ -52,6 +53,7 @@ ## for col in items(p.headers): ## echo "##", col, ":", p.rowEntry(col), "##" ## p.close() +## ``` ## ## See also ## ======== @@ -65,8 +67,10 @@ ## * `parsesql module <parsesql.html>`_ for a SQL parser ## * `other parsers <lib.html#pure-libraries-parsers>`_ for other parsers -import - lexbase, streams +import std/[lexbase, streams] + +when defined(nimPreviewSlimSystem): + import std/syncio type CsvRow* = seq[string] ## A row in a CSV file. @@ -97,10 +101,10 @@ proc raiseEInvalidCsv(filename: string, line, col: int, e.msg = filename & "(" & $line & ", " & $col & ") Error: " & msg raise e -proc error(my: CsvParser, pos: int, msg: string) = - raiseEInvalidCsv(my.filename, my.lineNumber, getColNumber(my, pos), msg) +proc error(self: CsvParser, pos: int, msg: string) = + raiseEInvalidCsv(self.filename, self.lineNumber, getColNumber(self, pos), msg) -proc open*(my: var CsvParser, input: Stream, filename: string, +proc open*(self: var CsvParser, input: Stream, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = ## Initializes the parser with an input stream. `Filename` is only used @@ -108,10 +112,10 @@ proc open*(my: var CsvParser, input: Stream, filename: string, ## the diverse optional parameters: ## - `separator`: character used to separate fields ## - `quote`: Used to quote fields containing special characters like - ## `separator`, `quote` or new-line characters. '\0' disables the parsing + ## `separator`, `quote` or new-line characters. '\\0' disables the parsing ## of quotes. ## - `escape`: removes any special meaning from the following character; - ## '\0' disables escaping; if escaping is disabled and `quote` is not '\0', + ## '\\0' disables escaping; if escaping is disabled and `quote` is not '\\0', ## two `quote` characters are parsed one literal `quote` character. ## - `skipInitialSpace`: If true, whitespace immediately following the ## `separator` is ignored. @@ -120,29 +124,27 @@ proc open*(my: var CsvParser, input: Stream, filename: string, ## * `open proc <#open,CsvParser,string,char,char,char>`_ which creates the ## file stream for you runnableExamples: - import streams + import std/streams var strm = newStringStream("One,Two,Three\n1,2,3\n10,20,30") var parser: CsvParser parser.open(strm, "tmp.csv") parser.close() strm.close() - lexbase.open(my, input) - my.filename = filename - my.sep = separator - my.quote = quote - my.esc = escape - my.skipWhite = skipInitialSpace - my.row = @[] - my.currRow = 0 + lexbase.open(self, input) + self.filename = filename + self.sep = separator + self.quote = quote + self.esc = escape + self.skipWhite = skipInitialSpace -proc open*(my: var CsvParser, filename: string, +proc open*(self: var CsvParser, filename: string, separator = ',', quote = '"', escape = '\0', skipInitialSpace = false) = ## Similar to the `other open proc<#open,CsvParser,Stream,string,char,char,char>`_, ## but creates the file stream for you. runnableExamples: - from os import removeFile + from std/os import removeFile writeFile("tmp.csv", "One,Two,Three\n1,2,3\n10,20,300") var parser: CsvParser parser.open("tmp.csv") @@ -150,60 +152,60 @@ proc open*(my: var CsvParser, filename: string, removeFile("tmp.csv") var s = newFileStream(filename, fmRead) - if s == nil: my.error(0, "cannot open: " & filename) - open(my, s, filename, separator, + if s == nil: self.error(0, "cannot open: " & filename) + open(self, s, filename, separator, quote, escape, skipInitialSpace) -proc parseField(my: var CsvParser, a: var string) = - var pos = my.bufpos - if my.skipWhite: - while my.buf[pos] in {' ', '\t'}: inc(pos) +proc parseField(self: var CsvParser, a: var string) = + var pos = self.bufpos + if self.skipWhite: + while self.buf[pos] in {' ', '\t'}: inc(pos) setLen(a, 0) # reuse memory - if my.buf[pos] == my.quote and my.quote != '\0': + if self.buf[pos] == self.quote and self.quote != '\0': inc(pos) while true: - let c = my.buf[pos] + let c = self.buf[pos] if c == '\0': - my.bufpos = pos # can continue after exception? - error(my, pos, my.quote & " expected") + self.bufpos = pos # can continue after exception? + error(self, pos, self.quote & " expected") break - elif c == my.quote: - if my.esc == '\0' and my.buf[pos+1] == my.quote: - add(a, my.quote) + elif c == self.quote: + if self.esc == '\0' and self.buf[pos + 1] == self.quote: + add(a, self.quote) inc(pos, 2) else: inc(pos) break - elif c == my.esc: - add(a, my.buf[pos+1]) + elif c == self.esc: + add(a, self.buf[pos + 1]) inc(pos, 2) else: case c of '\c': - pos = handleCR(my, pos) + pos = handleCR(self, pos) add(a, "\n") of '\l': - pos = handleLF(my, pos) + pos = handleLF(self, pos) add(a, "\n") else: add(a, c) inc(pos) else: while true: - let c = my.buf[pos] - if c == my.sep: break + let c = self.buf[pos] + if c == self.sep: break if c in {'\c', '\l', '\0'}: break add(a, c) inc(pos) - my.bufpos = pos + self.bufpos = pos -proc processedRows*(my: var CsvParser): int = +proc processedRows*(self: var CsvParser): int {.inline.} = ## Returns number of the processed rows. ## ## But even if `readRow <#readRow,CsvParser,int>`_ arrived at EOF then ## processed rows counter is incremented. runnableExamples: - import streams + import std/streams var strm = newStringStream("One,Two,Three\n1,2,3") var parser: CsvParser @@ -220,16 +222,16 @@ proc processedRows*(my: var CsvParser): int = parser.close() strm.close() - return my.currRow + self.currRow -proc readRow*(my: var CsvParser, columns = 0): bool = +proc readRow*(self: var CsvParser, columns = 0): bool = ## Reads the next row; if `columns` > 0, it expects the row to have ## exactly this many columns. Returns false if the end of the file ## has been encountered else true. ## ## Blank lines are skipped. runnableExamples: - import streams + import std/streams var strm = newStringStream("One,Two,Three\n1,2,3\n\n10,20,30") var parser: CsvParser parser.open(strm, "tmp.csv") @@ -251,52 +253,52 @@ proc readRow*(my: var CsvParser, columns = 0): bool = strm.close() var col = 0 # current column - let oldpos = my.bufpos + let oldpos = self.bufpos # skip initial empty lines #8365 while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break - while my.buf[my.bufpos] != '\0': - let oldlen = my.row.len - if oldlen < col+1: - setLen(my.row, col+1) - my.row[col] = "" - parseField(my, my.row[col]) + while self.buf[self.bufpos] != '\0': + let oldlen = self.row.len + if oldlen < col + 1: + setLen(self.row, col + 1) + self.row[col] = "" + parseField(self, self.row[col]) inc(col) - if my.buf[my.bufpos] == my.sep: - inc(my.bufpos) + if self.buf[self.bufpos] == self.sep: + inc(self.bufpos) else: - case my.buf[my.bufpos] + case self.buf[self.bufpos] of '\c', '\l': # skip empty lines: while true: - case my.buf[my.bufpos] - of '\c': my.bufpos = handleCR(my, my.bufpos) - of '\l': my.bufpos = handleLF(my, my.bufpos) + case self.buf[self.bufpos] + of '\c': self.bufpos = handleCR(self, self.bufpos) + of '\l': self.bufpos = handleLF(self, self.bufpos) else: break of '\0': discard - else: error(my, my.bufpos, my.sep & " expected") + else: error(self, self.bufpos, self.sep & " expected") break - setLen(my.row, col) + setLen(self.row, col) result = col > 0 if result and col != columns and columns > 0: - error(my, oldpos+1, $columns & " columns expected, but found " & + error(self, oldpos + 1, $columns & " columns expected, but found " & $col & " columns") - inc(my.currRow) + inc(self.currRow) -proc close*(my: var CsvParser) {.inline.} = - ## Closes the parser `my` and its associated input stream. - lexbase.close(my) +proc close*(self: var CsvParser) {.inline.} = + ## Closes the parser `self` and its associated input stream. + lexbase.close(self) -proc readHeaderRow*(my: var CsvParser) = +proc readHeaderRow*(self: var CsvParser) = ## Reads the first row and creates a look-up table for column numbers ## See also: ## * `rowEntry proc <#rowEntry,CsvParser,string>`_ runnableExamples: - import streams + import std/streams var strm = newStringStream("One,Two,Three\n1,2,3") var parser: CsvParser @@ -313,36 +315,41 @@ proc readHeaderRow*(my: var CsvParser) = parser.close() strm.close() - let present = my.readRow() + let present = self.readRow() if present: - my.headers = my.row + self.headers = self.row -proc rowEntry*(my: var CsvParser, entry: string): var string = +proc rowEntry*(self: var CsvParser, entry: string): var string = ## Accesses a specified `entry` from the current row. ## ## Assumes that `readHeaderRow <#readHeaderRow,CsvParser>`_ has already been ## called. + ## + ## If specified `entry` does not exist, raises KeyError. runnableExamples: - import streams + import std/streams var strm = newStringStream("One,Two,Three\n1,2,3\n\n10,20,30") var parser: CsvParser parser.open(strm, "tmp.csv") - ## Need calling `readHeaderRow`. + ## Requires calling `readHeaderRow`. parser.readHeaderRow() doAssert parser.readRow() doAssert parser.rowEntry("One") == "1" doAssert parser.rowEntry("Two") == "2" doAssert parser.rowEntry("Three") == "3" - ## `parser.rowEntry("NotExistEntry")` causes SIGSEGV fault. + doAssertRaises(KeyError): + discard parser.rowEntry("NonexistentEntry") parser.close() strm.close() - let index = my.headers.find(entry) + let index = self.headers.find(entry) if index >= 0: - result = my.row[index] + result = self.row[index] + else: + raise newException(KeyError, "Entry `" & entry & "` doesn't exist") when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: CsvParser @@ -352,35 +359,3 @@ when not defined(testing) and isMainModule: for val in items(x.row): echo "##", val, "##" close(x) - -when isMainModule: - import os - import strutils - block: # Tests for reading the header row - let content = "\nOne,Two,Three,Four\n1,2,3,4\n10,20,30,40,\n100,200,300,400\n" - writeFile("temp.csv", content) - - var p: CsvParser - p.open("temp.csv") - p.readHeaderRow() - while p.readRow(): - let zeros = repeat('0', p.currRow-2) - doAssert p.rowEntry("One") == "1" & zeros - doAssert p.rowEntry("Two") == "2" & zeros - doAssert p.rowEntry("Three") == "3" & zeros - doAssert p.rowEntry("Four") == "4" & zeros - p.close() - - when not defined(testing): - var parser: CsvParser - parser.open("temp.csv") - parser.readHeaderRow() - while parser.readRow(): - echo "new row: " - for col in items(parser.headers): - echo "##", col, ":", parser.rowEntry(col), "##" - parser.close() - removeFile("temp.csv") - - # Tidy up - removeFile("temp.csv") diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index 0d7d7093e..9292a8596 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -8,11 +8,14 @@ # ## This module implements a json parser. It is used -## and exported by the ``json`` standard library +## and exported by the `json` standard library ## module, but can also be used in its own right. -import - strutils, lexbase, streams, unicode +import std/[strutils, lexbase, streams, unicode] +import std/private/decode_helpers + +when defined(nimPreviewSlimSystem): + import std/assertions type JsonEventKind* = enum ## enumeration of all events that may occur when parsing @@ -21,13 +24,13 @@ type jsonString, ## a string literal jsonInt, ## an integer literal jsonFloat, ## a float literal - jsonTrue, ## the value ``true`` - jsonFalse, ## the value ``false`` - jsonNull, ## the value ``null`` - jsonObjectStart, ## start of an object: the ``{`` token - jsonObjectEnd, ## end of an object: the ``}`` token - jsonArrayStart, ## start of an array: the ``[`` token - jsonArrayEnd ## start of an array: the ``]`` token + jsonTrue, ## the value `true` + jsonFalse, ## the value `false` + jsonNull, ## the value `null` + jsonObjectStart, ## start of an object: the `{` token + jsonObjectEnd, ## end of an object: the `}` token + jsonArrayStart, ## start of an array: the `[` token + jsonArrayEnd ## end of an array: the `]` token TokKind* = enum # must be synchronized with TJsonEventKind! tkError, @@ -49,12 +52,12 @@ type errNone, ## no error errInvalidToken, ## invalid token errStringExpected, ## string expected - errColonExpected, ## ``:`` expected - errCommaExpected, ## ``,`` expected - errBracketRiExpected, ## ``]`` expected - errCurlyRiExpected, ## ``}`` expected - errQuoteExpected, ## ``"`` or ``'`` expected - errEOC_Expected, ## ``*/`` expected + errColonExpected, ## `:` expected + errCommaExpected, ## `,` expected + errBracketRiExpected, ## `]` expected + errCurlyRiExpected, ## `}` expected + errQuoteExpected, ## `"` or `'` expected + errEOC_Expected, ## `*/` expected errEofExpected, ## EOF expected errExprExpected ## expr expected @@ -71,7 +74,7 @@ type filename: string rawStringLiterals: bool - JsonKindError* = object of ValueError ## raised by the ``to`` macro if the + JsonKindError* = object of ValueError ## raised by the `to` macro if the ## JSON kind is incorrect. JsonParsingError* = object of ValueError ## is raised for a JSON error @@ -119,18 +122,18 @@ proc close*(my: var JsonParser) {.inline.} = lexbase.close(my) proc str*(my: JsonParser): string {.inline.} = - ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, - ## ``jsonString`` + ## returns the character data for the events: `jsonInt`, `jsonFloat`, + ## `jsonString` assert(my.kind in {jsonInt, jsonFloat, jsonString}) return my.a proc getInt*(my: JsonParser): BiggestInt {.inline.} = - ## returns the number for the event: ``jsonInt`` + ## returns the number for the event: `jsonInt` assert(my.kind == jsonInt) return parseBiggestInt(my.a) proc getFloat*(my: JsonParser): float {.inline.} = - ## returns the number for the event: ``jsonFloat`` + ## returns the number for the event: `jsonFloat` assert(my.kind == jsonFloat) return parseFloat(my.a) @@ -151,7 +154,7 @@ proc getFilename*(my: JsonParser): string {.inline.} = result = my.filename proc errorMsg*(my: JsonParser): string = - ## returns a helpful error message for the event ``jsonError`` + ## returns a helpful error message for the event `jsonError` assert(my.kind == jsonError) result = "$1($2, $3) Error: $4" % [ my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] @@ -162,18 +165,11 @@ proc errorMsgExpected*(my: JsonParser, e: string): string = result = "$1($2, $3) Error: $4" % [ my.filename, $getLine(my), $getColumn(my), e & " expected"] -proc handleHexChar(c: char, x: var int): bool = - result = true # Success - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: result = false # error - proc parseEscapedUTF16*(buf: cstring, pos: var int): int = result = 0 #UTF-16 escape is always 4 bytes. for _ in 0..3: + # if char in '0' .. '9', 'a' .. 'f', 'A' .. 'F' if handleHexChar(buf[pos], result): inc(pos) else: @@ -225,7 +221,7 @@ proc parseString(my: var JsonParser): TokKind = add(my.a, 'u') inc(pos, 2) var pos2 = pos - var r = parseEscapedUTF16(my.buf, pos) + var r = parseEscapedUTF16(cstring(my.buf), pos) if r < 0: my.err = errInvalidToken break @@ -235,7 +231,7 @@ proc parseString(my: var JsonParser): TokKind = my.err = errInvalidToken break inc(pos, 2) - var s = parseEscapedUTF16(my.buf, pos) + var s = parseEscapedUTF16(cstring(my.buf), pos) if (s and 0xfc00) == 0xdc00 and s > 0: r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) else: diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index a95a5b48d..03f151b66 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -14,21 +14,21 @@ ## Supported Syntax ## ================ ## -## The following syntax is supported when arguments for the ``shortNoVal`` and -## ``longNoVal`` parameters, which are -## `described later<#shortnoval-and-longnoval>`_, are not provided: +## The following syntax is supported when arguments for the `shortNoVal` and +## `longNoVal` parameters, which are +## `described later<#nimshortnoval-and-nimlongnoval>`_, are not provided: ## -## 1. Short options: ``-abcd``, ``-e:5``, ``-e=5`` -## 2. Long options: ``--foo:bar``, ``--foo=bar``, ``--foo`` -## 3. Arguments: everything that does not start with a ``-`` +## 1. Short options: `-abcd`, `-e:5`, `-e=5` +## 2. Long options: `--foo:bar`, `--foo=bar`, `--foo` +## 3. Arguments: everything that does not start with a `-` ## ## These three kinds of tokens are enumerated in the ## `CmdLineKind enum<#CmdLineKind>`_. ## ## When option values begin with ':' or '=', they need to be doubled up (as in -## ``--delim::``) or alternated (as in ``--delim=:``). +## `--delim::`) or alternated (as in `--delim=:`). ## -## The ``--`` option, commonly used to denote that every token that follows is +## The `--` option, commonly used to denote that every token that follows is ## an argument, is interpreted as a long option, and its name is the empty ## string. ## @@ -39,17 +39,17 @@ ## created with `initOptParser<#initOptParser,string,set[char],seq[string]>`_, ## and `next<#next,OptParser>`_ advances the parser by one token. ## -## For each token, the parser's ``kind``, ``key``, and ``val`` fields give -## information about that token. If the token is a long or short option, ``key`` -## is the option's name, and ``val`` is either the option's value, if provided, -## or the empty string. For arguments, the ``key`` field contains the argument -## itself, and ``val`` is unused. To check if the end of the command line has -## been reached, check if ``kind`` is equal to ``cmdEnd``. +## For each token, the parser's `kind`, `key`, and `val` fields give +## information about that token. If the token is a long or short option, `key` +## is the option's name, and `val` is either the option's value, if provided, +## or the empty string. For arguments, the `key` field contains the argument +## itself, and `val` is unused. To check if the end of the command line has +## been reached, check if `kind` is equal to `cmdEnd`. ## ## Here is an example: ## -## .. code-block:: -## import parseopt +## ```Nim +## import std/parseopt ## ## var p = initOptParser("-ab -e:5 --foo --bar=20 file.txt") ## while true: @@ -71,35 +71,59 @@ ## # Option: foo ## # Option and value: bar, 20 ## # Argument: file.txt +## ``` ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. ## -## ``shortNoVal`` and ``longNoVal`` -## ================================ +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. ## -## The optional ``shortNoVal`` and ``longNoVal`` parameters present in +## Here is an example: +## +## ```Nim +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName:<value> in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ``` +## +## `shortNoVal` and `longNoVal` +## ============================ +## +## The optional `shortNoVal` and `longNoVal` parameters present in ## `initOptParser<#initOptParser,string,set[char],seq[string]>`_ are for ## specifying which short and long options do not accept values. ## -## When ``shortNoVal`` is non-empty, users are not required to separate short +## When `shortNoVal` is non-empty, users are not required to separate short ## options and their values with a ':' or '=' since the parser knows which ## options accept values and which ones do not. This behavior also applies for -## long options if ``longNoVal`` is non-empty. For short options, ``-j4`` -## becomes supported syntax, and for long options, ``--foo bar`` becomes +## long options if `longNoVal` is non-empty. For short options, `-j4` +## becomes supported syntax, and for long options, `--foo bar` becomes ## supported. This is in addition to the `previously mentioned ## syntax<#supported-syntax>`_. Users can still separate options and their ## values with ':' or '=', but that becomes optional. ## ## As more options which do not accept values are added to your program, -## remember to amend ``shortNoVal`` and ``longNoVal`` accordingly. +## remember to amend `shortNoVal` and `longNoVal` accordingly. ## ## The following example illustrates the difference between having an empty -## ``shortNoVal`` and ``longNoVal``, which is the default, and providing +## `shortNoVal` and `longNoVal`, which is the default, and providing ## arguments for those two parameters: ## -## .. code-block:: -## import parseopt +## ```Nim +## import std/parseopt ## ## proc printToken(kind: CmdLineKind, key: string, val: string) = ## case kind @@ -132,6 +156,7 @@ ## # Output: ## # Option and value: j, 4 ## # Option and value: first, bar +## ``` ## ## See also ## ======== @@ -151,7 +176,8 @@ include "system/inclrtl" -import os +import std/strutils +import std/os type CmdLineKind* = enum ## The detected command line token. @@ -164,7 +190,7 @@ type ## ## To initialize it, use the ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. - pos*: int + pos: int inShortState: bool allowWhitespaceAfterColon: bool shortNoVal: set[char] @@ -172,7 +198,7 @@ type cmds: seq[string] idx: int kind*: CmdLineKind ## The detected command line token - key*, val*: TaintedString ## Key and value pair; the key is the option + key*, val*: string ## Key and value pair; the key is the option ## or the argument, and the value is not "" if ## the option was given a value @@ -192,101 +218,104 @@ proc parseWord(s: string, i: int, w: var string, add(w, s[result]) inc(result) -when declared(os.paramCount): - # we cannot provide this for NimRtl creation on Posix, because we can't - # access the command line arguments then! - - proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser = - ## Initializes the command line parser. - ## - ## If ``cmdline == ""``, the real command line as provided by the - ## ``os`` module is retrieved instead. - ## - ## ``shortNoVal`` and ``longNoVal`` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on - ## how this affects parsing. - ## - ## See also: - ## * `getopt iterator<#getopt.i,OptParser>`_ - runnableExamples: - var p = initOptParser() - p = initOptParser("--left --debug:3 -l -r:2") - p = initOptParser("--left --debug:3 -l -r:2", - shortNoVal = {'l'}, longNoVal = @["left"]) +proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline.len == 0`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## Behavior of the other parameters remains the same as in + ## `initOptParser(string, ...) + ## <#initOptParser,string,set[char],seq[string]>`_. + ## + ## See also: + ## * `getopt iterator<#getopt.i,seq[string],set[char],seq[string]>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"]) + p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"], + shortNoVal = {'l'}, longNoVal = @["left"]) - result.pos = 0 - result.idx = 0 - result.inShortState = false - result.shortNoVal = shortNoVal - result.longNoVal = longNoVal - result.allowWhitespaceAfterColon = allowWhitespaceAfterColon - if cmdline != "": - result.cmds = parseCmdLine(cmdline) + result.pos = 0 + result.idx = 0 + result.inShortState = false + result.shortNoVal = shortNoVal + result.longNoVal = longNoVal + result.allowWhitespaceAfterColon = allowWhitespaceAfterColon + if cmdline.len != 0: + result.cmds = newSeq[string](cmdline.len) + for i in 0..<cmdline.len: + result.cmds[i] = cmdline[i] + else: + when declared(paramCount): + when defined(nimscript): + var ctr = 0 + var firstNimsFound = false + for i in countup(0, paramCount()): + if firstNimsFound: + result.cmds[ctr] = paramStr(i) + inc ctr, 1 + if paramStr(i).endsWith(".nims") and not firstNimsFound: + firstNimsFound = true + result.cmds = newSeq[string](paramCount()-i) + else: + result.cmds = newSeq[string](paramCount()) + for i in countup(1, paramCount()): + result.cmds[i-1] = paramStr(i) else: - result.cmds = newSeq[string](os.paramCount()) - for i in countup(1, os.paramCount()): - result.cmds[i-1] = os.paramStr(i).string + # we cannot provide this for NimRtl creation on Posix, because we can't + # access the command line arguments then! + raiseAssert "empty command line given but" & + " real command line is not accessible" + result.kind = cmdEnd + result.key = "" + result.val = "" - result.kind = cmdEnd - result.key = TaintedString"" - result.val = TaintedString"" - - proc initOptParser*(cmdline: seq[TaintedString], shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser = - ## Initializes the command line parser. - ## - ## If ``cmdline.len == 0``, the real command line as provided by the - ## ``os`` module is retrieved instead. Behavior of the other parameters - ## remains the same as in `initOptParser(string, ...) - ## <#initOptParser,string,set[char],seq[string]>`_. - ## - ## See also: - ## * `getopt iterator<#getopt.i,seq[TaintedString],set[char],seq[string]>`_ - runnableExamples: - var p = initOptParser() - p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"]) - p = initOptParser(@["--left", "--debug:3", "-l", "-r:2"], - shortNoVal = {'l'}, longNoVal = @["left"]) +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline == ""`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) - result.pos = 0 - result.idx = 0 - result.inShortState = false - result.shortNoVal = shortNoVal - result.longNoVal = longNoVal - result.allowWhitespaceAfterColon = allowWhitespaceAfterColon - if cmdline.len != 0: - result.cmds = newSeq[string](cmdline.len) - for i in 0..<cmdline.len: - result.cmds[i] = cmdline[i].string - else: - result.cmds = newSeq[string](os.paramCount()) - for i in countup(1, os.paramCount()): - result.cmds[i-1] = os.paramStr(i).string - result.kind = cmdEnd - result.key = TaintedString"" - result.val = TaintedString"" + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption if i < cmd.len: - add(p.key.string, cmd[i]) + add(p.key, cmd[i]) inc(i) p.inShortState = true while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) p.inShortState = false - if i < cmd.len and cmd[i] in {':', '='} or - card(p.shortNoVal) > 0 and p.key.string[0] notin p.shortNoVal: + if i < cmd.len and (cmd[i] in {':', '='} or + card(p.shortNoVal) > 0 and p.key[0] notin p.shortNoVal): if i < cmd.len and cmd[i] in {':', '='}: inc(i) p.inShortState = false while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) - p.val = TaintedString substr(cmd, i) + p.val = substr(cmd, i) p.pos = 0 inc p.idx else: @@ -299,8 +328,8 @@ proc handleShortOption(p: var OptParser; cmd: string) = proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = ## Parses the next token. ## - ## ``p.kind`` describes what kind of token has been parsed. ``p.key`` and - ## ``p.val`` are set accordingly. + ## `p.kind` describes what kind of token has been parsed. `p.key` and + ## `p.val` are set accordingly. runnableExamples: var p = initOptParser("--left -r:2 file.txt") p.next() @@ -319,8 +348,8 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = var i = p.pos while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) p.pos = i - setLen(p.key.string, 0) - setLen(p.val.string, 0) + setLen(p.key, 0) + setLen(p.val, 0) if p.inShortState: p.inShortState = false if i >= p.cmds[p.idx].len: @@ -338,7 +367,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = if i < p.cmds[p.idx].len and p.cmds[p.idx][i] == '-': p.kind = cmdLongOption inc(i) - i = parseWord(p.cmds[p.idx], i, p.key.string, {' ', '\t', ':', '='}) + i = parseWord(p.cmds[p.idx], i, p.key, {' ', '\t', ':', '='}) while i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {'\t', ' '}: inc(i) if i < p.cmds[p.idx].len and p.cmds[p.idx][i] in {':', '='}: inc(i) @@ -349,12 +378,12 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = inc p.idx i = 0 if p.idx < p.cmds.len: - p.val = TaintedString p.cmds[p.idx].substr(i) - elif len(p.longNoVal) > 0 and p.key.string notin p.longNoVal and p.idx+1 < p.cmds.len: - p.val = TaintedString p.cmds[p.idx+1] + p.val = p.cmds[p.idx].substr(i) + elif len(p.longNoVal) > 0 and p.key notin p.longNoVal and p.idx+1 < p.cmds.len: + p.val = p.cmds[p.idx+1] inc p.idx else: - p.val = TaintedString"" + p.val = "" inc p.idx p.pos = 0 else: @@ -362,60 +391,60 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.key = TaintedString p.cmds[p.idx] + p.key = p.cmds[p.idx] inc p.idx p.pos = 0 -proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo$1".} = - ## Retrieves the rest of the command line that has not been parsed yet. - ## - ## See also: - ## * `remainingArgs proc<#remainingArgs,OptParser>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") - ## while true: - ## p.next() - ## if p.kind == cmdLongOption and p.key == "": # Look for "--" - ## break - ## else: continue - ## doAssert p.cmdLineRest == "foo.txt bar.txt" - result = p.cmds[p.idx .. ^1].quoteShellCommand.TaintedString +when declared(quoteShellCommand): + proc cmdLineRest*(p: OptParser): string {.rtl, extern: "npo$1".} = + ## Retrieves the rest of the command line that has not been parsed yet. + ## + ## See also: + ## * `remainingArgs proc<#remainingArgs,OptParser>`_ + ## + ## **Examples:** + ## ```Nim + ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") + ## while true: + ## p.next() + ## if p.kind == cmdLongOption and p.key == "": # Look for "--" + ## break + ## doAssert p.cmdLineRest == "foo.txt bar.txt" + ## ``` + result = p.cmds[p.idx .. ^1].quoteShellCommand -proc remainingArgs*(p: OptParser): seq[TaintedString] {.rtl, extern: "npo$1".} = +proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = ## Retrieves a sequence of the arguments that have not been parsed yet. ## ## See also: ## * `cmdLineRest proc<#cmdLineRest,OptParser>`_ ## ## **Examples:** - ## - ## .. code-block:: + ## ```Nim ## var p = initOptParser("--left -r:2 -- foo.txt bar.txt") ## while true: ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] + ## ``` result = @[] - for i in p.idx..<p.cmds.len: result.add TaintedString(p.cmds[i]) + for i in p.idx..<p.cmds.len: result.add p.cmds[i] iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, - val: TaintedString] = + val: string] = ## Convenience iterator for iterating over the given ## `OptParser<#OptParser>`_. ## - ## There is no need to check for ``cmdEnd`` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ ## ## **Examples:** ## - ## .. code-block:: + ## ```Nim ## # these are placeholders, of course ## proc writeHelp() = discard ## proc writeVersion() = discard @@ -435,6 +464,7 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## if filename == "": ## # no filename has been given, so we show the help ## writeHelp() + ## ``` p.pos = 0 p.idx = 0 while true: @@ -442,54 +472,54 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, if p.kind == cmdEnd: break yield (p.kind, p.key, p.val) -when declared(initOptParser): - iterator getopt*(cmdline: seq[TaintedString] = commandLineParams(), - shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]): - tuple[kind: CmdLineKind, key, val: TaintedString] = - ## Convenience iterator for iterating over command line arguments. - ## - ## This creates a new `OptParser<#OptParser>`_. If no command line - ## arguments are provided, the real command line as provided by the - ## ``os`` module is retrieved instead. - ## - ## ``shortNoVal`` and ``longNoVal`` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on - ## how this affects parsing. - ## - ## There is no need to check for ``cmdEnd`` while iterating. - ## - ## See also: - ## * `initOptParser proc<#initOptParser,seq[TaintedString],set[char],seq[string]>`_ - ## - ## **Examples:** - ## - ## .. code-block:: - ## - ## # these are placeholders, of course - ## proc writeHelp() = discard - ## proc writeVersion() = discard - ## - ## var filename: string - ## let params = @["--left", "--debug:3", "-l", "-r:2"] - ## - ## for kind, key, val in getopt(params): - ## case kind - ## of cmdArgument: - ## filename = key - ## of cmdLongOption, cmdShortOption: - ## case key - ## of "help", "h": writeHelp() - ## of "version", "v": writeVersion() - ## of cmdEnd: assert(false) # cannot happen - ## if filename == "": - ## # no filename has been written, so we show the help - ## writeHelp() - var p = initOptParser(cmdline, shortNoVal = shortNoVal, - longNoVal = longNoVal) - while true: - next(p) - if p.kind == cmdEnd: break - yield (p.kind, p.key, p.val) +iterator getopt*(cmdline: seq[string] = @[], + shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]): + tuple[kind: CmdLineKind, key, val: string] = + ## Convenience iterator for iterating over command line arguments. + ## + ## This creates a new `OptParser<#OptParser>`_. If no command line + ## arguments are provided, the real command line as provided by the + ## `os` module is retrieved instead. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. + ## + ## See also: + ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ + ## + ## **Examples:** + ## + ## ```Nim + ## # these are placeholders, of course + ## proc writeHelp() = discard + ## proc writeVersion() = discard + ## + ## var filename: string + ## let params = @["--left", "--debug:3", "-l", "-r:2"] + ## + ## for kind, key, val in getopt(params): + ## case kind + ## of cmdArgument: + ## filename = key + ## of cmdLongOption, cmdShortOption: + ## case key + ## of "help", "h": writeHelp() + ## of "version", "v": writeVersion() + ## of cmdEnd: assert(false) # cannot happen + ## if filename == "": + ## # no filename has been written, so we show the help + ## writeHelp() + ## ``` + var p = initOptParser(cmdline, shortNoVal = shortNoVal, + longNoVal = longNoVal) + while true: + next(p) + if p.kind == cmdEnd: break + yield (p.kind, p.key, p.val) {.pop.} diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index b84c1a744..a7c938d01 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -7,13 +7,16 @@ # distribution, for details about the copyright. # -## The ``parsesql`` module implements a high performance SQL file +## The `parsesql` module implements a high performance SQL file ## parser. It parses PostgreSQL syntax and the SQL ANSI standard. ## ## Unstable API. -import - strutils, lexbase +import std/[strutils, lexbase] +import std/private/decode_helpers + +when defined(nimPreviewSlimSystem): + import std/assertions # ------------------- scanner ------------------------------------------------- @@ -57,7 +60,7 @@ const reservedKeywords = @[ # statements - "select", "from", "where", "group", "limit", "having", + "select", "from", "where", "group", "limit", "offset", "having", # functions "count", ] @@ -72,20 +75,6 @@ proc getColumn(L: SqlLexer): int = proc getLine(L: SqlLexer): int = result = L.lineNumber -proc handleHexChar(c: var SqlLexer, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: - discard - proc handleOctChar(c: var SqlLexer, xi: var int) = if c.buf[c.bufpos] in {'0'..'7'}: xi = (xi shl 3) or (ord(c.buf[c.bufpos]) - ord('0')) @@ -130,8 +119,10 @@ proc getEscapedChar(c: var SqlLexer, tok: var Token) = of 'x', 'X': inc(c.bufpos) var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) add(tok.literal, chr(xi)) of '0'..'7': var xi = 0 @@ -518,6 +509,7 @@ type nkFromItemPair, nkGroup, nkLimit, + nkOffset, nkHaving, nkOrder, nkJoin, @@ -557,7 +549,14 @@ type tok: Token proc newNode*(k: SqlNodeKind): SqlNode = - result = SqlNode(kind: k) + when defined(js): # bug #14117 + case k + of LiteralNodes: + result = SqlNode(kind: k, strVal: "") + else: + result = SqlNode(kind: k, sons: @[]) + else: + result = SqlNode(kind: k) proc newNode*(k: SqlNodeKind, s: string): SqlNode = result = SqlNode(kind: k) @@ -672,8 +671,8 @@ proc getPrecedence(p: SqlParser): int = else: result = - 1 -proc parseExpr(p: var SqlParser): SqlNode -proc parseSelect(p: var SqlParser): SqlNode +proc parseExpr(p: var SqlParser): SqlNode {.gcsafe.} +proc parseSelect(p: var SqlParser): SqlNode {.gcsafe.} proc identOrLiteral(p: var SqlParser): SqlNode = case p.tok.kind @@ -987,7 +986,7 @@ proc parseInsert(p: var SqlParser): SqlNode = parseParIdentList(p, n) result.add n else: - result.add(nil) + result.add(newNode(nkNone)) if isKeyw(p, "default"): getTok(p) eat(p, "values") @@ -1022,7 +1021,7 @@ proc parseUpdate(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseDelete(p: var SqlParser): SqlNode = getTok(p) @@ -1034,7 +1033,7 @@ proc parseDelete(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseSelect(p: var SqlParser): SqlNode = getTok(p) @@ -1128,6 +1127,11 @@ proc parseSelect(p: var SqlParser): SqlNode = var l = newNode(nkLimit) l.add(parseExpr(p)) result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1184,9 +1188,12 @@ type proc add(s: var SqlWriter, thing: char) = s.buffer.add(thing) -proc add(s: var SqlWriter, thing: string) = +proc prepareAdd(s: var SqlWriter) {.inline.} = if s.buffer.len > 0 and s.buffer[^1] notin {' ', '\L', '(', '.'}: s.buffer.add(" ") + +proc add(s: var SqlWriter, thing: string) = + s.prepareAdd s.buffer.add(thing) proc addKeyw(s: var SqlWriter, thing: string) = @@ -1201,7 +1208,7 @@ proc addIden(s: var SqlWriter, thing: string) = iden = '"' & iden & '"' s.add(iden) -proc ra(n: SqlNode, s: var SqlWriter) +proc ra(n: SqlNode, s: var SqlWriter) {.gcsafe.} proc rs(n: SqlNode, s: var SqlWriter, prefix = "(", suffix = ")", sep = ", ") = if n.len > 0: @@ -1228,6 +1235,17 @@ proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = proc quoted(s: string): string = "\"" & replace(s, "\"", "\"\"") & "\"" +func escape(result: var string; s: string) = + result.add('\'') + for c in items(s): + case c + of '\0'..'\31': + result.add("\\x") + result.add(toHex(ord(c), 2)) + of '\'': result.add("''") + else: result.add(c) + result.add('\'') + proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind @@ -1240,7 +1258,8 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkQuotedIdent: s.add(quoted(n.strVal)) of nkStringLit: - s.add(escape(n.strVal, "'", "'")) + s.prepareAdd + s.buffer.escape(n.strVal) of nkBitStringLit: s.add("b'" & n.strVal & "'") of nkHexStringLit: @@ -1375,6 +1394,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkLimit: s.addKeyw("limit") s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: s.addKeyw("having") s.addMulti(n) @@ -1441,7 +1463,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("enum") rs(n, s) -proc renderSQL*(n: SqlNode, upperCase = false): string = +proc renderSql*(n: SqlNode, upperCase = false): string = ## Converts an SQL abstract syntax tree to its string representation. var s: SqlWriter s.buffer = "" @@ -1450,8 +1472,8 @@ proc renderSQL*(n: SqlNode, upperCase = false): string = return s.buffer proc `$`*(n: SqlNode): string = - ## an alias for `renderSQL`. - renderSQL(n) + ## an alias for `renderSql`. + renderSql(n) proc treeReprAux(s: SqlNode, level: int, result: var string) = result.add('\n') @@ -1469,34 +1491,33 @@ proc treeRepr*(s: SqlNode): string = result = newStringOfCap(128) treeReprAux(s, 0, result) -when not defined(js): - import streams +import std/streams - proc open(L: var SqlLexer, input: Stream, filename: string) = - lexbase.open(L, input) - L.filename = filename +proc open(L: var SqlLexer, input: Stream, filename: string) = + lexbase.open(L, input) + L.filename = filename - proc open(p: var SqlParser, input: Stream, filename: string) = - ## opens the parser `p` and assigns the input stream `input` to it. - ## `filename` is only used for error messages. - open(SqlLexer(p), input, filename) - p.tok.kind = tkInvalid - p.tok.literal = "" - getTok(p) +proc open(p: var SqlParser, input: Stream, filename: string) = + ## opens the parser `p` and assigns the input stream `input` to it. + ## `filename` is only used for error messages. + open(SqlLexer(p), input, filename) + p.tok.kind = tkInvalid + p.tok.literal = "" + getTok(p) - proc parseSQL*(input: Stream, filename: string): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `SqlParseError` exception. - var p: SqlParser - open(p, input, filename) - try: - result = parse(p) - finally: - close(p) - - proc parseSQL*(input: string, filename = ""): SqlNode = - ## parses the SQL from `input` into an AST and returns the AST. - ## `filename` is only used for error messages. - ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") +proc parseSql*(input: Stream, filename: string): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + var p: SqlParser + open(p, input, filename) + try: + result = parse(p) + finally: + close(p) + +proc parseSql*(input: string, filename = ""): SqlNode = + ## parses the SQL from `input` into an AST and returns the AST. + ## `filename` is only used for error messages. + ## Syntax errors raise an `SqlParseError` exception. + parseSql(newStringStream(input), "") diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 2c21c4952..2ca255fa0 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -12,29 +12,28 @@ ## ## To unpack raw bytes look at the `streams <streams.html>`_ module. ## -## .. code-block:: nim -## :test: +## ```nim test +## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] +## var outp: seq[string] ## -## let logs = @["2019-01-10: OK_", "2019-01-11: FAIL_", "2019-01: aaaa"] -## var outp: seq[string] +## for log in logs: +## var res: string +## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 +## outp.add(res & " - " & captureBetween(log, ' ', '_')) +## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ``` ## -## for log in logs: -## var res: string -## if parseUntil(log, res, ':') == 10: # YYYY-MM-DD == 10 -## outp.add(res & " - " & captureBetween(log, ' ', '_')) -## doAssert outp == @["2019-01-10 - OK", "2019-01-11 - FAIL"] +## ```nim test +## from std/strutils import Digits, parseInt ## -## .. code-block:: nim -## :test: -## from strutils import Digits, parseInt -## -## let -## input1 = "2019 school start" -## input2 = "3 years back" -## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 -## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 -## examYear = parseInt(startYear) + parseInt(yearsBack) -## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## let +## input1 = "2019 school start" +## input2 = "3 years back" +## startYear = input1[0 .. skipWhile(input1, Digits)-1] # 2019 +## yearsBack = input2[0 .. skipWhile(input2, Digits)-1] # 3 +## examYear = parseInt(startYear) + parseInt(yearsBack) +## doAssert "Examination is in " & $examYear == "Examination is in 2022" +## ``` ## ## **See also:** ## * `strutils module<strutils.html>`_ for combined and identical parsing proc's @@ -45,14 +44,13 @@ ## * `parsexml module<parsexml.html>`_ for a XML / HTML parser ## * `other parsers<lib.html#pure-libraries-parsers>`_ for other parsers - -{.deadCodeElim: on.} # dce option deprecated - {.push debugger: off.} # the user does not want to trace a part # of the standard library! include "system/inclrtl" +template toOa(s: string): openArray[char] = openArray[char](s) + const Whitespace = {' ', '\t', '\v', '\r', '\l', '\f'} IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} @@ -62,8 +60,7 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseBin*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a binary number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -92,7 +89,7 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 doAssert num64 == 336784608873 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -107,10 +104,9 @@ proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseOct*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses an octal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -139,7 +135,7 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseOct("2346475523464755", num64) == 16 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -154,10 +150,9 @@ proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, - maxLen = 0): int {.noSideEffect.} = +proc parseHex*[T: SomeInteger](s: openArray[char], number: var T, maxLen = 0): int {.noSideEffect.} = ## Parses a hexadecimal number and stores its value in ``number``. ## ## Returns the number of the parsed characters or 0 in case of an error. @@ -187,7 +182,7 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, var num64: int64 doAssert parseHex("4E69ED4E69ED", num64) == 12 doAssert num64 == 86216859871725 - var i = start + var i = 0 var output = T(0) var foundDigit = false let last = min(s.len, if maxLen == 0: s.len else: i + maxLen) @@ -209,11 +204,12 @@ proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, inc(i) if foundDigit: number = output - result = i - start + result = i -proc parseIdent*(s: string, ident: var string, start = 0): int = +proc parseIdent*(s: openArray[char], ident: var string): int = ## Parses an identifier and stores it in ``ident``. Returns ## the number of the parsed characters or 0 in case of an error. + ## If error, the value of `ident` is not changed. runnableExamples: var res: string doAssert parseIdent("Hello World", res, 0) == 5 @@ -222,14 +218,14 @@ proc parseIdent*(s: string, ident: var string, start = 0): int = doAssert res == "ello" doAssert parseIdent("Hello World", res, 6) == 5 doAssert res == "World" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - ident = substr(s, start, i-1) - result = i-start + ident = substr(s.toOpenArray(0, i-1)) + result = i -proc parseIdent*(s: string, start = 0): string = +proc parseIdent*(s: openArray[char]): string = ## Parses an identifier and returns it or an empty string in ## case of an error. runnableExamples: @@ -238,13 +234,27 @@ proc parseIdent*(s: string, start = 0): string = doAssert parseIdent("Hello World", 5) == "" doAssert parseIdent("Hello World", 6) == "World" result = "" - var i = start + var i = 0 if i < s.len and s[i] in IdentStartChars: inc(i) while i < s.len and s[i] in IdentChars: inc(i) - result = substr(s, start, i-1) + result = substr(s.toOpenArray(0, i - 1)) -proc skipWhitespace*(s: string, start = 0): int {.inline.} = +proc parseChar*(s: openArray[char], c: var char): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + if s.len > 0: + c = s[0] + result = 1 + +proc skipWhitespace*(s: openArray[char]): int {.inline.} = ## Skips the whitespace starting at ``s[start]``. Returns the number of ## skipped characters. runnableExamples: @@ -252,9 +262,10 @@ proc skipWhitespace*(s: string, start = 0): int {.inline.} = doAssert skipWhitespace(" Hello World", 0) == 1 doAssert skipWhitespace("Hello World", 5) == 1 doAssert skipWhitespace("Hello World", 5) == 2 - while start+result < s.len and s[start+result] in Whitespace: inc(result) + result = 0 + while result < s.len and s[result] in Whitespace: inc(result) -proc skip*(s, token: string, start = 0): int {.inline.} = +proc skip*(s, token: openArray[char]): int {.inline.} = ## Skips the `token` starting at ``s[start]``. Returns the length of `token` ## or 0 if there was no `token` at ``s[start]``. runnableExamples: @@ -263,21 +274,23 @@ proc skip*(s, token: string, start = 0): int {.inline.} = doAssert skip("2019-01-22", "19", 2) == 2 doAssert skip("CAPlow", "CAP", 0) == 3 doAssert skip("CAPlow", "cap", 0) == 0 - while start+result < s.len and result < token.len and - s[result+start] == token[result]: + result = 0 + while result < s.len and result < token.len and + s[result] == token[result]: inc(result) if result != token.len: result = 0 -proc skipIgnoreCase*(s, token: string, start = 0): int = +proc skipIgnoreCase*(s, token: openArray[char]): int = ## Same as `skip` but case is ignored for token matching. runnableExamples: doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 - while start+result < s.len and result < token.len and - toLower(s[result+start]) == toLower(token[result]): inc(result) + result = 0 + while result < s.len and result < token.len and + toLower(s[result]) == toLower(token[result]): inc(result) if result != token.len: result = 0 -proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: set[char]): int {.inline.} = ## Skips all characters until one char from the set `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -285,9 +298,10 @@ proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 doAssert skipUntil("Hello World", {'W'}, 0) == 6 doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 - while start+result < s.len and s[result+start] notin until: inc(result) + result = 0 + while result < s.len and s[result] notin until: inc(result) -proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = +proc skipUntil*(s: openArray[char], until: char): int {.inline.} = ## Skips all characters until the char `until` is found ## or the end is reached. ## Returns number of characters skipped. @@ -296,19 +310,24 @@ proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = doAssert skipUntil("Hello World", 'o', 4) == 0 doAssert skipUntil("Hello World", 'W', 0) == 6 doAssert skipUntil("Hello World", 'w', 0) == 11 - while start+result < s.len and s[result+start] != until: inc(result) + result = 0 + while result < s.len and s[result] != until: inc(result) -proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = - ## Skips all characters while one char from the set `token` is found. +proc skipWhile*(s: openArray[char], toSkip: set[char]): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. ## Returns number of characters skipped. runnableExamples: doAssert skipWhile("Hello World", {'H', 'e'}) == 2 doAssert skipWhile("Hello World", {'e'}) == 0 doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 - while start+result < s.len and s[result+start] in toSkip: inc(result) + result = 0 + while result < s.len and s[result] in toSkip: inc(result) -proc parseUntil*(s: string, token: var string, until: set[char], - start = 0): int {.inline.} = +proc fastSubstr(s: openArray[char]; token: var string; length: int) = + token.setLen length + for i in 0 ..< length: token[i] = s[i] + +proc parseUntil*(s: openArray[char], token: var string, until: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters notin `until`. @@ -320,13 +339,13 @@ proc parseUntil*(s: string, token: var string, until: set[char], doAssert myToken == "Hello " doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 doAssert myToken == "lo " - var i = start + var i = 0 while i < s.len and s[i] notin until: inc(i) - result = i-start - token = substr(s, start, i-1) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: char, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: char): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that is not the `until` character. @@ -338,13 +357,13 @@ proc parseUntil*(s: string, token: var string, until: char, doAssert myToken == "Hell" doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 doAssert myToken == "ll" - var i = start + var i = 0 while i < s.len and s[i] != until: inc(i) - result = i-start - token = substr(s, start, i-1) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc parseUntil*(s: string, token: var string, until: string, - start = 0): int {.inline.} = +proc parseUntil*(s: openArray[char], token: var string, until: string): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of any character that comes before the `until` token. @@ -354,22 +373,23 @@ proc parseUntil*(s: string, token: var string, until: string, doAssert myToken == "Hello " doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 doAssert myToken == "llo " - if until.len == 0: - token.setLen(0) - return 0 - var i = start + when (NimMajor, NimMinor) <= (1, 0): + if until.len == 0: + token.setLen(0) + return 0 + var i = 0 while i < s.len: - if s[i] == until[0]: + if until.len > 0 and s[i] == until[0]: var u = 1 while i+u < s.len and u < until.len and s[i+u] == until[u]: inc u if u >= until.len: break inc(i) - result = i-start - token = substr(s, start, i-1) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc parseWhile*(s: string, token: var string, validChars: set[char], - start = 0): int {.inline.} = +proc parseWhile*(s: openArray[char], token: var string, validChars: set[char]): int {.inline.} = ## Parses a token and stores it in ``token``. Returns ## the number of the parsed characters or 0 in case of an error. A token ## consists of the characters in `validChars`. @@ -379,21 +399,22 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], doAssert myToken.len() == 0 doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 doAssert myToken == "Wor" - var i = start + var i = 0 while i < s.len and s[i] in validChars: inc(i) - result = i-start - token = substr(s, start, i-1) + result = i + fastSubstr(s, token, result) + #token = substr(s, start, i-1) -proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = +proc captureBetween*(s: openArray[char], first: char, second = '\0'): string = ## Finds the first occurrence of ``first``, then returns everything from there ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). runnableExamples: doAssert captureBetween("Hello World", 'e') == "llo World" doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" - doAssert captureBetween("Hello World", 'l', start = 6) == "d" - var i = skipUntil(s, first, start)+1+start + doAssert captureBetween("Hello World".toOpenArray(6, "Hello World".high), 'l') == "d" + var i = skipUntil(s, first) + 1 result = "" - discard s.parseUntil(result, if second == '\0': first else: second, i) + discard parseUntil(s.toOpenArray(i, s.high), result, if second == '\0': first else: second) proc integerOutOfRangeError() {.noinline.} = raise newException(ValueError, "Parsed integer outside of valid range") @@ -402,10 +423,10 @@ proc integerOutOfRangeError() {.noinline.} = when defined(js): {.push overflowChecks: off.} -proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = +proc rawParseInt(s: openArray[char], b: var BiggestInt): int = var sign: BiggestInt = -1 - i = start + i = 0 if i < s.len: if s[i] == '+': inc(i) elif s[i] == '-': @@ -425,47 +446,47 @@ proc rawParseInt(s: string, b: var BiggestInt, start = 0): int = integerOutOfRangeError() else: b = b * sign - result = i - start + result = i when defined(js): {.pop.} # overflowChecks: off -proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {. +proc parseBiggestInt*(s: openArray[char], number: var BiggestInt): int {. rtl, extern: "npuParseBiggestInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: BiggestInt - doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert parseBiggestInt("9223372036854775807", res) == 19 doAssert res == 9223372036854775807 - var res: BiggestInt + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 + var res = BiggestInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseInt(s, res, start) + result = rawParseInt(s, res) if result != 0: number = res -proc parseInt*(s: string, number: var int, start = 0): int {. +proc parseInt*(s: openArray[char], number: var int): int {. rtl, extern: "npuParseInt", noSideEffect, raises: [ValueError].} = - ## Parses an integer starting at `start` and stores the value into `number`. + ## Parses an integer and stores the value into `number`. ## Result is the number of processed chars or 0 if there is no integer. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: var res: int - doAssert parseInt("2019", res, 0) == 4 - doAssert res == 2019 - doAssert parseInt("2019", res, 2) == 2 - doAssert res == 19 - var res: BiggestInt - result = parseBiggestInt(s, res, start) + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 + var res = BiggestInt(0) + result = parseBiggestInt(s, res) when sizeof(int) <= 4: if res < low(int) or res > high(int): integerOutOfRangeError() if result != 0: number = int(res) -proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. +proc parseSaturatedNatural*(s: openArray[char], b: var int): int {. raises: [].} = ## Parses a natural number into ``b``. This cannot raise an overflow ## error. ``high(int)`` is returned for an overflow. @@ -475,7 +496,7 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. var res = 0 discard parseSaturatedNatural("848", res) doAssert res == 848 - var i = start + var i = 0 if i < s.len and s[i] == '+': inc(i) if i < s.len and s[i] in {'0'..'9'}: b = 0 @@ -487,13 +508,13 @@ proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. b = high(int) inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored - result = i - start + result = i -proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = +proc rawParseUInt(s: openArray[char], b: var BiggestUInt): int = var res = 0.BiggestUInt prev = 0.BiggestUInt - i = start + i = 0 if i < s.len - 1 and s[i] == '-' and s[i + 1] in {'0'..'9'}: integerOutOfRangeError() if i < s.len and s[i] == '+': inc(i) # Allow @@ -507,11 +528,11 @@ proc rawParseUInt(s: string, b: var BiggestUInt, start = 0): int = inc(i) while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored b = res - result = i - start + result = i -proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. +proc parseBiggestUInt*(s: openArray[char], number: var BiggestUInt): int {. rtl, extern: "npuParseBiggestUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -520,16 +541,16 @@ proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {. doAssert res == 12 doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 doAssert res == 1111111111111111111'u64 - var res: BiggestUInt + var res = BiggestUInt(0) # use 'res' for exception safety (don't write to 'number' in case of an # overflow exception): - result = rawParseUInt(s, res, start) + result = rawParseUInt(s, res) if result != 0: number = res -proc parseUInt*(s: string, number: var uint, start = 0): int {. +proc parseUInt*(s: openArray[char], number: var uint): int {. rtl, extern: "npuParseUInt", noSideEffect, raises: [ValueError].} = - ## Parses an unsigned integer starting at `start` and stores the value + ## Parses an unsigned integer and stores the value ## into `number`. ## `ValueError` is raised if the parsed integer is out of the valid range. runnableExamples: @@ -538,23 +559,23 @@ proc parseUInt*(s: string, number: var uint, start = 0): int {. doAssert res == 3450 doAssert parseUInt("3450", res, 2) == 2 doAssert res == 50 - var res: BiggestUInt - result = parseBiggestUInt(s, res, start) + var res = BiggestUInt(0) + result = parseBiggestUInt(s, res) when sizeof(BiggestUInt) > sizeof(uint) and sizeof(uint) <= 4: if res > 0xFFFF_FFFF'u64: integerOutOfRangeError() if result != 0: number = uint(res) -proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. +proc parseBiggestFloat*(s: openArray[char], number: var BiggestFloat): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if a parsing error ## occurred. -proc parseFloat*(s: string, number: var float, start = 0): int {. +proc parseFloat*(s: openArray[char], number: var float): int {. rtl, extern: "npuParseFloat", noSideEffect.} = - ## Parses a float starting at `start` and stores the value into `number`. + ## Parses a float and stores the value into `number`. ## Result is the number of processed chars or 0 if there occurred a parsing ## error. runnableExamples: @@ -565,11 +586,76 @@ proc parseFloat*(s: string, number: var float, start = 0): int {. doAssert res == 32.57 doAssert parseFloat("32.57", res, 3) == 2 doAssert res == 57.00 - var bf: BiggestFloat - result = parseBiggestFloat(s, bf, start) + var bf = BiggestFloat(0.0) + result = parseBiggestFloat(s, bf) if result != 0: number = bf +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). + ## + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). + ## + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). + ## + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module<strutils.html>`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + type InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is @@ -579,7 +665,7 @@ type ikVar, ## ``var`` part of the interpolated string ikExpr ## ``expr`` part of the interpolated string -iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, +iterator interpolatedFragments*(s: openArray[char]): tuple[kind: InterpolatedKind, value: string] = ## Tokenizes the string `s` into substrings for interpolation purposes. ## @@ -614,7 +700,7 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, else: discard inc j raise newException(ValueError, - "Expected closing '}': " & substr(s, i, s.high)) + "Expected closing '}': " & substr(s.toOpenArray(i, s.high))) inc i, 2 # skip ${ kind = ikExpr elif j+1 < s.len and s[j+1] in IdentStartChars: @@ -628,58 +714,374 @@ iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, kind = ikDollar else: raise newException(ValueError, - "Unable to parse a variable name at " & substr(s, i, s.high)) + "Unable to parse a variable name at " & substr(s.toOpenArray(i, s.high))) else: while j < s.len and s[j] != '$': inc j kind = ikStr if j > i: # do not copy the trailing } for ikExpr: - yield (kind, substr(s, i, j-1-ord(kind == ikExpr))) + yield (kind, substr(s.toOpenArray(i, j-1-ord(kind == ikExpr)))) else: break i = j -when isMainModule: - import sequtils - let input = "$test{} $this is ${an{ example}} " - let expected = @[(ikVar, "test"), (ikStr, "{} "), (ikVar, "this"), - (ikStr, " is "), (ikExpr, "an{ example}"), (ikStr, " ")] - doAssert toSeq(interpolatedFragments(input)) == expected +{.pop.} - var value = 0 - discard parseHex("0x38", value) - doAssert value == 56 - value = -1 - doAssert(parseSaturatedNatural("848", value) == 3) - doAssert value == 848 +proc parseBin*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a binary number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-bin character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseBin("0100_1110_0110_1001_1110_1101", num) == 29 + doAssert num == 5138925 + doAssert parseBin("3", num) == 0 + var num8: int8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8) == 32 + doAssert num8 == 0b1110_1101'i8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8, 3, 9) == 9 + doAssert num8 == 0b0100_1110'i8 + var num8u: uint8 + doAssert parseBin("0b_0100_1110_0110_1001_1110_1101", num8u) == 32 + doAssert num8u == 237 + var num64: int64 + doAssert parseBin("0100111001101001111011010100111001101001", num64) == 40 + doAssert num64 == 336784608873 + parseBin(s.toOpenArray(start, s.high), number, maxLen) - value = -1 - discard parseSaturatedNatural("84899999999999999999324234243143142342135435342532453", value) - doAssert value == high(int) +proc parseOct*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses an octal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-oct character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseOct("0o23464755", num) == 10 + doAssert num == 5138925 + doAssert parseOct("8", num) == 0 + var num8: int8 + doAssert parseOct("0o_1464_755", num8) == 11 + doAssert num8 == -19 + doAssert parseOct("0o_1464_755", num8, 3, 3) == 3 + doAssert num8 == 102 + var num8u: uint8 + doAssert parseOct("1464755", num8u) == 7 + doAssert num8u == 237 + var num64: int64 + doAssert parseOct("2346475523464755", num64) == 16 + doAssert num64 == 86216859871725 + parseOct(s.toOpenArray(start, s.high), number, maxLen) - value = -1 - discard parseSaturatedNatural("9223372036854775808", value) - doAssert value == high(int) +proc parseHex*[T: SomeInteger](s: string, number: var T, start = 0, + maxLen = 0): int {.noSideEffect.} = + ## Parses a hexadecimal number and stores its value in ``number``. + ## + ## Returns the number of the parsed characters or 0 in case of an error. + ## If error, the value of ``number`` is not changed. + ## + ## If ``maxLen == 0``, the parsing continues until the first non-hex character + ## or to the end of the string. Otherwise, no more than ``maxLen`` characters + ## are parsed starting from the ``start`` position. + ## + ## It does not check for overflow. If the value represented by the string is + ## too big to fit into ``number``, only the value of last fitting characters + ## will be stored in ``number`` without producing an error. + runnableExamples: + var num: int + doAssert parseHex("4E_69_ED", num) == 8 + doAssert num == 5138925 + doAssert parseHex("X", num) == 0 + doAssert parseHex("#ABC", num) == 4 + var num8: int8 + doAssert parseHex("0x_4E_69_ED", num8) == 11 + doAssert num8 == 0xED'i8 + doAssert parseHex("0x_4E_69_ED", num8, 3, 2) == 2 + doAssert num8 == 0x4E'i8 + var num8u: uint8 + doAssert parseHex("0x_4E_69_ED", num8u) == 11 + doAssert num8u == 237 + var num64: int64 + doAssert parseHex("4E69ED4E69ED", num64) == 12 + doAssert num64 == 86216859871725 + parseHex(s.toOpenArray(start, s.high), number, maxLen) - value = -1 - discard parseSaturatedNatural("9223372036854775807", value) - doAssert value == high(int) +proc parseIdent*(s: string, ident: var string, start = 0): int = + ## Parses an identifier and stores it in ``ident``. Returns + ## the number of the parsed characters or 0 in case of an error. + ## If error, the value of `ident` is not changed. + runnableExamples: + var res: string + doAssert parseIdent("Hello World", res, 0) == 5 + doAssert res == "Hello" + doAssert parseIdent("Hello World", res, 1) == 4 + doAssert res == "ello" + doAssert parseIdent("Hello World", res, 6) == 5 + doAssert res == "World" + parseIdent(s.toOpenArray(start, s.high), ident) - value = -1 - discard parseSaturatedNatural("18446744073709551616", value) - doAssert value == high(int) +proc parseIdent*(s: string, start = 0): string = + ## Parses an identifier and returns it or an empty string in + ## case of an error. + runnableExamples: + doAssert parseIdent("Hello World", 0) == "Hello" + doAssert parseIdent("Hello World", 1) == "ello" + doAssert parseIdent("Hello World", 5) == "" + doAssert parseIdent("Hello World", 6) == "World" + parseIdent(s.toOpenArray(start, s.high)) - value = -1 - discard parseSaturatedNatural("18446744073709551615", value) - doAssert value == high(int) +proc parseChar*(s: string, c: var char, start = 0): int = + ## Parses a single character, stores it in `c` and returns 1. + ## In case of error (if start >= s.len) it returns 0 + ## and the value of `c` is unchanged. + runnableExamples: + var c: char + doAssert "nim".parseChar(c, 3) == 0 + doAssert c == '\0' + doAssert "nim".parseChar(c, 0) == 1 + doAssert c == 'n' + parseChar(s.toOpenArray(start, s.high), c) - value = -1 - doAssert(parseSaturatedNatural("1_000_000", value) == 9) - doAssert value == 1_000_000 +proc skipWhitespace*(s: string, start = 0): int {.inline.} = + ## Skips the whitespace starting at ``s[start]``. Returns the number of + ## skipped characters. + runnableExamples: + doAssert skipWhitespace("Hello World", 0) == 0 + doAssert skipWhitespace(" Hello World", 0) == 1 + doAssert skipWhitespace("Hello World", 5) == 1 + doAssert skipWhitespace("Hello World", 5) == 2 + skipWhitespace(s.toOpenArray(start, s.high)) - var i64Value: int64 - discard parseBiggestInt("9223372036854775807", i64Value) - doAssert i64Value == 9223372036854775807 +proc skip*(s, token: string, start = 0): int {.inline.} = + ## Skips the `token` starting at ``s[start]``. Returns the length of `token` + ## or 0 if there was no `token` at ``s[start]``. + runnableExamples: + doAssert skip("2019-01-22", "2019", 0) == 4 + doAssert skip("2019-01-22", "19", 0) == 0 + doAssert skip("2019-01-22", "19", 2) == 2 + doAssert skip("CAPlow", "CAP", 0) == 3 + doAssert skip("CAPlow", "cap", 0) == 0 + skip(s.toOpenArray(start, s.high), token) + +proc skipIgnoreCase*(s, token: string, start = 0): int = + ## Same as `skip` but case is ignored for token matching. + runnableExamples: + doAssert skipIgnoreCase("CAPlow", "CAP", 0) == 3 + doAssert skipIgnoreCase("CAPlow", "cap", 0) == 3 + skipIgnoreCase(s.toOpenArray(start, s.high), token) + +proc skipUntil*(s: string, until: set[char], start = 0): int {.inline.} = + ## Skips all characters until one char from the set `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", {'W', 'e'}, 0) == 1 + doAssert skipUntil("Hello World", {'W'}, 0) == 6 + doAssert skipUntil("Hello World", {'W', 'd'}, 0) == 6 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipUntil*(s: string, until: char, start = 0): int {.inline.} = + ## Skips all characters until the char `until` is found + ## or the end is reached. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipUntil("Hello World", 'o', 0) == 4 + doAssert skipUntil("Hello World", 'o', 4) == 0 + doAssert skipUntil("Hello World", 'W', 0) == 6 + doAssert skipUntil("Hello World", 'w', 0) == 11 + skipUntil(s.toOpenArray(start, s.high), until) + +proc skipWhile*(s: string, toSkip: set[char], start = 0): int {.inline.} = + ## Skips all characters while one char from the set `toSkip` is found. + ## Returns number of characters skipped. + runnableExamples: + doAssert skipWhile("Hello World", {'H', 'e'}) == 2 + doAssert skipWhile("Hello World", {'e'}) == 0 + doAssert skipWhile("Hello World", {'W', 'o', 'r'}, 6) == 3 + skipWhile(s.toOpenArray(start, s.high), toSkip) + +proc parseUntil*(s: string, token: var string, until: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters notin `until`. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, {'W', 'o', 'r'}) == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, {'W', 'r'}) == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, {'W', 'r'}, 3) == 3 + doAssert myToken == "lo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: char, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that is not the `until` character. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, 'W') == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, 'o') == 4 + doAssert myToken == "Hell" + doAssert parseUntil("Hello World", myToken, 'o', 2) == 2 + doAssert myToken == "ll" + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseUntil*(s: string, token: var string, until: string, + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of any character that comes before the `until` token. + runnableExamples: + var myToken: string + doAssert parseUntil("Hello World", myToken, "Wor") == 6 + doAssert myToken == "Hello " + doAssert parseUntil("Hello World", myToken, "Wor", 2) == 4 + doAssert myToken == "llo " + parseUntil(s.toOpenArray(start, s.high), token, until) + +proc parseWhile*(s: string, token: var string, validChars: set[char], + start = 0): int {.inline.} = + ## Parses a token and stores it in ``token``. Returns + ## the number of the parsed characters or 0 in case of an error. A token + ## consists of the characters in `validChars`. + runnableExamples: + var myToken: string + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 0) == 0 + doAssert myToken.len() == 0 + doAssert parseWhile("Hello World", myToken, {'W', 'o', 'r'}, 6) == 3 + doAssert myToken == "Wor" + parseWhile(s.toOpenArray(start, s.high), token, validChars) + +proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = + ## Finds the first occurrence of ``first``, then returns everything from there + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). + runnableExamples: + doAssert captureBetween("Hello World", 'e') == "llo World" + doAssert captureBetween("Hello World", 'e', 'r') == "llo Wo" + doAssert captureBetween("Hello World", 'l', start = 6) == "d" + captureBetween(s.toOpenArray(start, s.high), first, second) + +proc parseBiggestInt*(s: string, number: var BiggestInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestInt + doAssert parseBiggestInt("9223372036854775807", res, 0) == 19 + doAssert res == 9223372036854775807 + doAssert parseBiggestInt("-2024_05_09", res) == 11 + doAssert res == -20240509 + doAssert parseBiggestInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseBiggestInt(s.toOpenArray(start, s.high), number) + +proc parseInt*(s: string, number: var int, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: int + doAssert parseInt("-2024_05_02", res) == 11 + doAssert res == -20240502 + doAssert parseInt("-2024_05_02", res, 7) == 4 + doAssert res == 502 + parseInt(s.toOpenArray(start, s.high), number) + + +proc parseSaturatedNatural*(s: string, b: var int, start = 0): int {. + raises: [].} = + ## Parses a natural number into ``b``. This cannot raise an overflow + ## error. ``high(int)`` is returned for an overflow. + ## The number of processed character is returned. + ## This is usually what you really want to use instead of `parseInt`:idx:. + runnableExamples: + var res = 0 + discard parseSaturatedNatural("848", res) + doAssert res == 848 + parseSaturatedNatural(s.toOpenArray(start, s.high), b) + + +proc parseBiggestUInt*(s: string, number: var BiggestUInt, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: BiggestUInt + doAssert parseBiggestUInt("12", res, 0) == 2 + doAssert res == 12 + doAssert parseBiggestUInt("1111111111111111111", res, 0) == 19 + doAssert res == 1111111111111111111'u64 + parseBiggestUInt(s.toOpenArray(start, s.high), number) + +proc parseUInt*(s: string, number: var uint, start = 0): int {.noSideEffect, raises: [ValueError].} = + ## Parses an unsigned integer starting at `start` and stores the value + ## into `number`. + ## `ValueError` is raised if the parsed integer is out of the valid range. + runnableExamples: + var res: uint + doAssert parseUInt("3450", res) == 4 + doAssert res == 3450 + doAssert parseUInt("3450", res, 2) == 2 + doAssert res == 50 + parseUInt(s.toOpenArray(start, s.high), number) + +proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if a parsing error + ## occurred. + parseFloat(s.toOpenArray(start, s.high), number) + +proc parseFloat*(s: string, number: var float, start = 0): int {.noSideEffect.} = + ## Parses a float starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there occurred a parsing + ## error. + runnableExamples: + var res: float + doAssert parseFloat("32", res, 0) == 2 + doAssert res == 32.0 + doAssert parseFloat("32.57", res, 0) == 5 + doAssert res == 32.57 + doAssert parseFloat("32.57", res, 3) == 2 + doAssert res == 57.00 + parseFloat(s.toOpenArray(start, s.high), number) + +iterator interpolatedFragments*(s: string): tuple[kind: InterpolatedKind, + value: string] = + ## Tokenizes the string `s` into substrings for interpolation purposes. + ## + runnableExamples: + var outp: seq[tuple[kind: InterpolatedKind, value: string]] + for k, v in interpolatedFragments(" $this is ${an example} $$"): + outp.add (k, v) + doAssert outp == @[(ikStr, " "), + (ikVar, "this"), + (ikStr, " is "), + (ikExpr, "an example"), + (ikStr, " "), + (ikDollar, "$")] + for x in s.toOa.interpolatedFragments: + yield x -{.pop.} diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index a8e7aaab8..c760799a2 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -36,43 +36,43 @@ The file ``examples/htmltitle.nim`` demonstrates how to use the XML parser to accomplish a simple task: To determine the title of an HTML document. -.. code-block:: nim + ```nim + # Example program to show the parsexml module + # This program reads an HTML file and writes its title to stdout. + # Errors and whitespace are ignored. - # Example program to show the parsexml module - # This program reads an HTML file and writes its title to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + if paramCount() < 1: + quit("Usage: htmltitle filename[.html]") - if paramCount() < 1: - quit("Usage: htmltitle filename[.html]") - - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - while true: - x.next() - case x.kind - of xmlElementStart: - if cmpIgnoreCase(x.elementName, "title") == 0: - var title = "" - x.next() # skip "<title>" - while x.kind == xmlCharData: - title.add(x.charData) - x.next() - if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: - echo("Title: " & title) - quit(0) # Success! - else: - echo(x.errorMsgExpected("/title")) + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + while true: + x.next() + case x.kind + of xmlElementStart: + if cmpIgnoreCase(x.elementName, "title") == 0: + var title = "" + x.next() # skip "<title>" + while x.kind == xmlCharData: + title.add(x.charData) + x.next() + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0: + echo("Title: " & title) + quit(0) # Success! + else: + echo(x.errorMsgExpected("/title")) - of xmlEof: break # end of file reached - else: discard # ignore other events + of xmlEof: break # end of file reached + else: discard # ignore other events - x.close() - quit("Could not determine title!") + x.close() + quit("Could not determine title!") + ``` ]## @@ -85,69 +85,72 @@ The file ``examples/htmlrefs.nim`` demonstrates how to use the XML parser to accomplish another simple task: To determine all the links an HTML document contains. -.. code-block:: nim + ```nim + # Example program to show the new parsexml module + # This program reads an HTML file and writes all its used links to stdout. + # Errors and whitespace are ignored. - # Example program to show the new parsexml module - # This program reads an HTML file and writes all its used links to stdout. - # Errors and whitespace are ignored. + import std/[os, streams, parsexml, strutils] - import os, streams, parsexml, strutils + proc `=?=` (a, b: string): bool = + # little trick: define our own comparator that ignores case + return cmpIgnoreCase(a, b) == 0 - proc `=?=` (a, b: string): bool = - # little trick: define our own comparator that ignores case - return cmpIgnoreCase(a, b) == 0 + if paramCount() < 1: + quit("Usage: htmlrefs filename[.html]") - if paramCount() < 1: - quit("Usage: htmlrefs filename[.html]") - - var links = 0 # count the number of links - var filename = addFileExt(paramStr(1), "html") - var s = newFileStream(filename, fmRead) - if s == nil: quit("cannot open the file " & filename) - var x: XmlParser - open(x, s, filename) - next(x) # get first event - block mainLoop: - while true: - case x.kind - of xmlElementOpen: - # the <a href = "xyz"> tag we are interested in always has an attribute, - # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` - if x.elementName =?= "a": - x.next() - if x.kind == xmlAttribute: - if x.attrKey =?= "href": - var link = x.attrValue - inc(links) - # skip until we have an ``xmlElementClose`` event - while true: - x.next() - case x.kind - of xmlEof: break mainLoop - of xmlElementClose: break - else: discard - x.next() # skip ``xmlElementClose`` - # now we have the description for the ``a`` element - var desc = "" - while x.kind == xmlCharData: - desc.add(x.charData) - x.next() - echo(desc & ": " & link) - else: - x.next() - of xmlEof: break # end of file reached - of xmlError: - echo(errorMsg(x)) + var links = 0 # count the number of links + var filename = addFileExt(paramStr(1), "html") + var s = newFileStream(filename, fmRead) + if s == nil: quit("cannot open the file " & filename) + var x: XmlParser + open(x, s, filename) + next(x) # get first event + block mainLoop: + while true: + case x.kind + of xmlElementOpen: + # the <a href = "xyz"> tag we are interested in always has an attribute, + # thus we search for ``xmlElementOpen`` and not for ``xmlElementStart`` + if x.elementName =?= "a": x.next() - else: x.next() # skip other events + if x.kind == xmlAttribute: + if x.attrKey =?= "href": + var link = x.attrValue + inc(links) + # skip until we have an ``xmlElementClose`` event + while true: + x.next() + case x.kind + of xmlEof: break mainLoop + of xmlElementClose: break + else: discard + x.next() # skip ``xmlElementClose`` + # now we have the description for the ``a`` element + var desc = "" + while x.kind == xmlCharData: + desc.add(x.charData) + x.next() + echo(desc & ": " & link) + else: + x.next() + of xmlEof: break # end of file reached + of xmlError: + echo(errorMsg(x)) + x.next() + else: x.next() # skip other events - echo($links & " link(s) found!") - x.close() + echo($links & " link(s) found!") + x.close() + ``` ]## import - strutils, lexbase, streams, unicode + std/[strutils, lexbase, streams, unicode] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] # the parser treats ``<br />`` as ``<br></br>`` @@ -299,21 +302,15 @@ template piRest*(my: XmlParser): string = assert(my.kind == xmlPI) my.b -proc rawData*(my: var XmlParser): string {.inline.} = +proc rawData*(my: var XmlParser): lent string {.inline.} = ## returns the underlying 'data' string by reference. ## This is only used for speed hacks. - when defined(gcDestructors): - result = move(my.a) - else: - shallowCopy(result, my.a) + result = my.a -proc rawData2*(my: var XmlParser): string {.inline.} = +proc rawData2*(my: var XmlParser): lent string {.inline.} = ## returns the underlying second 'data' string by reference. ## This is only used for speed hacks. - when defined(gcDestructors): - result = move(my.b) - else: - shallowCopy(result, my.b) + result = my.b proc getColumn*(my: XmlParser): int {.inline.} = ## get the current column the parser has arrived at. @@ -672,6 +669,9 @@ proc parseAttribute(my: var XmlParser) = parseEntity(my, my.b) my.kind = xmlAttribute # parseEntity overwrites my.kind! pos = my.bufpos + elif c == '/': + pos = lexbase.handleRefillChar(my, pos) + add(my.b, '/') else: add(my.b, c) inc(pos) @@ -792,7 +792,7 @@ proc next*(my: var XmlParser) = my.state = stateNormal when not defined(testing) and isMainModule: - import os + import std/os var s = newFileStream(paramStr(1), fmRead) if s == nil: quit("cannot open the file" & paramStr(1)) var x: XmlParser diff --git a/lib/pure/pathnorm.nim b/lib/pure/pathnorm.nim index 407ec156c..4cdc02303 100644 --- a/lib/pure/pathnorm.nim +++ b/lib/pure/pathnorm.nim @@ -7,14 +7,14 @@ # distribution, for details about the copyright. # -## OS-Path normalization. Used by ``os.nim`` but also +## OS-Path normalization. Used by `os.nim` but also ## generally useful for dealing with paths. ## ## Unstable API. # Yes, this uses import here, not include so that # we don't end up exporting these symbols from pathnorm and os: -import "includes/osseps" +import std/private/osseps type PathIter* = object @@ -29,10 +29,6 @@ proc next*(it: var PathIter; x: string): (int, int) = if not it.notFirst and x[it.i] in {DirSep, AltSep}: # absolute path: inc it.i - when doslikeFileSystem: # UNC paths have leading `\\` - if hasNext(it, x) and x[it.i] == DirSep and - it.i+1 < x.len and x[it.i+1] != DirSep: - inc it.i else: while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i if it.i > it.prev: @@ -44,7 +40,7 @@ proc next*(it: var PathIter; x: string): (int, int) = it.notFirst = true iterator dirs(x: string): (int, int) = - var it: PathIter + var it = default PathIter while hasNext(it, x): yield next(it, x) proc isDot(x: string; bounds: (int, int)): bool = @@ -56,10 +52,23 @@ proc isDotDot(x: string; bounds: (int, int)): bool = proc isSlash(x: string; bounds: (int, int)): bool = bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep} +when doslikeFileSystem: + import std/private/ntpath + proc addNormalizePath*(x: string; result: var string; state: var int; dirSep = DirSep) = ## Low level proc. Undocumented. + when doslikeFileSystem: # Add Windows drive at start without normalization + var x = x + if result == "": + let (drive, file) = splitDrive(x) + x = file + result.add drive + for c in result.mitems: + if c in {DirSep, AltSep}: + c = dirSep + # state: 0th bit set if isAbsolute path. Other bits count # the number of path components. var it: PathIter @@ -69,40 +78,44 @@ proc addNormalizePath*(x: string; result: var string; state: var int; while hasNext(it, x): let b = next(it, x) if (state shr 1 == 0) and isSlash(x, b): - result.add dirSep + if result.len == 0 or result[result.len - 1] notin {DirSep, AltSep}: + result.add dirSep state = state or 1 elif isDotDot(x, b): if (state shr 1) >= 1: var d = result.len # f/.. + # We could handle stripping trailing sep here: foo// => foo like this: + # while (d-1) > (state and 1) and result[d-1] in {DirSep, AltSep}: dec d + # but right now we instead handle it inside os.joinPath + + # strip path component: foo/bar => foo while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}: dec d if d > 0: setLen(result, d-1) dec state, 2 else: - if result.len > 0 and result[^1] notin {DirSep, AltSep}: + if result.len > 0 and result[result.len - 1] notin {DirSep, AltSep}: result.add dirSep result.add substr(x, b[0], b[1]) elif isDot(x, b): discard "discard the dot" elif b[1] >= b[0]: - if result.len > 0 and result[^1] notin {DirSep, AltSep}: + if result.len > 0 and result[result.len - 1] notin {DirSep, AltSep}: result.add dirSep result.add substr(x, b[0], b[1]) inc state, 2 if result == "" and x != "": result = "." proc normalizePath*(path: string; dirSep = DirSep): string = - ## Example: - ## - ## .. code-block:: nim - ## assert normalizePath("./foo//bar/../baz") == "foo/baz" - ## - ## + runnableExamples: + when defined(posix): + doAssert normalizePath("./foo//bar/../baz") == "foo/baz" + ## - Turns multiple slashes into single slashes. - ## - Resolves '/foo/../bar' to '/bar'. - ## - Removes './' from the path (but "foo/.." becomes ".") + ## - Resolves `'/foo/../bar'` to `'/bar'`. + ## - Removes `'./'` from the path, but `"foo/.."` becomes `"."`. result = newStringOfCap(path.len) var state = 0 addNormalizePath(path, result, state, dirSep) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 3c3941285..2969fd6d7 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -16,14 +16,17 @@ ## include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] const useUnicode = true ## change this to deactivate proper UTF-8 support -import strutils, macros +import std/[strutils, macros] +import std/private/decode_helpers when useUnicode: - import unicode + import std/unicode export unicode.`==` const @@ -82,30 +85,30 @@ type of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] + of pkBackRef..pkBackRefIgnoreStyle: index: range[-MaxSubpatterns..MaxSubpatterns-1] else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj -proc kind*(p: Peg): PegKind = p.kind +func kind*(p: Peg): PegKind = p.kind ## Returns the *PegKind* of a given *Peg* object. -proc term*(p: Peg): string = p.term +func term*(p: Peg): string = p.term ## Returns the *string* representation of a given *Peg* variant object ## where present. -proc ch*(p: Peg): char = p.ch +func ch*(p: Peg): char = p.ch ## Returns the *char* representation of a given *Peg* variant object ## where present. -proc charChoice*(p: Peg): ref set[char] = p.charChoice +func charChoice*(p: Peg): ref set[char] = p.charChoice ## Returns the *charChoice* field of a given *Peg* variant object ## where present. -proc nt*(p: Peg): NonTerminal = p.nt +func nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[0..MaxSubpatterns] = p.index +func index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -119,59 +122,60 @@ iterator pairs*(p: Peg): (int, Peg) {.inline.} = for i in 0 ..< p.sons.len: yield (i, p.sons[i]) -proc name*(nt: NonTerminal): string = nt.name +func name*(nt: NonTerminal): string = nt.name ## Gets the name of the symbol represented by the parent *Peg* object variant ## of a given *NonTerminal*. -proc line*(nt: NonTerminal): int = nt.line +func line*(nt: NonTerminal): int = nt.line ## Gets the line number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc col*(nt: NonTerminal): int = nt.col +func col*(nt: NonTerminal): int = nt.col ## Gets the column number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags +func flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant ## object of a given *NonTerminal*. -proc rule*(nt: NonTerminal): Peg = nt.rule +func rule*(nt: NonTerminal): Peg = nt.rule ## Gets the *Peg* object representing the rule definition of the parent *Peg* ## object variant of a given *NonTerminal*. -proc term*(t: string): Peg {.noSideEffect, rtl, extern: "npegs$1Str".} = +func term*(t: string): Peg {.rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: result = Peg(kind: pkTerminal, term: t) else: result = Peg(kind: pkChar, ch: t[0]) -proc termIgnoreCase*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreCase*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore case for matching result = Peg(kind: pkTerminalIgnoreCase, term: t) -proc termIgnoreStyle*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreStyle*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore style for matching result = Peg(kind: pkTerminalIgnoreStyle, term: t) -proc term*(t: char): Peg {.noSideEffect, rtl, extern: "npegs$1Char".} = +func term*(t: char): Peg {.rtl, extern: "npegs$1Char".} = ## constructs a PEG from a terminal char assert t != '\0' result = Peg(kind: pkChar, ch: t) -proc charSet*(s: set[char]): Peg {.noSideEffect, rtl, extern: "npegs$1".} = +func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s result = Peg(kind: pkCharChoice) - new(result.charChoice) - result.charChoice[] = s + {.cast(noSideEffect).}: + new(result.charChoice) + result.charChoice[] = s -proc len(a: Peg): int {.inline.} = return a.sons.len -proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) +func len(a: Peg): int {.inline.} = return a.sons.len +func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) -proc addChoice(dest: var Peg, elem: Peg) = +func addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! @@ -194,12 +198,12 @@ template multipleOp(k: PegKind, localOpt: untyped) = if result.len == 1: result = result.sons[0] -proc `/`*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegsOrderedChoice".} = +func `/`*(a: varargs[Peg]): Peg {. + rtl, extern: "npegsOrderedChoice".} = ## constructs an ordered choice with the PEGs in `a` multipleOp(pkOrderedChoice, addChoice) -proc addSequence(dest: var Peg, elem: Peg) = +func addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! @@ -211,12 +215,12 @@ proc addSequence(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -proc sequence*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func sequence*(a: varargs[Peg]): Peg {. + rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) -proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = +func `?`*(a: Peg): Peg {.rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, pkGreedyRepSet}: @@ -226,7 +230,7 @@ proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = else: result = Peg(kind: pkOption, sons: @[a]) -proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = +func `*`*(a: Peg): Peg {.rtl, extern: "npegsGreedyRep".} = ## constructs a "greedy repetition" for the PEG `a` case a.kind of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: @@ -241,96 +245,99 @@ proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = else: result = Peg(kind: pkGreedyRep, sons: @[a]) -proc `!*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsSearch".} = +func `!*`*(a: Peg): Peg {.rtl, extern: "npegsSearch".} = ## constructs a "search" for the PEG `a` result = Peg(kind: pkSearch, sons: @[a]) -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +func `!*\`*(a: Peg): Peg {.rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` result = Peg(kind: pkCapturedSearch, sons: @[a]) -proc `+`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyPosRep".} = +func `+`*(a: Peg): Peg {.rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) -proc `&`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsAndPredicate".} = +func `&`*(a: Peg): Peg {.rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` result = Peg(kind: pkAndPredicate, sons: @[a]) -proc `!`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsNotPredicate".} = +func `!`*(a: Peg): Peg {.rtl, extern: "npegsNotPredicate".} = ## constructs a "not predicate" with the PEG `a` result = Peg(kind: pkNotPredicate, sons: @[a]) -proc any*: Peg {.inline.} = +func any*: Peg {.inline.} = ## constructs the PEG `any character`:idx: (``.``) result = Peg(kind: pkAny) -proc anyRune*: Peg {.inline.} = +func anyRune*: Peg {.inline.} = ## constructs the PEG `any rune`:idx: (``_``) result = Peg(kind: pkAnyRune) -proc newLine*: Peg {.inline.} = +func newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result = Peg(kind: pkNewLine) -proc unicodeLetter*: Peg {.inline.} = +func unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. result = Peg(kind: pkLetter) -proc unicodeLower*: Peg {.inline.} = +func unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. result = Peg(kind: pkLower) -proc unicodeUpper*: Peg {.inline.} = +func unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. result = Peg(kind: pkUpper) -proc unicodeTitle*: Peg {.inline.} = +func unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. result = Peg(kind: pkTitle) -proc unicodeWhitespace*: Peg {.inline.} = +func unicodeWhitespace*: Peg {.inline.} = ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. result = Peg(kind: pkWhitespace) -proc startAnchor*: Peg {.inline.} = +func startAnchor*: Peg {.inline.} = ## constructs the PEG ``^`` which matches the start of the input. result = Peg(kind: pkStartAnchor) -proc endAnchor*: Peg {.inline.} = +func endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result = Peg(kind: pkBackRef, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. + result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result = Peg(kind: pkBackRefIgnoreCase, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores case for matching. + result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result = Peg(kind: pkBackRefIgnoreStyle, index: index-1) + ## from 1. `reverse` specifies whether indexing starts from the end of the + ## capture list. Ignores style for matching. + result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) -proc spaceCost(n: Peg): int = +func spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: + pkAny..pkWhitespace, pkGreedyAny, pkBackRef..pkBackRefIgnoreStyle: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -340,8 +347,8 @@ proc spaceCost(n: Peg): int = inc(result, spaceCost(n.sons[i])) if result >= InlineThreshold: break -proc nonterminal*(n: NonTerminal): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func nonterminal*(n: NonTerminal): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: @@ -350,8 +357,8 @@ proc nonterminal*(n: NonTerminal): Peg {. else: result = Peg(kind: pkNonTerminal, nt: n) -proc newNonTerminal*(name: string, line, column: int): NonTerminal {. - noSideEffect, rtl, extern: "npegs$1".} = +func newNonTerminal*(name: string, line, column: int): NonTerminal {. + rtl, extern: "npegs$1".} = ## constructs a nonterminal symbol result = NonTerminal(name: name, line: line, col: column) @@ -386,7 +393,7 @@ template natural*: Peg = # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +func esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -402,14 +409,14 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = elif c in reserved: result = '\\' & c else: result = $c -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" +func singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +func singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' -proc charSetEscAux(cc: set[char]): string = +func charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 @@ -426,13 +433,13 @@ proc charSetEscAux(cc: set[char]): string = c1 = c2 inc(c1) -proc charSetEsc(cc: set[char]): string = +func charSetEsc(cc: set[char]): string = if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' else: result = '[' & charSetEscAux(cc) & ']' -proc toStrAux(r: Peg, res: var string) = +func toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -518,7 +525,7 @@ proc toStrAux(r: Peg, res: var string) = of pkStartAnchor: add(res, '^') -proc `$` *(r: Peg): string {.noSideEffect, rtl, extern: "npegsToString".} = +func `$` *(r: Peg): string {.rtl, extern: "npegsToString".} = ## converts a PEG to its string representation result = "" toStrAux(r, result) @@ -531,7 +538,7 @@ type ml: int origStart: int -proc bounds*(c: Captures, +func bounds*(c: Captures, i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] @@ -544,24 +551,26 @@ when not useUnicode: inc(i) template runeLenAt(s, i): untyped = 1 - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + func isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} + func isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + func isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + func isTitle(a: char): bool {.inline.} = return false + func isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} template matchOrParse(mopProc: untyped) = # Used to make the main matcher proc *rawMatch* as well as event parser # procs. For the former, *enter* and *leave* event handler code generators # are provided which just return *discard*. - proc mopProc(s: string, p: Peg, start: int, c: var Captures): int = + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.gcsafe, raises: [].} = proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for # *pkBackRef..pkBackRefIgnoreStyle* here. - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] + var index = p.index + if index < 0: index.inc(c.ml) + if index < 0 or index >= c.ml: return -1 + var (a, b) = c.matches[index] var n: Peg case p.kind of pkBackRef: @@ -821,15 +830,22 @@ template matchOrParse(mopProc: untyped) = leave(pkNotPredicate, s, p, start, result) of pkCapture: enter(pkCapture, s, p, start) - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = mopProc(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture + if p.sons.len == 0 or p.sons[0].kind == pkEmpty: + # empty capture removes last match + dec(c.ml) + c.matches[c.ml] = (0, 0) + result = 0 # match of length 0 else: - c.ml = idx + var idx = c.ml # reserve a slot for the subpattern + result = mopProc(s, p.sons[0], start, c) + if result >= 0: + if idx < MaxSubpatterns: + if idx != c.ml: + for i in countdown(c.ml, idx): + c.matches[i+1] = c.matches[i] + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture + inc(c.ml) leave(pkCapture, s, p, start, result) of pkBackRef: enter(pkBackRef, s, p, start) @@ -850,8 +866,8 @@ template matchOrParse(mopProc: untyped) = leave(pkStartAnchor, s, p, start, result) of pkRule, pkList: assert false -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int - {.noSideEffect, rtl, extern: "npegs$1".} = +func rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.rtl, extern: "npegs$1".} = ## low-level matching proc that implements the PEG interpreter. Use this ## for maximum efficiency (every other PEG operation ends up calling this ## proc). @@ -863,34 +879,39 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int template leave(pk, s, p, start, length) = discard matchOrParse(matchIt) - result = matchIt(s, p, start, c) + {.cast(noSideEffect).}: + # This cast is allowed because the `matchOrParse` template is used for + # both matching and parsing, but side effects are only possible when it's + # used by `eventParser`. + result = matchIt(s, p, start, c) macro mkHandlerTplts(handlers: untyped): untyped = # Transforms the handler spec in *handlers* into handler templates. # The AST structure of *handlers[0]*: # - # .. code-block:: - # StmtList - # Call - # Ident "pkNonTerminal" - # StmtList - # Call - # Ident "enter" - # StmtList - # <handler code block> - # Call - # Ident "leave" - # StmtList - # <handler code block> - # Call - # Ident "pkChar" - # StmtList - # Call - # Ident "leave" - # StmtList - # <handler code block> - # ... - proc mkEnter(hdName, body: NimNode): NimNode = + # ``` + # StmtList + # Call + # Ident "pkNonTerminal" + # StmtList + # Call + # Ident "enter" + # StmtList + # <handler code block> + # Call + # Ident "leave" + # StmtList + # <handler code block> + # Call + # Ident "pkChar" + # StmtList + # Call + # Ident "leave" + # StmtList + # <handler code block> + # ... + # ``` + func mkEnter(hdName, body: NimNode): NimNode = template helper(hdName, body) {.dirty.} = template hdName(s, p, start) = let s {.inject.} = s @@ -907,16 +928,16 @@ macro mkHandlerTplts(handlers: untyped): untyped = result = newStmtList() for topCall in handlers[0]: - if nnkCall != topCall.kind: + if topCall.kind notin nnkCallKinds: error("Call syntax expected.", topCall) let pegKind = topCall[0] - if nnkIdent != pegKind.kind: + if pegKind.kind notin {nnkIdent, nnkSym}: error("PegKind expected.", pegKind) if 2 == topCall.len: for hdDef in topCall[1]: - if nnkCall != hdDef.kind: + if hdDef.kind notin nnkCallKinds: error("Call syntax expected.", hdDef) - if nnkIdent != hdDef[0].kind: + if hdDef[0].kind notin {nnkIdent, nnkSym}: error("Handler identifier expected.", hdDef[0]) if 2 == hdDef.len: let hdPostf = substr(pegKind.strVal, 2) @@ -939,60 +960,61 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## match, else the length of the total match. The following example code ## evaluates an arithmetic expression defined by a simple PEG: ## - ## .. code-block:: nim - ## import strutils, pegs + ## ```nim + ## import std/[strutils, pegs] ## - ## let - ## pegAst = """ - ## Expr <- Sum - ## Sum <- Product (('+' / '-')Product)* - ## Product <- Value (('*' / '/')Value)* - ## Value <- [0-9]+ / '(' Expr ')' - ## """.peg - ## txt = "(5+3)/2-7*22" + ## let + ## pegAst = """ + ## Expr <- Sum + ## Sum <- Product (('+' / '-')Product)* + ## Product <- Value (('*' / '/')Value)* + ## Value <- [0-9]+ / '(' Expr ')' + ## """.peg + ## txt = "(5+3)/2-7*22" ## - ## var - ## pStack: seq[string] = @[] - ## valStack: seq[float] = @[] - ## opStack = "" - ## let - ## parseArithExpr = pegAst.eventParser: - ## pkNonTerminal: - ## enter: - ## pStack.add p.nt.name - ## leave: - ## pStack.setLen pStack.high - ## if length > 0: - ## let matchStr = s.substr(start, start+length-1) - ## case p.nt.name - ## of "Value": - ## try: - ## valStack.add matchStr.parseFloat - ## echo valStack - ## except ValueError: - ## discard - ## of "Sum", "Product": - ## try: - ## let val = matchStr.parseFloat - ## except ValueError: - ## if valStack.len > 1 and opStack.len > 0: - ## valStack[^2] = case opStack[^1] - ## of '+': valStack[^2] + valStack[^1] - ## of '-': valStack[^2] - valStack[^1] - ## of '*': valStack[^2] * valStack[^1] - ## else: valStack[^2] / valStack[^1] - ## valStack.setLen valStack.high - ## echo valStack - ## opStack.setLen opStack.high - ## echo opStack - ## pkChar: - ## leave: - ## if length == 1 and "Value" != pStack[^1]: - ## let matchChar = s[start] - ## opStack.add matchChar - ## echo opStack + ## var + ## pStack: seq[string] = @[] + ## valStack: seq[float] = @[] + ## opStack = "" + ## let + ## parseArithExpr = pegAst.eventParser: + ## pkNonTerminal: + ## enter: + ## pStack.add p.nt.name + ## leave: + ## pStack.setLen pStack.high + ## if length > 0: + ## let matchStr = s.substr(start, start+length-1) + ## case p.nt.name + ## of "Value": + ## try: + ## valStack.add matchStr.parseFloat + ## echo valStack + ## except ValueError: + ## discard + ## of "Sum", "Product": + ## try: + ## let val = matchStr.parseFloat + ## except ValueError: + ## if valStack.len > 1 and opStack.len > 0: + ## valStack[^2] = case opStack[^1] + ## of '+': valStack[^2] + valStack[^1] + ## of '-': valStack[^2] - valStack[^1] + ## of '*': valStack[^2] * valStack[^1] + ## else: valStack[^2] / valStack[^1] + ## valStack.setLen valStack.high + ## echo valStack + ## opStack.setLen opStack.high + ## echo opStack + ## pkChar: + ## leave: + ## if length == 1 and "Value" != pStack[^1]: + ## let matchChar = s[start] + ## opStack.add matchChar + ## echo opStack ## - ## let pLen = parseArithExpr(txt) + ## let pLen = parseArithExpr(txt) + ## ``` ## ## The *handlers* parameter consists of code blocks for *PegKinds*, ## which define the grammar elements of interest. Each block can contain @@ -1007,7 +1029,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## Symbols declared in an *enter* handler can be made visible in the ## corresponding *leave* handler by annotating them with an *inject* pragma. proc rawParse(s: string, p: Peg, start: int, c: var Captures): int - {.genSym.} = + {.gensym.} = # binding from *macros* bind strVal @@ -1022,10 +1044,10 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # by *mkHandlerTplts*. template mkDoEnter(hdPostf, s, pegNode, start) = when declared(`enter hdPostf`): - `enter hdPostf`(s, pegNode, start): + `enter hdPostf`(s, pegNode, start) else: discard - let hdPostf = ident(substr(strVal(pegKind), 2)) + let hdPostf = ident(substr($pegKind, 2)) getAst(mkDoEnter(hdPostf, s, pegNode, start)) macro leave(pegKind, s, pegNode, start, length: untyped): untyped = @@ -1033,16 +1055,16 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = # a grammar element of kind *pegKind*. template mkDoLeave(hdPostf, s, pegNode, start, length) = when declared(`leave hdPostf`): - `leave hdPostf`(s, pegNode, start, length): + `leave hdPostf`(s, pegNode, start, length) else: discard - let hdPostf = ident(substr(strVal(pegKind), 2)) + let hdPostf = ident(substr($pegKind, 2)) getAst(mkDoLeave(hdPostf, s, pegNode, start, length)) matchOrParse(parseIt) parseIt(s, p, start, c) - proc parser(s: string): int {.genSym.} = + proc parser(s: string): int {.gensym.} = # the proc to be returned var ms: array[MaxSubpatterns, (int, int)] @@ -1059,8 +1081,8 @@ template fillMatches(s, caps, c) = else: caps[k] = "" -proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func matchLen*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1070,8 +1092,8 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func matchLen*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1080,22 +1102,22 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. result = matchLen(s, pattern, matches, start) != -1 -proc match*(s: string, pattern: Peg, - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1".} = +func match*(s: string, pattern: Peg, + start = 0): bool {.rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. result = matchLen(s, pattern, start) != -1 -proc find*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func find*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and -1 is returned. @@ -1109,9 +1131,9 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return -1 # could also use the pattern here: (!P .)* P -proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], +func findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. - noSideEffect, rtl, extern: "npegs$1Capture".} = + rtl, extern: "npegs$1Capture".} = ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing @@ -1126,8 +1148,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], return (i, i+L-1) return (-1, 0) -proc find*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func find*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. var c: Captures @@ -1150,22 +1172,18 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = yield substr(s, i, i+L-1) inc(i, L) -proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. + rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. - ## If it does not match, @[] is returned. + ## If it does not match, `@[]` is returned. result = @[] for it in findAll(s, pattern, start): result.add it -when not defined(nimhygiene): - {.pragma: inject.} - template `=~`*(s: string, pattern: Peg): bool = ## This calls ``match`` with an implicit declared ``matches`` array that ## can be used in the scope of the ``=~`` call: ## - ## .. code-block:: nim - ## + ## ```nim ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## # matches a key=value pair: ## echo("Key: ", matches[0]) @@ -1177,50 +1195,51 @@ template `=~`*(s: string, pattern: Peg): bool = ## echo("comment: ", matches[0]) ## else: ## echo("syntax error") - ## + ## ``` bind MaxSubpatterns when not declaredInScope(matches): - var matches {.inject.}: array[0..MaxSubpatterns-1, string] + var matches {.inject.} = default(array[0..MaxSubpatterns-1, string]) match(s, pattern, matches) # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func contains*(s: string, pattern: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## same as ``find(s, pattern, start) >= 0`` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func contains*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## same as ``find(s, pattern, matches, start) >= 0`` return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func startsWith*(s: string, prefix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix, start) >= 0 -proc endsWith*(s: string, suffix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func endsWith*(s: string, suffix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` ends with the pattern `suffix` var c: Captures c.origStart = start for i in start .. s.len-1: if rawMatch(s, suffix, i, c) == s.len - i: return true -proc replacef*(s: string, sub: Peg, by: string): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replacef*(s: string, sub: Peg, by: string): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## - ## .. code-block:: nim + ## ```nim ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1<-keykey; val2<-key2key2" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1237,8 +1256,8 @@ proc replacef*(s: string, sub: Peg, by: string): string {. inc(i, x) add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, by = ""): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replace*(s: string, sub: Peg, by = ""): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed ## in `by`. result = "" @@ -1254,9 +1273,9 @@ proc replace*(s: string, sub: Peg, by = ""): string {. inc(i, x) add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: varargs[ +func parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - noSideEffect, rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -1278,16 +1297,18 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, cb: proc( +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +func replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. - rtl, extern: "npegs$1cb".} = + rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. ## The callback proc receives the index of the current match (starting with 0), ## the count of captures and an open array with the captures of each match. Examples: ## - ## .. code-block:: nim - ## - ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## ```nim + ## func handleMatches*(m: int, n: int, c: openArray[string]): string = ## result = "" ## if m > 0: ## result.add ", " @@ -1298,12 +1319,13 @@ proc replace*(s: string, sub: Peg, cb: proc( ## ## let s = "Var1=key1;var2=Key2; VAR3" ## echo s.replace(peg"{\ident}('='{\ident})* ';'* \s*", handleMatches) + ## ``` ## ## Results in: ## - ## .. code-block:: nim - ## + ## ```nim ## "var1: 'key1', var2: 'Key2', var3: ''" + ## ``` result = "" var i = 0 var caps: array[0..MaxSubpatterns-1, string] @@ -1331,7 +1353,7 @@ when not defined(js): ## error occurs. This is supposed to be used for quick scripting. ## ## **Note**: this proc does not exist while using the JS backend. - var x = readFile(infile).string + var x = readFile(infile) writeFile(outfile, x.parallelReplace(subs)) @@ -1341,18 +1363,19 @@ iterator split*(s: string, sep: Peg): string = ## Substrings are separated by the PEG `sep`. ## Examples: ## - ## .. code-block:: nim + ## ```nim ## for word in split("00232this02939is39an22example111", peg"\d+"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "this" ## "is" ## "an" ## "example" - ## + ## ``` var c: Captures var first = 0 @@ -1370,8 +1393,8 @@ iterator split*(s: string, sep: Peg): string = if first < last: yield substr(s, first, last-1) -proc split*(s: string, sep: Peg): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func split*(s: string, sep: Peg): seq[string] {. + rtl, extern: "npegs$1".} = ## Splits the string `s` into substrings. result = @[] for it in split(s, sep): result.add it @@ -1397,6 +1420,7 @@ type tkCurlyLe, ## '{' tkCurlyRi, ## '}' tkCurlyAt, ## '{@}' + tkEmptyCurl, ## '{}' tkArrow, ## '<-' tkBar, ## '/' tkStar, ## '*' @@ -1420,7 +1444,7 @@ type PegLexer {.inheritable.} = object ## the lexer object. bufpos: int ## the current position within the buffer - buf: cstring ## the buffer itself + buf: string ## the buffer itself lineNumber: int ## the current line number lineStart: int ## index of last line start in buffer colOffset: int ## column to add @@ -1429,25 +1453,25 @@ type const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", + "character set", "(", ")", "{", "}", "{@}", "{}", "<-", "/", "*", "+", "&", "!", "?", "@", "built-in", "escaped", "$", "$", "^" ] -proc handleCR(L: var PegLexer, pos: int): int = +func handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result -proc handleLF(L: var PegLexer, pos: int): int = +func handleLF(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\L') inc(L.lineNumber) result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +func init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1455,32 +1479,22 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +func getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = +func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc handleHexChar(c: var PegLexer, xi: var int) = - case c.buf[c.bufpos] - of '0'..'9': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) - inc(c.bufpos) - of 'a'..'f': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) - inc(c.bufpos) - of 'A'..'F': - xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) - inc(c.bufpos) - else: discard - -proc getEscapedChar(c: var PegLexer, tok: var Token) = +func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) + if c.bufpos >= len(c.buf): + tok.kind = tkInvalid + return case c.buf[c.bufpos] of 'r', 'R', 'c', 'C': add(tok.literal, '\c') @@ -1508,16 +1522,21 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) of 'x', 'X': inc(c.bufpos) + if c.bufpos >= len(c.buf): + tok.kind = tkInvalid + return var xi = 0 - handleHexChar(c, xi) - handleHexChar(c, xi) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) + if handleHexChar(c.buf[c.bufpos], xi): + inc(c.bufpos) if xi == 0: tok.kind = tkInvalid else: add(tok.literal, chr(xi)) of '0'..'9': var val = ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) var i = 1 - while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): + while (c.bufpos < len(c.buf)) and (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): val = val * 10 + ord(c.buf[c.bufpos]) - ord('0') inc(c.bufpos) inc(i) @@ -1531,7 +1550,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc skip(c: var PegLexer) = +func skip(c: var PegLexer) = var pos = c.bufpos while pos < c.buf.len: case c.buf[pos] @@ -1548,7 +1567,7 @@ proc skip(c: var PegLexer) = break # EndOfFile also leaves the loop c.bufpos = pos -proc getString(c: var PegLexer, tok: var Token) = +func getString(c: var PegLexer, tok: var Token) = tok.kind = tkStringLit var pos = c.bufpos + 1 var quote = c.buf[pos-1] @@ -1569,75 +1588,84 @@ proc getString(c: var PegLexer, tok: var Token) = inc(pos) c.bufpos = pos -proc getDollar(c: var PegLexer, tok: var Token) = +func getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 - if c.buf[pos] in {'0'..'9'}: + var neg = false + if pos < c.buf.len and c.buf[pos] == '^': + neg = true + inc(pos) + if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0') inc(pos) + if neg: + tok.index = -tok.index else: + if neg: + dec(pos) tok.kind = tkDollar c.bufpos = pos -proc getCharSet(c: var PegLexer, tok: var Token) = +func getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 var caret = false - if c.buf[pos] == '^': - inc(pos) - caret = true - while pos < c.buf.len: - var ch: char - case c.buf[pos] - of ']': - if pos < c.buf.len: inc(pos) - break - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: - ch = c.buf[pos] + if pos < c.buf.len: + if c.buf[pos] == '^': inc(pos) - incl(tok.charset, ch) - if c.buf[pos] == '-': - if pos+1 < c.buf.len and c.buf[pos+1] == ']': - incl(tok.charset, '-') - inc(pos) + caret = true + while pos < c.buf.len: + var ch: char + case c.buf[pos] + of ']': + if pos < c.buf.len: inc(pos) + break + of '\\': + c.bufpos = pos + getEscapedChar(c, tok) + pos = c.bufpos + ch = tok.literal[tok.literal.len-1] + of '\C', '\L', '\0': + tok.kind = tkInvalid + break else: - if pos+1 < c.buf.len: + ch = c.buf[pos] + inc(pos) + incl(tok.charset, ch) + if c.buf[pos] == '-': + if pos+1 < c.buf.len and c.buf[pos+1] == ']': + incl(tok.charset, '-') inc(pos) else: - break - var ch2: char - case c.buf[pos] - of '\\': - c.bufpos = pos - getEscapedChar(c, tok) - pos = c.bufpos - ch2 = tok.literal[tok.literal.len-1] - of '\C', '\L', '\0': - tok.kind = tkInvalid - break - else: if pos+1 < c.buf.len: - ch2 = c.buf[pos] inc(pos) else: break - for i in ord(ch)+1 .. ord(ch2): - incl(tok.charset, chr(i)) + var ch2: char + case c.buf[pos] + of '\\': + c.bufpos = pos + getEscapedChar(c, tok) + pos = c.bufpos + ch2 = tok.literal[tok.literal.len-1] + of '\C', '\L', '\0': + tok.kind = tkInvalid + break + else: + if pos+1 < c.buf.len: + ch2 = c.buf[pos] + inc(pos) + else: + break + for i in ord(ch)+1 .. ord(ch2): + incl(tok.charset, chr(i)) c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset -proc getSymbol(c: var PegLexer, tok: var Token) = +func getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos while pos < c.buf.len: add(tok.literal, c.buf[pos]) @@ -1646,7 +1674,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getBuiltin(c: var PegLexer, tok: var Token) = +func getBuiltin(c: var PegLexer, tok: var Token) = if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) @@ -1655,12 +1683,19 @@ proc getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) skip(c) + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" + add(tok.literal, '\0') + inc(c.bufpos) + return + case c.buf[c.bufpos] of '{': inc(c.bufpos) @@ -1669,6 +1704,10 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") + elif c.buf[c.bufpos] == '}' and c.bufpos < c.buf.len: + tok.kind = tkEmptyCurl + inc(c.bufpos) + add(tok.literal, "{}") else: tok.kind = tkCurlyLe add(tok.literal, '{') @@ -1700,9 +1739,11 @@ proc getTok(c: var PegLexer, tok: var Token) = of '$': getDollar(c, tok) of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) + if c.bufpos >= c.buf.len: + return if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and - c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos+1] in {'^', '0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1764,11 +1805,13 @@ proc getTok(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc arrowIsNextTok(c: PegLexer): bool = +func arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-' + if pos+1 >= c.buf.len: + return + result = c.buf[pos] == '<' and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- @@ -1783,23 +1826,21 @@ type identIsVerbatim: bool skip: Peg -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = - var e: ref EInvalidPeg - new(e) - e.msg = errorStr(p, msg, line, col) +func pegError(p: PegParser, msg: string, line = -1, col = -1) = + var e = (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) raise e -proc getTok(p: var PegParser) = +func getTok(p: var PegParser) = getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") -proc eat(p: var PegParser, kind: TokKind) = +func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -proc parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.gcsafe.} -proc getNonTerminal(p: var PegParser, name: string): NonTerminal = +func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): result = p.nonterms[i] if cmpIgnoreStyle(result.name, name) == 0: return @@ -1807,19 +1848,22 @@ proc getNonTerminal(p: var PegParser, name: string): NonTerminal = result = newNonTerminal(name, getLine(p), getColumn(p)) add(p.nonterms, result) -proc modifiedTerm(s: string, m: Modifier): Peg = +func modifiedTerm(s: string, m: Modifier): Peg = case m of modNone, modVerbatim: result = term(s) of modIgnoreCase: result = termIgnoreCase(s) of modIgnoreStyle: result = termIgnoreStyle(s) -proc modifiedBackref(s: int, m: Modifier): Peg = +func modifiedBackref(s: int, m: Modifier): Peg = + var + reverse = s < 0 + index = if reverse: -s else: s case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) + of modNone, modVerbatim: result = backref(index, reverse) + of modIgnoreCase: result = backrefIgnoreCase(index, reverse) + of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) -proc builtin(p: var PegParser): Peg = +func builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous case p.tok.literal of "n": result = newLine() @@ -1839,11 +1883,11 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -proc primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg = case p.tok.kind of tkAmp: getTok(p) @@ -1867,7 +1911,8 @@ proc primary(p: var PegParser): Peg = getTok(p) elif not arrowIsNextTok(p): var nt = getNonTerminal(p, p.tok.literal) - incl(nt.flags, ntUsed) + {.cast(noSideEffect).}: + incl(nt.flags, ntUsed) result = nonterminal(nt).token(p) getTok(p) else: @@ -1891,6 +1936,9 @@ proc primary(p: var PegParser): Peg = result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) + of tkEmptyCurl: + result = capture() + getTok(p) of tkAny: result = any().token(p) getTok(p) @@ -1910,11 +1958,11 @@ proc primary(p: var PegParser): Peg = result = startAnchor() getTok(p) of tkBackref: + if abs(p.tok.index) > p.captures or p.tok.index == 0: + pegError(p, "invalid back reference index: " & $p.tok.index) var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) @@ -1932,13 +1980,13 @@ proc primary(p: var PegParser): Peg = getTok(p) else: break -proc seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg = result = primary(p) while true: case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: + tkHat, tkCurlyAt, tkEmptyCurl: result = sequence(result, primary(p)) of tkIdentifier: if not arrowIsNextTok(p): @@ -1946,27 +1994,29 @@ proc seqExpr(p: var PegParser): Peg = else: break else: break -proc parseExpr(p: var PegParser): Peg = +func parseExpr(p: var PegParser): Peg = result = seqExpr(p) while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) -proc parseRule(p: var PegParser): NonTerminal = +func parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: pegError(p, "attempt to redefine: " & result.name) - result.line = getLine(p) - result.col = getColumn(p) + {.cast(noSideEffect).}: + result.line = getLine(p) + result.col = getColumn(p) getTok(p) eat(p, tkArrow) - result.rule = parseExpr(p) - incl(result.flags, ntDeclared) # NOW inlining may be attempted + {.cast(noSideEffect).}: + result.rule = parseExpr(p) + incl(result.flags, ntDeclared) # NOW inlining may be attempted else: pegError(p, "rule expected, but found: " & p.tok.literal) -proc rawParse(p: var PegParser): Peg = +func rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: case p.tok.literal @@ -1996,7 +2046,7 @@ proc rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -2011,14 +2061,14 @@ proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -proc peg*(pattern: string): Peg = +func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been - ## chosen to encourage its use as a raw string modifier:: + ## chosen to encourage its use as a raw string modifier: ## - ## peg"{\ident} \s* '=' \s* {.*}" + ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") -proc escapePeg*(s: string): string = +func escapePeg*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a peg. result = "" var inQuote = false @@ -2036,143 +2086,3 @@ proc escapePeg*(s: string): string = inQuote = true result.add(c) if inQuote: result.add('\'') - -when isMainModule: - assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27" - assert match("(a b c)", peg"'(' @ ')'") - assert match("W_HI_Le", peg"\y 'while'") - assert(not match("W_HI_L", peg"\y 'while'")) - assert(not match("W_HI_Le", peg"\y v'while'")) - assert match("W_HI_Le", peg"y'while'") - - assert($ +digits == $peg"\d+") - assert "0158787".match(peg"\d+") - assert "ABC 0232".match(peg"\w+\s+\d+") - assert "ABC".match(peg"\d+ / \w+") - - var accum: seq[string] = @[] - for word in split("00232this02939is39an22example111", peg"\d+"): - accum.add(word) - assert(accum == @["this", "is", "an", "example"]) - - assert matchLen("key", ident) == 3 - - var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) - assert matchLen("key1= cal9", pattern) == 11 - - var ws = newNonTerminal("ws", 1, 1) - ws.rule = *whitespace - - var expr = newNonTerminal("expr", 1, 1) - expr.rule = sequence(capture(ident), *sequence( - nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - - var c: Captures - var s = "a+b + c +d+e+f" - assert rawMatch(s, expr.rule, 0, c) == len(s) - var a = "" - for i in 0..c.ml-1: - a.add(substr(s, c.matches[i][0], c.matches[i][1])) - assert a == "abcdef" - #echo expr.rule - - #const filename = "lib/devel/peg/grammar.txt" - #var grammar = parsePeg(newFileStream(filename, fmRead), filename) - #echo "a <- [abc]*?".match(grammar) - assert find("_____abc_______", term("abc"), 2) == 5 - assert match("_______ana", peg"A <- 'ana' / . A") - assert match("abcs%%%", peg"A <- ..A / .A / '%'") - - var matches: array[0..MaxSubpatterns-1, string] - if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}": - assert matches[0] == "abc" - else: - assert false - - var g2 = peg"""S <- A B / C D - A <- 'a'+ - B <- 'b'+ - C <- 'c'+ - D <- 'd'+ - """ - assert($g2 == "((A B) / (C D))") - assert match("cccccdddddd", g2) - assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") == - "$1<-$2$2; $1<-$2$2") - assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}") - - if "aaaaaa" =~ peg"'aa' !. / ({'a'})+": - assert matches[0] == "a" - else: - assert false - - if match("abcdefg", peg"c {d} ef {g}", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - accum = @[] - for x in findAll("abcdef", peg".", 3): - accum.add(x) - assert(accum == @["d", "e", "f"]) - - for x in findAll("abcdef", peg"^{.}", 3): - assert x == "d" - - if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": - assert matches[0] == "f" - assert matches[1] == "a, b" - else: - assert false - - assert match("eine übersicht und außerdem", peg"(\letter \white*)+") - # ß is not a lower cased letter?! - assert match("eine übersicht und auerdem", peg"(\lower \white*)+") - assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") - assert(not match("456678", peg"(\letter)+")) - - assert("var1 = key; var2 = key2".replacef( - peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") == - "var1<-keykey;var2<-key2key2") - - assert match("prefix/start", peg"^start$", 7) - - if "foo" =~ peg"{'a'}?.*": - assert matches[0].len == 0 - else: assert false - - if "foo" =~ peg"{''}.*": - assert matches[0] == "" - else: assert false - - if "foo" =~ peg"{'foo'}": - assert matches[0] == "foo" - else: assert false - - let empty_test = peg"^\d*" - let str = "XYZ" - - assert(str.find(empty_test) == 0) - assert(str.match(empty_test)) - - proc handleMatches*(m: int, n: int, c: openArray[string]): string = - result = "" - - if m > 0: - result.add ", " - - result.add case n: - of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'" - of 1: toLowerAscii(c[0]) & ": ''" - else: "" - - assert("Var1=key1;var2=Key2; VAR3". - replace(peg"{\ident}('='{\ident})* ';'* \s*", - handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''") - - - doAssert "test1".match(peg"""{@}$""") - doAssert "test2".match(peg"""{(!$ .)*} $""") diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim new file mode 100644 index 000000000..9428f29eb --- /dev/null +++ b/lib/pure/prelude.nim @@ -0,0 +1,28 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when defined(nimdoc) and isMainModule: + from std/compileSettings import nil + when compileSettings.querySetting(compileSettings.SingleValueSetting.projectFull) == currentSourcePath: + ## This is an include file that simply imports common modules for your convenience. + runnableExamples: + include std/prelude + # same as: + # import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] + let x = 1 + assert "foo $# $#" % [$x, "bar"] == "foo 1 bar" + assert toSeq(1..3) == @[1, 2, 3] + when not defined(js) or defined(nodejs): + assert getCurrentDir().len > 0 + assert ($now()).startsWith "20" + + # xxx `nim doc -b:js -d:nodejs --doccmd:-d:nodejs lib/pure/prelude.nim` fails for some reason + # specific to `nim doc`, but the code otherwise works with nodejs. + +import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim deleted file mode 100644 index ce80a37fa..000000000 --- a/lib/pure/punycode.nim +++ /dev/null @@ -1,177 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2016 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements a representation of Unicode with the limited -## ASCII character subset. - -import strutils -import unicode - -# issue #3045 - -const - Base = 36 - TMin = 1 - TMax = 26 - Skew = 38 - Damp = 700 - InitialBias = 72 - InitialN = 128 - Delimiter = '-' - -type - PunyError* = object of ValueError - -proc decodeDigit(x: char): int {.raises: [PunyError].} = - if '0' <= x and x <= '9': - result = ord(x) - (ord('0') - 26) - elif 'A' <= x and x <= 'Z': - result = ord(x) - ord('A') - elif 'a' <= x and x <= 'z': - result = ord(x) - ord('a') - else: - raise newException(PunyError, "Bad input") - -proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = - if 0 <= digit and digit < 26: - result = Rune(digit + ord('a')) - elif 26 <= digit and digit < 36: - result = Rune(digit + (ord('0') - 26)) - else: - raise newException(PunyError, "internal error in punycode encoding") - -proc isBasic(c: char): bool = ord(c) < 0x80 -proc isBasic(r: Rune): bool = int(r) < 0x80 - -proc adapt(delta, numPoints: int, first: bool): int = - var d = if first: delta div Damp else: delta div 2 - d += d div numPoints - var k = 0 - while d > ((Base-TMin)*TMax) div 2: - d = d div (Base - TMin) - k += Base - result = k + (Base - TMin + 1) * d div (d + Skew) - -proc encode*(prefix, s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. - ## Prepend `prefix` to the result - result = prefix - var (d, n, bias) = (0, InitialN, InitialBias) - var (b, remaining) = (0, 0) - for r in s.runes: - if r.isBasic: - # basic Ascii character - inc b - result.add($r) - else: - # special character - inc remaining - - var h = b - if b > 0: - result.add(Delimiter) # we have some Ascii chars - while remaining != 0: - var m: int = high(int32) - for r in s.runes: - if m > int(r) and int(r) >= n: - m = int(r) - d += (m - n) * (h + 1) - if d < 0: - raise newException(PunyError, "invalid label " & s) - n = m - for r in s.runes: - if int(r) < n: - inc d - if d < 0: - raise newException(PunyError, "invalid label " & s) - continue - if int(r) > n: - continue - var q = d - var k = Base - while true: - var t = k - bias - if t < TMin: - t = TMin - elif t > TMax: - t = TMax - if q < t: - break - result.add($encodeDigit(t + (q - t) mod (Base - t))) - q = (q - t) div (Base - t) - k += Base - result.add($encodeDigit(q)) - bias = adapt(d, h + 1, h == b) - d = 0 - inc h - dec remaining - inc d - inc n - -proc encode*(s: string): string {.raises: [PunyError].} = - ## Encode a string that may contain Unicode. Prefix is empty. - result = encode("", s) - -proc decode*(encoded: string): string {.raises: [PunyError].} = - ## Decode a Punycode-encoded string - var - n = InitialN - i = 0 - bias = InitialBias - var d = rfind(encoded, Delimiter) - result = "" - - if d > 0: - # found Delimiter - for j in 0..<d: - var c = encoded[j] # char - if not c.isBasic: - raise newException(PunyError, "Encoded contains a non-basic char") - result.add(c) # add the character - inc d - else: - d = 0 # set to first index - - while (d < len(encoded)): - var oldi = i - var w = 1 - var k = Base - while true: - if d == len(encoded): - raise newException(PunyError, "Bad input: " & encoded) - var c = encoded[d]; inc d - var digit = int(decodeDigit(c)) - if digit > (high(int32) - i) div w: - raise newException(PunyError, "Too large a value: " & $digit) - i += digit * w - var t: int - if k <= bias: - t = TMin - elif k >= bias + TMax: - t = TMax - else: - t = k - bias - if digit < t: - break - w *= Base - t - k += Base - bias = adapt(i - oldi, runelen(result) + 1, oldi == 0) - - if i div (runelen(result) + 1) > high(int32) - n: - raise newException(PunyError, "Value too large") - - n += i div (runelen(result) + 1) - i = i mod (runelen(result) + 1) - insert(result, $Rune(n), i) - inc i - -when isMainModule: - assert(decode(encode("", "bücher")) == "bücher") - assert(decode(encode("münchen")) == "münchen") - assert encode("xn--", "münchen") == "xn--mnchen-3ya" diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 1237e5d0a..3ec77d37e 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -7,11 +7,11 @@ # distribution, for details about the copyright. # -## Nim's standard random number generator. +## Nim's standard random number generator (RNG). ## -## Its implementation is based on the ``xoroshiro128+`` +## Its implementation is based on the `xoroshiro128+` ## (xor/rotate/shift/rotate) library. -## * More information: http://xoroshiro.di.unimi.it/ +## * More information: http://xoroshiro.di.unimi.it ## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c ## ## **Do not use this module for cryptographic purposes!** @@ -19,88 +19,98 @@ ## Basic usage ## =========== ## -## To get started, here are some examples: -## -## .. code-block:: -## -## import random -## -## # Call randomize() once to initialize the default random number generator -## # If this is not called, the same results will occur every time these -## # examples are run -## randomize() -## -## # Pick a number between 0 and 100 -## let num = rand(100) -## echo num -## -## # Roll a six-sided die -## let roll = rand(1..6) -## echo roll -## -## # Pick a marble from a bag -## let marbles = ["red", "blue", "green", "yellow", "purple"] -## let pick = sample(marbles) -## echo pick -## -## # Shuffle some cards -## var cards = ["Ace", "King", "Queen", "Jack", "Ten"] -## shuffle(cards) -## echo cards -## -## These examples all use the default random number generator. The -## `Rand type<#Rand>`_ represents the state of a random number generator. +runnableExamples: + # Call randomize() once to initialize the default random number generator. + # If this is not called, the same results will occur every time these + # examples are run. + randomize() + + # Pick a number in 0..100. + let num = rand(100) + doAssert num in 0..100 + + # Roll a six-sided die. + let roll = rand(1..6) + doAssert roll in 1..6 + + # Pick a marble from a bag. + let marbles = ["red", "blue", "green", "yellow", "purple"] + let pick = sample(marbles) + doAssert pick in marbles + + # Shuffle some cards. + var cards = ["Ace", "King", "Queen", "Jack", "Ten"] + shuffle(cards) + doAssert cards.len == 5 + +## These examples all use the default RNG. The +## `Rand type <#Rand>`_ represents the state of an RNG. ## For convenience, this module contains a default Rand state that corresponds -## to the default random number generator. Most procs in this module which do +## to the default RNG. Most procs in this module which do ## not take in a Rand parameter, including those called in the above examples, ## use the default generator. Those procs are **not** thread-safe. ## ## Note that the default generator always starts in the same state. -## The `randomize proc<#randomize>`_ can be called to initialize the default +## The `randomize proc <#randomize>`_ can be called to initialize the default ## generator with a seed based on the current time, and it only needs to be ## called once before the first usage of procs from this module. If -## ``randomize`` is not called, then the default generator will always produce +## `randomize` is not called, the default generator will always produce ## the same results. ## -## Generators that are independent of the default one can be created with the -## `initRand proc<#initRand,int64>`_. +## RNGs that are independent of the default one can be created with the +## `initRand proc <#initRand,int64>`_. ## ## Again, it is important to remember that this module must **not** be used for ## cryptographic applications. ## ## See also ## ======== -## * `math module<math.html>`_ for basic math routines -## * `mersenne module<mersenne.html>`_ for the Mersenne Twister random number -## generator -## * `stats module<stats.html>`_ for statistical analysis -## * `list of cryptographic and hashing modules -## <lib.html#pure-libraries-hashing>`_ +## * `std/sysrand module <sysrand.html>`_ for a cryptographically secure pseudorandom number generator +## * `math module <math.html>`_ for basic math routines +## * `stats module <stats.html>`_ for statistical analysis +## * `list of cryptographic and hashing modules <lib.html#pure-libraries-hashing>`_ ## in the standard library -import algorithm #For upperBound +import std/[algorithm, math] +import std/private/[since, jsutils] + +when defined(nimPreviewSlimSystem): + import std/[assertions] -include "system/inclrtl" +include system/inclrtl {.push debugger: off.} +template whenHasBigInt64(yes64, no64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + yes64 -when defined(js): - type Ui = uint32 - const randMax = 4_294_967_295u32 -else: +whenHasBigInt64: type Ui = uint64 const randMax = 18_446_744_073_709_551_615u64 +do: + type Ui = uint32 + + const randMax = 4_294_967_295u32 + type Rand* = object ## State of a random number generator. ## - ## Create a new Rand state using the `initRand proc<#initRand,int64>`_. + ## Create a new Rand state using the `initRand proc <#initRand,int64>`_. ## ## The module contains a default Rand state for convenience. - ## It corresponds to the default random number generator's state. + ## It corresponds to the default RNG's state. ## The default Rand state always starts with the same values, but the - ## `randomize proc<#randomize>`_ can be used to seed the default generator + ## `randomize proc <#randomize>`_ can be used to seed the default generator ## with a value based on the current time. ## ## Many procs have two variations: one that takes in a Rand parameter and @@ -108,34 +118,50 @@ type ## generator are **not** thread-safe! a0, a1: Ui -when defined(js): +whenHasBigInt64: + const DefaultRandSeed = Rand( + a0: 0x69B4C98CB8530805u64, + a1: 0xFED1DD3004688D67CAu64) + + # racy for multi-threading but good enough for now: + var state = DefaultRandSeed # global for backwards compatibility +do: var state = Rand( a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility -else: - # racy for multi-threading but good enough for now: - var state = Rand( - a0: 0x69B4C98CB8530805u64, - a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + +func isValid(r: Rand): bool {.inline.} = + ## Check whether state of `r` is valid. + ## + ## In `xoroshiro128+`, if all bits of `a0` and `a1` are zero, + ## they are always zero after calling `next(r: var Rand)`. + not (r.a0 == 0 and r.a1 == 0) + +since (1, 5): + template randState*(): untyped = + ## Makes the default Rand state accessible from other modules. + ## Useful for module authors. + state proc rotl(x, k: Ui): Ui = result = (x shl k) or (x shr (Ui(64) - k)) proc next*(r: var Rand): uint64 = - ## Computes a random ``uint64`` number using the given state. + ## Computes a random `uint64` number using the given state. ## - ## See also: + ## **See also:** ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer between zero and ## a given upper bound ## * `rand proc<#rand,Rand,range[]>`_ that returns a float - ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type ## * `skipRandomNumbers proc<#skipRandomNumbers,Rand>`_ - runnableExamples: + runnableExamples("-r:off"): var r = initRand(2019) - doAssert r.next() == 138_744_656_611_299'u64 - doAssert r.next() == 979_810_537_855_049_344'u64 - doAssert r.next() == 3_628_232_584_225_300_704'u64 + assert r.next() == 13223559681708962501'u64 # implementation defined + assert r.next() == 7229677234260823147'u64 # ditto + let s0 = r.a0 var s1 = r.a1 result = s0 + s1 @@ -146,12 +172,12 @@ proc next*(r: var Rand): uint64 = proc skipRandomNumbers*(s: var Rand) = ## The jump function for the generator. ## - ## This proc is equivalent to 2^64 calls to `next<#next,Rand>`_, and it can - ## be used to generate 2^64 non-overlapping subsequences for parallel + ## This proc is equivalent to `2^64` calls to `next <#next,Rand>`_, and it can + ## be used to generate `2^64` non-overlapping subsequences for parallel ## computations. ## ## When multiple threads are generating random numbers, each thread must - ## own the `Rand<#Rand>`_ state it is using so that the thread can safely + ## own the `Rand <#Rand>`_ state it is using so that the thread can safely ## obtain random numbers. However, if each thread creates its own Rand state, ## the subsequences of random numbers that each thread generates may overlap, ## even if the provided seeds are unique. This is more likely to happen as the @@ -162,38 +188,43 @@ proc skipRandomNumbers*(s: var Rand) = ## Rand state to a thread, call this proc before passing it to the next one. ## By using the Rand state this way, the subsequences of random numbers ## generated in each thread will never overlap as long as no thread generates - ## more than 2^64 random numbers. - ## - ## The following example below demonstrates this pattern: - ## - ## .. code-block:: - ## # Compile this example with --threads:on - ## import random - ## import threadpool - ## - ## const spawns = 4 - ## const numbers = 100000 - ## - ## proc randomSum(rand: Rand): int = - ## var r = rand - ## for i in 1..numbers: - ## result += rand(1..10) - ## - ## var r = initRand(2019) - ## var vals: array[spawns, FlowVar[int]] - ## for val in vals.mitems: - ## val = spawn(randomSum(r)) - ## r.skipRandomNumbers() + ## more than `2^64` random numbers. ## - ## for val in vals: - ## echo ^val - ## - ## See also: + ## **See also:** ## * `next proc<#next,Rand>`_ - when defined(js): - const helper = [0xbeac0467u32, 0xd86b048bu32] - else: + runnableExamples("--threads:on"): + import std/random + + const numbers = 100000 + + var + thr: array[0..3, Thread[(Rand, int)]] + vals: array[0..3, int] + + proc randomSum(params: tuple[r: Rand, index: int]) {.thread.} = + var r = params.r + var s = 0 # avoid cache thrashing + for i in 1..numbers: + s += r.rand(0..10) + vals[params.index] = s + + var r = initRand(2019) + for i in 0..<thr.len: + createThread(thr[i], randomSum, (r, i)) + r.skipRandomNumbers() + + joinThreads(thr) + + for val in vals: + doAssert abs(val - numbers * 5) / numbers < 0.1 + + doAssert vals == [501737, 497901, 500683, 500157] + + + whenHasBigInt64: const helper = [0xbeac0467eba5facbu64, 0xd86b048b86aa9922u64] + do: + const helper = [0xbeac0467u32, 0xd86b048bu32] var s0 = Ui 0 s1 = Ui 0 @@ -206,89 +237,83 @@ proc skipRandomNumbers*(s: var Rand) = s.a0 = s0 s.a1 = s1 -proc random*(max: int): int {.benign, deprecated: - "Deprecated since v0.18.0; use 'rand' instead".} = - while true: - let x = next(state) - if x < randMax - (randMax mod Ui(max)): - return int(x mod uint64(max)) - -proc random*(max: float): float {.benign, deprecated: - "Deprecated since v0.18.0; use 'rand' instead".} = - let x = next(state) - when defined(js): - result = (float(x) / float(high(uint32))) * max +proc rand[T: uint | uint64](r: var Rand; max: T): T = + # xxx export in future work + if max == 0: return else: - let u = (0x3FFu64 shl 52u64) or (x shr 12u64) - result = (cast[float](u) - 1.0) * max - -proc random*[T](x: HSlice[T, T]): T {.deprecated: - "Deprecated since v0.18.0; use 'rand' instead".} = - result = T(random(x.b - x.a)) + x.a - -proc random*[T](a: openArray[T]): T {.deprecated: - "Deprecated since v0.18.0; use 'sample' instead".} = - result = a[random(a.low..a.len)] + let max = uint64(max) + when T.high.uint64 == uint64.high: + if max == uint64.high: return T(next(r)) + var iters = 0 + while true: + let x = next(r) + # avoid `mod` bias + if x <= randMax - (randMax mod max) or iters > 20: + return T(x mod (max + 1)) + else: + inc iters proc rand*(r: var Rand; max: Natural): int {.benign.} = ## Returns a random integer in the range `0..max` using the given state. ## - ## See also: - ## * `rand proc<#rand,int>`_ that returns an integer using the default - ## random number generator + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer using the default RNG ## * `rand proc<#rand,Rand,range[]>`_ that returns a float - ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(123) - doAssert r.rand(100) == 0 - doAssert r.rand(100) == 96 - doAssert r.rand(100) == 66 - if max == 0: return - while true: - let x = next(r) - if x <= randMax - (randMax mod Ui(max)): - return int(x mod (uint64(max)+1u64)) + if false: + assert r.rand(100) == 96 # implementation defined + # bootstrap: can't use `runnableExamples("-r:off")` + cast[int](rand(r, uint64(max))) + # xxx toUnsigned pending https://github.com/nim-lang/Nim/pull/18445 proc rand*(max: int): int {.benign.} = ## Returns a random integer in the range `0..max`. ## - ## If `randomize<#randomize>`_ has not been called, the sequence of random + ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer using a ## provided state ## * `rand proc<#rand,float>`_ that returns a float - ## * `rand proc<#rand,HSlice[T,T]>`_ that accepts a slice + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type - runnableExamples: + runnableExamples("-r:off"): randomize(123) - doAssert rand(100) == 0 - doAssert rand(100) == 96 - doAssert rand(100) == 66 + assert [rand(100), rand(100)] == [96, 63] # implementation defined + rand(state, max) proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} = ## Returns a random floating point number in the range `0.0..max` ## using the given state. ## - ## See also: - ## * `rand proc<#rand,float>`_ that returns a float using the default - ## random number generator + ## **See also:** + ## * `rand proc<#rand,float>`_ that returns a float using the default RNG ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer - ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(234) - let f = r.rand(1.0) - ## f = 8.717181376738381e-07 + let f = r.rand(1.0) # 8.717181376738381e-07 + let x = next(r) when defined(js): - result = (float(x) / float(high(uint32))) * max + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + result = (float(x) / float(high(uint64))) * max + else: + result = (float(x) / float(high(uint32))) * max + else: + result = (float(x) / float(high(uint32))) * max else: let u = (0x3FFu64 shl 52u64) or (x shr 12u64) result = (cast[float](u) - 1.0) * max @@ -296,22 +321,22 @@ proc rand*(r: var Rand; max: range[0.0 .. high(float)]): float {.benign.} = proc rand*(max: float): float {.benign.} = ## Returns a random floating point number in the range `0.0..max`. ## - ## If `randomize<#randomize>`_ has not been called, the sequence of random + ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `rand proc<#rand,Rand,range[]>`_ that returns a float using a ## provided state ## * `rand proc<#rand,int>`_ that returns an integer - ## * `rand proc<#rand,HSlice[T,T]>`_ that accepts a slice + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: randomize(234) - let f = rand(1.0) - ## f = 8.717181376738381e-07 + let f = rand(1.0) # 8.717181376738381e-07 + rand(state, max) proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T = @@ -320,100 +345,106 @@ proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T = ## ## Allowed types for `T` are integers, floats, and enums without holes. ## - ## See also: - ## * `rand proc<#rand,HSlice[T,T]>`_ that accepts a slice and uses the - ## default random number generator + ## **See also:** + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice and uses the default RNG ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer ## * `rand proc<#rand,Rand,range[]>`_ that returns a float ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: var r = initRand(345) - doAssert r.rand(1..6) == 4 - doAssert r.rand(1..6) == 4 - doAssert r.rand(1..6) == 6 - let f = r.rand(-1.0 .. 1.0) - ## f = 0.8741183448756229 + assert r.rand(1..5) <= 5 + assert r.rand(-1.1 .. 1.2) >= -1.1 + assert x.a <= x.b when T is SomeFloat: result = rand(r, x.b - x.a) + x.a else: # Integers and Enum types - result = T(rand(r, int(x.b) - int(x.a)) + int(x.a)) + whenJsNoBigInt64: + result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a)) + do: + result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a)) proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = ## For a slice `a..b`, returns a value in the range `a..b`. ## ## Allowed types for `T` are integers, floats, and enums without holes. ## - ## If `randomize<#randomize>`_ has not been called, the sequence of random + ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: - ## * `rand proc<#rand,Rand,HSlice[T,T]>`_ that accepts a slice and uses - ## a provided state + ## **See also:** + ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice and uses a provided state ## * `rand proc<#rand,int>`_ that returns an integer ## * `rand proc<#rand,float>`_ that returns a floating point number ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type runnableExamples: randomize(345) - doAssert rand(1..6) == 4 - doAssert rand(1..6) == 4 - doAssert rand(1..6) == 6 + assert rand(1..6) <= 6 + result = rand(state, x) -proc rand*[T](r: var Rand; a: openArray[T]): T {.deprecated: - "Deprecated since v0.20.0; use 'sample' instead".} = - result = a[rand(r, a.low..a.high)] +proc rand*[T: Ordinal](r: var Rand; t: typedesc[T]): T {.since: (1, 7, 1).} = + ## Returns a random Ordinal in the range `low(T)..high(T)`. + ## + ## If `randomize <#randomize>`_ has not been called, the sequence of random + ## numbers returned from this proc will always be the same. + ## + ## **See also:** + ## * `rand proc<#rand,int>`_ that returns an integer + ## * `rand proc<#rand,float>`_ that returns a floating point number + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice + when T is range or T is enum: + result = rand(r, low(T)..high(T)) + elif T is bool: + result = r.next < randMax div 2 + else: + whenJsNoBigInt64: + result = cast[T](r.next shr (sizeof(uint)*8 - sizeof(T)*8)) + do: + result = cast[T](r.next shr (sizeof(uint64)*8 - sizeof(T)*8)) -proc rand*[T: SomeInteger](t: typedesc[T]): T = - ## Returns a random integer in the range `low(T)..high(T)`. +proc rand*[T: Ordinal](t: typedesc[T]): T = + ## Returns a random Ordinal in the range `low(T)..high(T)`. ## - ## If `randomize<#randomize>`_ has not been called, the sequence of random + ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `rand proc<#rand,int>`_ that returns an integer ## * `rand proc<#rand,float>`_ that returns a floating point number - ## * `rand proc<#rand,HSlice[T,T]>`_ that accepts a slice + ## * `rand proc<#rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_ + ## that accepts a slice runnableExamples: randomize(567) - doAssert rand(int8) == 55 - doAssert rand(int8) == -42 - doAssert rand(int8) == 43 - doAssert rand(uint32) == 578980729'u32 - doAssert rand(uint32) == 4052940463'u32 - doAssert rand(uint32) == 2163872389'u32 - doAssert rand(range[1..16]) == 11 - doAssert rand(range[1..16]) == 4 - doAssert rand(range[1..16]) == 16 - when T is range: - result = rand(state, low(T)..high(T)) - else: - result = cast[T](state.next) + type E = enum a, b, c, d -proc rand*[T](a: openArray[T]): T {.deprecated: - "Deprecated since v0.20.0; use 'sample' instead".} = - result = a[rand(a.low..a.high)] + assert rand(E) in a..d + assert rand(char) in low(char)..high(char) + assert rand(int8) in low(int8)..high(int8) + assert rand(uint32) in low(uint32)..high(uint32) + assert rand(range[1..16]) in 1..16 + + result = rand(state, t) proc sample*[T](r: var Rand; s: set[T]): T = - ## Returns a random element from the set ``s`` using the given state. + ## Returns a random element from the set `s` using the given state. ## - ## See also: - ## * `sample proc<#sample,set[T]>`_ that uses the default random number - ## generator - ## * `sample proc<#sample,Rand,openArray[T]>`_ for openarrays + ## **See also:** + ## * `sample proc<#sample,set[T]>`_ that uses the default RNG + ## * `sample proc<#sample,Rand,openArray[T]>`_ for `openArray`s ## * `sample proc<#sample,Rand,openArray[T],openArray[U]>`_ that uses a ## cumulative distribution function runnableExamples: var r = initRand(987) let s = {1, 3, 5, 7, 9} - doAssert r.sample(s) == 5 - doAssert r.sample(s) == 7 - doAssert r.sample(s) == 1 + assert r.sample(s) in s + assert card(s) != 0 var i = rand(r, card(s) - 1) for e in s: @@ -421,54 +452,49 @@ proc sample*[T](r: var Rand; s: set[T]): T = dec(i) proc sample*[T](s: set[T]): T = - ## Returns a random element from the set ``s``. + ## Returns a random element from the set `s`. ## - ## If `randomize<#randomize>`_ has not been called, the order of outcomes + ## If `randomize <#randomize>`_ has not been called, the order of outcomes ## from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `sample proc<#sample,Rand,set[T]>`_ that uses a provided state - ## * `sample proc<#sample,openArray[T]>`_ for openarrays + ## * `sample proc<#sample,openArray[T]>`_ for `openArray`s ## * `sample proc<#sample,openArray[T],openArray[U]>`_ that uses a ## cumulative distribution function runnableExamples: randomize(987) let s = {1, 3, 5, 7, 9} - doAssert sample(s) == 5 - doAssert sample(s) == 7 - doAssert sample(s) == 1 + assert sample(s) in s + sample(state, s) proc sample*[T](r: var Rand; a: openArray[T]): T = - ## Returns a random element from ``a`` using the given state. + ## Returns a random element from `a` using the given state. ## - ## See also: - ## * `sample proc<#sample,openArray[T]>`_ that uses the default - ## random number generator + ## **See also:** + ## * `sample proc<#sample,openArray[T]>`_ that uses the default RNG ## * `sample proc<#sample,Rand,openArray[T],openArray[U]>`_ that uses a ## cumulative distribution function ## * `sample proc<#sample,Rand,set[T]>`_ for sets runnableExamples: let marbles = ["red", "blue", "green", "yellow", "purple"] var r = initRand(456) - doAssert r.sample(marbles) == "blue" - doAssert r.sample(marbles) == "yellow" - doAssert r.sample(marbles) == "red" + assert r.sample(marbles) in marbles + result = a[r.rand(a.low..a.high)] -proc sample*[T](a: openArray[T]): T = - ## Returns a random element from ``a``. +proc sample*[T](a: openArray[T]): lent T = + ## Returns a random element from `a`. ## - ## If `randomize<#randomize>`_ has not been called, the order of outcomes + ## If `randomize <#randomize>`_ has not been called, the order of outcomes ## from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `sample proc<#sample,Rand,openArray[T]>`_ that uses a provided state ## * `sample proc<#sample,openArray[T],openArray[U]>`_ that uses a ## cumulative distribution function @@ -476,133 +502,160 @@ proc sample*[T](a: openArray[T]): T = runnableExamples: let marbles = ["red", "blue", "green", "yellow", "purple"] randomize(456) - doAssert sample(marbles) == "blue" - doAssert sample(marbles) == "yellow" - doAssert sample(marbles) == "red" + assert sample(marbles) in marbles + result = a[rand(a.low..a.high)] proc sample*[T, U](r: var Rand; a: openArray[T]; cdf: openArray[U]): T = - ## Returns an element from ``a`` using a cumulative distribution function + ## Returns an element from `a` using a cumulative distribution function ## (CDF) and the given state. ## - ## The ``cdf`` argument does not have to be normalized, and it could contain - ## any type of elements that can be converted to a ``float``. It must be - ## the same length as ``a``. Each element in ``cdf`` should be greater than + ## The `cdf` argument does not have to be normalized, and it could contain + ## any type of elements that can be converted to a `float`. It must be + ## the same length as `a`. Each element in `cdf` should be greater than ## or equal to the previous element. ## ## The outcome of the `cumsum<math.html#cumsum,openArray[T]>`_ proc and the ## return value of the `cumsummed<math.html#cumsummed,openArray[T]>`_ proc, - ## which are both in the math module, can be used as the ``cdf`` argument. + ## which are both in the math module, can be used as the `cdf` argument. ## - ## See also: + ## **See also:** ## * `sample proc<#sample,openArray[T],openArray[U]>`_ that also utilizes - ## a CDF but uses the default random number generator + ## a CDF but uses the default RNG ## * `sample proc<#sample,Rand,openArray[T]>`_ that does not use a CDF ## * `sample proc<#sample,Rand,set[T]>`_ for sets runnableExamples: - from math import cumsummed + from std/math import cumsummed let marbles = ["red", "blue", "green", "yellow", "purple"] let count = [1, 6, 8, 3, 4] let cdf = count.cumsummed var r = initRand(789) - doAssert r.sample(marbles, cdf) == "red" - doAssert r.sample(marbles, cdf) == "green" - doAssert r.sample(marbles, cdf) == "blue" + assert r.sample(marbles, cdf) in marbles + assert(cdf.len == a.len) # Two basic sanity checks. assert(float(cdf[^1]) > 0.0) - #While we could check cdf[i-1] <= cdf[i] for i in 1..cdf.len, that could get - #awfully expensive even in debugging modes. + # While we could check cdf[i-1] <= cdf[i] for i in 1..cdf.len, that could get + # awfully expensive even in debugging modes. let u = r.rand(float(cdf[^1])) a[cdf.upperBound(U(u))] proc sample*[T, U](a: openArray[T]; cdf: openArray[U]): T = - ## Returns an element from ``a`` using a cumulative distribution function + ## Returns an element from `a` using a cumulative distribution function ## (CDF). ## ## This proc works similarly to - ## `sample[T, U](Rand, openArray[T], openArray[U]) - ## <#sample,Rand,openArray[T],openArray[U]>`_. + ## `sample <#sample,Rand,openArray[T],openArray[U]>`_. ## See that proc's documentation for more details. ## - ## If `randomize<#randomize>`_ has not been called, the order of outcomes + ## If `randomize <#randomize>`_ has not been called, the order of outcomes ## from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `sample proc<#sample,Rand,openArray[T],openArray[U]>`_ that also utilizes ## a CDF but uses a provided state ## * `sample proc<#sample,openArray[T]>`_ that does not use a CDF ## * `sample proc<#sample,set[T]>`_ for sets runnableExamples: - from math import cumsummed + from std/math import cumsummed let marbles = ["red", "blue", "green", "yellow", "purple"] let count = [1, 6, 8, 3, 4] let cdf = count.cumsummed randomize(789) - doAssert sample(marbles, cdf) == "red" - doAssert sample(marbles, cdf) == "green" - doAssert sample(marbles, cdf) == "blue" + assert sample(marbles, cdf) in marbles + state.sample(a, cdf) +proc gauss*(r: var Rand; mu = 0.0; sigma = 1.0): float {.since: (1, 3).} = + ## Returns a Gaussian random variate, + ## with mean `mu` and standard deviation `sigma` + ## using the given state. + # Ratio of uniforms method for normal + # https://www2.econ.osaka-u.ac.jp/~tanizaki/class/2013/econome3/13.pdf + const K = sqrt(2 / E) + var + a = 0.0 + b = 0.0 + while true: + a = rand(r, 1.0) + b = (2.0 * rand(r, 1.0) - 1.0) * K + if b * b <= -4.0 * a * a * ln(a): break + result = mu + sigma * (b / a) + +proc gauss*(mu = 0.0, sigma = 1.0): float {.since: (1, 3).} = + ## Returns a Gaussian random variate, + ## with mean `mu` and standard deviation `sigma`. + ## + ## If `randomize <#randomize>`_ has not been called, the order of outcomes + ## from this proc will always be the same. + ## + ## This proc uses the default RNG. Thus, it is **not** thread-safe. + result = gauss(state, mu, sigma) + proc initRand*(seed: int64): Rand = - ## Initializes a new `Rand<#Rand>`_ state using the given seed. + ## Initializes a new `Rand <#Rand>`_ state using the given seed. ## - ## `seed` must not be zero. Providing a specific seed will produce - ## the same results for that seed each time. + ## Providing a specific seed will produce the same results for that seed each time. ## - ## The resulting state is independent of the default random number - ## generator's state. + ## The resulting state is independent of the default RNG's state. When `seed == 0`, + ## we internally set the seed to an implementation defined non-zero value. ## - ## See also: - ## * `randomize proc<#randomize,int64>`_ that accepts a seed for the default - ## random number generator - ## * `randomize proc<#randomize>`_ that initializes the default random - ## number generator using the current time + ## **See also:** + ## * `initRand proc<#initRand>`_ that uses the current time + ## * `randomize proc<#randomize,int64>`_ that accepts a seed for the default RNG + ## * `randomize proc<#randomize>`_ that initializes the default RNG using the current time runnableExamples: - from times import getTime, toUnix, nanosecond + from std/times import getTime, toUnix, nanosecond var r1 = initRand(123) - let now = getTime() var r2 = initRand(now.toUnix * 1_000_000_000 + now.nanosecond) - doAssert seed != 0 # 0 causes `rand(int)` to always return 0 for example. + const seedFallback0 = int32.high # arbitrary + let seed = if seed != 0: seed else: seedFallback0 # because 0 is a fixed point result.a0 = Ui(seed shr 16) result.a1 = Ui(seed and 0xffff) + when not defined(nimLegacyRandomInitRand): + # calling `discard next(result)` (even a few times) would still produce + # skewed numbers for the 1st call to `rand()`. + skipRandomNumbers(result) discard next(result) proc randomize*(seed: int64) {.benign.} = ## Initializes the default random number generator with the given seed. ## - ## `seed` must not be zero. Providing a specific seed will produce - ## the same results for that seed each time. + ## Providing a specific seed will produce the same results for that seed each time. ## - ## See also: - ## * `initRand proc<#initRand,int64>`_ + ## **See also:** + ## * `initRand proc<#initRand,int64>`_ that initializes a Rand state + ## with a given seed ## * `randomize proc<#randomize>`_ that uses the current time instead + ## * `initRand proc<#initRand>`_ that initializes a Rand state using + ## the current time runnableExamples: - from times import getTime, toUnix, nanosecond + from std/times import getTime, toUnix, nanosecond randomize(123) let now = getTime() randomize(now.toUnix * 1_000_000_000 + now.nanosecond) + state = initRand(seed) proc shuffle*[T](r: var Rand; x: var openArray[T]) = ## Shuffles a sequence of elements in-place using the given state. ## - ## See also: - ## * `shuffle proc<#shuffle,openArray[T]>`_ that uses the default - ## random number generator + ## **See also:** + ## * `shuffle proc<#shuffle,openArray[T]>`_ that uses the default RNG runnableExamples: var cards = ["Ace", "King", "Queen", "Jack", "Ten"] var r = initRand(678) r.shuffle(cards) - doAssert cards == ["King", "Ace", "Queen", "Ten", "Jack"] + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] + for i in countdown(x.high, 1): let j = r.rand(i) swap(x[i], x[j]) @@ -610,83 +663,104 @@ proc shuffle*[T](r: var Rand; x: var openArray[T]) = proc shuffle*[T](x: var openArray[T]) = ## Shuffles a sequence of elements in-place. ## - ## If `randomize<#randomize>`_ has not been called, the order of outcomes + ## If `randomize <#randomize>`_ has not been called, the order of outcomes ## from this proc will always be the same. ## - ## This proc uses the default random number generator. Thus, it is **not** - ## thread-safe. + ## This proc uses the default RNG. Thus, it is **not** thread-safe. ## - ## See also: + ## **See also:** ## * `shuffle proc<#shuffle,Rand,openArray[T]>`_ that uses a provided state runnableExamples: var cards = ["Ace", "King", "Queen", "Jack", "Ten"] randomize(678) shuffle(cards) - doAssert cards == ["King", "Ace", "Queen", "Ten", "Jack"] + import std/algorithm + assert cards.sorted == @["Ace", "Jack", "King", "Queen", "Ten"] + shuffle(state, x) -when not defined(nimscript) and not defined(standalone): - import times +when not defined(standalone): + when defined(js): + import std/times + else: + when defined(nimscript): + import std/hashes + else: + import std/[hashes, os, sysrand, monotimes] - proc randomize*() {.benign.} = - ## Initializes the default random number generator with a value based on - ## the current time. + when compileOption("threads"): + import std/locks + var baseSeedLock: Lock + baseSeedLock.initLock + + var baseState: Rand + + proc initRand(): Rand = + ## Initializes a new Rand state. ## - ## This proc only needs to be called once, and it should be called before - ## the first usage of procs from this module that use the default random - ## number generator. + ## The resulting state is independent of the default RNG's state. ## - ## **Note:** Does not work for NimScript. + ## **Note:** Does not work for the compile-time VM. ## ## See also: - ## * `randomize proc<#randomize,int64>`_ that accepts a seed - ## * `initRand proc<#initRand,int64>`_ + ## * `initRand proc<#initRand,int64>`_ that accepts a seed for a new Rand state + ## * `randomize proc<#randomize>`_ that initializes the default RNG using the current time + ## * `randomize proc<#randomize,int64>`_ that accepts a seed for the default RNG when defined(js): let time = int64(times.epochTime() * 1000) and 0x7fff_ffff - randomize(time) + result = initRand(time) else: - let now = times.getTime() - randomize(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) + proc getRandomState(): Rand = + when defined(nimscript): + result = Rand( + a0: CompileTime.hash.Ui, + a1: CompileDate.hash.Ui) + if not result.isValid: + result = DefaultRandSeed + else: + var urand: array[sizeof(Rand), byte] + + for i in 0 .. 7: + if sysrand.urandom(urand): + copyMem(result.addr, urand[0].addr, sizeof(Rand)) + if result.isValid: + break + + if not result.isValid: + # Don't try to get alternative random values from other source like time or process/thread id, + # because such code would be never tested and is a liability for security. + quit("Failed to initializes baseState in random module as sysrand.urandom doesn't work.") + + when compileOption("threads"): + baseSeedLock.withLock: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + else: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + + since (1, 5, 1): + export initRand -{.pop.} + proc randomize*() {.benign.} = + ## Initializes the default random number generator with a seed based on + ## random number source. + ## + ## This proc only needs to be called once, and it should be called before + ## the first usage of procs from this module that use the default RNG. + ## + ## **Note:** Does not work for the compile-time VM. + ## + ## **See also:** + ## * `randomize proc<#randomize,int64>`_ that accepts a seed + ## * `initRand proc<#initRand>`_ that initializes a Rand state using + ## the current time + ## * `initRand proc<#initRand,int64>`_ that initializes a Rand state + ## with a given seed + state = initRand() -when isMainModule: - proc main = - var occur: array[1000, int] - - var x = 8234 - for i in 0..100_000: - x = rand(high(occur)) - inc occur[x] - for i, oc in occur: - if oc < 69: - doAssert false, "too few occurrences of " & $i - elif oc > 150: - doAssert false, "too many occurrences of " & $i - - var a = [0, 1] - shuffle(a) - doAssert a[0] == 1 - doAssert a[1] == 0 - - doAssert rand(0) == 0 - doAssert sample("a") == 'a' - - when compileOption("rangeChecks"): - try: - discard rand(-1) - doAssert false - except RangeError: - discard - - try: - discard rand(-1.0) - doAssert false - except RangeError: - discard - - - # don't use causes integer overflow - doAssert compiles(random[int](low(int) .. high(int))) - - main() +{.pop.} diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index b668a9f71..5f806bd70 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -8,56 +8,95 @@ # -## This module implements rational numbers, consisting of a numerator `num` and -## a denominator `den`, both of type int. The denominator can not be 0. +## This module implements rational numbers, consisting of a numerator and +## a denominator. The denominator can not be 0. -import math -import hashes +runnableExamples: + let + r1 = 1 // 2 + r2 = -3 // 4 + + doAssert r1 + r2 == -1 // 4 + doAssert r1 - r2 == 5 // 4 + doAssert r1 * r2 == -3 // 8 + doAssert r1 / r2 == -2 // 3 + +import std/[math, hashes] +when defined(nimPreviewSlimSystem): + import std/assertions type Rational*[T] = object - ## a rational number, consisting of a numerator and denominator + ## A rational number, consisting of a numerator `num` and a denominator `den`. num*, den*: T -proc initRational*[T: SomeInteger](num, den: T): Rational[T] = - ## Create a new rational number. - assert(den != 0, "a denominator of zero value is invalid") +func reduce*[T: SomeInteger](x: var Rational[T]) = + ## Reduces the rational number `x`, so that the numerator and denominator + ## have no common divisors other than 1 (and -1). + ## If `x` is 0, raises `DivByZeroDefect`. + ## + ## **Note:** This is called automatically by the various operations on rationals. + runnableExamples: + var r = Rational[int](num: 2, den: 4) # 1/2 + reduce(r) + doAssert r.num == 1 + doAssert r.den == 2 + if x.den == 0: + raise newException(DivByZeroDefect, "division by zero") + let common = gcd(x.num, x.den) + if x.den > 0: + x.num = x.num div common + x.den = x.den div common + when T isnot SomeUnsignedInt: + if x.den < 0: + x.num = -x.num div common + x.den = -x.den div common + +func initRational*[T: SomeInteger](num, den: T): Rational[T] = + ## Creates a new rational number with numerator `num` and denominator `den`. + ## `den` must not be 0. + ## + ## **Note:** `den != 0` is not checked when assertions are turned off. + assert(den != 0, "a denominator of zero is invalid") result.num = num result.den = den + reduce(result) -proc `//`*[T](num, den: T): Rational[T] = initRational[T](num, den) - ## A friendlier version of `initRational`. Example usage: - ## - ## .. code-block:: nim - ## var x = 1//3 + 1//5 +func `//`*[T](num, den: T): Rational[T] = + ## A friendlier version of `initRational <#initRational,T,T>`_. + runnableExamples: + let x = 1 // 3 + 1 // 5 + doAssert x == 8 // 15 + + initRational[T](num, den) + +func `$`*[T](x: Rational[T]): string = + ## Turns a rational number into a string. + runnableExamples: + doAssert $(1 // 2) == "1/2" -proc `$`*[T](x: Rational[T]): string = - ## Turn a rational number into a string. result = $x.num & "/" & $x.den -proc toRational*[T: SomeInteger](x: T): Rational[T] = - ## Convert some integer `x` to a rational number. +func toRational*[T: SomeInteger](x: T): Rational[T] = + ## Converts some integer `x` to a rational number. + runnableExamples: + doAssert toRational(42) == 42 // 1 + result.num = x result.den = 1 -proc toRational*(x: float, +func toRational*(x: float, n: int = high(int) shr (sizeof(int) div 2 * 8)): Rational[int] = - ## Calculates the best rational numerator and denominator - ## that approximates to `x`, where the denominator is - ## smaller than `n` (default is the largest possible - ## int to give maximum resolution). + ## Calculates the best rational approximation of `x`, + ## where the denominator is smaller than `n` + ## (default is the largest possible `int` for maximal resolution). ## ## The algorithm is based on the theory of continued fractions. - ## - ## .. code-block:: Nim - ## import math, rationals - ## for i in 1..10: - ## let t = (10 ^ (i+3)).int - ## let x = toRational(PI, t) - ## let newPI = x.num / x.den - ## echo x, " ", newPI, " error: ", PI - newPI, " ", t - # David Eppstein / UC Irvine / 8 Aug 1993 # With corrections from Arno Formella, May 2008 + runnableExamples: + let x = 1.2 + doAssert x.toRational.toFloat == x + var m11, m22 = 1 m12, m21 = 0 @@ -69,124 +108,113 @@ proc toRational*(x: float, m11 = m12 * ai + m11 m21 = m22 * ai + m21 if x == float(ai): break # division by zero - x = 1/(x - float(ai)) + x = 1 / (x - float(ai)) if x > float(high(int32)): break # representation failure ai = int(x) result = m11 // m21 -proc toFloat*[T](x: Rational[T]): float = - ## Convert a rational number `x` to a float. +func toFloat*[T](x: Rational[T]): float = + ## Converts a rational number `x` to a `float`. x.num / x.den -proc toInt*[T](x: Rational[T]): int = - ## Convert a rational number `x` to an int. Conversion rounds towards 0 if +func toInt*[T](x: Rational[T]): int = + ## Converts a rational number `x` to an `int`. Conversion rounds towards 0 if ## `x` does not contain an integer value. x.num div x.den -proc reduce*[T: SomeInteger](x: var Rational[T]) = - ## Reduce rational `x`. - let common = gcd(x.num, x.den) - if x.den > 0: - x.num = x.num div common - x.den = x.den div common - elif x.den < 0: - x.num = -x.num div common - x.den = -x.den div common - else: - raise newException(DivByZeroError, "division by zero") - -proc `+` *[T](x, y: Rational[T]): Rational[T] = - ## Add two rational numbers. +func `+`*[T](x, y: Rational[T]): Rational[T] = + ## Adds two rational numbers. let common = lcm(x.den, y.den) result.num = common div x.den * x.num + common div y.den * y.num result.den = common reduce(result) -proc `+` *[T](x: Rational[T], y: T): Rational[T] = - ## Add rational `x` to int `y`. +func `+`*[T](x: Rational[T], y: T): Rational[T] = + ## Adds the rational `x` to the int `y`. result.num = x.num + y * x.den result.den = x.den -proc `+` *[T](x: T, y: Rational[T]): Rational[T] = - ## Add int `x` to rational `y`. +func `+`*[T](x: T, y: Rational[T]): Rational[T] = + ## Adds the int `x` to the rational `y`. result.num = x * y.den + y.num result.den = y.den -proc `+=` *[T](x: var Rational[T], y: Rational[T]) = - ## Add rational `y` to rational `x`. +func `+=`*[T](x: var Rational[T], y: Rational[T]) = + ## Adds the rational `y` to the rational `x` in-place. let common = lcm(x.den, y.den) x.num = common div x.den * x.num + common div y.den * y.num x.den = common reduce(x) -proc `+=` *[T](x: var Rational[T], y: T) = - ## Add int `y` to rational `x`. +func `+=`*[T](x: var Rational[T], y: T) = + ## Adds the int `y` to the rational `x` in-place. x.num += y * x.den -proc `-` *[T](x: Rational[T]): Rational[T] = +func `-`*[T](x: Rational[T]): Rational[T] = ## Unary minus for rational numbers. result.num = -x.num result.den = x.den -proc `-` *[T](x, y: Rational[T]): Rational[T] = - ## Subtract two rational numbers. +func `-`*[T](x, y: Rational[T]): Rational[T] = + ## Subtracts two rational numbers. let common = lcm(x.den, y.den) result.num = common div x.den * x.num - common div y.den * y.num result.den = common reduce(result) -proc `-` *[T](x: Rational[T], y: T): Rational[T] = - ## Subtract int `y` from rational `x`. +func `-`*[T](x: Rational[T], y: T): Rational[T] = + ## Subtracts the int `y` from the rational `x`. result.num = x.num - y * x.den result.den = x.den -proc `-` *[T](x: T, y: Rational[T]): Rational[T] = - ## Subtract rational `y` from int `x`. +func `-`*[T](x: T, y: Rational[T]): Rational[T] = + ## Subtracts the rational `y` from the int `x`. result.num = x * y.den - y.num result.den = y.den -proc `-=` *[T](x: var Rational[T], y: Rational[T]) = - ## Subtract rational `y` from rational `x`. +func `-=`*[T](x: var Rational[T], y: Rational[T]) = + ## Subtracts the rational `y` from the rational `x` in-place. let common = lcm(x.den, y.den) x.num = common div x.den * x.num - common div y.den * y.num x.den = common reduce(x) -proc `-=` *[T](x: var Rational[T], y: T) = - ## Subtract int `y` from rational `x`. +func `-=`*[T](x: var Rational[T], y: T) = + ## Subtracts the int `y` from the rational `x` in-place. x.num -= y * x.den -proc `*` *[T](x, y: Rational[T]): Rational[T] = - ## Multiply two rational numbers. +func `*`*[T](x, y: Rational[T]): Rational[T] = + ## Multiplies two rational numbers. result.num = x.num * y.num result.den = x.den * y.den reduce(result) -proc `*` *[T](x: Rational[T], y: T): Rational[T] = - ## Multiply rational `x` with int `y`. +func `*`*[T](x: Rational[T], y: T): Rational[T] = + ## Multiplies the rational `x` with the int `y`. result.num = x.num * y result.den = x.den reduce(result) -proc `*` *[T](x: T, y: Rational[T]): Rational[T] = - ## Multiply int `x` with rational `y`. +func `*`*[T](x: T, y: Rational[T]): Rational[T] = + ## Multiplies the int `x` with the rational `y`. result.num = x * y.num result.den = y.den reduce(result) -proc `*=` *[T](x: var Rational[T], y: Rational[T]) = - ## Multiply rationals `y` to `x`. +func `*=`*[T](x: var Rational[T], y: Rational[T]) = + ## Multiplies the rational `x` by `y` in-place. x.num *= y.num x.den *= y.den reduce(x) -proc `*=` *[T](x: var Rational[T], y: T) = - ## Multiply int `y` to rational `x`. +func `*=`*[T](x: var Rational[T], y: T) = + ## Multiplies the rational `x` by the int `y` in-place. x.num *= y reduce(x) -proc reciprocal*[T](x: Rational[T]): Rational[T] = - ## Calculate the reciprocal of `x`. (1/x) +func reciprocal*[T](x: Rational[T]): Rational[T] = + ## Calculates the reciprocal of `x` (`1/x`). + ## If `x` is 0, raises `DivByZeroDefect`. if x.num > 0: result.num = x.den result.den = x.num @@ -194,83 +222,94 @@ proc reciprocal*[T](x: Rational[T]): Rational[T] = result.num = -x.den result.den = -x.num else: - raise newException(DivByZeroError, "division by zero") + raise newException(DivByZeroDefect, "division by zero") -proc `/`*[T](x, y: Rational[T]): Rational[T] = - ## Divide rationals `x` by `y`. +func `/`*[T](x, y: Rational[T]): Rational[T] = + ## Divides the rational `x` by the rational `y`. result.num = x.num * y.den result.den = x.den * y.num reduce(result) -proc `/`*[T](x: Rational[T], y: T): Rational[T] = - ## Divide rational `x` by int `y`. +func `/`*[T](x: Rational[T], y: T): Rational[T] = + ## Divides the rational `x` by the int `y`. result.num = x.num result.den = x.den * y reduce(result) -proc `/`*[T](x: T, y: Rational[T]): Rational[T] = - ## Divide int `x` by Rational `y`. +func `/`*[T](x: T, y: Rational[T]): Rational[T] = + ## Divides the int `x` by the rational `y`. result.num = x * y.den result.den = y.num reduce(result) -proc `/=`*[T](x: var Rational[T], y: Rational[T]) = - ## Divide rationals `x` by `y` in place. +func `/=`*[T](x: var Rational[T], y: Rational[T]) = + ## Divides the rational `x` by the rational `y` in-place. x.num *= y.den x.den *= y.num reduce(x) -proc `/=`*[T](x: var Rational[T], y: T) = - ## Divide rational `x` by int `y` in place. +func `/=`*[T](x: var Rational[T], y: T) = + ## Divides the rational `x` by the int `y` in-place. x.den *= y reduce(x) -proc cmp*(x, y: Rational): int {.procvar.} = - ## Compares two rationals. +func cmp*(x, y: Rational): int = + ## Compares two rationals. Returns + ## * a value less than zero, if `x < y` + ## * a value greater than zero, if `x > y` + ## * zero, if `x == y` (x - y).num -proc `<` *(x, y: Rational): bool = +func `<`*(x, y: Rational): bool = + ## Returns true if `x` is less than `y`. (x - y).num < 0 -proc `<=` *(x, y: Rational): bool = +func `<=`*(x, y: Rational): bool = + ## Returns tue if `x` is less than or equal to `y`. (x - y).num <= 0 -proc `==` *(x, y: Rational): bool = +func `==`*(x, y: Rational): bool = + ## Compares two rationals for equality. (x - y).num == 0 -proc abs*[T](x: Rational[T]): Rational[T] = +func abs*[T](x: Rational[T]): Rational[T] = + ## Returns the absolute value of `x`. + runnableExamples: + doAssert abs(1 // 2) == 1 // 2 + doAssert abs(-1 // 2) == 1 // 2 + result.num = abs x.num result.den = abs x.den -proc `div`*[T: SomeInteger](x, y: Rational[T]): T = +func `div`*[T: SomeInteger](x, y: Rational[T]): T = ## Computes the rational truncated division. (x.num * y.den) div (y.num * x.den) -proc `mod`*[T: SomeInteger](x, y: Rational[T]): Rational[T] = +func `mod`*[T: SomeInteger](x, y: Rational[T]): Rational[T] = ## Computes the rational modulo by truncated division (remainder). - ## This is same as ``x - (x div y) * y``. + ## This is same as `x - (x div y) * y`. result = ((x.num * y.den) mod (y.num * x.den)) // (x.den * y.den) reduce(result) -proc floorDiv*[T: SomeInteger](x, y: Rational[T]): T = +func floorDiv*[T: SomeInteger](x, y: Rational[T]): T = ## Computes the rational floor division. ## - ## Floor division is conceptually defined as ``floor(x / y)``. - ## This is different from the ``div`` operator, which is defined - ## as ``trunc(x / y)``. That is, ``div`` rounds towards ``0`` and ``floorDiv`` + ## Floor division is conceptually defined as `floor(x / y)`. + ## This is different from the `div` operator, which is defined + ## as `trunc(x / y)`. That is, `div` rounds towards 0 and `floorDiv` ## rounds down. floorDiv(x.num * y.den, y.num * x.den) -proc floorMod*[T: SomeInteger](x, y: Rational[T]): Rational[T] = +func floorMod*[T: SomeInteger](x, y: Rational[T]): Rational[T] = ## Computes the rational modulo by floor division (modulo). ## - ## This is same as ``x - floorDiv(x, y) * y``. - ## This proc behaves the same as the ``%`` operator in python. + ## This is same as `x - floorDiv(x, y) * y`. + ## This func behaves the same as the `%` operator in Python. result = floorMod(x.num * y.den, y.num * x.den) // (x.den * y.den) reduce(result) -proc hash*[T](x: Rational[T]): Hash = - ## Computes hash for rational `x` +func hash*[T](x: Rational[T]): Hash = + ## Computes the hash for the rational `x`. # reduce first so that hash(x) == hash(y) for x == y var copy = x reduce(copy) @@ -280,99 +319,22 @@ proc hash*[T](x: Rational[T]): Hash = h = h !& hash(copy.den) result = !$h -when isMainModule: - var - z = Rational[int](num: 0, den: 1) - o = initRational(num = 1, den = 1) - a = initRational(1, 2) - b = -1 // -2 - m1 = -1 // 1 - tt = 10 // 2 - - assert(a == a) - assert( (a-a) == z) - assert( (a+b) == o) - assert( (a/b) == o) - assert( (a*b) == 1 // 4) - assert( (3/a) == 6 // 1) - assert( (a/3) == 1 // 6) - assert(a*b == 1 // 4) - assert(tt*z == z) - assert(10*a == tt) - assert(a*10 == tt) - assert(tt/10 == a) - assert(a-m1 == 3 // 2) - assert(a+m1 == -1 // 2) - assert(m1+tt == 16 // 4) - assert(m1-tt == 6 // -1) - - assert(z < o) - assert(z <= o) - assert(z == z) - assert(cmp(z, o) < 0) - assert(cmp(o, z) > 0) - - assert(o == o) - assert(o >= o) - assert(not(o > o)) - assert(cmp(o, o) == 0) - assert(cmp(z, z) == 0) - assert(hash(o) == hash(o)) - - assert(a == b) - assert(a >= b) - assert(not(b > a)) - assert(cmp(a, b) == 0) - assert(hash(a) == hash(b)) - - var x = 1//3 - - x *= 5//1 - assert(x == 5//3) - x += 2 // 9 - assert(x == 17//9) - x -= 9//18 - assert(x == 25//18) - x /= 1//2 - assert(x == 50//18) - - var y = 1//3 - - y *= 4 - assert(y == 4//3) - y += 5 - assert(y == 19//3) - y -= 2 - assert(y == 13//3) - y /= 9 - assert(y == 13//27) - - assert toRational(5) == 5//1 - assert abs(toFloat(y) - 0.4814814814814815) < 1.0e-7 - assert toInt(z) == 0 - - when sizeof(int) == 8: - assert toRational(0.98765432) == 2111111029 // 2137499919 - assert toRational(PI) == 817696623 // 260280919 - when sizeof(int) == 4: - assert toRational(0.98765432) == 80 // 81 - assert toRational(PI) == 355 // 113 - - assert toRational(0.1) == 1 // 10 - assert toRational(0.9) == 9 // 10 - - assert toRational(0.0) == 0 // 1 - assert toRational(-0.25) == 1 // -4 - assert toRational(3.2) == 16 // 5 - assert toRational(0.33) == 33 // 100 - assert toRational(0.22) == 11 // 50 - assert toRational(10.0) == 10 // 1 - - assert (1//1) div (3//10) == 3 - assert (-1//1) div (3//10) == -3 - assert (3//10) mod (1//1) == 3//10 - assert (-3//10) mod (1//1) == -3//10 - assert floorDiv(1//1, 3//10) == 3 - assert floorDiv(-1//1, 3//10) == -4 - assert floorMod(3//10, 1//1) == 3//10 - assert floorMod(-3//10, 1//1) == 7//10 +func `^`*[T: SomeInteger](x: Rational[T], y: T): Rational[T] = + ## Computes `x` to the power of `y`. + ## + ## The exponent `y` must be an integer. Negative exponents are supported + ## but floating point exponents are not. + runnableExamples: + doAssert (-3 // 5) ^ 0 == (1 // 1) + doAssert (-3 // 5) ^ 1 == (-3 // 5) + doAssert (-3 // 5) ^ 2 == (9 // 25) + doAssert (-3 // 5) ^ -2 == (25 // 9) + + if y >= 0: + result.num = x.num ^ y + result.den = x.den ^ y + else: + result.num = x.den ^ -y + result.den = x.num ^ -y + # Note that all powers of reduced rationals are already reduced, + # so we don't need to call reduce() here diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim index 85a6e324f..ffa0128dc 100644 --- a/lib/pure/reservedmem.nim +++ b/lib/pure/reservedmem.nim @@ -9,7 +9,7 @@ ## :Authors: Zahary Karadjov ## -## This module provides utilities for reserving a portions of the +## This module provides utilities for reserving portions of the ## address space of a program without consuming physical memory. ## It can be used to implement a dynamically resizable buffer that ## is guaranteed to remain in the same memory location. The buffer @@ -18,7 +18,10 @@ ## ## Unstable API. -from ospaths import raiseOSError, osLastError +from std/oserrors import raiseOSError, osLastError + +when defined(nimPreviewSlimSystem): + import std/assertions template distance*(lhs, rhs: pointer): int = cast[int](rhs) - cast[int](lhs) @@ -41,26 +44,11 @@ type mem: ReservedMem when defined(windows): - import winlean - - type - SYSTEM_INFO {.final, pure.} = object - u1: uint32 - dwPageSize: uint32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr uint32 - dwNumberOfProcessors: uint32 - dwProcessorType: uint32 - dwAllocationGranularity: uint32 - wProcessorLevel: uint16 - wProcessorRevision: uint16 - - proc getSystemInfo(lpSystemInfo: ptr SYSTEM_INFO) {.stdcall, - dynlib: "kernel32", importc: "GetSystemInfo".} + import std/winlean + import std/private/win_getsysteminfo proc getAllocationGranularity: uint = - var sysInfo: SYSTEM_INFO + var sysInfo: SystemInfo getSystemInfo(addr sysInfo) return uint(sysInfo.dwAllocationGranularity) @@ -76,11 +64,11 @@ when defined(windows): template check(expr) = let r = expr - if r == cast[type(r)](0): + if r == cast[typeof(r)](0): raiseOSError(osLastError()) else: - import posix + import std/posix let allocationGranularity = sysconf(SC_PAGESIZE) diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 1205584db..8750aca87 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -8,18 +8,19 @@ # ## This module contains support for a `rope`:idx: data type. -## Ropes can represent very long strings efficiently; especially concatenation +## Ropes can represent very long strings efficiently; in particular, concatenation ## is done in O(1) instead of O(n). They are essentially concatenation ## trees that are only flattened when converting to a native Nim -## string. The empty string is represented by ``nil``. Ropes are immutable and +## string. The empty string is represented by `nil`. Ropes are immutable and ## subtrees can be shared without copying. ## Leaves can be cached for better memory efficiency at the cost of ## runtime efficiency. -include "system/inclrtl" -import streams +include system/inclrtl +import std/streams -{.deadCodeElim: on.} # dce option deprecated +when defined(nimPreviewSlimSystem): + import std/[syncio, formatfloat, assertions] {.push debugger: off.} # the user does not want to trace a part # of the standard library! @@ -31,11 +32,11 @@ var cacheEnabled = false type - Rope* = ref RopeObj ## empty rope is represented by nil - RopeObj {.acyclic.} = object + Rope* {.acyclic.} = ref object + ## A rope data type. The empty rope is represented by `nil`. left, right: Rope length: int - data: string # != nil if a leaf + data: string # not empty if a leaf # Note that the left and right pointers are not needed for leafs. # Leaves have relatively high memory overhead (~30 bytes on a 32 @@ -46,9 +47,8 @@ type # pointers. proc len*(a: Rope): int {.rtl, extern: "nro$1".} = - ## the rope's length - if a == nil: result = 0 - else: result = a.length + ## The rope's length. + if a == nil: 0 else: a.length proc newRope(): Rope = new(result) proc newRope(data: string): Rope = @@ -129,6 +129,10 @@ proc insertInCache(s: string, tree: Rope): Rope = proc rope*(s: string = ""): Rope {.rtl, extern: "nro$1Str".} = ## Converts a string to a rope. + runnableExamples: + let r = rope("I'm a rope") + doAssert $r == "I'm a rope" + if s.len == 0: result = nil else: @@ -144,10 +148,18 @@ proc rope*(s: string = ""): Rope {.rtl, extern: "nro$1Str".} = proc rope*(i: BiggestInt): Rope {.rtl, extern: "nro$1BiggestInt".} = ## Converts an int to a rope. + runnableExamples: + let r = rope(429) + doAssert $r == "429" + result = rope($i) proc rope*(f: BiggestFloat): Rope {.rtl, extern: "nro$1BiggestFloat".} = ## Converts a float to a rope. + runnableExamples: + let r = rope(4.29) + doAssert $r == "4.29" + result = rope($f) proc enableCache*() {.rtl, extern: "nro$1".} = @@ -156,12 +168,16 @@ proc enableCache*() {.rtl, extern: "nro$1".} = cacheEnabled = true proc disableCache*() {.rtl, extern: "nro$1".} = - ## the cache is discarded and disabled. The GC will reuse its used memory. + ## The cache is discarded and disabled. The GC will reuse its used memory. cache = nil cacheEnabled = false proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = - ## the concatenation operator for ropes. + ## The concatenation operator for ropes. + runnableExamples: + let r = rope("Hello, ") & rope("Nim!") + doAssert $r == "Hello, Nim!" + if a == nil: result = b elif b == nil: @@ -173,44 +189,81 @@ proc `&`*(a, b: Rope): Rope {.rtl, extern: "nroConcRopeRope".} = result.right = b proc `&`*(a: Rope, b: string): Rope {.rtl, extern: "nroConcRopeStr".} = - ## the concatenation operator for ropes. + ## The concatenation operator for ropes. + runnableExamples: + let r = rope("Hello, ") & "Nim!" + doAssert $r == "Hello, Nim!" + result = a & rope(b) proc `&`*(a: string, b: Rope): Rope {.rtl, extern: "nroConcStrRope".} = - ## the concatenation operator for ropes. + ## The concatenation operator for ropes. + runnableExamples: + let r = "Hello, " & rope("Nim!") + doAssert $r == "Hello, Nim!" + result = rope(a) & b proc `&`*(a: openArray[Rope]): Rope {.rtl, extern: "nroConcOpenArray".} = - ## the concatenation operator for an openarray of ropes. - for i in countup(0, high(a)): result = result & a[i] + ## The concatenation operator for an `openArray` of ropes. + runnableExamples: + let r = &[rope("Hello, "), rope("Nim"), rope("!")] + doAssert $r == "Hello, Nim!" + + for item in a: result = result & item proc add*(a: var Rope, b: Rope) {.rtl, extern: "nro$1Rope".} = - ## adds `b` to the rope `a`. + ## Adds `b` to the rope `a`. + runnableExamples: + var r = rope("Hello, ") + r.add(rope("Nim!")) + doAssert $r == "Hello, Nim!" + a = a & b proc add*(a: var Rope, b: string) {.rtl, extern: "nro$1Str".} = - ## adds `b` to the rope `a`. + ## Adds `b` to the rope `a`. + runnableExamples: + var r = rope("Hello, ") + r.add("Nim!") + doAssert $r == "Hello, Nim!" + a = a & b proc `[]`*(r: Rope, i: int): char {.rtl, extern: "nroCharAt".} = - ## returns the character at position `i` in the rope `r`. This is quite - ## expensive! Worst-case: O(n). If ``i >= r.len``, ``\0`` is returned. + ## Returns the character at position `i` in the rope `r`. This is quite + ## expensive! Worst-case: O(n). If `i >= r.len or i < 0`, `\0` is returned. + runnableExamples: + let r = rope("Hello, Nim!") + + doAssert r[0] == 'H' + doAssert r[7] == 'N' + doAssert r[22] == '\0' + var x = r var j = i - if x == nil: return + if x == nil or i < 0 or i >= r.len: return while true: if x != nil and x.data.len > 0: - if j < x.data.len: return x.data[j] - return '\0' + # leaf + return x.data[j] else: if x.left.length > j: x = x.left else: + dec(j, x.left.length) x = x.right - dec(j, x.len) iterator leaves*(r: Rope): string = - ## iterates over any leaf string in the rope `r`. + ## Iterates over any leaf string in the rope `r`. + runnableExamples: + let r = rope("Hello") & rope(", Nim!") + let s = ["Hello", ", Nim!"] + var index = 0 + for leave in r.leaves: + doAssert leave == s[index] + inc(index) + if r != nil: var stack = @[r] while stack.len > 0: @@ -223,27 +276,36 @@ iterator leaves*(r: Rope): string = yield it.data iterator items*(r: Rope): char = - ## iterates over any character in the rope `r`. + ## Iterates over any character in the rope `r`. for s in leaves(r): for c in items(s): yield c proc write*(f: File, r: Rope) {.rtl, extern: "nro$1".} = - ## writes a rope to a file. + ## Writes a rope to a file. for s in leaves(r): write(f, s) proc write*(s: Stream, r: Rope) {.rtl, extern: "nroWriteStream".} = - ## writes a rope to a stream. + ## Writes a rope to a stream. for rs in leaves(r): write(s, rs) proc `$`*(r: Rope): string {.rtl, extern: "nroToString".} = - ## converts a rope back to a string. + ## Converts a rope back to a string. result = newStringOfCap(r.len) for s in leaves(r): add(result, s) -proc `%`*(frmt: string, args: openArray[Rope]): Rope {. - rtl, extern: "nroFormat".} = - ## `%` substitution operator for ropes. Does not support the ``$identifier`` - ## nor ``${identifier}`` notations. +proc `%`*(frmt: string, args: openArray[Rope]): Rope {.rtl, extern: "nroFormat".} = + ## `%` substitution operator for ropes. Does not support the `$identifier` + ## nor `${identifier}` notations. + runnableExamples: + let r1 = "$1 $2 $3" % [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r1 == "Nim is a great language" + + let r2 = "$# $# $#" % [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r2 == "Nim is a great language" + + let r3 = "${1} ${2} ${3}" % [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r3 == "Nim is a great language" + var i = 0 var length = len(frmt) result = nil @@ -264,7 +326,7 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {. while true: j = j * 10 + ord(frmt[i]) - ord('0') inc(i) - if frmt[i] notin {'0'..'9'}: break + if i >= frmt.len or frmt[i] notin {'0'..'9'}: break add(result, args[j-1]) of '{': inc(i) @@ -284,51 +346,54 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {. if i - 1 >= start: add(result, substr(frmt, start, i - 1)) -proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {. - rtl, extern: "nro$1".} = - ## shortcut for ``add(c, frmt % args)``. - add(c, frmt % args) - -const - bufSize = 1024 # 1 KB is reasonable +proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {.rtl, extern: "nro$1".} = + ## Shortcut for `add(c, frmt % args)`. + runnableExamples: + var r = rope("Dash: ") + r.addf "$1 $2 $3", [rope("Nim"), rope("is"), rope("a great language")] + doAssert $r == "Dash: Nim is a great language" -proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} = - ## returns true if the contents of the file `f` equal `r`. - var - buf: array[bufSize, char] - bpos = buf.len - blen = buf.len + add(c, frmt % args) - for s in leaves(r): - var spos = 0 - let slen = s.len - while spos < slen: - if bpos == blen: - # Read more data - bpos = 0 - blen = readBuffer(f, addr(buf[0]), buf.len) - if blen == 0: # no more data in file - result = false - return - let n = min(blen - bpos, slen - spos) - # TODO There's gotta be a better way of comparing here... - if not equalMem(addr(buf[bpos]), - cast[pointer](cast[int](cstring(s))+spos), n): - result = false - return - spos += n - bpos += n - - result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all - -proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} = - ## returns true if the contents of the file `f` equal `r`. If `f` does not - ## exist, false is returned. - var f: File - result = open(f, filename) - if result: - result = equalsFile(r, f) - close(f) +when not defined(js) and not defined(nimscript): + const + bufSize = 1024 # 1 KB is reasonable + + proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} = + ## Returns true if the contents of the file `f` equal `r`. + var + buf: array[bufSize, char] + bpos = buf.len + blen = buf.len + + for s in leaves(r): + var spos = 0 + let slen = s.len + while spos < slen: + if bpos == blen: + # Read more data + bpos = 0 + blen = readBuffer(f, addr(buf[0]), buf.len) + if blen == 0: # no more data in file + return false + let n = min(blen - bpos, slen - spos) + # TODO: There's gotta be a better way of comparing here... + if not equalMem(addr(buf[bpos]), + cast[pointer](cast[int](cstring(s)) + spos), n): + return false + spos += n + bpos += n + + result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all + + proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} = + ## Returns true if the contents of the file `f` equal `r`. If `f` does not + ## exist, false is returned. + var f: File + result = open(f, filename) + if result: + result = equalsFile(r, f) + close(f) new(N) # init dummy node for splay algorithm diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim index 2378fce17..65b059e86 100644 --- a/lib/pure/segfaults.nim +++ b/lib/pure/segfaults.nim @@ -8,27 +8,29 @@ # ## This modules registers a signal handler that turns access violations / -## segfaults into a ``NilAccessError`` exception. To be able to catch -## a NilAccessError all you have to do is to import this module. +## segfaults into a ``NilAccessDefect`` exception. To be able to catch +## a NilAccessDefect all you have to do is to import this module. ## ## Tested on these OSes: Linux, Windows, OSX +# xxx possibly broken on arm64, see bug #17178 + {.used.} # do allocate memory upfront: -var se: ref NilAccessError +var se: ref NilAccessDefect new(se) -se.name = "NilAccessError" +se.name = "NilAccessDefect" se.msg = "Could not access value because it is nil." when defined(windows): include "../system/ansi_c" - import winlean + import std/winlean const EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) - EXCEPTION_CONTINUE_SEARCH = Long(0) + EXCEPTION_CONTINUE_SEARCH = LONG(0) type PEXCEPTION_RECORD = ptr object @@ -63,7 +65,7 @@ when defined(windows): c_signal(SIGSEGV, segfaultHandler) else: - import posix + import std/posix var sa: Sigaction diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 0ab1e7754..ac180e2bd 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -9,11 +9,11 @@ ## This module allows high-level and efficient I/O multiplexing. ## -## Supported OS primitives: ``epoll``, ``kqueue``, ``poll`` and -## Windows ``select``. +## Supported OS primitives: `epoll`, `kqueue`, `poll` and +## Windows `select`. ## ## To use threadsafe version of this module, it needs to be compiled -## with both ``-d:threadsafe`` and ``--threads:on`` options. +## with both `-d:threadsafe` and `--threads:on` options. ## ## Supported features: files, sockets, pipes, timers, processes, signals ## and user events. @@ -25,18 +25,22 @@ ## Solaris (files, sockets, handles and user events). ## Android (files, sockets, handles and user events). ## -## TODO: ``/dev/poll``, ``event ports`` and filesystem events. +## TODO: `/dev/poll`, `event ports` and filesystem events. -import os, strutils, nativesockets +import std/nativesockets +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/assertions const hasThreadSupport = compileOption("threads") and defined(threadsafe) const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly) or - (defined(linux) and not defined(android)) + defined(dragonfly) or defined(nuttx) or + (defined(linux) and not defined(android) and not defined(emscripten)) ## This constant is used to determine whether the destination platform is - ## fully supported by ``ioselectors`` module. + ## fully supported by `ioselectors` module. const bsdPlatform = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or @@ -47,6 +51,9 @@ when defined(nimdoc): Selector*[T] = ref object ## An object which holds descriptors to be checked for read/write status + IOSelectorsException* = object of CatchableError + ## Exception that is raised if an IOSelectors error occurs. + Event* {.pure.} = enum ## An enum which hold event types Read, ## Descriptor is available for read @@ -83,62 +90,62 @@ when defined(nimdoc): proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = - ## Registers file/socket descriptor ``fd`` to selector ``s`` - ## with events set in ``events``. The ``data`` is application-defined + ## Registers file/socket descriptor `fd` to selector `s` + ## with events set in `events`. The `data` is application-defined ## data, which will be passed when an event is triggered. proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = - ## Update file/socket descriptor ``fd``, registered in selector - ## ``s`` with new events set ``event``. + ## Update file/socket descriptor `fd`, registered in selector + ## `s` with new events set `event`. proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, data: T): int {.discardable.} = - ## Registers timer notification with ``timeout`` (in milliseconds) - ## to selector ``s``. + ## Registers timer notification with `timeout` (in milliseconds) + ## to selector `s`. ## - ## If ``oneshot`` is ``true``, timer will be notified only once. + ## If `oneshot` is `true`, timer will be notified only once. ## - ## Set ``oneshot`` to ``false`` if you want periodic notifications. + ## Set `oneshot` to `false` if you want periodic notifications. ## - ## The ``data`` is application-defined data, which will be passed, when + ## The `data` is application-defined data, which will be passed, when ## the timer is triggered. ## ## Returns the file descriptor for the registered timer. proc registerSignal*[T](s: Selector[T], signal: int, data: T): int {.discardable.} = - ## Registers Unix signal notification with ``signal`` to selector - ## ``s``. + ## Registers Unix signal notification with `signal` to selector + ## `s`. ## - ## The ``data`` is application-defined data, which will be + ## The `data` is application-defined data, which will be ## passed when signal raises. ## ## Returns the file descriptor for the registered signal. ## - ## **Note:** This function is not supported on ``Windows``. + ## **Note:** This function is not supported on `Windows`. proc registerProcess*[T](s: Selector[T], pid: int, data: T): int {.discardable.} = ## Registers a process id (pid) notification (when process has - ## exited) in selector ``s``. + ## exited) in selector `s`. ## - ## The ``data`` is application-defined data, which will be passed when - ## process with ``pid`` has exited. + ## The `data` is application-defined data, which will be passed when + ## process with `pid` has exited. ## ## Returns the file descriptor for the registered signal. proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = - ## Registers selector event ``ev`` in selector ``s``. + ## Registers selector event `ev` in selector `s`. ## - ## The ``data`` is application-defined data, which will be passed when - ## ``ev`` happens. + ## The `data` is application-defined data, which will be passed when + ## `ev` happens. proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], data: T) = ## Registers selector BSD/MacOSX specific vnode events for file - ## descriptor ``fd`` and events ``events``. - ## ``data`` application-defined data, which to be passed, when + ## descriptor `fd` and events `events`. + ## `data` application-defined data, which to be passed, when ## vnode event happens. ## ## **Note:** This function is supported only by BSD and MacOSX. @@ -147,79 +154,77 @@ when defined(nimdoc): ## Creates a new user-defined event. proc trigger*(ev: SelectEvent) = - ## Trigger event ``ev``. + ## Trigger event `ev`. proc close*(ev: SelectEvent) = - ## Closes user-defined event ``ev``. + ## Closes user-defined event `ev`. proc unregister*[T](s: Selector[T], ev: SelectEvent) = - ## Unregisters user-defined event ``ev`` from selector ``s``. + ## Unregisters user-defined event `ev` from selector `s`. proc unregister*[T](s: Selector[T], fd: int|SocketHandle|cint) = - ## Unregisters file/socket descriptor ``fd`` from selector ``s``. + ## Unregisters file/socket descriptor `fd` from selector `s`. proc selectInto*[T](s: Selector[T], timeout: int, - results: var openarray[ReadyKey]): int = - ## Waits for events registered in selector ``s``. + results: var openArray[ReadyKey]): int = + ## Waits for events registered in selector `s`. ## - ## The ``timeout`` argument specifies the maximum number of milliseconds + ## The `timeout` argument specifies the maximum number of milliseconds ## the function will be blocked for if no events are ready. Specifying a - ## timeout of ``-1`` causes the function to block indefinitely. - ## All available events will be stored in ``results`` array. + ## timeout of `-1` causes the function to block indefinitely. + ## All available events will be stored in `results` array. ## ## Returns number of triggered events. proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = - ## Waits for events registered in selector ``s``. + ## Waits for events registered in selector `s`. ## - ## The ``timeout`` argument specifies the maximum number of milliseconds + ## The `timeout` argument specifies the maximum number of milliseconds ## the function will be blocked for if no events are ready. Specifying a - ## timeout of ``-1`` causes the function to block indefinitely. + ## timeout of `-1` causes the function to block indefinitely. ## ## Returns a list of triggered events. proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = - ## Retrieves application-defined ``data`` associated with descriptor ``fd``. - ## If specified descriptor ``fd`` is not registered, empty/default value + ## Retrieves application-defined `data` associated with descriptor `fd`. + ## If specified descriptor `fd` is not registered, empty/default value ## will be returned. proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: var T): bool = - ## Associate application-defined ``data`` with descriptor ``fd``. + ## Associate application-defined `data` with descriptor `fd`. ## - ## Returns ``true``, if data was successfully updated, ``false`` otherwise. + ## Returns `true`, if data was successfully updated, `false` otherwise. template isEmpty*[T](s: Selector[T]): bool = # TODO: Why is this a template? - ## Returns ``true``, if there are no registered events or descriptors + ## Returns `true`, if there are no registered events or descriptors ## in selector. template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body: untyped) = - ## Retrieves the application-data assigned with descriptor ``fd`` - ## to ``value``. This ``value`` can be modified in the scope of - ## the ``withData`` call. - ## - ## .. code-block:: nim + ## Retrieves the application-data assigned with descriptor `fd` + ## to `value`. This `value` can be modified in the scope of + ## the `withData` call. ## + ## ```nim ## s.withData(fd, value) do: - ## # block is executed only if ``fd`` registered in selector ``s`` + ## # block is executed only if `fd` registered in selector `s` ## value.uid = 1000 - ## + ## ``` template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = - ## Retrieves the application-data assigned with descriptor ``fd`` - ## to ``value``. This ``value`` can be modified in the scope of - ## the ``withData`` call. - ## - ## .. code-block:: nim + ## Retrieves the application-data assigned with descriptor `fd` + ## to `value`. This `value` can be modified in the scope of + ## the `withData` call. ## + ## ```nim ## s.withData(fd, value) do: - ## # block is executed only if ``fd`` registered in selector ``s``. + ## # block is executed only if `fd` registered in selector `s`. ## value.uid = 1000 ## do: - ## # block is executed if ``fd`` not registered in selector ``s``. + ## # block is executed if `fd` not registered in selector `s`. ## raise - ## + ## ``` proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. @@ -227,11 +232,12 @@ when defined(nimdoc): proc getFd*[T](s: Selector[T]): int = ## Retrieves the underlying selector's file descriptor. ## - ## For *poll* and *select* selectors ``-1`` is returned. + ## For *poll* and *select* selectors `-1` is returned. else: + import std/strutils when hasThreadSupport: - import locks + import std/locks type SharedArray[T] = UncheckedArray[T] @@ -239,8 +245,8 @@ else: proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) - proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] = - result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize)) + proc reallocSharedArray[T](sa: ptr SharedArray[T], oldsize, nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared0(sa, oldsize * sizeof(T), sizeof(T) * nsize)) proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) @@ -251,10 +257,10 @@ else: VnodeRename, VnodeRevoke type - IOSelectorsException* = object of Exception + IOSelectorsException* = object of CatchableError ReadyKey* = object - fd* : int + fd*: int events*: set[Event] errorCode*: OSErrorCode @@ -282,14 +288,14 @@ else: setBlocking(fd.SocketHandle, false) when not defined(windows): - import posix + import std/posix template setKey(s, pident, pevents, pparam, pdata: untyped) = var skey = addr(s.fds[pident]) skey.ident = pident skey.events = pevents skey.param = pparam - skey.data = data + skey.data = pdata when ioselSupportedPlatform: template blockSignals(newmask: var Sigset, oldmask: var Sigset) = @@ -317,9 +323,37 @@ else: proc verifySelectParams(timeout: int) = # Timeout of -1 means: wait forever # Anything higher is the time to wait in milliseconds. - doAssert(timeout >= -1, "Cannot select with a negative value, got " & $timeout) - - when defined(linux): + doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) + + when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(haiku): + template maxDescriptors*(): int = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. + when defined(windows): + 16_700_000 + elif defined(zephyr) or defined(freertos): + FD_MAX + else: + var fdLim: RLimit + var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) + if res >= 0: + res = int(fdLim.rlim_cur) - 1 + res + + when defined(nimIoselector): + when nimIoselector == "epoll": + include ioselects/ioselectors_epoll + elif nimIoselector == "kqueue": + include ioselects/ioselectors_kqueue + elif nimIoselector == "poll": + include ioselects/ioselectors_poll + elif nimIoselector == "select": + include ioselects/ioselectors_select + else: + {.fatal: "Unknown nimIoselector specified by define.".} + elif defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll elif bsdPlatform: include ioselects/ioselectors_kqueue @@ -331,24 +365,11 @@ else: include ioselects/ioselectors_select # TODO: use the native VFS layer elif defined(nintendoswitch): include ioselects/ioselectors_select + elif defined(freertos) or defined(lwip): + include ioselects/ioselectors_select + elif defined(zephyr): + include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: include ioselects/ioselectors_poll - -proc register*[T](s: Selector[T], fd: int | SocketHandle, - events: set[Event], data: T) {.deprecated: "use registerHandle instead".} = - ## **Deprecated since v0.18.0:** Use ``registerHandle`` instead. - s.registerHandle(fd, events, data) - -proc setEvent*(ev: SelectEvent) {.deprecated: "use trigger instead".} = - ## Trigger event ``ev``. - ## - ## **Deprecated since v0.18.0:** Use ``trigger`` instead. - ev.trigger() - -proc update*[T](s: Selector[T], fd: int | SocketHandle, - events: set[Event]) {.deprecated: "use updateHandle instead".} = - ## Update file/socket descriptor ``fd``, registered in selector - ## ``s`` with new events set ``event``. - ## - ## **Deprecated since v0.18.0:** Use ``updateHandle`` instead. - s.updateHandle() diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim deleted file mode 100644 index 1a9e22029..000000000 --- a/lib/pure/smtp.nim +++ /dev/null @@ -1,279 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements the SMTP client protocol as specified by RFC 5321, -## this can be used to send mail to any SMTP Server. -## -## This module also implements the protocol used to format messages, -## as specified by RFC 2822. -## -## Example gmail use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(useSsl = true, debug=true) -## smtpConn.connect("smtp.gmail.com", Port 465) -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## Example for startTls use: -## -## -## .. code-block:: Nim -## var msg = createMessage("Hello from Nim's SMTP", -## "Hello!.\n Is this awesome or what?", -## @["foo@gmail.com"]) -## let smtpConn = newSmtp(debug=true) -## smtpConn.connect("smtp.mailtrap.io", Port 2525) -## smtpConn.startTls() -## smtpConn.auth("username", "password") -## smtpConn.sendmail("username@gmail.com", @["foo@gmail.com"], $msg) -## -## -## For SSL support this module relies on OpenSSL. If you want to -## enable SSL, compile with ``-d:ssl``. - -import net, strutils, strtabs, base64, os -import asyncnet, asyncdispatch - -export Port - -type - Message* = object - msgTo: seq[string] - msgCc: seq[string] - msgSubject: string - msgOtherHeaders: StringTableRef - msgBody: string - - ReplyError* = object of IOError - - SmtpBase[SocketType] = ref object - sock: SocketType - debug: bool - - Smtp* = SmtpBase[Socket] - AsyncSmtp* = SmtpBase[AsyncSocket] - -proc debugSend(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} = - if smtp.debug: - echo("C:" & cmd) - await smtp.sock.send(cmd) - -proc debugRecv(smtp: Smtp | AsyncSmtp): Future[TaintedString] {.multisync.} = - result = await smtp.sock.recvLine() - if smtp.debug: - echo("S:" & result.string) - -proc quitExcpt(smtp: Smtp, msg: string) = - smtp.debugSend("QUIT") - raise newException(ReplyError, msg) - -const compiledWithSsl = defined(ssl) - -when not defined(ssl): - type PSSLContext = ref object - let defaultSSLContext: PSSLContext = nil -else: - var defaultSSLContext {.threadvar.}: SSLContext - - proc getSSLContext(): SSLContext = - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) - result = defaultSSLContext - -proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string], - otherHeaders: openarray[tuple[name, value: string]]): Message = - ## Creates a new MIME compliant message. - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - for n, v in items(otherHeaders): - result.msgOtherHeaders[n] = v - -proc createMessage*(mSubject, mBody: string, mTo, - mCc: seq[string] = @[]): Message = - ## Alternate version of the above. - result.msgTo = mTo - result.msgCc = mCc - result.msgSubject = mSubject - result.msgBody = mBody - result.msgOtherHeaders = newStringTable() - -proc `$`*(msg: Message): string = - ## stringify for ``Message``. - result = "" - if msg.msgTo.len() > 0: - result = "TO: " & msg.msgTo.join(", ") & "\c\L" - if msg.msgCc.len() > 0: - result.add("CC: " & msg.msgCc.join(", ") & "\c\L") - # TODO: Folding? i.e when a line is too long, shorten it... - result.add("Subject: " & msg.msgSubject & "\c\L") - for key, value in pairs(msg.msgOtherHeaders): - result.add(key & ": " & value & "\c\L") - - result.add("\c\L") - result.add(msg.msgBody) - -proc newSmtp*(useSsl = false, debug = false, - sslContext: SSLContext = nil): Smtp = - ## Creates a new ``Smtp`` instance. - new result - result.debug = debug - result.sock = newSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc newAsyncSmtp*(useSsl = false, debug = false, - sslContext: SSLContext = nil): AsyncSmtp = - ## Creates a new ``AsyncSmtp`` instance. - new result - result.debug = debug - - result.sock = newAsyncSocket() - if useSsl: - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapSocket(result.sock) - else: - sslContext.wrapSocket(result.sock) - else: - {.error: "SMTP module compiled without SSL support".} - -proc quitExcpt(smtp: AsyncSmtp, msg: string): Future[void] = - var retFuture = newFuture[void]() - var sendFut = smtp.debugSend("QUIT") - sendFut.callback = - proc () = - retFuture.fail(newException(ReplyError, msg)) - return retFuture - -proc checkReply(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = - var line = await smtp.debugRecv() - if not line.startswith(reply): - await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) - -proc connect*(smtp: Smtp | AsyncSmtp, - address: string, port: Port) {.multisync.} = - ## Establishes a connection with a SMTP server. - ## May fail with ReplyError or with a socket error. - await smtp.sock.connect(address, port) - - await smtp.checkReply("220") - await smtp.debugSend("HELO " & address & "\c\L") - await smtp.checkReply("250") - -proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SSLContext = nil) {.multisync.} = - ## Put the SMTP connection in TLS (Transport Layer Security) mode. - ## May fail with ReplyError - await smtp.debugSend("STARTTLS\c\L") - await smtp.checkReply("220") - when compiledWithSsl: - if sslContext == nil: - getSSLContext().wrapConnectedSocket(smtp.sock, handshakeAsClient) - else: - sslContext.wrapConnectedSocket(smtp.sock, handshakeAsClient) - else: - {.error: "SMTP module compiled without SSL support".} - -proc auth*(smtp: Smtp | AsyncSmtp, username, password: string) {.multisync.} = - ## Sends an AUTH command to the server to login as the `username` - ## using `password`. - ## May fail with ReplyError. - - await smtp.debugSend("AUTH LOGIN\c\L") - await smtp.checkReply("334") # TODO: Check whether it's asking for the "Username:" - # i.e "334 VXNlcm5hbWU6" - await smtp.debugSend(encode(username) & "\c\L") - await smtp.checkReply("334") # TODO: Same as above, only "Password:" (I think?) - - await smtp.debugSend(encode(password) & "\c\L") - await smtp.checkReply("235") # Check whether the authentication was successful. - -proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string, - toAddrs: seq[string], msg: string) {.multisync.} = - ## Sends ``msg`` from ``fromAddr`` to the addresses specified in ``toAddrs``. - ## Messages may be formed using ``createMessage`` by converting the - ## Message into a string. - - await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L") - await smtp.checkReply("250") - for address in items(toAddrs): - await smtp.debugSend("RCPT TO:<" & address & ">\c\L") - await smtp.checkReply("250") - - # Send the message - await smtp.debugSend("DATA " & "\c\L") - await smtp.checkReply("354") - await smtp.sock.send(msg & "\c\L") - await smtp.debugSend(".\c\L") - await smtp.checkReply("250") - -proc close*(smtp: Smtp | AsyncSmtp) {.multisync.} = - ## Disconnects from the SMTP server and closes the socket. - await smtp.debugSend("QUIT\c\L") - smtp.sock.close() - -when not defined(testing) and isMainModule: - # To test with a real SMTP service, create a smtp.ini file, e.g.: - # username = "" - # password = "" - # smtphost = "smtp.gmail.com" - # port = 465 - # use_tls = true - # sender = "" - # recipient = "" - - import parsecfg - - proc `[]`(c: Config, key: string): string = c.getSectionValue("", key) - - let - conf = loadConfig("smtp.ini") - msg = createMessage("Hello from Nim's SMTP!", - "Hello!\n Is this awesome or what?", @[conf["recipient"]]) - - assert conf["smtphost"] != "" - - proc async_test() {.async.} = - let client = newAsyncSmtp( - conf["use_tls"].parseBool, - debug = true - ) - await client.connect(conf["smtphost"], conf["port"].parseInt.Port) - await client.auth(conf["username"], conf["password"]) - await client.sendMail(conf["sender"], @[conf["recipient"]], $msg) - await client.close() - echo "async email sent" - - proc sync_test() = - var smtpConn = newSmtp( - conf["use_tls"].parseBool, - debug = true - ) - smtpConn.connect(conf["smtphost"], conf["port"].parseInt.Port) - smtpConn.auth(conf["username"], conf["password"]) - smtpConn.sendMail(conf["sender"], @[conf["recipient"]], $msg) - smtpConn.close() - echo "sync email sent" - - waitFor async_test() - sync_test() diff --git a/lib/pure/smtp.nim.cfg b/lib/pure/smtp.nim.cfg deleted file mode 100644 index 521e21de4..000000000 --- a/lib/pure/smtp.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:ssl diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim new file mode 100644 index 000000000..d60cd22eb --- /dev/null +++ b/lib/pure/ssl_certs.nim @@ -0,0 +1,172 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Scan for SSL/TLS CA certificates on disk +## The default locations can be overridden using the SSL_CERT_FILE and +## SSL_CERT_DIR environment variables. + +import std/[os, strutils] + +# FWIW look for files before scanning entire dirs. + +when defined(macosx): + const certificatePaths = [ + "/etc/ssl/cert.pem", + "/System/Library/OpenSSL/certs/cert.pem" + ] +elif defined(linux): + const certificatePaths = [ + # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo + # NetBSD (security/mozilla-rootcerts) + # SLES10/SLES11, https://golang.org/issue/12139 + "/etc/ssl/certs/ca-certificates.crt", + # OpenSUSE + "/etc/ssl/ca-bundle.pem", + # Red Hat 5+, Fedora, Centos + "/etc/pki/tls/certs/ca-bundle.crt", + # Red Hat 4 + "/usr/share/ssl/certs/ca-bundle.crt", + # Fedora/RHEL + "/etc/pki/tls/certs", + # Android + "/data/data/com.termux/files/usr/etc/tls/cert.pem", + "/system/etc/security/cacerts", + ] +elif defined(bsd): + const certificatePaths = [ + # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo + # NetBSD (security/mozilla-rootcerts) + # SLES10/SLES11, https://golang.org/issue/12139 + "/etc/ssl/certs/ca-certificates.crt", + # FreeBSD (security/ca-root-nss package) + "/usr/local/share/certs/ca-root-nss.crt", + # OpenBSD, FreeBSD (optional symlink) + "/etc/ssl/cert.pem", + # FreeBSD + "/usr/local/share/certs", + # NetBSD + "/etc/openssl/certs", + ] +else: + const certificatePaths = [ + # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo + # NetBSD (security/mozilla-rootcerts) + # SLES10/SLES11, https://golang.org/issue/12139 + "/etc/ssl/certs/ca-certificates.crt", + # OpenSUSE + "/etc/ssl/ca-bundle.pem", + # Red Hat 5+, Fedora, Centos + "/etc/pki/tls/certs/ca-bundle.crt", + # Red Hat 4 + "/usr/share/ssl/certs/ca-bundle.crt", + # FreeBSD (security/ca-root-nss package) + "/usr/local/share/certs/ca-root-nss.crt", + # CentOS/RHEL 7 + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + # OpenBSD, FreeBSD (optional symlink) + "/etc/ssl/cert.pem", + # Fedora/RHEL + "/etc/pki/tls/certs", + # Android + "/system/etc/security/cacerts", + # FreeBSD + "/usr/local/share/certs", + # NetBSD + "/etc/openssl/certs", + ] + +when defined(haiku): + const + B_FIND_PATH_EXISTING_ONLY = 0x4 + B_FIND_PATH_DATA_DIRECTORY = 6 + + proc find_paths_etc(architecture: cstring, baseDirectory: cint, + subPath: cstring, flags: uint32, + paths: var ptr UncheckedArray[cstring], + pathCount: var csize_t): int32 + {.importc, header: "<FindDirectory.h>".} + proc free(p: pointer) {.importc, header: "<stdlib.h>".} + +iterator scanSSLCertificates*(useEnvVars = false): string = + ## Scan for SSL/TLS CA certificates on disk. + ## + ## if `useEnvVars` is true, the SSL_CERT_FILE and SSL_CERT_DIR + ## environment variables can be used to override the certificate + ## directories to scan or specify a CA certificate file. + if useEnvVars and existsEnv("SSL_CERT_FILE"): + yield getEnv("SSL_CERT_FILE") + + elif useEnvVars and existsEnv("SSL_CERT_DIR"): + let p = getEnv("SSL_CERT_DIR") + for fn in joinPath(p, "*").walkFiles(): + yield fn + + else: + when defined(windows): + const cacert = "cacert.pem" + let pem = getAppDir() / cacert + if fileExists(pem): + yield pem + else: + let path = getEnv("PATH") + for candidate in split(path, PathSep): + if candidate.len != 0: + let x = (if candidate[0] == '"' and candidate[^1] == '"': + substr(candidate, 1, candidate.len-2) else: candidate) / cacert + if fileExists(x): + yield x + elif not defined(haiku): + for p in certificatePaths: + if p.endsWith(".pem") or p.endsWith(".crt"): + if fileExists(p): + yield p + elif dirExists(p): + # check if it's a dir where each cert is one file + # named by it's hasg + for fn in joinPath(p, "*.0").walkFiles: + yield p.normalizePathEnd(true) + break + for fn in joinPath(p, "*").walkFiles(): + + yield fn + else: + var + paths: ptr UncheckedArray[cstring] + size: csize_t + let err = find_paths_etc( + nil, B_FIND_PATH_DATA_DIRECTORY, "ssl/CARootCertificates.pem", + B_FIND_PATH_EXISTING_ONLY, paths, size + ) + if err == 0: + defer: free(paths) + for i in 0 ..< size: + yield $paths[i] + +# Certificates management on windows +# when defined(windows) or defined(nimdoc): +# +# import std/openssl +# +# type +# PCCertContext {.final, pure.} = pointer +# X509 {.final, pure.} = pointer +# CertStore {.final, pure.} = pointer +# +# # OpenSSL cert store +# +# {.push stdcall, dynlib: "kernel32", importc.} +# +# proc CertOpenSystemStore*(hprov: pointer=nil, szSubsystemProtocol: cstring): CertStore +# +# proc CertEnumCertificatesInStore*(hCertStore: CertStore, pPrevCertContext: PCCertContext): pointer +# +# proc CertFreeCertificateContext*(pContext: PCCertContext): bool +# +# proc CertCloseStore*(hCertStore:CertStore, flags:cint): bool +# +# {.pop.} diff --git a/lib/pure/ssl_config.nim b/lib/pure/ssl_config.nim new file mode 100644 index 000000000..14f66ede4 --- /dev/null +++ b/lib/pure/ssl_config.nim @@ -0,0 +1,51 @@ +# This file was automatically generated by tools/ssl_config_parser on 2020-06-03T22:02:05Z. DO NOT EDIT. + +## This module contains SSL configuration parameters obtained from +## `Mozilla OpSec <https://wiki.mozilla.org/Security/Server_Side_TLS>`_. +## +## The configuration file used to generate this module: https://ssl-config.mozilla.org/guidelines/5.4.json + +const CiphersModern* = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" + ## An OpenSSL-compatible list of secure ciphers for ``modern`` compatibility + ## per Mozilla's recommendations. + ## + ## Oldest clients supported by this list: + ## * Firefox 63 + ## * Android 10.0 + ## * Chrome 70 + ## * Edge 75 + ## * Java 11 + ## * OpenSSL 1.1.1 + ## * Opera 57 + ## * Safari 12.1 + +const CiphersIntermediate* = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" + ## An OpenSSL-compatible list of secure ciphers for ``intermediate`` compatibility + ## per Mozilla's recommendations. + ## + ## Oldest clients supported by this list: + ## * Firefox 27 + ## * Android 4.4.2 + ## * Chrome 31 + ## * Edge + ## * IE 11 on Windows 7 + ## * Java 8u31 + ## * OpenSSL 1.0.1 + ## * Opera 20 + ## * Safari 9 + +const CiphersOld* = "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" + ## An OpenSSL-compatible list of secure ciphers for ``old`` compatibility + ## per Mozilla's recommendations. + ## + ## Oldest clients supported by this list: + ## * Firefox 1 + ## * Android 2.3 + ## * Chrome 1 + ## * Edge 12 + ## * IE8 on Windows XP + ## * Java 6 + ## * OpenSSL 0.9.8 + ## * Opera 5 + ## * Safari 1 + diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index d62276da9..6a4fd8f01 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -9,67 +9,77 @@ ## Statistical analysis framework for performing ## basic statistical analysis of data. -## The data is analysed in a single pass, when a data value -## is pushed to the ``RunningStat`` or ``RunningRegress`` objects +## The data is analysed in a single pass, when it +## is pushed to a `RunningStat` or `RunningRegress` object. ## -## ``RunningStat`` calculates for a single data set +## `RunningStat` calculates for a single data set ## - n (data count) -## - min (smallest value) -## - max (largest value) +## - min (smallest value) +## - max (largest value) ## - sum ## - mean ## - variance -## - varianceS (sample var) +## - varianceS (sample variance) ## - standardDeviation -## - standardDeviationS (sample stddev) +## - standardDeviationS (sample standard deviation) ## - skewness (the third statistical moment) ## - kurtosis (the fourth statistical moment) ## -## ``RunningRegress`` calculates for two sets of data -## - n +## `RunningRegress` calculates for two sets of data +## - n (data count) ## - slope ## - intercept ## - correlation ## -## Procs have been provided to calculate statistics on arrays and sequences. +## Procs are provided to calculate statistics on `openArray`s. ## ## However, if more than a single statistical calculation is required, it is more -## efficient to push the data once to the RunningStat object, and -## call the numerous statistical procs for the RunningStat object. -## -## .. code-block:: Nim -## -## var rs: RunningStat -## rs.push(MySeqOfData) -## rs.mean() -## rs.variance() -## rs.skewness() -## rs.kurtosis() +## efficient to push the data once to a `RunningStat` object and then +## call the numerous statistical procs for the `RunningStat` object: + +runnableExamples: + from std/math import almostEqual + + template `~=`(a, b: float): bool = almostEqual(a, b) + + var statistics: RunningStat # must be var + statistics.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) + doAssert statistics.n == 8 + doAssert statistics.mean() ~= 2.0 + doAssert statistics.variance() ~= 1.5 + doAssert statistics.varianceS() ~= 1.714285714285715 + doAssert statistics.skewness() ~= 0.8164965809277261 + doAssert statistics.skewnessS() ~= 1.018350154434631 + doAssert statistics.kurtosis() ~= -1.0 + doAssert statistics.kurtosisS() ~= -0.7000000000000008 + +from std/math import FloatClass, sqrt, pow, round -from math import FloatClass, sqrt, pow, round +when defined(nimPreviewSlimSystem): + import std/[assertions, formatfloat] {.push debugger: off.} # the user does not want to trace a part # of the standard library! {.push checks: off, line_dir: off, stack_trace: off.} type - RunningStat* = object ## an accumulator for statistical data - n*: int ## number of pushed data + RunningStat* = object ## An accumulator for statistical data. + n*: int ## amount of pushed data min*, max*, sum*: float ## self-explaining mom1, mom2, mom3, mom4: float ## statistical moments, mom1 is mean - - RunningRegress* = object ## an accumulator for regression calculations - n*: int ## number of pushed data - x_stats*: RunningStat ## stats for first set of data - y_stats*: RunningStat ## stats for second set of data + RunningRegress* = object ## An accumulator for regression calculations. + n*: int ## amount of pushed data + x_stats*: RunningStat ## stats for the first set of data + y_stats*: RunningStat ## stats for the second set of data s_xy: float ## accumulated data for combined xy # ----------- RunningStat -------------------------- + proc clear*(s: var RunningStat) = - ## reset `s` + ## Resets `s`. s.n = 0 - s.min = toBiggestFloat(int.high) + s.min = 0.0 s.max = 0.0 s.sum = 0.0 s.mom1 = 0.0 @@ -78,12 +88,15 @@ proc clear*(s: var RunningStat) = s.mom4 = 0.0 proc push*(s: var RunningStat, x: float) = - ## pushes a value `x` for processing - if s.n == 0: s.min = x + ## Pushes a value `x` for processing. + if s.n == 0: + s.min = x + s.max = x + else: + if s.min > x: s.min = x + if s.max < x: s.max = x inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.min > x: s.min = x - if s.max < x: s.max = x s.sum += x let n = toFloat(s.n) let delta = x - s.mom1 @@ -97,63 +110,63 @@ proc push*(s: var RunningStat, x: float) = s.mom1 += delta_n proc push*(s: var RunningStat, x: int) = - ## pushes a value `x` for processing. + ## Pushes a value `x` for processing. ## - ## `x` is simply converted to ``float`` + ## `x` is simply converted to `float` ## and the other push operation is called. s.push(toFloat(x)) -proc push*(s: var RunningStat, x: openarray[float|int]) = - ## pushes all values of `x` for processing. +proc push*(s: var RunningStat, x: openArray[float|int]) = + ## Pushes all values of `x` for processing. ## - ## Int values of `x` are simply converted to ``float`` and + ## Int values of `x` are simply converted to `float` and ## the other push operation is called. for val in x: s.push(val) proc mean*(s: RunningStat): float = - ## computes the current mean of `s` + ## Computes the current mean of `s`. result = s.mom1 proc variance*(s: RunningStat): float = - ## computes the current population variance of `s` + ## Computes the current population variance of `s`. result = s.mom2 / toFloat(s.n) proc varianceS*(s: RunningStat): float = - ## computes the current sample variance of `s` + ## Computes the current sample variance of `s`. if s.n > 1: result = s.mom2 / toFloat(s.n - 1) proc standardDeviation*(s: RunningStat): float = - ## computes the current population standard deviation of `s` + ## Computes the current population standard deviation of `s`. result = sqrt(variance(s)) proc standardDeviationS*(s: RunningStat): float = - ## computes the current sample standard deviation of `s` + ## Computes the current sample standard deviation of `s`. result = sqrt(varianceS(s)) proc skewness*(s: RunningStat): float = - ## computes the current population skewness of `s` + ## Computes the current population skewness of `s`. result = sqrt(toFloat(s.n)) * s.mom3 / pow(s.mom2, 1.5) proc skewnessS*(s: RunningStat): float = - ## computes the current sample skewness of `s` + ## Computes the current sample skewness of `s`. let s2 = skewness(s) result = sqrt(toFloat(s.n*(s.n-1)))*s2 / toFloat(s.n-2) proc kurtosis*(s: RunningStat): float = - ## computes the current population kurtosis of `s` + ## Computes the current population kurtosis of `s`. result = toFloat(s.n) * s.mom4 / (s.mom2 * s.mom2) - 3.0 proc kurtosisS*(s: RunningStat): float = - ## computes the current sample kurtosis of `s` + ## Computes the current sample kurtosis of `s`. result = toFloat(s.n-1) / toFloat((s.n-2)*(s.n-3)) * (toFloat(s.n+1)*kurtosis(s) + 6) proc `+`*(a, b: RunningStat): RunningStat = - ## combine two RunningStats. + ## Combines two `RunningStat`s. ## - ## Useful if performing parallel analysis of data series - ## and need to re-combine parallel result sets + ## Useful when performing parallel analysis of data series + ## and needing to re-combine parallel result sets. result.clear() result.n = a.n + b.n @@ -178,11 +191,11 @@ proc `+`*(a, b: RunningStat): RunningStat = result.min = min(a.min, b.min) proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = - ## add a second RunningStats `b` to `a` + ## Adds the `RunningStat` `b` to `a`. a = a + b proc `$`*(a: RunningStat): string = - ## produces a string representation of the ``RunningStat``. The exact + ## Produces a string representation of the `RunningStat`. The exact ## format is currently unspecified and subject to change. Currently ## it contains: ## @@ -199,56 +212,57 @@ proc `$`*(a: RunningStat): string = result.add ")" # ---------------------- standalone array/seq stats --------------------- + proc mean*[T](x: openArray[T]): float = - ## computes the mean of `x` + ## Computes the mean of `x`. var rs: RunningStat rs.push(x) result = rs.mean() proc variance*[T](x: openArray[T]): float = - ## computes the population variance of `x` + ## Computes the population variance of `x`. var rs: RunningStat rs.push(x) result = rs.variance() proc varianceS*[T](x: openArray[T]): float = - ## computes the sample variance of `x` + ## Computes the sample variance of `x`. var rs: RunningStat rs.push(x) result = rs.varianceS() proc standardDeviation*[T](x: openArray[T]): float = - ## computes the population standardDeviation of `x` + ## Computes the population standard deviation of `x`. var rs: RunningStat rs.push(x) result = rs.standardDeviation() proc standardDeviationS*[T](x: openArray[T]): float = - ## computes the sample standardDeviation of `x` + ## Computes the sample standard deviation of `x`. var rs: RunningStat rs.push(x) result = rs.standardDeviationS() proc skewness*[T](x: openArray[T]): float = - ## computes the population skewness of `x` + ## Computes the population skewness of `x`. var rs: RunningStat rs.push(x) result = rs.skewness() proc skewnessS*[T](x: openArray[T]): float = - ## computes the sample skewness of `x` + ## Computes the sample skewness of `x`. var rs: RunningStat rs.push(x) result = rs.skewnessS() proc kurtosis*[T](x: openArray[T]): float = - ## computes the population kurtosis of `x` + ## Computes the population kurtosis of `x`. var rs: RunningStat rs.push(x) result = rs.kurtosis() proc kurtosisS*[T](x: openArray[T]): float = - ## computes the sample kurtosis of `x` + ## Computes the sample kurtosis of `x`. var rs: RunningStat rs.push(x) result = rs.kurtosisS() @@ -256,14 +270,14 @@ proc kurtosisS*[T](x: openArray[T]): float = # ---------------------- Running Regression ----------------------------- proc clear*(r: var RunningRegress) = - ## reset `r` + ## Resets `r`. r.x_stats.clear() r.y_stats.clear() r.s_xy = 0.0 r.n = 0 proc push*(r: var RunningRegress, x, y: float) = - ## pushes two values `x` and `y` for processing + ## Pushes two values `x` and `y` for processing. r.s_xy += (r.x_stats.mean() - x)*(r.y_stats.mean() - y) * toFloat(r.n) / toFloat(r.n + 1) r.x_stats.push(x) @@ -271,38 +285,38 @@ proc push*(r: var RunningRegress, x, y: float) = inc(r.n) proc push*(r: var RunningRegress, x, y: int) {.inline.} = - ## pushes two values `x` and `y` for processing. + ## Pushes two values `x` and `y` for processing. ## - ## `x` and `y` are converted to ``float`` + ## `x` and `y` are converted to `float` ## and the other push operation is called. r.push(toFloat(x), toFloat(y)) -proc push*(r: var RunningRegress, x, y: openarray[float|int]) = - ## pushes two sets of values `x` and `y` for processing. +proc push*(r: var RunningRegress, x, y: openArray[float|int]) = + ## Pushes two sets of values `x` and `y` for processing. assert(x.len == y.len) for i in 0..<x.len: r.push(x[i], y[i]) proc slope*(r: RunningRegress): float = - ## computes the current slope of `r` + ## Computes the current slope of `r`. let s_xx = r.x_stats.varianceS()*toFloat(r.n - 1) result = r.s_xy / s_xx proc intercept*(r: RunningRegress): float = - ## computes the current intercept of `r` + ## Computes the current intercept of `r`. result = r.y_stats.mean() - r.slope()*r.x_stats.mean() proc correlation*(r: RunningRegress): float = - ## computes the current correlation of the two data - ## sets pushed into `r` + ## Computes the current correlation of the two data + ## sets pushed into `r`. let t = r.x_stats.standardDeviation() * r.y_stats.standardDeviation() result = r.s_xy / (toFloat(r.n) * t) proc `+`*(a, b: RunningRegress): RunningRegress = - ## combine two `RunningRegress` objects. + ## Combines two `RunningRegress` objects. ## - ## Useful if performing parallel analysis of data series - ## and need to re-combine parallel result sets + ## Useful when performing parallel analysis of data series + ## and needing to re-combine parallel result sets result.clear() result.x_stats = a.x_stats + b.x_stats result.y_stats = a.y_stats + b.y_stats @@ -314,54 +328,8 @@ proc `+`*(a, b: RunningRegress): RunningRegress = toFloat(a.n*b.n)*delta_x*delta_y/toFloat(result.n) proc `+=`*(a: var RunningRegress, b: RunningRegress) = - ## add RunningRegress `b` to `a` + ## Adds the `RunningRegress` `b` to `a`. a = a + b {.pop.} {.pop.} - -when isMainModule: - proc clean(x: float): float = - result = round(1.0e8*x).float * 1.0e-8 - - var rs: RunningStat - rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) - doAssert(rs.n == 8) - doAssert(clean(rs.mean) == 2.0) - doAssert(clean(rs.variance()) == 1.5) - doAssert(clean(rs.varianceS()) == 1.71428571) - doAssert(clean(rs.skewness()) == 0.81649658) - doAssert(clean(rs.skewnessS()) == 1.01835015) - doAssert(clean(rs.kurtosis()) == -1.0) - doAssert(clean(rs.kurtosisS()) == -0.7000000000000001) - - var rs1, rs2: RunningStat - rs1.push(@[1.0, 2.0, 1.0, 4.0]) - rs2.push(@[1.0, 4.0, 1.0, 2.0]) - let rs3 = rs1 + rs2 - doAssert(clean(rs3.mom2) == clean(rs.mom2)) - doAssert(clean(rs3.mom3) == clean(rs.mom3)) - doAssert(clean(rs3.mom4) == clean(rs.mom4)) - rs1 += rs2 - doAssert(clean(rs1.mom2) == clean(rs.mom2)) - doAssert(clean(rs1.mom3) == clean(rs.mom3)) - doAssert(clean(rs1.mom4) == clean(rs.mom4)) - rs1.clear() - rs1.push(@[1.0, 2.2, 1.4, 4.9]) - doAssert(rs1.sum == 9.5) - doAssert(rs1.mean() == 2.375) - - when not defined(cpu32): - # XXX For some reason on 32bit CPUs these results differ - var rr: RunningRegress - rr.push(@[0.0, 1.0, 2.8, 3.0, 4.0], @[0.0, 1.0, 2.3, 3.0, 4.0]) - doAssert(rr.slope() == 0.9695585996955861) - doAssert(rr.intercept() == -0.03424657534246611) - doAssert(rr.correlation() == 0.9905100362239381) - var rr1, rr2: RunningRegress - rr1.push(@[0.0, 1.0], @[0.0, 1.0]) - rr2.push(@[2.8, 3.0, 4.0], @[2.3, 3.0, 4.0]) - let rr3 = rr1 + rr2 - doAssert(rr3.correlation() == rr.correlation()) - doAssert(clean(rr3.slope()) == clean(rr.slope())) - doAssert(clean(rr3.intercept()) == clean(rr.intercept())) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index cdb881960..56f49d7b1 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -15,6 +15,11 @@ ## Other modules may provide other implementations for this standard ## stream interface. ## +## .. warning:: Due to the use of `pointer`, the `readData`, `peekData` and +## `writeData` interfaces are not available on the compile-time VM, and must +## be cast from a `ptr string` on the JS backend. However, `readDataStr` is +## available generally in place of `readData`. +## ## Basic usage ## =========== ## @@ -27,76 +32,78 @@ ## StringStream example ## -------------------- ## -## .. code-block:: Nim -## -## import streams +## ```Nim +## import std/streams ## -## var strm = newStringStream("""The first line -## the second line -## the third line""") +## var strm = newStringStream("""The first line +## the second line +## the third line""") ## -## var line = "" +## var line = "" ## -## while strm.readLine(line): -## echo line +## while strm.readLine(line): +## echo line ## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line ## -## strm.close() +## strm.close() +## ``` ## ## FileStream example ## ------------------ ## ## Write file stream example: ## -## .. code-block:: Nim -## -## import streams +## ```Nim +## import std/streams ## -## var strm = newFileStream("somefile.txt", fmWrite) -## var line = "" +## var strm = newFileStream("somefile.txt", fmWrite) +## var line = "" ## -## if not isNil(strm): -## strm.writeLine("The first line") -## strm.writeLine("the second line") -## strm.writeLine("the third line") -## strm.close() +## if not isNil(strm): +## strm.writeLine("The first line") +## strm.writeLine("the second line") +## strm.writeLine("the third line") +## strm.close() ## -## # Output (somefile.txt): -## # The first line -## # the second line -## # the third line +## # Output (somefile.txt): +## # The first line +## # the second line +## # the third line +## ``` ## ## Read file stream example: ## -## .. code-block:: Nim +## ```Nim +## import std/streams ## -## import streams +## var strm = newFileStream("somefile.txt", fmRead) +## var line = "" ## -## var strm = newFileStream("somefile.txt", fmRead) -## var line = "" +## if not isNil(strm): +## while strm.readLine(line): +## echo line +## strm.close() ## -## if not isNil(strm): -## while strm.readLine(line): -## echo line -## strm.close() -## -## # Output: -## # The first line -## # the second line -## # the third line +## # Output: +## # The first line +## # the second line +## # the third line +## ``` ## ## See also ## ======== ## * `asyncstreams module <asyncstreams.html>`_ -## * `io module <io.html>`_ for `FileMode enum <io.html#FileMode>`_ +## * `io module <syncio.html>`_ for `FileMode enum <syncio.html#FileMode>`_ -include "system/inclrtl" +import std/private/since -const taintMode = compileOption("taintmode") +when defined(nimPreviewSlimSystem): + import std/syncio + export FileMode proc newEIO(msg: string): owned(ref IOError) = new(result) @@ -113,7 +120,7 @@ type ## * That these fields here shouldn't be used directly. ## They are accessible so that a stream implementation can override them. closeImpl*: proc (s: Stream) - {.nimcall, raises: [Exception, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} atEndImpl*: proc (s: Stream): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} setPositionImpl*: proc (s: Stream, pos: int) @@ -124,7 +131,7 @@ type readDataStrImpl*: proc (s: Stream, buffer: var string, slice: Slice[int]): int {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} - readLineImpl*: proc(s: Stream, line: var TaintedString): bool + readLineImpl*: proc(s: Stream, line: var string): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} readDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int @@ -146,7 +153,7 @@ proc flush*(s: Stream) = ## See also: ## * `close proc <#close,Stream>`_ runnableExamples: - from os import removeFile + from std/os import removeFile var strm = newFileStream("somefile.txt", fmWrite) @@ -172,10 +179,20 @@ proc close*(s: Stream) = ## See also: ## * `flush proc <#flush,Stream>`_ runnableExamples: - var strm = newStringStream("The first line\nthe second line\nthe third line") - ## do something... - strm.close() - if not isNil(s.closeImpl): s.closeImpl(s) + block: + let strm = newStringStream("The first line\nthe second line\nthe third line") + ## do something... + strm.close() + + block: + let strm = newFileStream("amissingfile.txt") + # deferring works even if newFileStream fails + defer: strm.close() + if not isNil(strm): + ## do something... + + if not isNil(s) and not isNil(s.closeImpl): + s.closeImpl(s) proc atEnd*(s: Stream): bool = ## Checks if more data can be read from `s`. Returns ``true`` if all data has @@ -216,6 +233,9 @@ proc getPosition*(s: Stream): int = proc readData*(s: Stream, buffer: pointer, bufLen: int): int = ## Low level proc that reads data into an untyped `buffer` of `bufLen` size. + ## + ## **JS note:** `buffer` is treated as a ``ptr string`` and written to between + ## ``0..<bufLen``. runnableExamples: var strm = newStringStream("abcde") var buffer: array[6, char] @@ -239,13 +259,26 @@ proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = result = s.readDataStrImpl(s, buffer, slice) else: # fallback - result = s.readData(addr buffer[0], buffer.len) + when declared(prepareMutation): + # buffer might potentially be a CoW literal with ARC + prepareMutation(buffer) + result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) -when not defined(js): +template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = + when nimvm: + block: + caseJsOrVm + else: + block: + when defined(js) or defined(nimscript): + # nimscript has to be here to avoid semantic checking of caseElse + caseJsOrVm + else: + caseElse + +when (NimMajor, NimMinor) >= (1, 3) or not defined(js): proc readAll*(s: Stream): string = ## Reads all available data. - ## - ## **Note:** Not available for JS backend. runnableExamples: var strm = newStringStream("The first line\nthe second line\nthe third line") doAssert strm.readAll() == "The first line\nthe second line\nthe third line" @@ -253,7 +286,7 @@ when not defined(js): strm.close() const bufferSize = 1024 - when nimvm: + jsOrVmBlock: var buffer2: string buffer2.setLen(bufferSize) while true: @@ -265,7 +298,7 @@ when not defined(js): result[prevLen..<prevLen+readBytes] = buffer2[0..<readBytes] if readBytes < bufferSize: break - else: + do: # not JS or VM var buffer {.noinit.}: array[bufferSize, char] while true: let readBytes = readData(s, addr(buffer[0]), bufferSize) @@ -280,6 +313,9 @@ when not defined(js): proc peekData*(s: Stream, buffer: pointer, bufLen: int): int = ## Low level proc that reads data into an untyped `buffer` of `bufLen` size ## without moving stream position. + ## + ## **JS note:** `buffer` is treated as a ``ptr string`` and written to between + ## ``0..<bufLen``. runnableExamples: var strm = newStringStream("abcde") var buffer: array[6, char] @@ -293,6 +329,9 @@ proc peekData*(s: Stream, buffer: pointer, bufLen: int): int = proc writeData*(s: Stream, buffer: pointer, bufLen: int) = ## Low level proc that writes an untyped `buffer` of `bufLen` size ## to the stream `s`. + ## + ## **JS note:** `buffer` is treated as a ``ptr string`` and read between + ## ``0..<bufLen``. runnableExamples: ## writeData var strm = newStringStream("") @@ -311,9 +350,12 @@ proc writeData*(s: Stream, buffer: pointer, bufLen: int) = proc write*[T](s: Stream, x: T) = ## Generic write procedure. Writes `x` to the stream `s`. Implementation: ## - ## .. code-block:: Nim + ## **Note:** Not available for JS backend. Use `write(Stream, string) + ## <#write,Stream,string>`_ for now. ## - ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ```Nim + ## s.writeData(s, unsafeAddr(x), sizeof(x)) + ## ``` runnableExamples: var strm = newStringStream("") strm.write("abcde") @@ -324,7 +366,7 @@ proc write*[T](s: Stream, x: T) = writeData(s, unsafeAddr(x), sizeof(x)) proc write*(s: Stream, x: string) = - ## Writes the string `x` to the the stream `s`. No length field or + ## Writes the string `x` to the stream `s`. No length field or ## terminating zero is written. runnableExamples: var strm = newStringStream("") @@ -336,7 +378,12 @@ proc write*(s: Stream, x: string) = when nimvm: writeData(s, cstring(x), x.len) else: - if x.len > 0: writeData(s, cstring(x), x.len) + if x.len > 0: + when defined(js): + var x = x + writeData(s, addr(x), x.len) + else: + writeData(s, cstring(x), x.len) proc write*(s: Stream, args: varargs[string, `$`]) = ## Writes one or more strings to the the stream. No length fields or @@ -366,6 +413,8 @@ proc writeLine*(s: Stream, args: varargs[string, `$`]) = proc read*[T](s: Stream, result: var T) = ## Generic read procedure. Reads `result` from the stream `s`. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream("012") ## readInt @@ -383,6 +432,8 @@ proc read*[T](s: Stream, result: var T) = proc peek*[T](s: Stream, result: var T) = ## Generic peek procedure. Peeks `result` from the stream `s`. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream("012") ## peekInt @@ -412,11 +463,11 @@ proc readChar*(s: Stream): char = doAssert strm.readChar() == '\x00' strm.close() - when nimvm: + jsOrVmBlock: var str = " " - if readDataStr(s, str, 0..<sizeof(result)) != 1: result = '\0' + if readDataStr(s, str, 0..0) != 1: result = '\0' else: result = str[0] - else: + do: if readData(s, addr(result), sizeof(result)) != 1: result = '\0' proc peekChar*(s: Stream): char = @@ -430,7 +481,12 @@ proc peekChar*(s: Stream): char = doAssert strm.peekChar() == '\x00' strm.close() - if peekData(s, addr(result), sizeof(result)) != 1: result = '\0' + when defined(js): + var str = " " + if peekData(s, addr(str), sizeof(result)) != 1: result = '\0' + else: result = str[0] + else: + if peekData(s, addr(result), sizeof(result)) != 1: result = '\0' proc readBool*(s: Stream): bool = ## Reads a bool from the stream `s`. @@ -438,6 +494,8 @@ proc readBool*(s: Stream): bool = ## A bool is one byte long and it is `true` for every non-zero ## (`0000_0000`) value. ## Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -461,6 +519,8 @@ proc peekBool*(s: Stream): bool = ## A bool is one byte long and it is `true` for every non-zero ## (`0000_0000`) value. ## Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -482,6 +542,8 @@ proc peekBool*(s: Stream): bool = proc readInt8*(s: Stream): int8 = ## Reads an int8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -499,6 +561,8 @@ proc readInt8*(s: Stream): int8 = proc peekInt8*(s: Stream): int8 = ## Peeks an int8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -518,6 +582,8 @@ proc peekInt8*(s: Stream): int8 = proc readInt16*(s: Stream): int16 = ## Reads an int16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -535,6 +601,8 @@ proc readInt16*(s: Stream): int16 = proc peekInt16*(s: Stream): int16 = ## Peeks an int16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -554,6 +622,8 @@ proc peekInt16*(s: Stream): int16 = proc readInt32*(s: Stream): int32 = ## Reads an int32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -571,6 +641,8 @@ proc readInt32*(s: Stream): int32 = proc peekInt32*(s: Stream): int32 = ## Peeks an int32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -590,6 +662,8 @@ proc peekInt32*(s: Stream): int32 = proc readInt64*(s: Stream): int64 = ## Reads an int64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -607,6 +681,8 @@ proc readInt64*(s: Stream): int64 = proc peekInt64*(s: Stream): int64 = ## Peeks an int64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -626,6 +702,8 @@ proc peekInt64*(s: Stream): int64 = proc readUint8*(s: Stream): uint8 = ## Reads an uint8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -643,6 +721,8 @@ proc readUint8*(s: Stream): uint8 = proc peekUint8*(s: Stream): uint8 = ## Peeks an uint8 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -662,6 +742,8 @@ proc peekUint8*(s: Stream): uint8 = proc readUint16*(s: Stream): uint16 = ## Reads an uint16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -679,6 +761,8 @@ proc readUint16*(s: Stream): uint16 = proc peekUint16*(s: Stream): uint16 = ## Peeks an uint16 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -698,6 +782,8 @@ proc peekUint16*(s: Stream): uint16 = proc readUint32*(s: Stream): uint32 = ## Reads an uint32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -716,6 +802,8 @@ proc readUint32*(s: Stream): uint32 = proc peekUint32*(s: Stream): uint32 = ## Peeks an uint32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -735,6 +823,8 @@ proc peekUint32*(s: Stream): uint32 = proc readUint64*(s: Stream): uint64 = ## Reads an uint64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -752,6 +842,8 @@ proc readUint64*(s: Stream): uint64 = proc peekUint64*(s: Stream): uint64 = ## Peeks an uint64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -771,6 +863,8 @@ proc peekUint64*(s: Stream): uint64 = proc readFloat32*(s: Stream): float32 = ## Reads a float32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -788,6 +882,8 @@ proc readFloat32*(s: Stream): float32 = proc peekFloat32*(s: Stream): float32 = ## Peeks a float32 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -807,6 +903,8 @@ proc peekFloat32*(s: Stream): float32 = proc readFloat64*(s: Stream): float64 = ## Reads a float64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `readStr <#readStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -824,6 +922,8 @@ proc readFloat64*(s: Stream): float64 = proc peekFloat64*(s: Stream): float64 = ## Peeks a float64 from the stream `s`. Raises `IOError` if an error occurred. + ## + ## **Note:** Not available for JS backend. Use `peekStr <#peekStr,Stream,int>`_ for now. runnableExamples: var strm = newStringStream() ## setup for reading data @@ -841,7 +941,24 @@ proc peekFloat64*(s: Stream): float64 = peek(s, result) -proc readStr*(s: Stream, length: int): TaintedString = +proc readStrPrivate(s: Stream, length: int, str: var string) = + if length > len(str): setLen(str, length) + var L: int + when nimvm: + L = readDataStr(s, str, 0..length-1) + else: + when defined(js): + L = readData(s, addr(str), length) + else: + L = readData(s, cstring(str), length) + if L != len(str): setLen(str, L) + +proc readStr*(s: Stream, length: int, str: var string) {.since: (1, 3).} = + ## Reads a string of length `length` from the stream `s`. Raises `IOError` if + ## an error occurred. + readStrPrivate(s, length, str) + +proc readStr*(s: Stream, length: int): string = ## Reads a string of length `length` from the stream `s`. Raises `IOError` if ## an error occurred. runnableExamples: @@ -851,12 +968,23 @@ proc readStr*(s: Stream, length: int): TaintedString = doAssert strm.readStr(2) == "e" doAssert strm.readStr(2) == "" strm.close() + result = newString(length) + readStrPrivate(s, length, result) - result = newString(length).TaintedString - var L = readData(s, cstring(result), length) - if L != length: setLen(result.string, L) +proc peekStrPrivate(s: Stream, length: int, str: var string) = + if length > len(str): setLen(str, length) + when defined(js): + let L = peekData(s, addr(str), length) + else: + let L = peekData(s, cstring(str), length) + if L != len(str): setLen(str, L) -proc peekStr*(s: Stream, length: int): TaintedString = +proc peekStr*(s: Stream, length: int, str: var string) {.since: (1, 3).} = + ## Peeks a string of length `length` from the stream `s`. Raises `IOError` if + ## an error occurred. + peekStrPrivate(s, length, str) + +proc peekStr*(s: Stream, length: int): string = ## Peeks a string of length `length` from the stream `s`. Raises `IOError` if ## an error occurred. runnableExamples: @@ -867,12 +995,10 @@ proc peekStr*(s: Stream, length: int): TaintedString = doAssert strm.readStr(2) == "ab" doAssert strm.peekStr(2) == "cd" strm.close() + result = newString(length) + peekStrPrivate(s, length, result) - result = newString(length).TaintedString - var L = peekData(s, cstring(result), length) - if L != length: setLen(result.string, L) - -proc readLine*(s: Stream, line: var TaintedString): bool = +proc readLine*(s: Stream, line: var string): bool = ## Reads a line of text from the stream `s` into `line`. `line` must not be ## ``nil``! May throw an IO exception. ## @@ -884,7 +1010,7 @@ proc readLine*(s: Stream, line: var TaintedString): bool = ## See also: ## * `readLine(Stream) proc <#readLine,Stream>`_ ## * `peekLine(Stream) proc <#peekLine,Stream>`_ - ## * `peekLine(Stream, TaintedString) proc <#peekLine,Stream,TaintedString>`_ + ## * `peekLine(Stream, string) proc <#peekLine,Stream,string>`_ runnableExamples: var strm = newStringStream("The first line\nthe second line\nthe third line") var line = "" @@ -902,13 +1028,7 @@ proc readLine*(s: Stream, line: var TaintedString): bool = result = s.readLineImpl(s, line) else: # fallback - when nimvm: #Bug #12282 - when taintMode: - line.string.setLen(0) - else: - line.setLen(0) - else: - line.string.setLen(0) + line.setLen(0) while true: var c = readChar(s) if c == '\c': @@ -918,16 +1038,10 @@ proc readLine*(s: Stream, line: var TaintedString): bool = elif c == '\0': if line.len > 0: break else: return false - when nimvm: #Bug #12282 - when taintMode: - line.string.add(c) - else: - line.add(c) - else: - line.string.add(c) + line.add(c) result = true -proc peekLine*(s: Stream, line: var TaintedString): bool = +proc peekLine*(s: Stream, line: var string): bool = ## Peeks a line of text from the stream `s` into `line`. `line` must not be ## ``nil``! May throw an IO exception. ## @@ -938,7 +1052,7 @@ proc peekLine*(s: Stream, line: var TaintedString): bool = ## ## See also: ## * `readLine(Stream) proc <#readLine,Stream>`_ - ## * `readLine(Stream, TaintedString) proc <#readLine,Stream,TaintedString>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ ## * `peekLine(Stream) proc <#peekLine,Stream>`_ runnableExamples: var strm = newStringStream("The first line\nthe second line\nthe third line") @@ -958,15 +1072,15 @@ proc peekLine*(s: Stream, line: var TaintedString): bool = defer: setPosition(s, pos) result = readLine(s, line) -proc readLine*(s: Stream): TaintedString = +proc readLine*(s: Stream): string = ## Reads a line from a stream `s`. Raises `IOError` if an error occurred. ## ## **Note:** This is not very efficient. ## ## See also: - ## * `readLine(Stream, TaintedString) proc <#readLine,Stream,TaintedString>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ ## * `peekLine(Stream) proc <#peekLine,Stream>`_ - ## * `peekLine(Stream, TaintedString) proc <#peekLine,Stream,TaintedString>`_ + ## * `peekLine(Stream, string) proc <#peekLine,Stream,string>`_ runnableExamples: var strm = newStringStream("The first line\nthe second line\nthe third line") doAssert strm.readLine() == "The first line" @@ -975,7 +1089,7 @@ proc readLine*(s: Stream): TaintedString = doAssertRaises(IOError): discard strm.readLine() strm.close() - result = TaintedString"" + result = "" if s.atEnd: raise newEIO("cannot read from stream") while true: @@ -986,17 +1100,17 @@ proc readLine*(s: Stream): TaintedString = if c == '\L' or c == '\0': break else: - result.string.add(c) + result.add(c) -proc peekLine*(s: Stream): TaintedString = +proc peekLine*(s: Stream): string = ## Peeks a line from a stream `s`. Raises `IOError` if an error occurred. ## ## **Note:** This is not very efficient. ## ## See also: ## * `readLine(Stream) proc <#readLine,Stream>`_ - ## * `readLine(Stream, TaintedString) proc <#readLine,Stream,TaintedString>`_ - ## * `peekLine(Stream, TaintedString) proc <#peekLine,Stream,TaintedString>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ + ## * `peekLine(Stream, string) proc <#peekLine,Stream,string>`_ runnableExamples: var strm = newStringStream("The first line\nthe second line\nthe third line") doAssert strm.peekLine() == "The first line" @@ -1010,13 +1124,13 @@ proc peekLine*(s: Stream): TaintedString = defer: setPosition(s, pos) result = readLine(s) -iterator lines*(s: Stream): TaintedString = +iterator lines*(s: Stream): string = ## Iterates over every line in the stream. ## The iteration is based on ``readLine``. ## ## See also: ## * `readLine(Stream) proc <#readLine,Stream>`_ - ## * `readLine(Stream, TaintedString) proc <#readLine,Stream,TaintedString>`_ + ## * `readLine(Stream, string) proc <#readLine,Stream,string>`_ runnableExamples: var strm = newStringStream("The first line\nthe second line\nthe third line") var lines: seq[string] @@ -1025,24 +1139,20 @@ iterator lines*(s: Stream): TaintedString = doAssert lines == @["The first line", "the second line", "the third line"] strm.close() - var line: TaintedString + var line: string while s.readLine(line): yield line type StringStream* = ref StringStreamObj ## A stream that encapsulates a string. - ## - ## **Note:** Not available for JS backend. StringStreamObj* = object of StreamObj ## A string stream object. - ## - ## **Note:** Not available for JS backend. data*: string ## A string data. ## This is updated when called `writeLine` etc. pos: int -when defined(js): #This section exists so that string streams work at compile time for the js backend +when (NimMajor, NimMinor) < (1, 3) and defined(js): proc ssAtEnd(s: Stream): bool {.compileTime.} = var s = StringStream(s) return s.pos >= s.data.len @@ -1066,10 +1176,7 @@ when defined(js): #This section exists so that string streams work at compile ti proc ssClose(s: Stream) {.compileTime.} = var s = StringStream(s) - when defined(nimNoNilSeqs): - s.data = "" - else: - s.data = nil + s.data = "" proc newStringStream*(s: string = ""): owned StringStream {.compileTime.} = new(result) @@ -1095,7 +1202,7 @@ when defined(js): #This section exists so that string streams work at compile ti if readBytes < bufferSize: break -else: +else: # after 1.3 or JS not defined proc ssAtEnd(s: Stream): bool = var s = StringStream(s) return s.pos >= s.data.len @@ -1110,11 +1217,16 @@ else: proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = var s = StringStream(s) + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(buffer) # buffer might potentially be a CoW literal with ARC result = min(slice.b + 1 - slice.a, s.data.len - s.pos) if result > 0: - when nimvm: + jsOrVmBlock: buffer[slice.a..<slice.a+result] = s.data[s.pos..<s.pos+result] - else: + do: copyMem(unsafeAddr buffer[slice.a], addr s.data[s.pos], result) inc(s.pos, result) else: @@ -1124,7 +1236,14 @@ else: var s = StringStream(s) result = min(bufLen, s.data.len - s.pos) if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) + when defined(js): + try: + cast[ptr string](buffer)[][0..<result] = s.data[s.pos..<s.pos+result] + except: + raise newException(Defect, "could not read string stream, " & + "did you use a non-string buffer pointer?", getCurrentException()) + elif not defined(nimscript): + copyMem(buffer, addr(s.data[s.pos]), result) inc(s.pos, result) else: result = 0 @@ -1133,7 +1252,14 @@ else: var s = StringStream(s) result = min(bufLen, s.data.len - s.pos) if result > 0: - copyMem(buffer, addr(s.data[s.pos]), result) + when defined(js): + try: + cast[ptr string](buffer)[][0..<result] = s.data[s.pos..<s.pos+result] + except: + raise newException(Defect, "could not peek string stream, " & + "did you use a non-string buffer pointer?", getCurrentException()) + elif not defined(nimscript): + copyMem(buffer, addr(s.data[s.pos]), result) else: result = 0 @@ -1143,21 +1269,23 @@ else: return if s.pos + bufLen > s.data.len: setLen(s.data, s.pos + bufLen) - copyMem(addr(s.data[s.pos]), buffer, bufLen) + when defined(js): + try: + s.data[s.pos..<s.pos+bufLen] = cast[ptr string](buffer)[][0..<bufLen] + except: + raise newException(Defect, "could not write to string stream, " & + "did you use a non-string buffer pointer?", getCurrentException()) + elif not defined(nimscript): + copyMem(addr(s.data[s.pos]), buffer, bufLen) inc(s.pos, bufLen) proc ssClose(s: Stream) = var s = StringStream(s) - when defined(nimNoNilSeqs): - s.data = "" - else: - s.data = nil + s.data = "" - proc newStringStream*(s: string = ""): owned StringStream = + proc newStringStream*(s: sink string = ""): owned StringStream = ## Creates a new stream from the string `s`. ## - ## **Note:** Not available for JS backend. - ## ## See also: ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from ## opened File. @@ -1174,168 +1302,176 @@ else: new(result) result.data = s + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd result.setPositionImpl = ssSetPosition result.getPositionImpl = ssGetPosition - result.readDataImpl = ssReadData - result.peekDataImpl = ssPeekData - result.writeDataImpl = ssWriteData result.readDataStrImpl = ssReadDataStr + when nimvm: + discard + else: + result.readDataImpl = ssReadData + result.peekDataImpl = ssPeekData + result.writeDataImpl = ssWriteData - type - FileStream* = ref FileStreamObj - ## A stream that encapsulates a `File`. - ## - ## **Note:** Not available for JS backend. - FileStreamObj* = object of Stream - ## A file stream object. - ## - ## **Note:** Not available for JS backend. - f: File - - proc fsClose(s: Stream) = - if FileStream(s).f != nil: - close(FileStream(s).f) - FileStream(s).f = nil - proc fsFlush(s: Stream) = flushFile(FileStream(s).f) - proc fsAtEnd(s: Stream): bool = return endOfFile(FileStream(s).f) - proc fsSetPosition(s: Stream, pos: int) = setFilePos(FileStream(s).f, pos) - proc fsGetPosition(s: Stream): int = return int(getFilePos(FileStream(s).f)) - - proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int = - result = readBuffer(FileStream(s).f, buffer, bufLen) - - proc fsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = - result = readBuffer(FileStream(s).f, addr buffer[slice.a], slice.b + 1 - slice.a) - - proc fsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let pos = fsGetPosition(s) - defer: fsSetPosition(s, pos) - result = readBuffer(FileStream(s).f, buffer, bufLen) - - proc fsWriteData(s: Stream, buffer: pointer, bufLen: int) = - if writeBuffer(FileStream(s).f, buffer, bufLen) != bufLen: - raise newEIO("cannot write to stream") - - proc fsReadLine(s: Stream, line: var TaintedString): bool = - result = readLine(FileStream(s).f, line) - - proc newFileStream*(f: File): owned FileStream = - ## Creates a new stream from the file `f`. +type + FileStream* = ref FileStreamObj + ## A stream that encapsulates a `File`. ## ## **Note:** Not available for JS backend. + FileStreamObj* = object of Stream + ## A file stream object. ## - ## See also: - ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream - ## from string. - ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same - ## as using `open proc <io.html#open,File,string,FileMode,int>`_ - ## on Examples. - ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a - ## file stream from the file name and the mode. - runnableExamples: - ## Input (somefile.txt): + ## **Note:** Not available for JS backend. + f: File + +proc fsClose(s: Stream) = + if FileStream(s).f != nil: + close(FileStream(s).f) + FileStream(s).f = nil +proc fsFlush(s: Stream) = flushFile(FileStream(s).f) +proc fsAtEnd(s: Stream): bool = return endOfFile(FileStream(s).f) +proc fsSetPosition(s: Stream, pos: int) = setFilePos(FileStream(s).f, pos) +proc fsGetPosition(s: Stream): int = return int(getFilePos(FileStream(s).f)) + +proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = readBuffer(FileStream(s).f, buffer, bufLen) + +proc fsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = + result = readBuffer(FileStream(s).f, addr buffer[slice.a], slice.b + 1 - slice.a) + +proc fsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let pos = fsGetPosition(s) + defer: fsSetPosition(s, pos) + result = readBuffer(FileStream(s).f, buffer, bufLen) + +proc fsWriteData(s: Stream, buffer: pointer, bufLen: int) = + if writeBuffer(FileStream(s).f, buffer, bufLen) != bufLen: + raise newEIO("cannot write to stream") + +proc fsReadLine(s: Stream, line: var string): bool = + result = readLine(FileStream(s).f, line) + +proc newFileStream*(f: File): owned FileStream = + ## Creates a new stream from the file `f`. + ## + ## **Note:** Not available for JS backend. + ## + ## See also: + ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream + ## from string. + ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same + ## as using `open proc <syncio.html#open,File,string,FileMode,int>`_ + ## on Examples. + ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + ## Input (somefile.txt): + ## The first line + ## the second line + ## the third line + var f: File + if open(f, "somefile.txt", fmRead, -1): + var strm = newFileStream(f) + var line = "" + while strm.readLine(line): + echo line + ## Output: ## The first line ## the second line ## the third line - var f: File - if open(f, "somefile.txt", fmRead, -1): - var strm = newFileStream(f) - var line = "" - while strm.readLine(line): - echo line - ## Output: - ## The first line - ## the second line - ## the third line - strm.close() + strm.close() - new(result) - result.f = f - result.closeImpl = fsClose - result.atEndImpl = fsAtEnd - result.setPositionImpl = fsSetPosition - result.getPositionImpl = fsGetPosition - result.readDataStrImpl = fsReadDataStr - result.readDataImpl = fsReadData - result.readLineImpl = fsReadLine - result.peekDataImpl = fsPeekData - result.writeDataImpl = fsWriteData - result.flushImpl = fsFlush - - proc newFileStream*(filename: string, mode: FileMode = fmRead, - bufSize: int = -1): owned FileStream = - ## Creates a new stream from the file named `filename` with the mode `mode`. - ## - ## If the file cannot be opened, `nil` is returned. See the `io module - ## <io.html>`_ for a list of available `FileMode enums <io.html#FileMode>`_. - ## - ## **Note:** - ## * **This function returns nil in case of failure.** - ## To prevent unexpected behavior and ensure proper error handling, - ## use `openFileStream proc <#openFileStream,string,FileMode,int>`_ - ## instead. - ## * Not available for JS backend. - ## - ## See also: - ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream - ## from string. - ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from - ## opened File. - ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a - ## file stream from the file name and the mode. - runnableExamples: - from os import removeFile - var strm = newFileStream("somefile.txt", fmWrite) - if not isNil(strm): - strm.writeLine("The first line") - strm.writeLine("the second line") - strm.writeLine("the third line") - strm.close() - ## Output (somefile.txt) - ## The first line - ## the second line - ## the third line - removeFile("somefile.txt") + new(result) + result.f = f + result.closeImpl = fsClose + result.atEndImpl = fsAtEnd + result.setPositionImpl = fsSetPosition + result.getPositionImpl = fsGetPosition + result.readDataStrImpl = fsReadDataStr + result.readDataImpl = fsReadData + result.readLineImpl = fsReadLine + result.peekDataImpl = fsPeekData + result.writeDataImpl = fsWriteData + result.flushImpl = fsFlush + +proc newFileStream*(filename: string, mode: FileMode = fmRead, + bufSize: int = -1): owned FileStream = + ## Creates a new stream from the file named `filename` with the mode `mode`. + ## + ## If the file cannot be opened, `nil` is returned. See the `io module + ## <syncio.html>`_ for a list of available `FileMode enums <syncio.html#FileMode>`_. + ## + ## **Note:** + ## * **This function returns nil in case of failure.** + ## To prevent unexpected behavior and ensure proper error handling, + ## use `openFileStream proc <#openFileStream,string,FileMode,int>`_ + ## instead. + ## * Not available for JS backend. + ## + ## See also: + ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream + ## from string. + ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from + ## opened File. + ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + from std/os import removeFile + var strm = newFileStream("somefile.txt", fmWrite) + if not isNil(strm): + strm.writeLine("The first line") + strm.writeLine("the second line") + strm.writeLine("the third line") + strm.close() + ## Output (somefile.txt) + ## The first line + ## the second line + ## the third line + removeFile("somefile.txt") - var f: File - if open(f, filename, mode, bufSize): result = newFileStream(f) + var f: File + if open(f, filename, mode, bufSize): result = newFileStream(f) - proc openFileStream*(filename: string, mode: FileMode = fmRead, - bufSize: int = -1): owned FileStream = - ## Creates a new stream from the file named `filename` with the mode `mode`. - ## If the file cannot be opened, an IO exception is raised. - ## - ## **Note:** Not available for JS backend. - ## - ## See also: - ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream - ## from string. - ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from - ## opened File. - ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ creates a - ## file stream from the file name and the mode. - runnableExamples: - try: - ## Input (somefile.txt): - ## The first line - ## the second line - ## the third line - var strm = openFileStream("somefile.txt") - echo strm.readLine() - ## Output: - ## The first line - strm.close() - except: - stderr.write getCurrentExceptionMsg() +proc openFileStream*(filename: string, mode: FileMode = fmRead, + bufSize: int = -1): owned FileStream = + ## Creates a new stream from the file named `filename` with the mode `mode`. + ## If the file cannot be opened, an IO exception is raised. + ## + ## **Note:** Not available for JS backend. + ## + ## See also: + ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream + ## from string. + ## * `newFileStream proc <#newFileStream,File>`_ creates a file stream from + ## opened File. + ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ creates a + ## file stream from the file name and the mode. + runnableExamples: + try: + ## Input (somefile.txt): + ## The first line + ## the second line + ## the third line + var strm = openFileStream("somefile.txt") + echo strm.readLine() + ## Output: + ## The first line + strm.close() + except: + stderr.write getCurrentExceptionMsg() - var f: File - if open(f, filename, mode, bufSize): - return newFileStream(f) - else: - raise newEIO("cannot open file stream: " & filename) + var f: File + if open(f, filename, mode, bufSize): + return newFileStream(f) + else: + raise newEIO("cannot open file stream: " & filename) when false: type @@ -1355,7 +1491,7 @@ when false: # do not import windows as this increases compile times: discard else: - import posix + import std/posix proc hsSetPosition(s: FileHandleStream, pos: int) = discard lseek(s.handle, pos, SEEK_SET) @@ -1403,20 +1539,7 @@ when false: of fmReadWrite: flags = O_RDWR or int(O_CREAT) of fmReadWriteExisting: flags = O_RDWR of fmAppend: flags = O_WRONLY or int(O_CREAT) or O_APPEND + static: raiseAssert "unreachable" # handle bug #17888 var handle = open(filename, flags) if handle < 0: raise newEOS("posix.open() call failed") result = newFileHandleStream(handle) - -when isMainModule and defined(testing): - var ss = newStringStream("The quick brown fox jumped over the lazy dog.\nThe lazy dog ran") - assert(ss.getPosition == 0) - assert(ss.peekStr(5) == "The q") - assert(ss.getPosition == 0) # haven't moved - assert(ss.readStr(5) == "The q") - assert(ss.getPosition == 5) # did move - assert(ss.peekLine() == "uick brown fox jumped over the lazy dog.") - assert(ss.getPosition == 5) # haven't moved - var str = newString(100) - assert(ss.peekLine(str)) - assert(str == "uick brown fox jumped over the lazy dog.") - assert(ss.getPosition == 5) # haven't moved diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim new file mode 100644 index 000000000..99752a9ab --- /dev/null +++ b/lib/pure/streamwrapper.nim @@ -0,0 +1,121 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements stream wrapper. +## +## **Since** version 1.2. + +import std/[deques, streams] + +when defined(nimPreviewSlimSystem): + import std/assertions + + +type + PipeOutStream*[T] = ref object of T + # When stream peek operation is called, it reads from base stream + # type using `baseReadDataImpl` and stores the content to this buffer. + # Next stream read operation returns data in the buffer so that previus peek + # operation looks like didn't changed read positon. + # When stream read operation that returns N byte data is called and the size is smaller than buffer size, + # first N elements are removed from buffer. + # Deque type can do such operation more efficiently than seq type. + buffer: Deque[char] + baseReadLineImpl: typeof(StreamObj.readLineImpl) + baseReadDataImpl: typeof(StreamObj.readDataImpl) + +proc posReadLine[T](s: Stream, line: var string): bool = + var s = PipeOutStream[T](s) + assert s.baseReadLineImpl != nil + + let n = s.buffer.len + line.setLen(0) + for i in 0..<n: + var c = s.buffer.popFirst + if c == '\c': + c = readChar(s) + return true + elif c == '\L': return true + elif c == '\0': + return line.len > 0 + line.add(c) + + var line2: string + result = s.baseReadLineImpl(s, line2) + line.add line2 + +proc posReadData[T](s: Stream, buffer: pointer, bufLen: int): int = + var s = PipeOutStream[T](s) + assert s.baseReadDataImpl != nil + + let + dest = cast[ptr UncheckedArray[char]](buffer) + n = min(s.buffer.len, bufLen) + result = n + for i in 0..<n: + dest[i] = s.buffer.popFirst + if bufLen > n: + result += s.baseReadDataImpl(s, addr dest[n], bufLen - n) + +proc posReadDataStr[T](s: Stream, buffer: var string, slice: Slice[int]): int = + posReadData[T](s, addr buffer[slice.a], slice.len) + +proc posPeekData[T](s: Stream, buffer: pointer, bufLen: int): int = + var s = PipeOutStream[T](s) + assert s.baseReadDataImpl != nil + + let + dest = cast[ptr UncheckedArray[char]](buffer) + n = min(s.buffer.len, bufLen) + + result = n + for i in 0..<n: + dest[i] = s.buffer[i] + + if bufLen > n: + let + newDataNeeded = bufLen - n + numRead = s.baseReadDataImpl(s, addr dest[n], newDataNeeded) + result += numRead + for i in 0..<numRead: + s.buffer.addLast dest[n + i] + +proc newPipeOutStream*[T](s: sink (ref T)): owned PipeOutStream[T] = + ## Wrap pipe for reading with PipeOutStream so that you can use peek* procs and generate runtime error + ## when setPosition/getPosition is called or write operation is performed. + ## + ## Example: + ## ```Nim + ## import std/[osproc, streamwrapper] + ## var + ## p = startProcess(exePath) + ## outStream = p.outputStream().newPipeOutStream() + ## echo outStream.peekChar + ## p.close() + ## ``` + + assert s.readDataImpl != nil + + new(result) + for dest, src in fields((ref T)(result)[], s[]): + dest = src + wasMoved(s[]) + if result.readLineImpl != nil: + result.baseReadLineImpl = result.readLineImpl + result.readLineImpl = posReadLine[T] + result.baseReadDataImpl = result.readDataImpl + result.readDataImpl = posReadData[T] + result.readDataStrImpl = posReadDataStr[T] + result.peekDataImpl = posPeekData[T] + + # Set nil to anything you may not call. + result.setPositionImpl = nil + result.getPositionImpl = nil + result.writeDataImpl = nil + result.flushImpl = nil diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index bc0d4b069..7d093ebb3 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -9,262 +9,325 @@ ##[ String `interpolation`:idx: / `format`:idx: inspired by -Python's ``f``-strings. +Python's f-strings. -``fmt`` vs. ``&`` -================= +# `fmt` vs. `&` -You can use either ``fmt`` or the unary ``&`` operator for formatting. The +You can use either `fmt` or the unary `&` operator for formatting. The difference between them is subtle but important. -The ``fmt"{expr}"`` syntax is more aesthetically pleasing, but it hides a small +The `fmt"{expr}"` syntax is more aesthetically pleasing, but it hides a small gotcha. The string is a `generalized raw string literal <manual.html#lexical-analysis-generalized-raw-string-literals>`_. This has some surprising effects: +]## -.. code-block:: nim - - import strformat - let msg = "hello" - doAssert fmt"{msg}\n" == "hello\\n" +runnableExamples: + let msg = "hello" + assert fmt"{msg}\n" == "hello\\n" -Because the literal is a raw string literal, the ``\n`` is not interpreted as +##[ +Because the literal is a raw string literal, the `\n` is not interpreted as an escape sequence. -There are multiple ways to get around this, including the use of the ``&`` -operator: - -.. code-block:: nim +There are multiple ways to get around this, including the use of the `&` operator: +]## - import strformat - let msg = "hello" +runnableExamples: + let msg = "hello" - doAssert &"{msg}\n" == "hello\n" + assert &"{msg}\n" == "hello\n" - doAssert fmt"{msg}{'\n'}" == "hello\n" - doAssert fmt("{msg}\n") == "hello\n" - doAssert "{msg}\n".fmt == "hello\n" + assert fmt"{msg}{'\n'}" == "hello\n" + assert fmt("{msg}\n") == "hello\n" + assert "{msg}\n".fmt == "hello\n" +##[ The choice of style is up to you. -Formatting strings -================== +# Formatting strings +]## + +runnableExamples: + assert &"""{"abc":>4}""" == " abc" + assert &"""{"abc":<4}""" == "abc " + +##[ +# Formatting floats +]## + +runnableExamples: + assert fmt"{-12345:08}" == "-0012345" + assert fmt"{-1:3}" == " -1" + assert fmt"{-1:03}" == "-01" + assert fmt"{16:#X}" == "0x10" -.. code-block:: nim + assert fmt"{123.456}" == "123.456" + assert fmt"{123.456:>9.3f}" == " 123.456" + assert fmt"{123.456:9.3f}" == " 123.456" + assert fmt"{123.456:9.4f}" == " 123.4560" + assert fmt"{123.456:>9.0f}" == " 123." + assert fmt"{123.456:<9.4f}" == "123.4560 " - import strformat + assert fmt"{123.456:e}" == "1.234560e+02" + assert fmt"{123.456:>13e}" == " 1.234560e+02" + assert fmt"{123.456:13e}" == " 1.234560e+02" - doAssert &"""{"abc":>4}""" == " abc" - doAssert &"""{"abc":<4}""" == "abc " +##[ +# Expressions +]## +runnableExamples: + let x = 3.14 + assert fmt"{(if x!=0: 1.0/x else: 0):.5}" == "0.31847" + assert fmt"""{(block: + var res: string + for i in 1..15: + res.add (if i mod 15 == 0: "FizzBuzz" + elif i mod 5 == 0: "Buzz" + elif i mod 3 == 0: "Fizz" + else: $i) & " " + res)}""" == "1 2 Fizz 4 Buzz Fizz 7 8 Fizz Buzz 11 Fizz 13 14 FizzBuzz " +##[ +# Debugging strings -Formatting floats -================= +`fmt"{expr=}"` expands to `fmt"expr={expr}"` namely the text of the expression, +an equal sign and the results of evaluated expression. +]## -.. code-block:: nim +runnableExamples: + assert fmt"{123.456=}" == "123.456=123.456" + assert fmt"{123.456=:>9.3f}" == "123.456= 123.456" - import strformat + let x = "hello" + assert fmt"{x=}" == "x=hello" + assert fmt"{x =}" == "x =hello" - doAssert fmt"{-12345:08}" == "-0012345" - doAssert fmt"{-1:3}" == " -1" - doAssert fmt"{-1:03}" == "-01" - doAssert fmt"{16:#X}" == "0x10" + let y = 3.1415926 + assert fmt"{y=:.2f}" == fmt"y={y:.2f}" + assert fmt"{y=}" == fmt"y={y}" + assert fmt"{y = : <8}" == fmt"y = 3.14159 " - doAssert fmt"{123.456}" == "123.456" - doAssert fmt"{123.456:>9.3f}" == " 123.456" - doAssert fmt"{123.456:9.3f}" == " 123.456" - doAssert fmt"{123.456:9.4f}" == " 123.4560" - doAssert fmt"{123.456:>9.0f}" == " 123." - doAssert fmt"{123.456:<9.4f}" == "123.4560 " + proc hello(a: string, b: float): int = 12 + assert fmt"{hello(x, y) = }" == "hello(x, y) = 12" + assert fmt"{x.hello(y) = }" == "x.hello(y) = 12" + assert fmt"{hello x, y = }" == "hello x, y = 12" - doAssert fmt"{123.456:e}" == "1.234560e+02" - doAssert fmt"{123.456:>13e}" == " 1.234560e+02" - doAssert fmt"{123.456:13e}" == " 1.234560e+02" +##[ +Note that it is space sensitive: +]## +runnableExamples: + let x = "12" + assert fmt"{x=}" == "x=12" + assert fmt"{x =:}" == "x =12" + assert fmt"{x =}" == "x =12" + assert fmt"{x= :}" == "x= 12" + assert fmt"{x= }" == "x= 12" + assert fmt"{x = :}" == "x = 12" + assert fmt"{x = }" == "x = 12" + assert fmt"{x = :}" == "x = 12" + assert fmt"{x = }" == "x = 12" -Implementation details -====================== +##[ +# Implementation details -An expression like ``&"{key} is {value:arg} {{z}}"`` is transformed into: +An expression like `&"{key} is {value:arg} {{z}}"` is transformed into: -.. code-block:: nim + ```nim var temp = newStringOfCap(educatedCapGuess) - temp.formatValue key, "" - temp.add " is " - temp.formatValue value, arg - temp.add " {z}" + temp.formatValue(key, "") + temp.add(" is ") + temp.formatValue(value, arg) + temp.add(" {z}") temp + ``` Parts of the string that are enclosed in the curly braces are interpreted -as Nim code, to escape an ``{`` or ``}`` double it. +as Nim code. To escape a `{` or `}`, double it. + +Within a curly expression, however, `{`, `}`, must be escaped with a backslash. + +To enable evaluating Nim expressions within curlies, colons inside parentheses +do not need to be escaped. +]## -``&`` delegates most of the work to an open overloaded set -of ``formatValue`` procs. The required signature for a type ``T`` that supports -formatting is usually ``proc formatValue(result: var string; x: T; specifier: string)``. +runnableExamples: + let x = "hello" + assert fmt"""{ "\{(" & x & ")\}" }""" == "{(hello)}" + assert fmt"""{{({ x })}}""" == "{(hello)}" + assert fmt"""{ $(\{x:1,"world":2\}) }""" == """[("hello", 1), ("world", 2)]""" + +##[ +`&` delegates most of the work to an open overloaded set +of `formatValue` procs. The required signature for a type `T` that supports +formatting is usually `proc formatValue(result: var string; x: T; specifier: string)`. The subexpression after the colon -(``arg`` in ``&"{key} is {value:arg} {{z}}"``) is optional. It will be passed as -the last argument to ``formatValue``. When the colon with the subexpression it is +(`arg` in `&"{key} is {value:arg} {{z}}"`) is optional. It will be passed as +the last argument to `formatValue`. When the colon with the subexpression it is left out, an empty string will be taken instead. For strings and numeric types the optional argument is a so-called "standard format specifier". +# Standard format specifiers for strings, integers and floats -Standard format specifier for strings, integers and floats -========================================================== - +The general form of a standard format specifier is: -The general form of a standard format specifier is:: + [[fill]align][sign][#][0][minimumwidth][.precision][type] - [[fill]align][sign][#][0][minimumwidth][.precision][type] +The square brackets `[]` indicate an optional element. -The square brackets ``[]`` indicate an optional element. +The optional `align` flag can be one of the following: -The optional align flag can be one of the following: - -'<' - Forces the field to be left-aligned within the available +`<` +: Forces the field to be left-aligned within the available space. (This is the default for strings.) -'>' - Forces the field to be right-aligned within the available space. +`>` +: Forces the field to be right-aligned within the available space. (This is the default for numbers.) -'^' - Forces the field to be centered within the available space. +`^` +: Forces the field to be centered within the available space. Note that unless a minimum field width is defined, the field width will always be the same size as the data to fill it, so that the alignment option has no meaning in this case. -The optional 'fill' character defines the character to be used to pad +The optional `fill` character defines the character to be used to pad the field to the minimum width. The fill character, if present, must be followed by an alignment flag. -The 'sign' option is only valid for numeric types, and can be one of the following: +The `sign` option is only valid for numeric types, and can be one of the following: ================= ==================================================== Sign Meaning ================= ==================================================== -``+`` Indicates that a sign should be used for both +`+` Indicates that a sign should be used for both positive as well as negative numbers. -``-`` Indicates that a sign should be used only for +`-` Indicates that a sign should be used only for negative numbers (this is the default behavior). (space) Indicates that a leading space should be used on positive numbers. ================= ==================================================== -If the '#' character is present, integers use the 'alternate form' for formatting. -This means that binary, octal, and hexadecimal output will be prefixed -with '0b', '0o', and '0x', respectively. +If the `#` character is present, integers use the 'alternate form' for formatting. +This means that binary, octal and hexadecimal output will be prefixed +with `0b`, `0o` and `0x`, respectively. -'width' is a decimal integer defining the minimum field width. If not specified, +`width` is a decimal integer defining the minimum field width. If not specified, then the field width will be determined by the content. -If the width field is preceded by a zero ('0') character, this enables +If the width field is preceded by a zero (`0`) character, this enables zero-padding. -The 'precision' is a decimal number indicating how many digits should be displayed +The `precision` is a decimal number indicating how many digits should be displayed after the decimal point in a floating point conversion. For non-numeric types the field indicates the maximum field size - in other words, how many characters will be used from the field content. The precision is ignored for integer conversions. -Finally, the 'type' determines how the data should be presented. +Finally, the `type` determines how the data should be presented. The available integer presentation types are: - ================= ==================================================== Type Result ================= ==================================================== -``b`` Binary. Outputs the number in base 2. -``d`` Decimal Integer. Outputs the number in base 10. -``o`` Octal format. Outputs the number in base 8. -``x`` Hex format. Outputs the number in base 16, using +`b` Binary. Outputs the number in base 2. +`d` Decimal Integer. Outputs the number in base 10. +`o` Octal format. Outputs the number in base 8. +`x` Hex format. Outputs the number in base 16, using lower-case letters for the digits above 9. -``X`` Hex format. Outputs the number in base 16, using +`X` Hex format. Outputs the number in base 16, using uppercase letters for the digits above 9. -(None) the same as 'd' +(None) The same as `d`. ================= ==================================================== - The available floating point presentation types are: ================= ==================================================== Type Result ================= ==================================================== -``e`` Exponent notation. Prints the number in scientific - notation using the letter 'e' to indicate the +`e` Exponent notation. Prints the number in scientific + notation using the letter `e` to indicate the exponent. -``E`` Exponent notation. Same as 'e' except it converts +`E` Exponent notation. Same as `e` except it converts the number to uppercase. -``f`` Fixed point. Displays the number as a fixed-point +`f` Fixed point. Displays the number as a fixed-point number. -``F`` Fixed point. Same as 'f' except it converts the +`F` Fixed point. Same as `f` except it converts the number to uppercase. -``g`` General format. This prints the number as a +`g` General format. This prints the number as a fixed-point number, unless the number is too - large, in which case it switches to 'e' + large, in which case it switches to `e` exponent notation. -``G`` General format. Same as 'g' except switches to 'E' +`G` General format. Same as `g` except it switches to `E` if the number gets to large. -(None) similar to 'g', except that it prints at least one +`i` Complex General format. This is only supported for + complex numbers, which it prints using the mathematical + (RE+IMj) format. The real and imaginary parts are printed + using the general format `g` by default, but it is + possible to combine this format with one of the other + formats (e.g `jf`). +(None) Similar to `g`, except that it prints at least one digit after the decimal point. ================= ==================================================== - -Limitations -=========== +# Limitations Because of the well defined order how templates and macros are expanded, strformat cannot expand template arguments: -.. code-block:: nim + ```nim template myTemplate(arg: untyped): untyped = echo "arg is: ", arg echo &"--- {arg} ---" let x = "abc" myTemplate(x) + ``` -First the template ``myTemplate`` is expanded, where every identifier -``arg`` is substituted with its argument. The ``arg`` inside the +First the template `myTemplate` is expanded, where every identifier +`arg` is substituted with its argument. The `arg` inside the format string is not seen by this process, because it is part of a quoted string literal. It is not an identifier yet. Then the strformat -macro creates the ``arg`` identifier from the string literal. An +macro creates the `arg` identifier from the string literal, an identifier that cannot be resolved anymore. The workaround for this is to bind the template argument to a new local variable. -.. code-block:: nim - + ```nim template myTemplate(arg: untyped): untyped = block: let arg1 {.inject.} = arg echo "arg is: ", arg1 echo &"--- {arg1} ---" + ``` -The use of ``{.inject.}`` here is necessary again because of template +The use of `{.inject.}` here is necessary again because of template expansion order and hygienic templates. But since we generally want to -keep the hygienicness of ``myTemplate``, and we do not want ``arg1`` -to be injected into the context where ``myTemplate`` is expanded, -everything is wrapped in a ``block``. +keep the hygiene of `myTemplate`, and we do not want `arg1` +to be injected into the context where `myTemplate` is expanded, +everything is wrapped in a `block`. -Future directions -================= +# Future directions -A curly expression with commas in it like ``{x, argA, argB}`` could be -transformed to ``formatValue(result, x, argA, argB)`` in order to support +A curly expression with commas in it like `{x, argA, argB}` could be +transformed to `formatValue(result, x, argA, argB)` in order to support formatters that do not need to parse a custom language within a custom -language but instead prefer to use Nim's existing syntax. This also -helps in readability since there is only so much you can cram into +language but instead prefer to use Nim's existing syntax. This would also +help with readability, since there is only so much you can cram into single letter DSLs. - ]## -import macros, parseutils, unicode -import strutils except format +import std/[macros, parseutils, unicode] +import std/strutils except format + +when defined(nimPreviewSlimSystem): + import std/assertions + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) @@ -273,10 +336,9 @@ proc mkDigit(v: int, typ: char): string {.inline.} = else: result = $chr(ord(if typ == 'x': 'a' else: 'A') + v - 10) -proc alignString*(s: string, minimumWidth: int; align = '\0'; - fill = ' '): string = - ## Aligns ``s`` using ``fill`` char. - ## This is only of interest if you want to write a custom ``format`` proc that +proc alignString*(s: string, minimumWidth: int; align = '\0'; fill = ' '): string = + ## Aligns `s` using the `fill` char. + ## This is only of interest if you want to write a custom `format` proc that ## should support the standard format specifiers. if minimumWidth == 0: result = s @@ -298,28 +360,30 @@ type fill*, align*: char ## Desired fill and alignment. sign*: char ## Desired sign. alternateForm*: bool ## Whether to prefix binary, octal and hex numbers - ## with ``0b``, ``0o``, ``0x``. + ## with `0b`, `0o`, `0x`. padWithZero*: bool ## Whether to pad with zeros rather than spaces. minimumWidth*, precision*: int ## Desired minimum width and precision. typ*: char ## Type like 'f', 'g' or 'd'. endPosition*: int ## End position in the format specifier after - ## ``parseStandardFormatSpecifier`` returned. - -proc formatInt(n: SomeNumber; radix: int; - spec: StandardFormatSpecifier): string = - ## Converts ``n`` to string. If ``n`` is `SomeFloat`, it casts to `int64`. - ## Conversion is done using ``radix``. If result's length is lesser than - ## ``minimumWidth``, it aligns result to the right or left (depending on ``a``) - ## with ``fill`` char. + ## `parseStandardFormatSpecifier` returned. + +proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string = + ## Converts `n` to a string. If `n` is `SomeFloat`, it casts to `int64`. + ## Conversion is done using `radix`. If result's length is less than + ## `minimumWidth`, it aligns result to the right or left (depending on `a`) + ## with the `fill` char. when n is SomeUnsignedInt: var v = n.uint64 let negative = false else: - var v = n.int64 - let negative = v.int64 < 0 - if negative: - # FIXME: overflow error for low(int64) - v = v * -1 + let n = n.int64 + let negative = n < 0 + var v = + if negative: + # `uint64(-n)`, but accounts for `n == low(int64)` + uint64(not n) + 1 + else: + uint64(n) var xx = "" if spec.alternateForm: @@ -334,9 +398,9 @@ proc formatInt(n: SomeNumber; radix: int; result = "0" else: result = "" - while v > type(v)(0): - let d = v mod type(v)(radix) - v = v div type(v)(radix) + while v > typeof(v)(0): + let d = v mod typeof(v)(radix) + v = v div typeof(v)(radix) result.add(mkDigit(d.int, spec.typ)) for idx in 0..<(result.len div 2): swap result[idx], result[result.len - idx - 1] @@ -368,13 +432,13 @@ proc formatInt(n: SomeNumber; radix: int; proc parseStandardFormatSpecifier*(s: string; start = 0; ignoreUnknownSuffix = false): StandardFormatSpecifier = ## An exported helper proc that parses the "standard format specifiers", - ## as specified by the grammar:: + ## as specified by the grammar: ## - ## [[fill]align][sign][#][0][minimumwidth][.precision][type] + ## [[fill]align][sign][#][0][minimumwidth][.precision][type] ## - ## This is only of interest if you want to write a custom ``format`` proc that - ## should support the standard format specifiers. If ``ignoreUnknownSuffix`` is true, - ## an unknown suffix after the ``type`` field is not an error. + ## This is only of interest if you want to write a custom `format` proc that + ## should support the standard format specifiers. If `ignoreUnknownSuffix` is true, + ## an unknown suffix after the `type` field is not an error. const alignChars = {'<', '>', '^'} result.fill = ' ' result.align = '\0' @@ -396,7 +460,7 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; result.alternateForm = true inc i - if i+1 < s.len and s[i] == '0' and s[i+1] in {'0'..'9'}: + if i + 1 < s.len and s[i] == '0' and s[i+1] in {'0'..'9'}: result.padWithZero = true inc i @@ -417,50 +481,48 @@ proc parseStandardFormatSpecifier*(s: string; start = 0; raise newException(ValueError, "invalid format string, cannot parse: " & s[i..^1]) +proc toRadix(typ: char): int = + case typ + of 'x', 'X': 16 + of 'd', '\0': 10 + of 'o': 8 + of 'b': 2 + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'x', 'X', 'b', 'd', 'o' but got: " & typ) + proc formatValue*[T: SomeInteger](result: var string; value: T; - specifier: string) = - ## Standard format implementation for ``SomeInteger``. It makes little + specifier: static string) = + ## Standard format implementation for `SomeInteger`. It makes little ## sense to call this directly, but it is required to exist - ## by the ``&`` macro. - if specifier.len == 0: + ## by the `&` macro. + when specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - var radix = 10 - case spec.typ - of 'x', 'X': radix = 16 - of 'd', '\0': discard - of 'b': radix = 2 - of 'o': radix = 8 else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ) - result.add formatInt(value, radix, spec) + const + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) -proc formatValue*(result: var string; value: SomeFloat; specifier: string) = - ## Standard format implementation for ``SomeFloat``. It makes little + result.add formatInt(value, radix, spec) + +proc formatValue*[T: SomeInteger](result: var string; value: T; + specifier: string) = + ## Standard format implementation for `SomeInteger`. It makes little ## sense to call this directly, but it is required to exist - ## by the ``&`` macro. + ## by the `&` macro. if specifier.len == 0: result.add $value - return - let spec = parseStandardFormatSpecifier(specifier) - - var fmode = ffDefault - case spec.typ - of 'e', 'E': - fmode = ffScientific - of 'f', 'F': - fmode = ffDecimal - of 'g', 'G': - fmode = ffDefault - of '\0': discard else: - raise newException(ValueError, - "invalid type in format string for number, expected one " & - " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ) + let + spec = parseStandardFormatSpecifier(specifier) + radix = toRadix(spec.typ) + + result.add formatInt(value, radix, spec) +proc formatFloat( + result: var string, value: SomeFloat, fmode: FloatFormatMode, + spec: StandardFormatSpecifier) = var f = formatBiggestFloat(value, fmode, spec.precision) var sign = false if value >= 0.0: @@ -495,25 +557,84 @@ proc formatValue*(result: var string; value: SomeFloat; specifier: string) = else: result.add res +proc toFloatFormatMode(typ: char): FloatFormatMode = + case typ + of 'e', 'E': ffScientific + of 'f', 'F': ffDecimal + of 'g', 'G': ffDefault + of '\0': ffDefault + else: + raise newException(ValueError, + "invalid type in format string for number, expected one " & + " of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & typ) + +proc formatValue*(result: var string; value: SomeFloat; specifier: static string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + when specifier.len == 0: + result.add $value + else: + const + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: SomeFloat; specifier: string) = + ## Standard format implementation for `SomeFloat`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + if specifier.len == 0: + result.add $value + else: + let + spec = parseStandardFormatSpecifier(specifier) + fmode = toFloatFormatMode(spec.typ) + + formatFloat(result, value, fmode, spec) + +proc formatValue*(result: var string; value: string; specifier: static string) = + ## Standard format implementation for `string`. It makes little + ## sense to call this directly, but it is required to exist + ## by the `&` macro. + const spec = parseStandardFormatSpecifier(specifier) + var value = + when spec.typ in {'s', '\0'}: value + else: static: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) + when spec.precision != -1: + if spec.precision < runeLen(value): + const precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) + proc formatValue*(result: var string; value: string; specifier: string) = - ## Standard format implementation for ``string``. It makes little + ## Standard format implementation for `string`. It makes little ## sense to call this directly, but it is required to exist - ## by the ``&`` macro. + ## by the `&` macro. let spec = parseStandardFormatSpecifier(specifier) - var value = value - case spec.typ - of 's', '\0': discard - else: - raise newException(ValueError, - "invalid type in format string for string, expected 's', but got " & - spec.typ) + var value = + if spec.typ in {'s', '\0'}: value + else: + raise newException(ValueError, + "invalid type in format string for string, expected 's', but got " & + spec.typ) if spec.precision != -1: if spec.precision < runeLen(value): - setLen(value, runeOffset(value, spec.precision)) + let precision = cast[Natural](spec.precision) + setLen(value, Natural(runeOffset(value, precision))) + result.add alignString(value, spec.minimumWidth, spec.align, spec.fill) -proc formatValue[T: not SomeInteger](result: var string; value: T; - specifier: string) = +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: static string) = + mixin `$` + formatValue(result, $value, specifier) + +proc formatValue[T: not SomeInteger](result: var string; value: T; specifier: string) = mixin `$` formatValue(result, $value, specifier) @@ -523,19 +644,20 @@ template formatValue(result: var string; value: char; specifier: string) = template formatValue(result: var string; value: cstring; specifier: string) = result.add value -proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = - if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "string formatting (fmt(), &) only works with string literals", pattern +proc strformatImpl(f: string; openChar, closeChar: char, + lineInfoNode: NimNode = nil): NimNode = + template missingCloseChar = + error("invalid format string: missing closing character '" & closeChar & "'") + if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" - let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") - result = newNimNode(nnkStmtListExpr, lineInfoFrom = pattern) + result = newNimNode(nnkStmtListExpr, lineInfoNode) # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: - let expectedGrowth = when defined(useNimRtl): 0 else: count(f, '{') * 10 + let expectedGrowth = when defined(useNimRtl): 0 else: count(f, openChar) * 10 result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + expectedGrowth))) var strlit = "" @@ -551,18 +673,46 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = strlit = "" var subexpr = "" - while i < f.len and f[i] != closeChar and f[i] != ':': + var inParens = 0 + var inSingleQuotes = false + var inDoubleQuotes = false + template notEscaped:bool = f[i-1]!='\\' + while i < f.len and f[i] != closeChar and (f[i] != ':' or inParens != 0): + case f[i] + of '\\': + if i < f.len-1 and f[i+1] in {openChar,closeChar,':'}: inc i + of '\'': + if not inDoubleQuotes and notEscaped: inSingleQuotes = not inSingleQuotes + of '\"': + if notEscaped: inDoubleQuotes = not inDoubleQuotes + of '(': + if not (inSingleQuotes or inDoubleQuotes): inc inParens + of ')': + if not (inSingleQuotes or inDoubleQuotes): dec inParens + of '=': + let start = i + inc i + i += f.skipWhitespace(i) + if i == f.len: + missingCloseChar + if f[i] == closeChar or f[i] == ':': + result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) + else: + subexpr.add f[start ..< i] + continue + else: discard subexpr.add f[i] inc i + + if i == f.len: + missingCloseChar + var x: NimNode try: x = parseExpr(subexpr) - except ValueError: - when declared(getCurrentExceptionMsg): - let msg = getCurrentExceptionMsg() - error("could not parse `" & subexpr & "`.\n" & msg, pattern) - else: - error("could not parse `" & subexpr & "`.\n", pattern) + except ValueError as e: + error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) + x.copyLineInfo(lineInfoNode) let formatSym = bindSym("formatValue", brOpen) var options = "" if f[i] == ':': @@ -570,182 +720,71 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = while i < f.len and f[i] != closeChar: options.add f[i] inc i + if i == f.len: + missingCloseChar if f[i] == closeChar: inc i - else: - doAssert false, "invalid format string: missing '}'" result.add newCall(formatSym, res, x, newLit(options)) elif f[i] == closeChar: - if f[i+1] == closeChar: + if i<f.len-1 and f[i+1] == closeChar: strlit.add closeChar inc i, 2 else: - doAssert false, "invalid format string: '}' instead of '}}'" - inc i + raiseAssert "invalid format string: '$1' instead of '$1$1'" % $closeChar else: strlit.add f[i] inc i if strlit.len > 0: result.add newCall(bindSym"add", res, newLit(strlit)) result.add res + # workaround for #20381 + var blockExpr = newNimNode(nnkBlockExpr, lineInfoNode) + blockExpr.add(newEmptyNode()) + blockExpr.add(result) + result = blockExpr when defined(debugFmtDsl): echo repr result -macro `&`*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## For a specification of the ``&`` macro, see the module level documentation. +macro fmt(pattern: static string; openChar: static char, closeChar: static char, lineInfoNode: untyped): string = + ## version of `fmt` with dummy untyped param for line info + strformatImpl(pattern, openChar, closeChar, lineInfoNode) -macro fmt*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## An alias for ``&``. +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} -macro fmt*(pattern: string; openChar, closeChar: char): untyped = - ## Use ``openChar`` instead of '{' and ``closeChar`` instead of '}' +template fmt*(pattern: static string; openChar: static char, closeChar: static char): string {.callsite.} = + ## Interpolates `pattern` using symbols in scope. + runnableExamples: + let x = 7 + assert "var is {x * 2}".fmt == "var is 14" + assert "var is {{x}}".fmt == "var is {x}" # escape via doubling + const s = "foo: {x}" + assert s.fmt == "foo: 7" # also works with const strings + + assert fmt"\n" == r"\n" # raw string literal + assert "\n".fmt == "\n" # regular literal (likewise with `fmt("\n")` or `fmt "\n"`) + runnableExamples: + # custom `openChar`, `closeChar` + let x = 7 + assert "<x>".fmt('<', '>') == "7" + assert "<<<x>>>".fmt('<', '>') == "<7>" + assert "`x`".fmt('`', '`') == "7" + fmt(pattern, openChar, closeChar, dummyForLineInfo) + +template fmt*(pattern: static string): untyped {.callsite.} = + ## Alias for `fmt(pattern, '{', '}')`. + fmt(pattern, '{', '}', dummyForLineInfo) + +template `&`*(pattern: string{lit}): string {.callsite.} = + ## `&pattern` is the same as `pattern.fmt`. + ## For a specification of the `&` macro, see the module level documentation. + # pending bug #18275, bug #18278, use `pattern: static string` + # consider deprecating this, it's redundant with `fmt` and `fmt` is strictly + # more flexible, readable (no confusion with the binary `&`), self-documenting, + # not to mention #18275, bug #18278. runnableExamples: - let testInt = 123 - doAssert "<testInt>".fmt('<', '>') == "123" - doAssert """(()"foo" & "bar"())""".fmt(')', '(') == "(foobar)" - doAssert """ ""{"123+123"}"" """.fmt('"', '"') == " \"{246}\" " - strformatImpl(pattern, openChar.intVal.char, closeChar.intVal.char) - -when isMainModule: - template check(actual, expected: string) = - doAssert actual == expected - - from strutils import toUpperAscii, repeat - - # Basic tests - let s = "string" - check &"{0} {s}", "0 string" - check &"{s[0..2].toUpperAscii}", "STR" - check &"{-10:04}", "-010" - check &"{-10:<04}", "-010" - check &"{-10:>04}", "-010" - check &"0x{10:02X}", "0x0A" - - check &"{10:#04X}", "0x0A" - - check &"""{"test":#>5}""", "#test" - check &"""{"test":>5}""", " test" - - check &"""{"test":#^7}""", "#test##" - - check &"""{"test": <5}""", "test " - check &"""{"test":<5}""", "test " - check &"{1f:.3f}", "1.000" - check &"Hello, {s}!", "Hello, string!" - - # Tests for identifiers without parenthesis - check &"{s} works{s}", "string worksstring" - check &"{s:>7}", " string" - doAssert(not compiles(&"{s_works}")) # parsed as identifier `s_works` - - # Misc general tests - check &"{{}}", "{}" - check &"{0}%", "0%" - check &"{0}%asdf", "0%asdf" - check &("\n{\"\\n\"}\n"), "\n\n\n" - check &"""{"abc"}s""", "abcs" - - # String tests - check &"""{"abc"}""", "abc" - check &"""{"abc":>4}""", " abc" - check &"""{"abc":<4}""", "abc " - check &"""{"":>4}""", " " - check &"""{"":<4}""", " " - - # Int tests - check &"{12345}", "12345" - check &"{ - 12345}", "-12345" - check &"{12345:6}", " 12345" - check &"{12345:>6}", " 12345" - check &"{12345:4}", "12345" - check &"{12345:08}", "00012345" - check &"{-12345:08}", "-0012345" - check &"{0:0}", "0" - check &"{0:02}", "00" - check &"{-1:3}", " -1" - check &"{-1:03}", "-01" - check &"{10}", "10" - check &"{16:#X}", "0x10" - check &"{16:^#7X}", " 0x10 " - check &"{16:^+#7X}", " +0x10 " - - # Hex tests - check &"{0:x}", "0" - check &"{-0:x}", "0" - check &"{255:x}", "ff" - check &"{255:X}", "FF" - check &"{-255:x}", "-ff" - check &"{-255:X}", "-FF" - check &"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe" - check &"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe" - check &"{255:4x}", " ff" - check &"{255:04x}", "00ff" - check &"{-255:4x}", " -ff" - check &"{-255:04x}", "-0ff" - - # Float tests - check &"{123.456}", "123.456" - check &"{-123.456}", "-123.456" - check &"{123.456:.3f}", "123.456" - check &"{123.456:+.3f}", "+123.456" - check &"{-123.456:+.3f}", "-123.456" - check &"{-123.456:.3f}", "-123.456" - check &"{123.456:1g}", "123.456" - check &"{123.456:.1f}", "123.5" - check &"{123.456:.0f}", "123" - check &"{123.456:>9.3f}", " 123.456" - check &"{123.456:9.3f}", " 123.456" - check &"{123.456:>9.4f}", " 123.4560" - check &"{123.456:>9.0f}", " 123" - check &"{123.456:<9.4f}", "123.4560 " - - # Float (scientific) tests - check &"{123.456:e}", "1.234560e+02" - check &"{123.456:>13e}", " 1.234560e+02" - check &"{123.456:<13e}", "1.234560e+02 " - check &"{123.456:.1e}", "1.2e+02" - check &"{123.456:.2e}", "1.23e+02" - check &"{123.456:.3e}", "1.235e+02" - - # Note: times.format adheres to the format protocol. Test that this - # works: - import times - - var dt = initDateTime(01, mJan, 2000, 00, 00, 00) - check &"{dt:yyyy-MM-dd}", "2000-01-01" - - var tm = fromUnix(0) - discard &"{tm}" - - var noww = now() - check &"{noww}", $noww - - # Unicode string tests - check &"""{"αβγ"}""", "αβγ" - check &"""{"αβγ":>5}""", " αβγ" - check &"""{"αβγ":<5}""", "αβγ " - check &"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈" - check &"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 " - # Invalid unicode sequences should be handled as plain strings. - # Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173 - let invalidUtf8 = [ - "\xc3\x28", "\xa0\xa1", - "\xe2\x28\xa1", "\xe2\x82\x28", - "\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28" - ] - for s in invalidUtf8: - check &"{s:>5}", repeat(" ", 5-s.len) & s - - # bug #11089 - let flfoo: float = 1.0 - check &"{flfoo}", "1.0" - - # bug #11092 - check &"{high(int64)}", "9223372036854775807" - check &"{low(int64)}", "-9223372036854775808" - - import json - - doAssert fmt"{'a'} {'b'}" == "a b" - - echo("All tests ok") + let x = 7 + assert &"{x}\n" == "7\n" # regular string literal + assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal + fmt(pattern, '{', '}', dummyForLineInfo) diff --git a/lib/pure/strmisc.nim b/lib/pure/strmisc.nim index 43437ddb4..a3e539e7e 100644 --- a/lib/pure/strmisc.nim +++ b/lib/pure/strmisc.nim @@ -8,15 +8,12 @@ # ## This module contains various string utility routines that are uncommonly -## used in comparison to `strutils <strutils.html>`_. +## used in comparison to the ones in `strutils <strutils.html>`_. -import strutils +import std/strutils -{.deadCodeElim: on.} # dce option deprecated - -proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, - procvar.} = - ## Expand tab characters in `s` replacing them by spaces. +func expandTabs*(s: string, tabSize: int = 8): string = + ## Expands tab characters in `s`, replacing them by spaces. ## ## The amount of inserted spaces for each tab character is the difference ## between the current column number and the next tab position. Tab positions @@ -27,25 +24,20 @@ proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, runnableExamples: doAssert expandTabs("\t", 4) == " " doAssert expandTabs("\tfoo\t", 4) == " foo " - doAssert expandTabs("\tfoo\tbar", 4) == " foo bar" - doAssert expandTabs("\tfoo\tbar\t", 4) == " foo bar " - doAssert expandTabs("ab\tcd\n\txy\t", 3) == "ab cd\n xy " + doAssert expandTabs("a\tb\n\txy\t", 3) == "a b\n xy " result = newStringOfCap(s.len + s.len shr 2) - var pos = 0 template addSpaces(n) = - for j in 0 ..< n: + for _ in 1..n: result.add(' ') - pos += 1 + pos += n - for i in 0 ..< len(s): - let c = s[i] + var pos = 0 + let denominator = if tabSize > 0: tabSize else: 1 + for c in s: if c == '\t': - let - denominator = if tabSize > 0: tabSize else: 1 - numSpaces = tabSize - pos mod denominator - + let numSpaces = tabSize - pos mod denominator addSpaces(numSpaces) else: result.add(c) @@ -53,58 +45,39 @@ proc expandTabs*(s: string, tabSize: int = 8): string {.noSideEffect, if c == '\l': pos = 0 -proc partition*(s: string, sep: string, - right: bool = false): (string, string, string) - {.noSideEffect, procvar.} = - ## Split the string at the first or last occurrence of `sep` into a 3-tuple +func partition*(s: string, sep: string, + right: bool = false): (string, string, string) = + ## Splits the string at the first (if `right` is false) + ## or last (if `right` is true) occurrence of `sep` into a 3-tuple. ## - ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or - ## (`s`, "", "") if `sep` is not found and `right` is false or - ## ("", "", `s`) if `sep` is not found and `right` is true + ## Returns a 3-tuple of strings, `(beforeSep, sep, afterSep)` or + ## `(s, "", "")` if `sep` is not found and `right` is false or + ## `("", "", s)` if `sep` is not found and `right` is true. + ## + ## **See also:** + ## * `rpartition proc <#rpartition,string,string>`_ runnableExamples: - doAssert partition("foo:bar", ":") == ("foo", ":", "bar") - doAssert partition("foobarbar", "bar") == ("foo", "bar", "bar") - doAssert partition("foobarbar", "bank") == ("foobarbar", "", "") - doAssert partition("foobarbar", "foo") == ("", "foo", "barbar") - doAssert partition("foofoobar", "bar") == ("foofoo", "bar", "") + doAssert partition("foo:bar:baz", ":") == ("foo", ":", "bar:baz") + doAssert partition("foo:bar:baz", ":", right = true) == ("foo:bar", ":", "baz") + doAssert partition("foobar", ":") == ("foobar", "", "") + doAssert partition("foobar", ":", right = true) == ("", "", "foobar") let position = if right: s.rfind(sep) else: s.find(sep) if position != -1: return (s[0 ..< position], sep, s[position + sep.len ..< s.len]) return if right: ("", "", s) else: (s, "", "") -proc rpartition*(s: string, sep: string): (string, string, string) - {.noSideEffect, procvar.} = - ## Split the string at the last occurrence of `sep` into a 3-tuple +func rpartition*(s: string, sep: string): (string, string, string) = + ## Splits the string at the last occurrence of `sep` into a 3-tuple. + ## + ## Returns a 3-tuple of strings, `(beforeSep, sep, afterSep)` or + ## `("", "", s)` if `sep` is not found. This is the same as + ## `partition(s, sep, right = true)`. ## - ## Returns a 3 string tuple of (beforeSep, `sep`, afterSep) or - ## ("", "", `s`) if `sep` is not found + ## **See also:** + ## * `partition proc <#partition,string,string,bool>`_ runnableExamples: - doAssert rpartition("foo:bar", ":") == ("foo", ":", "bar") - doAssert rpartition("foobarbar", "bar") == ("foobar", "bar", "") - doAssert rpartition("foobarbar", "bank") == ("", "", "foobarbar") - doAssert rpartition("foobarbar", "foo") == ("", "foo", "barbar") - doAssert rpartition("foofoobar", "bar") == ("foofoo", "bar", "") - - return partition(s, sep, right = true) - -when isMainModule: - doAssert expandTabs("\t", 4) == " " - doAssert expandTabs("\tfoo\t", 4) == " foo " - doAssert expandTabs("\tfoo\tbar", 4) == " foo bar" - doAssert expandTabs("\tfoo\tbar\t", 4) == " foo bar " - doAssert expandTabs("", 4) == "" - doAssert expandTabs("", 0) == "" - doAssert expandTabs("\t\t\t", 0) == "" - - doAssert partition("foo:bar", ":") == ("foo", ":", "bar") - doAssert partition("foobarbar", "bar") == ("foo", "bar", "bar") - doAssert partition("foobarbar", "bank") == ("foobarbar", "", "") - doAssert partition("foobarbar", "foo") == ("", "foo", "barbar") - doAssert partition("foofoobar", "bar") == ("foofoo", "bar", "") + doAssert rpartition("foo:bar:baz", ":") == ("foo:bar", ":", "baz") + doAssert rpartition("foobar", ":") == ("", "", "foobar") - doAssert rpartition("foo:bar", ":") == ("foo", ":", "bar") - doAssert rpartition("foobarbar", "bar") == ("foobar", "bar", "") - doAssert rpartition("foobarbar", "bank") == ("", "", "foobarbar") - doAssert rpartition("foobarbar", "foo") == ("", "foo", "barbar") - doAssert rpartition("foofoobar", "bar") == ("foofoo", "bar", "") + partition(s, sep, right = true) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 0125a1926..16ef9e642 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -12,7 +12,7 @@ This module contains a `scanf`:idx: macro that can be used for extracting substrings from an input string. This is often easier than regular expressions. Some examples as an appetizer: -.. code-block:: nim + ```nim # check if input string matches a triple of integers: const input = "(1,2,4)" var x, y, z: int @@ -26,6 +26,7 @@ Some examples as an appetizer: var myfloat: float if scanf(input, "$i-$i-$i $w$s$f", year, month, day, identifier, myfloat): echo "yes, we have a match!" + ``` As can be seen from the examples, strings are matched verbatim except for substrings starting with ``$``. These constructions are available: @@ -35,8 +36,9 @@ substrings starting with ``$``. These constructions are available: ``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. ``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. ``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. -``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$f`` Matches a floating-point number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Za-z_][A-Za-z_0-9]*``. +``$c`` Matches a single ASCII character. ``$s`` Skips optional whitespace. ``$$`` Matches a single dollar sign. ``$.`` Matches if the end of the input string has been reached. @@ -51,7 +53,7 @@ substrings starting with ``$``. These constructions are available: ================= ======================================================== Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*`` -and ``.+`` they work quite differently, there is no non-deterministic +and ``.+``, they work quite differently. There is no non-deterministic state machine involved and the matches are non-greedy. ``[$*]`` matches ``[xyz]`` via ``parseutils.parseUntil``. @@ -82,8 +84,7 @@ matches optional tokens without any result binding. In this example, we define a helper proc ``someSep`` that skips some separators which we then use in our scanf pattern to help us in the matching process: -.. code-block:: nim - + ```nim proc someSep(input: string; start: int; seps: set[char] = {':','-','.'}): int = # Note: The parameters and return value must match to what ``scanf`` requires result = 0 @@ -91,11 +92,11 @@ which we then use in our scanf pattern to help us in the matching process: if scanf(input, "$w$[someSep]$w", key, value): ... + ``` -It also possible to pass arguments to a user definable matcher: - -.. code-block:: nim +It is also possible to pass arguments to a user definable matcher: + ```nim proc ndigits(input: string; intVal: var int; start: int; n: int): int = # matches exactly ``n`` digits. Matchers need to return 0 if nothing # matched or otherwise the number of processed chars. @@ -114,6 +115,7 @@ It also possible to pass arguments to a user definable matcher: var year, month, day: int if scanf("2013-01-03", "${ndigits(4)}-${ndigits(2)}-${ndigits(2)}$.", year, month, day): ... + ``` The scanp macro @@ -144,8 +146,7 @@ not implemented. Simple example that parses the ``/etc/passwd`` file line by line: -.. code-block:: nim - + ```nim const etc_passwd = """root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/bin/sh @@ -164,17 +165,17 @@ Simple example that parses the ``/etc/passwd`` file line by line: result.add entry else: break + ``` The ``scanp`` maps the grammar code into Nim code that performs the parsing. The parsing is performed with the help of 3 helper templates that that can be implemented for a custom type. These templates need to be named ``atom`` and ``nxt``. ``atom`` should be -overloaded to handle both single characters and sets of character. +overloaded to handle both `char` and `set[char]`. -.. code-block:: nim - - import streams + ```nim + import std/streams template atom(input: Stream; idx: int; c: char): bool = ## Used in scanp for the matching of atoms (usually chars). @@ -189,11 +190,11 @@ overloaded to handle both single characters and sets of character. if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'): result.add entry + ``` Calling ordinary Nim procs inside the macro is possible: -.. code-block:: nim - + ```nim proc digits(s: string; intVal: var int; start: int): int = var x = 0 while result+start < s.len and s[result+start] in {'0'..'9'} and s[result+start] != ':': @@ -219,12 +220,12 @@ Calling ordinary Nim procs inside the macro is possible: result.add login & " " & homedir else: break + ``` When used for matching, keep in mind that likewise scanf, no backtracking is performed. -.. code-block:: nim - + ```nim proc skipUntil(s: string; until: string; unless = '\0'; start: int): int = # Skips all characters until the string `until` is found. Returns 0 # if the char `unless` is found first or the end is reached. @@ -255,12 +256,12 @@ is performed. for r in collectLinks(body): echo r + ``` In this example both macros are combined seamlessly in order to maximise efficiency and perform different checks. -.. code-block:: nim - + ```nim iterator parseIps*(soup: string): string = ## ipv4 only! const digits = {'0'..'9'} @@ -278,11 +279,16 @@ efficiency and perform different checks. yield buf buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage idx.inc - + ``` ]## -import macros, parseutils +import std/[macros, parseutils] +import std/private/since + +when defined(nimPreviewSlimSystem): + import std/assertions + proc conditionsToIfChain(n, idx, res: NimNode; start: int): NimNode = assert n.kind == nnkStmtList @@ -345,6 +351,12 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b else: matchError inc i + of 'c': + if i < results.len and getType(results[i]).typeKind == ntyChar: + matchBind "parseChar" + else: + matchError + inc i of 'b': if i < results.len and getType(results[i]).typeKind == ntyInt: matchBind "parseBin" @@ -396,7 +408,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b var resLen = genSym(nskLet, "resLen") conds.add newLetStmt(resLen, newCall(bindSym"parseUntil", inp, results[i], newLit(token), idx)) - conds.add newCall(bindSym"!=", resLen, newLit min) + conds.add newCall(bindSym">=", resLen, newLit min) conds.add resLen else: matchError @@ -458,6 +470,54 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b else: result.add res +macro scanTuple*(input: untyped; pattern: static[string]; matcherTypes: varargs[untyped]): untyped {.since: (1, 5).}= + ## Works identically as scanf, but instead of predeclaring variables it returns a tuple. + ## Tuple is started with a bool which indicates if the scan was successful + ## followed by the requested data. + ## If using a user defined matcher, provide the types in order they appear after pattern: + ## `line.scanTuple("${yourMatcher()}", int)` + runnableExamples: + let (success, year, month, day, time) = scanTuple("1000-01-01 00:00:00", "$i-$i-$i$s$+") + if success: + assert year == 1000 + assert month == 1 + assert day == 1 + assert time == "00:00:00" + var + p = 0 + userMatches = 0 + arguments: seq[NimNode] + result = newStmtList() + template addVar(typ: string) = + let varIdent = ident("temp" & $arguments.len) + result.add(newNimNode(nnkVarSection).add(newIdentDefs(varIdent, ident(typ), newEmptyNode()))) + arguments.add(varIdent) + while p < pattern.len: + if pattern[p] == '$': + inc p + case pattern[p] + of 'w', '*', '+': + addVar("string") + of 'c': + addVar("char") + of 'b', 'o', 'i', 'h': + addVar("int") + of 'f': + addVar("float") + of '{': + if userMatches < matcherTypes.len: + let varIdent = ident("temp" & $arguments.len) + result.add(newNimNode(nnkVarSection).add(newIdentDefs(varIdent, matcherTypes[userMatches], newEmptyNode()))) + arguments.add(varIdent) + inc userMatches + else: discard + inc p + result.add nnkTupleConstr.newTree(newCall(ident("scanf"), input, newStrLitNode(pattern))) + for arg in arguments: + result[^1][0].add arg + result[^1].add arg + result = newBlockStmt(result) + template atom*(input: string; idx: int; c: char): bool = ## Used in scanp for the matching of atoms (usually chars). ## EOF is matched as ``'\0'``. @@ -521,13 +581,14 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = of nnkCallKinds: # *{'A'..'Z'} !! s.add(!_) template buildWhile(input, idx, init, cond, action): untyped = + mixin hasNxt while hasNxt(input, idx): init if not cond: break action # (x) a # bind action a to (x) - if it[0].kind == nnkPar and it.len == 2: + if it[0].kind in {nnkPar, nnkTupleConstr} and it.len == 2: result = atm(it[0], input, idx, placeholder(it[1], input, idx)) elif it.kind == nnkInfix and it[0].eqIdent"->": # bind matching to some action: @@ -633,115 +694,3 @@ macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool = result.add res when defined(debugScanp): echo repr result - - -when isMainModule: - proc twoDigits(input: string; x: var int; start: int): int = - if start+1 < input.len and input[start] == '0' and input[start+1] == '0': - result = 2 - x = 13 - else: - result = 0 - - proc someSep(input: string; start: int; seps: set[char] = {';', ',', '-', '.'}): int = - result = 0 - while start+result < input.len and input[start+result] in seps: inc result - - proc demangle(s: string; res: var string; start: int): int = - while result+start < s.len and s[result+start] in {'_', '@'}: inc result - res = "" - while result+start < s.len and s[result+start] > ' ' and s[result+start] != '_': - res.add s[result+start] - inc result - while result+start < s.len and s[result+start] > ' ': - inc result - - proc parseGDB(resp: string): seq[string] = - const - digits = {'0'..'9'} - hexdigits = digits + {'a'..'f', 'A'..'F'} - whites = {' ', '\t', '\C', '\L'} - result = @[] - var idx = 0 - while true: - var prc = "" - var info = "" - if scanp(resp, idx, *`whites`, '#', *`digits`, +`whites`, ?("0x", *`hexdigits`, " in "), - demangle($input, prc, $index), *`whites`, '(', * ~ ')', ')', - *`whites`, "at ", +(~{'\C', '\L'} -> info.add($_))): - result.add prc & " " & info - else: - break - - var key, val: string - var intval: int - var floatval: float - doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intval, floatVal) - doAssert key == "abc" - doAssert val == "xyz" - doAssert intval == 89 - doAssert floatVal == 33.25 - - var binval: int - var octval: int - var hexval: int - doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) - doAssert binval == 0b0101 - doAssert octval == 0o1234 - doAssert hexval == 0xabcd - - let xx = scanf("$abc", "$$$i", intval) - doAssert xx == false - - - let xx2 = scanf("$1234", "$$$i", intval) - doAssert xx2 - - let yy = scanf(";.--Breakpoint00 [output]", - "$[someSep]Breakpoint${twoDigits}$[someSep({';','.','-'})] [$+]$.", - intVal, key) - doAssert yy - doAssert key == "output" - doAssert intVal == 13 - - var ident = "" - var idx = 0 - let zz = scanp("foobar x x x xWZ", idx, +{'a'..'z'} -> add(ident, $_), *(*{ - ' ', '\t'}, "x"), ~'U', "Z") - doAssert zz - doAssert ident == "foobar" - - const digits = {'0'..'9'} - var year = 0 - var idx2 = 0 - if scanp("201655-8-9", idx2, `digits`{4, 6} -> (year = year * 10 + ord($_) - - ord('0')), "-8", "-9"): - doAssert year == 201655 - - const gdbOut = """ - #0 @foo_96013_1208911747@8 (x0=...) - at c:/users/anwender/projects/nim/temp.nim:11 - #1 0x00417754 in tempInit000 () at c:/users/anwender/projects/nim/temp.nim:13 - #2 0x0041768d in NimMainInner () - at c:/users/anwender/projects/nim/lib/system.nim:2605 - #3 0x004176b1 in NimMain () - at c:/users/anwender/projects/nim/lib/system.nim:2613 - #4 0x004176db in main (argc=1, args=0x712cc8, env=0x711ca8) - at c:/users/anwender/projects/nim/lib/system.nim:2620""" - const result = @["foo c:/users/anwender/projects/nim/temp.nim:11", - "tempInit000 c:/users/anwender/projects/nim/temp.nim:13", - "NimMainInner c:/users/anwender/projects/nim/lib/system.nim:2605", - "NimMain c:/users/anwender/projects/nim/lib/system.nim:2613", - "main c:/users/anwender/projects/nim/lib/system.nim:2620"] - #doAssert parseGDB(gdbOut) == result - - # bug #6487 - var count = 0 - - proc test(): string = - inc count - result = ",123123" - - var a: int - discard scanf(test(), ",$i", a) - doAssert count == 1 diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index c6b44a3ca..4b07aca5a 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -48,15 +48,20 @@ runnableExamples: ## * `json module<json.html>`_ for table-like structure which allows ## heterogeneous members +import std/private/since import - hashes, strutils + std/[hashes, strutils] -when defined(js): +when defined(nimPreviewSlimSystem): + import std/assertions + + +when defined(js) or defined(nimscript) or defined(Standalone): {.pragma: rtlFunc.} else: {.pragma: rtlFunc, rtl.} - import os + import std/envvars include "system/inclrtl" @@ -88,6 +93,7 @@ const growthFactor = 2 startSize = 64 +proc mode*(t: StringTableRef): StringTableMode {.inline.} = t.mode iterator pairs*(t: StringTableRef): tuple[key, value: string] = ## Iterates over every `(key, value)` pair in the table `t`. @@ -139,10 +145,8 @@ template get(t: StringTableRef, key: string) = var index = rawGet(t, key) if index >= 0: result = t.data[index].val else: - when compiles($key): - raise newException(KeyError, "key not found: " & $key) - else: - raise newException(KeyError, "key not found") + raise newException(KeyError, "key not found: " & key) + proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = ## Returns the number of keys in `t`. @@ -227,11 +231,11 @@ proc enlarge(t: StringTableRef) = var n: KeyValuePairSeq newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): - if t.data[i].hasValue: rawInsert(t, n, t.data[i].key, t.data[i].val) + if t.data[i].hasValue: rawInsert(t, n, move t.data[i].key, move t.data[i].val) swap(t.data, n) proc `[]=`*(t: StringTableRef, key, val: string) {. - rtlFunc, extern: "nstPut", noSideEffect.} = + rtlFunc, extern: "nstPut".} = ## Inserts a `(key, value)` pair into `t`. ## ## See also: @@ -251,20 +255,17 @@ proc `[]=`*(t: StringTableRef, key, val: string) {. inc(t.counter) proc newStringTable*(mode: StringTableMode): owned(StringTableRef) {. - rtlFunc, extern: "nst$1".} = + rtlFunc, extern: "nst$1", noSideEffect.} = ## Creates a new empty string table. ## ## See also: ## * `newStringTable(keyValuePairs) proc ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) + result = StringTableRef(mode: mode, counter: 0, data: newSeq[KeyValuePair](startSize)) proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): owned(StringTableRef) {. - rtlFunc, extern: "nst$1WithPairs".} = + rtlFunc, extern: "nst$1WithPairs", noSideEffect.} = ## Creates a new string table with given `key, value` string pairs. ## ## `StringTableMode` must be specified. @@ -275,12 +276,13 @@ proc newStringTable*(keyValuePairs: varargs[string], result = newStringTable(mode) var i = 0 while i < high(keyValuePairs): - result[keyValuePairs[i]] = keyValuePairs[i + 1] + {.noSideEffect.}: + result[keyValuePairs[i]] = keyValuePairs[i + 1] inc(i, 2) proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], mode: StringTableMode = modeCaseSensitive): owned(StringTableRef) {. - rtlFunc, extern: "nst$1WithTableConstr".} = + rtlFunc, extern: "nst$1WithTableConstr", noSideEffect.} = ## Creates a new string table with given `(key, value)` tuple pairs. ## ## The default mode is case sensitive. @@ -290,18 +292,19 @@ proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], mytab2 = newStringTable([("key3", "val3"), ("key4", "val4")]) result = newStringTable(mode) - for key, val in items(keyValuePairs): result[key] = val + for key, val in items(keyValuePairs): + {.noSideEffect.}: + result[key] = val proc raiseFormatException(s: string) = raise newException(ValueError, "format string: key not found: " & s) proc getValue(t: StringTableRef, flags: set[FormatFlag], key: string): string = if hasKey(t, key): return t.getOrDefault(key) - # hm difficult: assume safety in taint mode here. XXX This is dangerous! - when defined(js): + when defined(js) or defined(nimscript) or defined(Standalone): result = "" else: - if useEnvironment in flags: result = os.getEnv(key).string + if useEnvironment in flags: result = getEnv(key) else: result = "" if result.len == 0: if useKey in flags: result = '$' & key @@ -417,16 +420,3 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. else: add(result, f[i]) inc(i) - - -when isMainModule: - var x = {"k": "v", "11": "22", "565": "67"}.newStringTable - assert x["k"] == "v" - assert x["11"] == "22" - assert x["565"] == "67" - x["11"] = "23" - assert x["11"] == "23" - - x.clear(modeCaseInsensitive) - x["11"] = "22" - assert x["11"] == "22" diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 535af2b31..81be7db17 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -9,46 +9,43 @@ ## The system module defines several common functions for working with strings, ## such as: -## * ``$`` for converting other data-types to strings -## * ``&`` for string concatenation -## * ``add`` for adding a new character or a string to the existing one -## * ``in`` (alias for ``contains``) and ``notin`` for checking if a character +## * `$` for converting other data-types to strings +## * `&` for string concatenation +## * `add` for adding a new character or a string to the existing one +## * `in` (alias for `contains`) and `notin` for checking if a character ## is in a string ## ## This module builds upon that, providing additional functionality in form of ## procedures, iterators and templates for strings. -## -## .. code-block:: -## import strutils -## -## let -## numbers = @[867, 5309] -## multiLineString = "first line\nsecond line\nthird line" -## -## let jenny = numbers.join("-") -## assert jenny == "867-5309" -## -## assert splitLines(multiLineString) == -## @["first line", "second line", "third line"] -## assert split(multiLineString) == @["first", "line", "second", -## "line", "third", "line"] -## assert indent(multiLineString, 4) == -## " first line\n second line\n third line" -## assert 'z'.repeat(5) == "zzzzz" -## + +runnableExamples: + let + numbers = @[867, 5309] + multiLineString = "first line\nsecond line\nthird line" + + let jenny = numbers.join("-") + assert jenny == "867-5309" + + assert splitLines(multiLineString) == + @["first line", "second line", "third line"] + assert split(multiLineString) == @["first", "line", "second", + "line", "third", "line"] + assert indent(multiLineString, 4) == + " first line\n second line\n third line" + assert 'z'.repeat(5) == "zzzzz" + ## The chaining of functions is possible thanks to the ## `method call syntax<manual.html#procedures-method-call-syntax>`_: -## -## .. code-block:: -## import strutils -## from sequtils import map -## -## let jenny = "867-5309" -## assert jenny.split('-').map(parseInt) == @[867, 5309] -## -## assert "Beetlejuice".indent(1).repeat(3).strip == -## "Beetlejuice Beetlejuice Beetlejuice" -## + +runnableExamples: + from std/sequtils import map + + let jenny = "867-5309" + assert jenny.split('-').map(parseInt) == @[867, 5309] + + assert "Beetlejuice".indent(1).repeat(3).strip == + "Beetlejuice Beetlejuice Beetlejuice" + ## This module is available for the `JavaScript target ## <backends.html#backends-the-javascript-target>`_. ## @@ -59,67 +56,86 @@ ## * `unicode module<unicode.html>`_ for Unicode UTF-8 handling ## * `sequtils module<sequtils.html>`_ for operations on container ## types (including strings) +## * `parsecsv module<parsecsv.html>`_ for a high-performance CSV parser ## * `parseutils module<parseutils.html>`_ for lower-level parsing of tokens, ## numbers, identifiers, etc. ## * `parseopt module<parseopt.html>`_ for command-line parsing +## * `pegs module<pegs.html>`_ for PEG (Parsing Expression Grammar) support ## * `strtabs module<strtabs.html>`_ for efficient hash tables ## (dictionaries, in some programming languages) mapping from strings to strings -## * `pegs module<pegs.html>`_ for PEG (Parsing Expression Grammar) support ## * `ropes module<ropes.html>`_ for rope data type, which can represent very ## long strings efficiently ## * `re module<re.html>`_ for regular expression (regex) support -## * `strscans<strscans.html>`_ for ``scanf`` and ``scanp`` macros, which offer +## * `strscans<strscans.html>`_ for `scanf` and `scanp` macros, which offer ## easier substring extraction than regular expressions -import parseutils -from math import pow, floor, log10 -from algorithm import reverse +import std/parseutils +from std/math import pow, floor, log10 +from std/algorithm import fill, reverse +import std/enumutils -when defined(nimVmExportFixed): - from unicode import toLower, toUpper - export toLower, toUpper +from std/unicode import toLower, toUpper +export toLower, toUpper include "system/inclrtl" +import std/private/[since, jsutils] +from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, + startsWithImpl, endsWithImpl + +when defined(nimPreviewSlimSystem): + import std/assertions + const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} ## All the characters that count as whitespace (space, tab, vertical tab, - ## carriage return, new line, form feed) + ## carriage return, new line, form feed). Letters* = {'A'..'Z', 'a'..'z'} - ## the set of letters + ## The set of letters. + + UppercaseLetters* = {'A'..'Z'} + ## The set of uppercase ASCII letters. + + LowercaseLetters* = {'a'..'z'} + ## The set of lowercase ASCII letters. + + PunctuationChars* = {'!'..'/', ':'..'@', '['..'`', '{'..'~'} + ## The set of all ASCII punctuation characters. Digits* = {'0'..'9'} - ## the set of digits + ## The set of digits. HexDigits* = {'0'..'9', 'A'..'F', 'a'..'f'} - ## the set of hexadecimal digits + ## The set of hexadecimal digits. IdentChars* = {'a'..'z', 'A'..'Z', '0'..'9', '_'} - ## the set of characters an identifier can consist of + ## The set of characters an identifier can consist of. IdentStartChars* = {'a'..'z', 'A'..'Z', '_'} - ## the set of characters an identifier can start with + ## The set of characters an identifier can start with. Newlines* = {'\13', '\10'} - ## the set of characters a newline terminator can start with (carriage - ## return, line feed) + ## The set of characters a newline terminator can start with (carriage + ## return, line feed). + + PrintableChars* = Letters + Digits + PunctuationChars + Whitespace + ## The set of all printable ASCII characters (letters, digits, whitespace, and punctuation characters). AllChars* = {'\x00'..'\xFF'} ## A set with all the possible characters. ## ## Not very useful by its own, you can use it to create *inverted* sets to - ## make the `find proc<#find,string,set[char],Natural,int>`_ + ## make the `find func<#find,string,set[char],Natural,int>`_ ## find **invalid** characters in strings. Example: - ## - ## .. code-block:: nim + ## ```nim ## let invalid = AllChars - Digits ## doAssert "01234".find(invalid) == -1 ## doAssert "01A34".find(invalid) == 2 + ## ``` -proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaAsciiChar".} = +func isAlphaAscii*(c: char): bool {.rtl, extern: "nsuIsAlphaAsciiChar".} = ## Checks whether or not character `c` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -130,8 +146,7 @@ proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, doAssert isAlphaAscii('8') == false return c in Letters -proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericChar".} = +func isAlphaNumeric*(c: char): bool {.rtl, extern: "nsuIsAlphaNumericChar".} = ## Checks whether or not `c` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. @@ -141,8 +156,7 @@ proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar, doAssert isAlphaNumeric(' ') == false return c in Letters+Digits -proc isDigit*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitChar".} = +func isDigit*(c: char): bool {.rtl, extern: "nsuIsDigitChar".} = ## Checks whether or not `c` is a number. ## ## This checks 0-9 ASCII characters only. @@ -151,8 +165,7 @@ proc isDigit*(c: char): bool {.noSideEffect, procvar, doAssert isDigit('8') == true return c in Digits -proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiChar".} = +func isSpaceAscii*(c: char): bool {.rtl, extern: "nsuIsSpaceAsciiChar".} = ## Checks whether or not `c` is a whitespace character. runnableExamples: doAssert isSpaceAscii('n') == false @@ -160,53 +173,49 @@ proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, doAssert isSpaceAscii('\t') == true return c in Whitespace -proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiChar".} = +func isLowerAscii*(c: char): bool {.rtl, extern: "nsuIsLowerAsciiChar".} = ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. ## Use `Unicode module<unicode.html>`_ for UTF-8 support. ## ## See also: - ## * `toLowerAscii proc<#toLowerAscii,char>`_ + ## * `toLowerAscii func<#toLowerAscii,char>`_ runnableExamples: doAssert isLowerAscii('e') == true doAssert isLowerAscii('E') == false doAssert isLowerAscii('7') == false - return c in {'a'..'z'} + return c in LowercaseLetters -proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiChar".} = +func isUpperAscii*(c: char): bool {.rtl, extern: "nsuIsUpperAsciiChar".} = ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. ## Use `Unicode module<unicode.html>`_ for UTF-8 support. ## ## See also: - ## * `toUpperAscii proc<#toUpperAscii,char>`_ + ## * `toUpperAscii func<#toUpperAscii,char>`_ runnableExamples: doAssert isUpperAscii('e') == false doAssert isUpperAscii('E') == true doAssert isUpperAscii('7') == false - return c in {'A'..'Z'} + return c in UppercaseLetters - -proc toLowerAscii*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToLowerAsciiChar".} = - ## Returns the lower case version of character ``c``. +func toLowerAscii*(c: char): char {.rtl, extern: "nsuToLowerAsciiChar".} = + ## Returns the lower case version of character `c`. ## - ## This works only for the letters ``A-Z``. See `unicode.toLower + ## This works only for the letters `A-Z`. See `unicode.toLower ## <unicode.html#toLower,Rune>`_ for a version that works for any Unicode ## character. ## ## See also: - ## * `isLowerAscii proc<#isLowerAscii,char>`_ - ## * `toLowerAscii proc<#toLowerAscii,string>`_ for converting a string + ## * `isLowerAscii func<#isLowerAscii,char>`_ + ## * `toLowerAscii func<#toLowerAscii,string>`_ for converting a string runnableExamples: doAssert toLowerAscii('A') == 'a' doAssert toLowerAscii('e') == 'e' - if c in {'A'..'Z'}: - result = chr(ord(c) + (ord('a') - ord('A'))) + if c in UppercaseLetters: + result = char(uint8(c) xor 0b0010_0000'u8) else: result = c @@ -215,85 +224,106 @@ template toImpl(call) = for i in 0..len(s) - 1: result[i] = call(s[i]) -proc toLowerAscii*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToLowerAsciiStr".} = +func toLowerAscii*(s: string): string {.rtl, extern: "nsuToLowerAsciiStr".} = ## Converts string `s` into lower case. ## - ## This works only for the letters ``A-Z``. See `unicode.toLower + ## This works only for the letters `A-Z`. See `unicode.toLower ## <unicode.html#toLower,string>`_ for a version that works for any Unicode ## character. ## ## See also: - ## * `normalize proc<#normalize,string>`_ + ## * `normalize func<#normalize,string>`_ runnableExamples: doAssert toLowerAscii("FooBar!") == "foobar!" toImpl toLowerAscii -proc toUpperAscii*(c: char): char {.noSideEffect, procvar, - rtl, extern: "nsuToUpperAsciiChar".} = +func toUpperAscii*(c: char): char {.rtl, extern: "nsuToUpperAsciiChar".} = ## Converts character `c` into upper case. ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## This works only for the letters `A-Z`. See `unicode.toUpper ## <unicode.html#toUpper,Rune>`_ for a version that works for any Unicode ## character. ## ## See also: - ## * `isLowerAscii proc<#isLowerAscii,char>`_ - ## * `toUpperAscii proc<#toUpperAscii,string>`_ for converting a string - ## * `capitalizeAscii proc<#capitalizeAscii,string>`_ + ## * `isUpperAscii func<#isUpperAscii,char>`_ + ## * `toUpperAscii func<#toUpperAscii,string>`_ for converting a string + ## * `capitalizeAscii func<#capitalizeAscii,string>`_ runnableExamples: doAssert toUpperAscii('a') == 'A' doAssert toUpperAscii('E') == 'E' - if c in {'a'..'z'}: - result = chr(ord(c) - (ord('a') - ord('A'))) + if c in LowercaseLetters: + result = char(uint8(c) xor 0b0010_0000'u8) else: result = c -proc toUpperAscii*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuToUpperAsciiStr".} = +func toUpperAscii*(s: string): string {.rtl, extern: "nsuToUpperAsciiStr".} = ## Converts string `s` into upper case. ## - ## This works only for the letters ``A-Z``. See `unicode.toUpper + ## This works only for the letters `A-Z`. See `unicode.toUpper ## <unicode.html#toUpper,string>`_ for a version that works for any Unicode ## character. ## ## See also: - ## * `capitalizeAscii proc<#capitalizeAscii,string>`_ + ## * `capitalizeAscii func<#capitalizeAscii,string>`_ runnableExamples: doAssert toUpperAscii("FooBar!") == "FOOBAR!" toImpl toUpperAscii -proc capitalizeAscii*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuCapitalizeAscii".} = +func capitalizeAscii*(s: string): string {.rtl, extern: "nsuCapitalizeAscii".} = ## Converts the first character of string `s` into upper case. ## - ## This works only for the letters ``A-Z``. + ## This works only for the letters `A-Z`. ## Use `Unicode module<unicode.html>`_ for UTF-8 support. ## ## See also: - ## * `toUpperAscii proc<#toUpperAscii,char>`_ + ## * `toUpperAscii func<#toUpperAscii,char>`_ runnableExamples: doAssert capitalizeAscii("foo") == "Foo" doAssert capitalizeAscii("-bar") == "-bar" if s.len == 0: result = "" else: result = toUpperAscii(s[0]) & substr(s, 1) -proc normalize*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuNormalize".} = +func nimIdentNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## That means to convert to lower case and remove any '_' on all characters + ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## <dochelpers.html#nimIdentBackticksNormalize,string>`_ for + ## an alternative approach. + runnableExamples: + doAssert nimIdentNormalize("Foo_bar") == "Foobar" + result = newString(s.len) + if s.len == 0: + return + result[0] = s[0] + var j = 1 + for i in 1..len(s) - 1: + if s[i] in UppercaseLetters: + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + inc j + elif s[i] != '_': + result[j] = s[i] + inc j + if j != s.len: setLen(result, j) + +func normalize*(s: string): string {.rtl, extern: "nsuNormalize".} = ## Normalizes the string `s`. ## ## That means to convert it to lower case and remove any '_'. This ## should NOT be used to normalize Nim identifier names. ## ## See also: - ## * `toLowerAscii proc<#toLowerAscii,string>`_ + ## * `toLowerAscii func<#toLowerAscii,string>`_ runnableExamples: doAssert normalize("Foo_bar") == "foobar" doAssert normalize("Foo Bar") == "foo bar" result = newString(s.len) var j = 0 for i in 0..len(s) - 1: - if s[i] in {'A'..'Z'}: + if s[i] in UppercaseLetters: result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) inc j elif s[i] != '_': @@ -301,72 +331,49 @@ proc normalize*(s: string): string {.noSideEffect, procvar, inc j if j != s.len: setLen(result, j) -proc cmpIgnoreCase*(a, b: string): int {.noSideEffect, - rtl, extern: "nsuCmpIgnoreCase", procvar.} = +func cmpIgnoreCase*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreCase".} = ## Compares two strings in a case insensitive manner. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreCase("FooBar", "foobar") == 0 doAssert cmpIgnoreCase("bar", "Foo") < 0 doAssert cmpIgnoreCase("Foo5", "foo4") > 0 - var i = 0 - var m = min(a.len, b.len) - while i < m: - result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) - if result != 0: return - inc(i) - result = a.len - b.len + cmpIgnoreCaseImpl(a, b) {.push checks: off, line_trace: off.} # this is a hot-spot in the compiler! # thus we compile without checks here -proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect, - rtl, extern: "nsuCmpIgnoreStyle", procvar.} = - ## Semantically the same as ``cmp(normalize(a), normalize(b))``. It +func cmpIgnoreStyle*(a, b: string): int {.rtl, extern: "nsuCmpIgnoreStyle".} = + ## Semantically the same as `cmp(normalize(a), normalize(b))`. It ## is just optimized to not allocate temporary strings. This should ## NOT be used to compare Nim identifier names. ## Use `macros.eqIdent<macros.html#eqIdent,string,string>`_ for that. ## ## Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b runnableExamples: doAssert cmpIgnoreStyle("foo_bar", "FooBar") == 0 doAssert cmpIgnoreStyle("foo_bar_5", "FooBar4") > 0 - var i = 0 - var j = 0 - while true: - while i < a.len and a[i] == '_': inc i - while j < b.len and b[j] == '_': inc j - var aa = if i < a.len: toLowerAscii(a[i]) else: '\0' - var bb = if j < b.len: toLowerAscii(b[j]) else: '\0' - result = ord(aa) - ord(bb) - if result != 0: return result - # the characters are identical: - if i >= a.len: - # both cursors at the end: - if j >= b.len: return 0 - # not yet at the end of 'b': - return -1 - elif j >= b.len: - return 1 - inc i - inc j + cmpIgnoreStyleImpl(a, b) {.pop.} # --------- Private templates for different split separators ----------- -proc substrEq(s: string, pos: int, substr: string): bool = - var i = 0 +func substrEq(s: string, pos: int, substr: string): bool = + # Always returns false for empty `substr` var length = substr.len - while i < length and pos+i < s.len and s[pos+i] == substr[i]: - inc i - return i == length + if length > 0: + var i = 0 + while i < length and pos+i < s.len and s[pos+i] == substr[i]: + inc i + i == length + else: false template stringHasSep(s: string, index: int, seps: set[char]): bool = s[index] in seps @@ -416,14 +423,12 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## ## Substrings are separated by the character `sep`. ## The code: - ## - ## .. code-block:: nim + ## ```nim ## for word in split(";;this;is;an;;example;;;", ';'): ## writeLine(stdout, word) - ## + ## ``` ## Results in: - ## - ## .. code-block:: + ## ``` ## "" ## "" ## "this" @@ -434,12 +439,13 @@ iterator split*(s: string, sep: char, maxsplit: int = -1): string = ## "" ## "" ## "" + ## ``` ## ## See also: ## * `rsplit iterator<#rsplit.i,string,char,int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `split proc<#split,string,char,int>`_ + ## * `split func<#split,string,char,int>`_ splitCommon(s, sep, maxsplit, 1) iterator split*(s: string, seps: set[char] = Whitespace, @@ -448,47 +454,55 @@ iterator split*(s: string, seps: set[char] = Whitespace, ## ## Substrings are separated by a substring containing only `seps`. ## - ## .. code-block:: nim + ## ```nim ## for word in split("this\lis an\texample"): ## writeLine(stdout, word) + ## ``` ## ## ...generates this output: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "an" ## "example" + ## ``` ## ## And the following code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("this:is;an$example", {';', ':', '$'}): ## writeLine(stdout, word) + ## ``` ## ## ...produces the same output as the first example. The code: ## - ## .. code-block:: nim + ## ```nim ## let date = "2012-11-20T22:08:08.398990" ## let separators = {' ', '-', ':', 'T'} ## for number in split(date, separators): ## writeLine(stdout, number) + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## "2012" ## "11" ## "20" ## "22" ## "08" ## "08.398990" + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,set[char],int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `split proc<#split,string,set[char],int>`_ + ## * `split func<#split,string,set[char],int>`_ splitCommon(s, seps, maxsplit, 1) iterator split*(s: string, sep: string, maxsplit: int = -1): string = @@ -497,23 +511,30 @@ iterator split*(s: string, sep: string, maxsplit: int = -1): string = ## Substrings are separated by the string `sep`. ## The code: ## - ## .. code-block:: nim + ## ```nim ## for word in split("thisDATAisDATAcorrupted", "DATA"): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: + ## ``` ## "this" ## "is" ## "corrupted" + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator<#rsplit.i,string,string,int,bool>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `split proc<#split,string,string,int>`_ - splitCommon(s, sep, maxsplit, sep.len) + ## * `split func<#split,string,string,int>`_ + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + splitCommon(s, sep, maxsplit, sepLen) template rsplitCommon(s, sep, maxsplit, sepLen) = @@ -544,17 +565,19 @@ iterator rsplit*(s: string, sep: char, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo:bar".rsplit(':'): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the char `sep`. ## @@ -562,76 +585,89 @@ iterator rsplit*(s: string, sep: char, ## * `split iterator<#split.i,string,char,int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `rsplit proc<#rsplit,string,char,int>`_ + ## * `rsplit func<#rsplit,string,char,int>`_ rsplitCommon(s, sep, maxsplit, 1) iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,char,int>`_ except in reverse order. + ## <#split.i,string,char,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foo bar".rsplit(WhiteSpace): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the set of chars `seps` ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,set[char],int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `rsplit proc<#rsplit,string,set[char],int>`_ + ## * `rsplit func<#rsplit,string,set[char],int>`_ rsplitCommon(s, seps, maxsplit, 1) iterator rsplit*(s: string, sep: string, maxsplit: int = -1, keepSeparators: bool = false): string = ## Splits the string `s` into substrings from the right using a ## string separator. Works exactly the same as `split iterator - ## <#split.i,string,string,int>`_ except in reverse order. + ## <#split.i,string,string,int>`_ except in **reverse** order. ## - ## .. code-block:: nim + ## ```nim ## for piece in "foothebar".rsplit("the"): ## echo piece + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ``` ## "bar" ## "foo" + ## ``` ## ## Substrings are separated from the right by the string `sep` ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator<#split.i,string,string,int>`_ ## * `splitLines iterator<#splitLines.i,string>`_ ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `rsplit proc<#rsplit,string,string,int>`_ - rsplitCommon(s, sep, maxsplit, sep.len) + ## * `rsplit func<#rsplit,string,string,int>`_ + let sepLen = if sep.len == 0: 1 # prevents infinite loop + else: sep.len + rsplitCommon(s, sep, maxsplit, sepLen) iterator splitLines*(s: string, keepEol = false): string = ## Splits the string `s` into its containing lines. ## ## Every `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. The result strings - ## contain no trailing end of line characters unless parameter ``keepEol`` - ## is set to ``true``. + ## contain no trailing end of line characters unless the parameter `keepEol` + ## is set to `true`. ## ## Example: ## - ## .. code-block:: nim + ## ```nim ## for line in splitLines("\nthis\nis\nan\n\nexample\n"): ## writeLine(stdout, line) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "" ## "this" ## "is" @@ -639,10 +675,11 @@ iterator splitLines*(s: string, keepEol = false): string = ## "" ## "example" ## "" + ## ``` ## ## See also: ## * `splitWhitespace iterator<#splitWhitespace.i,string,int>`_ - ## * `splitLines proc<#splitLines,string>`_ + ## * `splitLines func<#splitLines,string>`_ var first = 0 var last = 0 var eolpos = 0 @@ -665,22 +702,23 @@ iterator splitLines*(s: string, keepEol = false): string = first = last iterator splitWhitespace*(s: string, maxsplit: int = -1): string = - ## Splits the string ``s`` at whitespace stripping leading and trailing - ## whitespace if necessary. If ``maxsplit`` is specified and is positive, - ## no more than ``maxsplit`` splits is made. + ## Splits the string `s` at whitespace stripping leading and trailing + ## whitespace if necessary. If `maxsplit` is specified and is positive, + ## no more than `maxsplit` splits is made. ## ## The following code: ## - ## .. code-block:: nim + ## ```nim ## let s = " foo \t bar baz " ## for ms in [-1, 1, 2, 3]: ## echo "------ maxsplit = ", ms, ":" ## for item in s.splitWhitespace(maxsplit=ms): ## echo '"', item, '"' + ## ``` ## ## ...results in: ## - ## .. code-block:: + ## ``` ## ------ maxsplit = -1: ## "foo" ## "bar" @@ -696,56 +734,64 @@ iterator splitWhitespace*(s: string, maxsplit: int = -1): string = ## "foo" ## "bar" ## "baz" + ## ``` ## ## See also: ## * `splitLines iterator<#splitLines.i,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ oldSplit(s, Whitespace, maxsplit) -proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitChar".} = +func split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuSplitChar".} = ## The same as the `split iterator <#split.i,string,char,int>`_ (see its - ## documentation), but is a proc that returns a sequence of substrings. + ## documentation), but is a func that returns a sequence of substrings. ## ## See also: ## * `split iterator <#split.i,string,char,int>`_ - ## * `rsplit proc<#rsplit,string,char,int>`_ - ## * `splitLines proc<#splitLines,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `rsplit func<#rsplit,string,char,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a,b,c".split(',') == @["a", "b", "c"] doAssert "".split(' ') == @[""] accResult(split(s, sep, maxsplit)) -proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] {. - noSideEffect, rtl, extern: "nsuSplitCharSet".} = +func split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[ + string] {.rtl, extern: "nsuSplitCharSet".} = ## The same as the `split iterator <#split.i,string,set[char],int>`_ (see its - ## documentation), but is a proc that returns a sequence of substrings. + ## documentation), but is a func that returns a sequence of substrings. + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `split iterator <#split.i,string,set[char],int>`_ - ## * `rsplit proc<#rsplit,string,set[char],int>`_ - ## * `splitLines proc<#splitLines,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `rsplit func<#rsplit,string,set[char],int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] doAssert "".split({' '}) == @[""] + doAssert "empty seps return unsplit s".split({}) == @["empty seps return unsplit s"] accResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitString".} = +func split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuSplitString".} = ## Splits the string `s` into substrings using a string separator. ## ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". + ## ## See also: ## * `split iterator <#split.i,string,string,int>`_ - ## * `rsplit proc<#rsplit,string,string,int>`_ - ## * `splitLines proc<#splitLines,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `rsplit func<#rsplit,string,string,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a,b,c".split(",") == @["a", "b", "c"] doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"] @@ -753,14 +799,13 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] doAssert "a largely spaced sentence".split(" ", maxsplit = 1) == @["a", " largely spaced sentence"] - doAssert(sep.len > 0) - + doAssert "empty sep returns unsplit s".split("") == @["empty sep returns unsplit s"] accResult(split(s, sep, maxsplit)) -proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] - {.noSideEffect, rtl, extern: "nsuRSplitChar".} = - ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a proc - ## that returns a sequence of substrings. +func rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuRSplitChar".} = + ## The same as the `rsplit iterator <#rsplit.i,string,char,int>`_, but is a func + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -768,27 +813,29 @@ proc rsplit*(s: string, sep: char, maxsplit: int = -1): seq[string] ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", '#', maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` ## ## See also: ## * `rsplit iterator <#rsplit.i,string,char,int>`_ - ## * `split proc<#split,string,char,int>`_ - ## * `splitLines proc<#splitLines,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `split func<#split,string,char,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ accResult(rsplit(s, sep, maxsplit)) result.reverse() -proc rsplit*(s: string, seps: set[char] = Whitespace, +func rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[string] - {.noSideEffect, rtl, extern: "nsuRSplitCharSet".} = + {.rtl, extern: "nsuRSplitCharSet".} = ## The same as the `rsplit iterator <#rsplit.i,string,set[char],int>`_, but is a - ## proc that returns a sequence of substrings. + ## func that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -796,26 +843,31 @@ proc rsplit*(s: string, seps: set[char] = Whitespace, ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", {'#'}, maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator set results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,set[char],int>`_ - ## * `split proc<#split,string,set[char],int>`_ - ## * `splitLines proc<#splitLines,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `split func<#split,string,set[char],int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ accResult(rsplit(s, seps, maxsplit)) result.reverse() -proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] - {.noSideEffect, rtl, extern: "nsuRSplitString".} = - ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a proc - ## that returns a sequence of substrings. +func rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuRSplitString".} = + ## The same as the `rsplit iterator <#rsplit.i,string,string,int,bool>`_, but is a func + ## that returns a sequence of substrings in original order. ## ## A possible common use case for `rsplit` is path manipulation, ## particularly on systems that don't use a common delimiter. @@ -823,19 +875,24 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] ## For example, if a system had `#` as a delimiter, you could ## do the following to get the tail of the path: ## - ## .. code-block:: nim + ## ```nim ## var tailSplit = rsplit("Root#Object#Method#Index", "#", maxsplit=1) + ## ``` ## ## Results in `tailSplit` containing: ## - ## .. code-block:: nim + ## ```nim ## @["Root#Object#Method", "Index"] + ## ``` + ## + ## .. note:: Empty separator string results in returning an original string, + ## following the interpretation "split by no element". ## ## See also: ## * `rsplit iterator <#rsplit.i,string,string,int,bool>`_ - ## * `split proc<#split,string,string,int>`_ - ## * `splitLines proc<#splitLines,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ + ## * `split func<#split,string,string,int>`_ + ## * `splitLines func<#splitLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ runnableExamples: doAssert "a largely spaced sentence".rsplit(" ", maxsplit = 1) == @[ "a largely spaced", "sentence"] @@ -845,35 +902,35 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] doAssert "".rsplit("Elon Musk") == @[""] doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + doAssert "empty sep returns unsplit s".rsplit("") == @["empty sep returns unsplit s"] accResult(rsplit(s, sep, maxsplit)) result.reverse() -proc splitLines*(s: string, keepEol = false): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitLines".} = +func splitLines*(s: string, keepEol = false): seq[string] {.rtl, + extern: "nsuSplitLines".} = ## The same as the `splitLines iterator<#splitLines.i,string>`_ (see its - ## documentation), but is a proc that returns a sequence of substrings. + ## documentation), but is a func that returns a sequence of substrings. ## ## See also: ## * `splitLines iterator<#splitLines.i,string>`_ - ## * `splitWhitespace proc<#splitWhitespace,string,int>`_ - ## * `countLines proc<#countLines,string>`_ + ## * `splitWhitespace func<#splitWhitespace,string,int>`_ + ## * `countLines func<#countLines,string>`_ accResult(splitLines(s, keepEol = keepEol)) -proc splitWhitespace*(s: string, maxsplit: int = -1): seq[string] {.noSideEffect, - rtl, extern: "nsuSplitWhitespace".} = +func splitWhitespace*(s: string, maxsplit: int = -1): seq[string] {.rtl, + extern: "nsuSplitWhitespace".} = ## The same as the `splitWhitespace iterator <#splitWhitespace.i,string,int>`_ - ## (see its documentation), but is a proc that returns a sequence of substrings. + ## (see its documentation), but is a func that returns a sequence of substrings. ## ## See also: ## * `splitWhitespace iterator <#splitWhitespace.i,string,int>`_ - ## * `splitLines proc<#splitLines,string>`_ + ## * `splitLines func<#splitLines,string>`_ accResult(splitWhitespace(s, maxsplit)) -proc toBin*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToBin".} = +func toBin*(x: BiggestInt, len: Positive): string {.rtl, extern: "nsuToBin".} = ## Converts `x` into its binary representation. ## - ## The resulting string is always `len` characters long. No leading ``0b`` + ## The resulting string is always `len` characters long. No leading `0b` ## prefix is generated. runnableExamples: let @@ -892,14 +949,13 @@ proc toBin*(x: BiggestInt, len: Positive): string {.noSideEffect, inc shift mask = mask shl BiggestUInt(1) -proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToOct".} = +func toOct*(x: BiggestInt, len: Positive): string {.rtl, extern: "nsuToOct".} = ## Converts `x` into its octal representation. ## - ## The resulting string is always `len` characters long. No leading ``0o`` + ## The resulting string is always `len` characters long. No leading `0o` ## prefix is generated. ## - ## Do not confuse it with `toOctal proc<#toOctal,char>`_. + ## Do not confuse it with `toOctal func<#toOctal,char>`_. runnableExamples: let a = 62 @@ -917,44 +973,60 @@ proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect, inc shift, 3 mask = mask shl BiggestUInt(3) -proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect, - rtl, extern: "nsuToHex".} = +func toHexImpl(x: BiggestUInt, len: Positive, handleNegative: bool): string = + const + HexChars = "0123456789ABCDEF" + var n = x + result = newString(len) + for j in countdown(len-1, 0): + result[j] = HexChars[int(n and 0xF)] + n = n shr 4 + # handle negative overflow + if n == 0 and handleNegative: n = not(BiggestUInt 0) + +func toHex*[T: SomeInteger](x: T, len: Positive): string = ## Converts `x` to its hexadecimal representation. ## ## The resulting string will be exactly `len` characters long. No prefix like - ## ``0x`` is generated. `x` is treated as an unsigned value. + ## `0x` is generated. `x` is treated as an unsigned value. runnableExamples: let - a = 62 - b = 4097 + a = 62'u64 + b = 4097'u64 doAssert a.toHex(3) == "03E" doAssert b.toHex(3) == "001" doAssert b.toHex(4) == "1001" - const - HexChars = "0123456789ABCDEF" - var - n = x - result = newString(len) - for j in countdown(len-1, 0): - result[j] = HexChars[int(n and 0xF)] - n = n shr 4 - # handle negative overflow - if n == 0 and x < 0: n = -1 + doAssert toHex(62, 3) == "03E" + doAssert toHex(-8, 6) == "FFFFF8" + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), len, x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) -proc toHex*[T: SomeInteger](x: T): string = - ## Shortcut for ``toHex(x, T.sizeof * 2)`` +func toHex*[T: SomeInteger](x: T): string = + ## Shortcut for `toHex(x, T.sizeof * 2)` runnableExamples: doAssert toHex(1984'i64) == "00000000000007C0" - toHex(BiggestInt(x), T.sizeof * 2) + doAssert toHex(1984'i16) == "07C0" + whenJsNoBigInt64: + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + do: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) -proc toHex*(s: string): string {.noSideEffect, rtl.} = +func toHex*(s: string): string {.rtl.} = ## Converts a bytes string to its hexadecimal representation. ## ## The output is twice the input long. No prefix like - ## ``0x`` is generated. + ## `0x` is generated. ## ## See also: - ## * `parseHexStr proc<#parseHexStr,string>`_ for the reverse operation + ## * `parseHexStr func<#parseHexStr,string>`_ for the reverse operation runnableExamples: let a = "1" @@ -972,13 +1044,13 @@ proc toHex*(s: string): string {.noSideEffect, rtl.} = n = n shr 4 result[pos * 2] = HexChars[n] -proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = +func toOctal*(c: char): string {.rtl, extern: "nsuToOctal".} = ## Converts a character `c` to its octal representation. ## ## The resulting string may not have a leading zero. Its length is always ## exactly 3. ## - ## Do not confuse it with `toOct proc<#toOct,BiggestInt,Positive>`_. + ## Do not confuse it with `toOct func<#toOct,BiggestInt,Positive>`_. runnableExamples: doAssert toOctal('1') == "061" doAssert toOctal('A') == "101" @@ -991,7 +1063,7 @@ proc toOctal*(c: char): string {.noSideEffect, rtl, extern: "nsuToOctal".} = result[i] = chr(val mod 8 + ord('0')) val = val div 8 -proc fromBin*[T: SomeInteger](s: string): T = +func fromBin*[T: SomeInteger](s: string): T = ## Parses a binary integer value from a string `s`. ## ## If `s` is not a valid binary integer, `ValueError` is raised. `s` can have @@ -1014,7 +1086,7 @@ proc fromBin*[T: SomeInteger](s: string): T = if p != s.len or p == 0: raise newException(ValueError, "invalid binary integer: " & s) -proc fromOct*[T: SomeInteger](s: string): T = +func fromOct*[T: SomeInteger](s: string): T = ## Parses an octal integer value from a string `s`. ## ## If `s` is not a valid octal integer, `ValueError` is raised. `s` can have @@ -1037,7 +1109,7 @@ proc fromOct*[T: SomeInteger](s: string): T = if p != s.len or p == 0: raise newException(ValueError, "invalid oct integer: " & s) -proc fromHex*[T: SomeInteger](s: string): T = +func fromHex*[T: SomeInteger](s: string): T = ## Parses a hex integer value from a string `s`. ## ## If `s` is not a valid hex integer, `ValueError` is raised. `s` can have @@ -1060,8 +1132,8 @@ proc fromHex*[T: SomeInteger](s: string): T = if p != s.len or p == 0: raise newException(ValueError, "invalid hex integer: " & s) -proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, - rtl, extern: "nsuIntToStr".} = +func intToStr*(x: int, minchars: Positive = 1): string {.rtl, + extern: "nsuIntToStr".} = ## Converts `x` to its decimal representation. ## ## The resulting string will be minimally `minchars` characters long. This is @@ -1075,63 +1147,64 @@ proc intToStr*(x: int, minchars: Positive = 1): string {.noSideEffect, if x < 0: result = '-' & result -proc parseInt*(s: string): int {.noSideEffect, procvar, - rtl, extern: "nsuParseInt".} = +func parseInt*(s: string): int {.rtl, extern: "nsuParseInt".} = ## Parses a decimal integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. runnableExamples: doAssert parseInt("-0042") == -42 + result = 0 let L = parseutils.parseInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) -proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, - rtl, extern: "nsuParseBiggestInt".} = +func parseBiggestInt*(s: string): BiggestInt {.rtl, + extern: "nsuParseBiggestInt".} = ## Parses a decimal integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. + result = BiggestInt(0) let L = parseutils.parseBiggestInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) -proc parseUInt*(s: string): uint {.noSideEffect, procvar, - rtl, extern: "nsuParseUInt".} = +func parseUInt*(s: string): uint {.rtl, extern: "nsuParseUInt".} = ## Parses a decimal unsigned integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. + result = uint(0) let L = parseutils.parseUInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid unsigned integer: " & s) -proc parseBiggestUInt*(s: string): BiggestUInt {.noSideEffect, procvar, - rtl, extern: "nsuParseBiggestUInt".} = +func parseBiggestUInt*(s: string): BiggestUInt {.rtl, + extern: "nsuParseBiggestUInt".} = ## Parses a decimal unsigned integer value contained in `s`. ## ## If `s` is not a valid integer, `ValueError` is raised. + result = BiggestUInt(0) let L = parseutils.parseBiggestUInt(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid unsigned integer: " & s) -proc parseFloat*(s: string): float {.noSideEffect, procvar, - rtl, extern: "nsuParseFloat".} = +func parseFloat*(s: string): float {.rtl, extern: "nsuParseFloat".} = ## Parses a decimal floating point value contained in `s`. ## ## If `s` is not a valid floating point number, `ValueError` is raised. - ##``NAN``, ``INF``, ``-INF`` are also supported (case insensitive comparison). + ##`NAN`, `INF`, `-INF` are also supported (case insensitive comparison). runnableExamples: doAssert parseFloat("3.14") == 3.14 doAssert parseFloat("inf") == 1.0/0 + result = 0.0 let L = parseutils.parseFloat(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid float: " & s) -proc parseBinInt*(s: string): int {.noSideEffect, procvar, - rtl, extern: "nsuParseBinInt".} = +func parseBinInt*(s: string): int {.rtl, extern: "nsuParseBinInt".} = ## Parses a binary integer value contained in `s`. ## ## If `s` is not a valid binary integer, `ValueError` is raised. `s` can have - ## one of the following optional prefixes: ``0b``, ``0B``. Underscores within + ## one of the following optional prefixes: `0b`, `0B`. Underscores within ## `s` are ignored. runnableExamples: let @@ -1140,56 +1213,56 @@ proc parseBinInt*(s: string): int {.noSideEffect, procvar, doAssert a.parseBinInt() == 53 doAssert b.parseBinInt() == 7 + result = 0 let L = parseutils.parseBin(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid binary integer: " & s) -proc parseOctInt*(s: string): int {.noSideEffect, - rtl, extern: "nsuParseOctInt".} = +func parseOctInt*(s: string): int {.rtl, extern: "nsuParseOctInt".} = ## Parses an octal integer value contained in `s`. ## ## If `s` is not a valid oct integer, `ValueError` is raised. `s` can have one - ## of the following optional prefixes: ``0o``, ``0O``. Underscores within + ## of the following optional prefixes: `0o`, `0O`. Underscores within ## `s` are ignored. + result = 0 let L = parseutils.parseOct(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid oct integer: " & s) -proc parseHexInt*(s: string): int {.noSideEffect, procvar, - rtl, extern: "nsuParseHexInt".} = +func parseHexInt*(s: string): int {.rtl, extern: "nsuParseHexInt".} = ## Parses a hexadecimal integer value contained in `s`. ## ## If `s` is not a valid hex integer, `ValueError` is raised. `s` can have one - ## of the following optional prefixes: ``0x``, ``0X``, ``#``. Underscores + ## of the following optional prefixes: `0x`, `0X`, `#`. Underscores ## within `s` are ignored. + result = 0 let L = parseutils.parseHex(s, result, 0) if L != s.len or L == 0: raise newException(ValueError, "invalid hex integer: " & s) -proc generateHexCharToValueMap(): string = - ## Generate a string to map a hex digit to uint value +func generateHexCharToValueMap(): string = + ## Generates a string to map a hex digit to uint value. result = "" for inp in 0..255: let ch = chr(inp) let o = - case ch: - of '0'..'9': inp - ord('0') - of 'a'..'f': inp - ord('a') + 10 - of 'A'..'F': inp - ord('A') + 10 - else: 17 # indicates an invalid hex char + case ch + of '0'..'9': inp - ord('0') + of 'a'..'f': inp - ord('a') + 10 + of 'A'..'F': inp - ord('A') + 10 + else: 17 # indicates an invalid hex char result.add chr(o) const hexCharToValueMap = generateHexCharToValueMap() -proc parseHexStr*(s: string): string {.noSideEffect, procvar, - rtl, extern: "nsuParseHexStr".} = - ## Convert hex-encoded string to byte string, e.g.: +func parseHexStr*(s: string): string {.rtl, extern: "nsuParseHexStr".} = + ## Converts hex-encoded string to byte string, e.g.: ## - ## Raises ``ValueError`` for an invalid hex values. The comparison is + ## Raises `ValueError` for an invalid hex values. The comparison is ## case-insensitive. ## ## See also: - ## * `toHex proc<#toHex,string>`_ for the reverse operation + ## * `toHex func<#toHex,string>`_ for the reverse operation runnableExamples: let a = "41" @@ -1213,13 +1286,13 @@ proc parseHexStr*(s: string): string {.noSideEffect, procvar, else: result[pos div 2] = chr(val + buf shl 4) -proc parseBool*(s: string): bool = +func parseBool*(s: string): bool = ## Parses a value into a `bool`. ## - ## If ``s`` is one of the following values: ``y, yes, true, 1, on``, then - ## returns `true`. If ``s`` is one of the following values: ``n, no, false, - ## 0, off``, then returns `false`. If ``s`` is something else a - ## ``ValueError`` exception is raised. + ## If `s` is one of the following values: `y, yes, true, 1, on`, then + ## returns `true`. If `s` is one of the following values: `n, no, false, + ## 0, off`, then returns `false`. If `s` is something else a + ## `ValueError` exception is raised. runnableExamples: let a = "n" doAssert parseBool(a) == false @@ -1229,11 +1302,12 @@ proc parseBool*(s: string): bool = of "n", "no", "false", "0", "off": result = false else: raise newException(ValueError, "cannot interpret as a bool: " & s) -proc parseEnum*[T: enum](s: string): T = - ## Parses an enum ``T``. +func parseEnum*[T: enum](s: string): T = + ## Parses an enum `T`. This errors at compile time, if the given enum + ## type contains multiple fields with the same string value. ## - ## Raises ``ValueError`` for an invalid value in `s`. The comparison is - ## done in a style insensitive way. + ## Raises `ValueError` for an invalid value in `s`. The comparison is + ## done in a style insensitive way (first letter is still case-sensitive). runnableExamples: type MyEnum = enum @@ -1246,16 +1320,14 @@ proc parseEnum*[T: enum](s: string): T = doAssertRaises(ValueError): echo parseEnum[MyEnum]("third") - for e in low(T)..high(T): - if cmpIgnoreStyle(s, $e) == 0: - return e - raise newException(ValueError, "invalid enum value: " & s) + genEnumCaseStmt(T, s, default = nil, ord(low(T)), ord(high(T)), nimIdentNormalize) -proc parseEnum*[T: enum](s: string, default: T): T = - ## Parses an enum ``T``. +func parseEnum*[T: enum](s: string, default: T): T = + ## Parses an enum `T`. This errors at compile time, if the given enum + ## type contains multiple fields with the same string value. ## ## Uses `default` for an invalid value in `s`. The comparison is done in a - ## style insensitive way. + ## style insensitive way (first letter is still case-sensitive). runnableExamples: type MyEnum = enum @@ -1267,13 +1339,9 @@ proc parseEnum*[T: enum](s: string, default: T): T = doAssert parseEnum[MyEnum]("second") == second doAssert parseEnum[MyEnum]("last", third) == third - for e in low(T)..high(T): - if cmpIgnoreStyle(s, $e) == 0: - return e - result = default + genEnumCaseStmt(T, s, default, ord(low(T)), ord(high(T)), nimIdentNormalize) -proc repeat*(c: char, count: Natural): string {.noSideEffect, - rtl, extern: "nsuRepeatChar".} = +func repeat*(c: char, count: Natural): string {.rtl, extern: "nsuRepeatChar".} = ## Returns a string of length `count` consisting only of ## the character `c`. runnableExamples: @@ -1282,8 +1350,7 @@ proc repeat*(c: char, count: Natural): string {.noSideEffect, result = newString(count) for i in 0..count-1: result[i] = c -proc repeat*(s: string, n: Natural): string {.noSideEffect, - rtl, extern: "nsuRepeatStr".} = +func repeat*(s: string, n: Natural): string {.rtl, extern: "nsuRepeatStr".} = ## Returns string `s` concatenated `n` times. runnableExamples: doAssert "+ foo +".repeat(3) == "+ foo ++ foo ++ foo +" @@ -1291,15 +1358,15 @@ proc repeat*(s: string, n: Natural): string {.noSideEffect, result = newStringOfCap(n * s.len) for i in 1..n: result.add(s) -proc spaces*(n: Natural): string {.inline.} = - ## Returns a string with `n` space characters. You can use this proc +func spaces*(n: Natural): string {.inline.} = + ## Returns a string with `n` space characters. You can use this func ## to left align strings. ## ## See also: - ## * `align proc<#align,string,Natural,char>`_ - ## * `alignLeft proc<#alignLeft,string,Natural,char>`_ - ## * `indent proc<#indent,string,Natural,string>`_ - ## * `center proc<#center,string,int,char>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `indent func<#indent,string,Natural,string>`_ + ## * `center func<#center,string,int,char>`_ runnableExamples: let width = 15 @@ -1311,20 +1378,20 @@ proc spaces*(n: Natural): string {.inline.} = "This is a very long string|" repeat(' ', n) -proc align*(s: string, count: Natural, padding = ' '): string {. - noSideEffect, rtl, extern: "nsuAlignString".} = +func align*(s: string, count: Natural, padding = ' '): string {.rtl, + extern: "nsuAlignString".} = ## Aligns a string `s` with `padding`, so that it is of length `count`. ## ## `padding` characters (by default spaces) are added before `s` resulting in - ## right alignment. If ``s.len >= count``, no spaces are added and `s` is + ## right alignment. If `s.len >= count`, no spaces are added and `s` is ## returned unchanged. If you need to left align a string use the `alignLeft - ## proc <#alignLeft,string,Natural,char>`_. + ## func<#alignLeft,string,Natural,char>`_. ## ## See also: - ## * `alignLeft proc<#alignLeft,string,Natural,char>`_ - ## * `spaces proc<#spaces,Natural>`_ - ## * `indent proc<#indent,string,Natural,string>`_ - ## * `center proc<#center,string,int,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + ## * `center func<#center,string,int,char>`_ runnableExamples: assert align("abc", 4) == " abc" assert align("a", 0) == "a" @@ -1338,20 +1405,19 @@ proc align*(s: string, count: Natural, padding = ' '): string {. else: result = s -proc alignLeft*(s: string, count: Natural, padding = ' '): string {. - noSideEffect.} = +func alignLeft*(s: string, count: Natural, padding = ' '): string = ## Left-Aligns a string `s` with `padding`, so that it is of length `count`. ## ## `padding` characters (by default spaces) are added after `s` resulting in - ## left alignment. If ``s.len >= count``, no spaces are added and `s` is + ## left alignment. If `s.len >= count`, no spaces are added and `s` is ## returned unchanged. If you need to right align a string use the `align - ## proc <#align,string,Natural,char>`_. + ## func<#align,string,Natural,char>`_. ## ## See also: - ## * `align proc<#align,string,Natural,char>`_ - ## * `spaces proc<#spaces,Natural>`_ - ## * `indent proc<#indent,string,Natural,string>`_ - ## * `center proc<#center,string,int,char>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ + ## * `center func<#center,string,int,char>`_ runnableExamples: assert alignLeft("abc", 4) == "abc " assert alignLeft("a", 0) == "a" @@ -1366,8 +1432,8 @@ proc alignLeft*(s: string, count: Natural, padding = ' '): string {. else: result = s -proc center*(s: string, width: int, fillChar: char = ' '): string {. - noSideEffect, rtl, extern: "nsuCenterString".} = +func center*(s: string, width: int, fillChar: char = ' '): string {.rtl, + extern: "nsuCenterString".} = ## Return the contents of `s` centered in a string `width` long using ## `fillChar` (default: space) as padding. ## @@ -1375,10 +1441,10 @@ proc center*(s: string, width: int, fillChar: char = ' '): string {. ## to `s.len`. ## ## See also: - ## * `align proc<#align,string,Natural,char>`_ - ## * `alignLeft proc<#alignLeft,string,Natural,char>`_ - ## * `spaces proc<#spaces,Natural>`_ - ## * `indent proc<#indent,string,Natural,string>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ runnableExamples: let a = "foo" doAssert a.center(2) == "foo" @@ -1401,17 +1467,18 @@ proc center*(s: string, width: int, fillChar: char = ' '): string {. # the string s should go result[i] = fillChar -proc indent*(s: string, count: Natural, padding: string = " "): string - {.noSideEffect, rtl, extern: "nsuIndent".} = - ## Indents each line in ``s`` by ``count`` amount of ``padding``. +func indent*(s: string, count: Natural, padding: string = " "): string {.rtl, + extern: "nsuIndent".} = + ## Indents each line in `s` by `count` amount of `padding`. ## - ## **Note:** This does not preserve the new line characters used in ``s``. + ## **Note:** This does not preserve the new line characters used in `s`. ## ## See also: - ## * `align proc<#align,string,Natural,char>`_ - ## * `alignLeft proc<#alignLeft,string,Natural,char>`_ - ## * `spaces proc<#spaces,Natural>`_ - ## * `unindent proc<#unindent,string,Natural,string>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `unindent func<#unindent,string,Natural,string>`_ + ## * `dedent func<#dedent,string,Natural>`_ runnableExamples: doAssert indent("First line\c\l and second line.", 2) == " First line\l and second line." @@ -1425,21 +1492,25 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, count: Natural, padding: string = " "): string - {.noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents each line in ``s`` by ``count`` amount of ``padding``. - ## Sometimes called `dedent`:idx: +func unindent*(s: string, count: Natural = int.high, + padding: string = " "): string {.rtl, extern: "nsuUnindent".} = + ## Unindents each line in `s` by `count` amount of `padding`. ## - ## **Note:** This does not preserve the new line characters used in ``s``. + ## **Note:** This does not preserve the new line characters used in `s`. ## ## See also: - ## * `align proc<#align,string,Natural,char>`_ - ## * `alignLeft proc<#alignLeft,string,Natural,char>`_ - ## * `spaces proc<#spaces,Natural>`_ - ## * `indent proc<#indent,string,Natural,string>`_ + ## * `dedent func<#dedent,string,Natural>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ runnableExamples: - doAssert unindent(" First line\l and second line", 3) == - "First line\land second line" + let x = """ + Hello + There + """.unindent() + + doAssert x == "Hello\nThere\n" result = "" var i = 0 for line in s.splitLines(): @@ -1454,31 +1525,78 @@ proc unindent*(s: string, count: Natural, padding: string = " "): string result.add(line[indentCount*padding.len .. ^1]) i.inc -proc unindent*(s: string): string - {.noSideEffect, rtl, extern: "nsuUnindentAll".} = - ## Removes all indentation composed of whitespace from each line in ``s``. +func indentation*(s: string): Natural {.since: (1, 3).} = + ## Returns the amount of indentation all lines of `s` have in common, + ## ignoring lines that consist only of whitespace. + result = int.high + for line in s.splitLines: + for i, c in line: + if i >= result: break + elif c != ' ': + result = i + break + if result == int.high: + result = 0 + +func dedent*(s: string, count: Natural = indentation(s)): string {.rtl, + extern: "nsuDedent", since: (1, 3).} = + ## Unindents each line in `s` by `count` amount of `padding`. + ## The only difference between this and the + ## `unindent func<#unindent,string,Natural,string>`_ is that this by default + ## only cuts off the amount of indentation that all lines of `s` share as + ## opposed to all indentation. It only supports spaces as padding. + ## + ## **Note:** This does not preserve the new line characters used in `s`. ## ## See also: - ## * `align proc<#align,string,Natural,char>`_ - ## * `alignLeft proc<#alignLeft,string,Natural,char>`_ - ## * `spaces proc<#spaces,Natural>`_ - ## * `indent proc<#indent,string,Natural,string>`_ + ## * `unindent func<#unindent,string,Natural,string>`_ + ## * `align func<#align,string,Natural,char>`_ + ## * `alignLeft func<#alignLeft,string,Natural,char>`_ + ## * `spaces func<#spaces,Natural>`_ + ## * `indent func<#indent,string,Natural,string>`_ runnableExamples: let x = """ Hello - There - """.unindent() - - doAssert x == "Hello\nThere\n" - unindent(s, 1000) # TODO: Passing a 1000 is a bit hackish. + There + """.dedent() + + doAssert x == "Hello\n There\n" + unindent(s, count, " ") + +func delete*(s: var string, slice: Slice[int]) = + ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains + ## elements out of range. + ## + ## This operation moves all elements after `s[slice]` in linear time, and + ## is the string analog to `sequtils.delete`. + runnableExamples: + var a = "abcde" + doAssertRaises(IndexDefect): a.delete(4..5) + assert a == "abcde" + a.delete(4..4) + assert a == "abcd" + a.delete(1..2) + assert a == "ad" + a.delete(1..<1) # empty slice + assert a == "ad" + when compileOption("boundChecks"): + if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len): + raise newException(IndexDefect, $(slice: slice, len: s.len)) + if slice.b >= slice.a: + var i = slice.a + var j = slice.b + 1 + var newLen = s.len - j + i + # if j < s.len: moveMem(addr s[i], addr s[j], s.len - j) # pending benchmark + while i < newLen: + s[i] = s[j] + inc(i) + inc(j) + setLen(s, newLen) -proc delete*(s: var string, first, last: int) {.noSideEffect, - rtl, extern: "nsuDelete".} = - ## Deletes in `s` (must be declared as ``var``) the characters at positions - ## ``first ..last`` (both ends included). - ## - ## This modifies `s` itself, it does not return a copy. - runnableExamples: +func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete", + deprecated: "use `delete(s, first..last)`".} = + ## Deletes in `s` the characters at positions `first .. last` (both ends included). + runnableExamples("--warning:deprecated:off"): var a = "abracadabra" a.delete(4, 5) @@ -1499,83 +1617,71 @@ proc delete*(s: var string, first, last: int) {.noSideEffect, inc(j) setLen(s, newLen) - -proc startsWith*(s: string, prefix: char): bool {.noSideEffect, inline.} = - ## Returns true if ``s`` starts with character ``prefix``. +func startsWith*(s: string, prefix: char): bool {.inline.} = + ## Returns true if `s` starts with character `prefix`. ## ## See also: - ## * `endsWith proc<#endsWith,string,char>`_ - ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ - ## * `removePrefix proc<#removePrefix,string,char>`_ + ## * `endsWith func<#endsWith,string,char>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removePrefix func<#removePrefix,string,char>`_ runnableExamples: let a = "abracadabra" doAssert a.startsWith('a') == true doAssert a.startsWith('b') == false result = s.len > 0 and s[0] == prefix -proc startsWith*(s, prefix: string): bool {.noSideEffect, - rtl, extern: "nsuStartsWith".} = - ## Returns true if ``s`` starts with string ``prefix``. +func startsWith*(s, prefix: string): bool {.rtl, extern: "nsuStartsWith".} = + ## Returns true if `s` starts with string `prefix`. ## - ## If ``prefix == ""`` true is returned. + ## If `prefix == ""` true is returned. ## ## See also: - ## * `endsWith proc<#endsWith,string,string>`_ - ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ - ## * `removePrefix proc<#removePrefix,string,string>`_ + ## * `endsWith func<#endsWith,string,string>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removePrefix func<#removePrefix,string,string>`_ runnableExamples: let a = "abracadabra" doAssert a.startsWith("abra") == true doAssert a.startsWith("bra") == false - var i = 0 - while true: - if i >= prefix.len: return true - if i >= s.len or s[i] != prefix[i]: return false - inc(i) + startsWithImpl(s, prefix) -proc endsWith*(s: string, suffix: char): bool {.noSideEffect, inline.} = - ## Returns true if ``s`` ends with ``suffix``. +func endsWith*(s: string, suffix: char): bool {.inline.} = + ## Returns true if `s` ends with `suffix`. ## ## See also: - ## * `startsWith proc<#startsWith,string,char>`_ - ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ - ## * `removeSuffix proc<#removeSuffix,string,char>`_ + ## * `startsWith func<#startsWith,string,char>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removeSuffix func<#removeSuffix,string,char>`_ runnableExamples: let a = "abracadabra" doAssert a.endsWith('a') == true doAssert a.endsWith('b') == false result = s.len > 0 and s[s.high] == suffix -proc endsWith*(s, suffix: string): bool {.noSideEffect, - rtl, extern: "nsuEndsWith".} = - ## Returns true if ``s`` ends with ``suffix``. +func endsWith*(s, suffix: string): bool {.rtl, extern: "nsuEndsWith".} = + ## Returns true if `s` ends with `suffix`. ## - ## If ``suffix == ""`` true is returned. + ## If `suffix == ""` true is returned. ## ## See also: - ## * `startsWith proc<#startsWith,string,string>`_ - ## * `continuesWith proc<#continuesWith,string,string,Natural>`_ - ## * `removeSuffix proc<#removeSuffix,string,string>`_ + ## * `startsWith func<#startsWith,string,string>`_ + ## * `continuesWith func<#continuesWith,string,string,Natural>`_ + ## * `removeSuffix func<#removeSuffix,string,string>`_ runnableExamples: let a = "abracadabra" doAssert a.endsWith("abra") == true doAssert a.endsWith("dab") == false - var i = 0 - var j = len(s) - len(suffix) - while i+j >= 0 and i+j < s.len: - if s[i+j] != suffix[i]: return false - inc(i) - if i >= suffix.len: return true + endsWithImpl(s, suffix) -proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, - rtl, extern: "nsuContinuesWith".} = - ## Returns true if ``s`` continues with ``substr`` at position ``start``. +func continuesWith*(s, substr: string, start: Natural): bool {.rtl, + extern: "nsuContinuesWith".} = + ## Returns true if `s` continues with `substr` at position `start`. ## - ## If ``substr == ""`` true is returned. + ## If `substr == ""` true is returned. ## ## See also: - ## * `startsWith proc<#startsWith,string,string>`_ - ## * `endsWith proc<#endsWith,string,string>`_ + ## * `startsWith func<#startsWith,string,string>`_ + ## * `endsWith func<#endsWith,string,string>`_ runnableExamples: let a = "abracadabra" doAssert a.continuesWith("ca", 4) == true @@ -1588,13 +1694,13 @@ proc continuesWith*(s, substr: string, start: Natural): bool {.noSideEffect, inc(i) -proc removePrefix*(s: var string, chars: set[char] = Newlines) {. - rtl, extern: "nsuRemovePrefixCharSet".} = +func removePrefix*(s: var string, chars: set[char] = Newlines) {.rtl, + extern: "nsuRemovePrefixCharSet".} = ## Removes all characters from `chars` from the start of the string `s` ## (in-place). ## ## See also: - ## * `removeSuffix proc<#removeSuffix,string,set[char]>`_ + ## * `removeSuffix func<#removeSuffix,string,set[char]>`_ runnableExamples: var userInput = "\r\n*~Hello World!" userInput.removePrefix @@ -1608,43 +1714,43 @@ proc removePrefix*(s: var string, chars: set[char] = Newlines) {. var start = 0 while start < s.len and s[start] in chars: start += 1 - if start > 0: s.delete(0, start - 1) + if start > 0: s.delete(0..start - 1) -proc removePrefix*(s: var string, c: char) {. - rtl, extern: "nsuRemovePrefixChar".} = +func removePrefix*(s: var string, c: char) {.rtl, + extern: "nsuRemovePrefixChar".} = ## Removes all occurrences of a single character (in-place) from the start ## of a string. ## ## See also: - ## * `removeSuffix proc<#removeSuffix,string,char>`_ - ## * `startsWith proc<#startsWith,string,char>`_ + ## * `removeSuffix func<#removeSuffix,string,char>`_ + ## * `startsWith func<#startsWith,string,char>`_ runnableExamples: var ident = "pControl" ident.removePrefix('p') doAssert ident == "Control" removePrefix(s, chars = {c}) -proc removePrefix*(s: var string, prefix: string) {. - rtl, extern: "nsuRemovePrefixString".} = +func removePrefix*(s: var string, prefix: string) {.rtl, + extern: "nsuRemovePrefixString".} = ## Remove the first matching prefix (in-place) from a string. ## ## See also: - ## * `removeSuffix proc<#removeSuffix,string,string>`_ - ## * `startsWith proc<#startsWith,string,string>`_ + ## * `removeSuffix func<#removeSuffix,string,string>`_ + ## * `startsWith func<#startsWith,string,string>`_ runnableExamples: var answers = "yesyes" answers.removePrefix("yes") doAssert answers == "yes" - if s.startsWith(prefix): - s.delete(0, prefix.len - 1) + if s.startsWith(prefix) and prefix.len > 0: + s.delete(0..prefix.len - 1) -proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. - rtl, extern: "nsuRemoveSuffixCharSet".} = +func removeSuffix*(s: var string, chars: set[char] = Newlines) {.rtl, + extern: "nsuRemoveSuffixCharSet".} = ## Removes all characters from `chars` from the end of the string `s` ## (in-place). ## ## See also: - ## * `removePrefix proc<#removePrefix,string,set[char]>`_ + ## * `removePrefix func<#removePrefix,string,set[char]>`_ runnableExamples: var userInput = "Hello World!*~\r\n" userInput.removeSuffix @@ -1661,14 +1767,14 @@ proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. while last > -1 and s[last] in chars: last -= 1 s.setLen(last + 1) -proc removeSuffix*(s: var string, c: char) {. - rtl, extern: "nsuRemoveSuffixChar".} = +func removeSuffix*(s: var string, c: char) {.rtl, + extern: "nsuRemoveSuffixChar".} = ## Removes all occurrences of a single character (in-place) from the end ## of a string. ## ## See also: - ## * `removePrefix proc<#removePrefix,string,char>`_ - ## * `endsWith proc<#endsWith,string,char>`_ + ## * `removePrefix func<#removePrefix,string,char>`_ + ## * `endsWith func<#endsWith,string,char>`_ runnableExamples: var table = "users" table.removeSuffix('s') @@ -1680,13 +1786,13 @@ proc removeSuffix*(s: var string, c: char) {. removeSuffix(s, chars = {c}) -proc removeSuffix*(s: var string, suffix: string) {. - rtl, extern: "nsuRemoveSuffixString".} = +func removeSuffix*(s: var string, suffix: string) {.rtl, + extern: "nsuRemoveSuffixString".} = ## Remove the first matching suffix (in-place) from a string. ## ## See also: - ## * `removePrefix proc<#removePrefix,string,string>`_ - ## * `endsWith proc<#endsWith,string,string>`_ + ## * `removePrefix func<#removePrefix,string,string>`_ + ## * `endsWith func<#endsWith,string,string>`_ runnableExamples: var answers = "yeses" answers.removeSuffix("es") @@ -1697,14 +1803,14 @@ proc removeSuffix*(s: var string, suffix: string) {. s.setLen(newLen) -proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) - {.noSideEffect, inline.} = +func addSep*(dest: var string, sep = ", ", startLen: Natural = 0) {.inline.} = ## Adds a separator to `dest` only if its length is bigger than `startLen`. ## ## A shorthand for: ## - ## .. code-block:: nim + ## ```nim ## if dest.len > startLen: add(dest, sep) + ## ``` ## ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than @@ -1720,7 +1826,7 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) if dest.len > startLen: add(dest, sep) -proc allCharsInSet*(s: string, theSet: set[char]): bool = +func allCharsInSet*(s: string, theSet: set[char]): bool = ## Returns true if every character of `s` is in the set `theSet`. runnableExamples: doAssert allCharsInSet("aeea", {'a', 'e'}) == true @@ -1730,9 +1836,9 @@ proc allCharsInSet*(s: string, theSet: set[char]): bool = if c notin theSet: return false return true -proc abbrev*(s: string, possibilities: openArray[string]): int = - ## Returns the index of the first item in ``possibilities`` which starts - ## with ``s``, if not ambiguous. +func abbrev*(s: string, possibilities: openArray[string]): int = + ## Returns the index of the first item in `possibilities` which starts + ## with `s`, if not ambiguous. ## ## Returns -1 if no item has been found and -2 if multiple items match. runnableExamples: @@ -1752,8 +1858,8 @@ proc abbrev*(s: string, possibilities: openArray[string]): int = # --------------------------------------------------------------------------- -proc join*(a: openArray[string], sep: string = ""): string {. - noSideEffect, rtl, extern: "nsuJoinSep".} = +func join*(a: openArray[string], sep: string = ""): string {.rtl, + extern: "nsuJoinSep".} = ## Concatenates all strings in the container `a`, separating them with `sep`. runnableExamples: doAssert join(["A", "B", "Conclusion"], " -> ") == "A -> B -> Conclusion" @@ -1769,8 +1875,7 @@ proc join*(a: openArray[string], sep: string = ""): string {. else: result = "" -proc join*[T: not string](a: openArray[T], sep: string = ""): string {. - noSideEffect, rtl.} = +proc join*[T: not string](a: openArray[T], sep: string = ""): string = ## Converts all elements in the container `a` to strings using `$`, ## and concatenates them with `sep`. runnableExamples: @@ -1783,36 +1888,44 @@ proc join*[T: not string](a: openArray[T], sep: string = ""): string {. add(result, $x) type - SkipTable* = array[char, int] + SkipTable* = array[char, int] ## Character table for efficient substring search. -proc initSkipTable*(a: var SkipTable, sub: string) - {.noSideEffect, rtl, extern: "nsuInitSkipTable".} = - ## Preprocess table `a` for `sub`. +func initSkipTable*(a: var SkipTable, sub: string) {.rtl, + extern: "nsuInitSkipTable".} = + ## Initializes table `a` for efficient search of substring `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + # TODO: this should be the `default()` initializer for the type. let m = len(sub) - var i = 0 - while i <= 0xff-7: - a[chr(i + 0)] = m - a[chr(i + 1)] = m - a[chr(i + 2)] = m - a[chr(i + 3)] = m - a[chr(i + 4)] = m - a[chr(i + 5)] = m - a[chr(i + 6)] = m - a[chr(i + 7)] = m - i += 8 + fill(a, m) for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i -proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int - {.noSideEffect, rtl, extern: "nsuFindStrA".} = - ## Searches for `sub` in `s` inside range `start`..`last` using preprocessed +func initSkipTable*(sub: string): SkipTable {.noinit, rtl, + extern: "nsuInitNewSkipTable".} = + ## Returns a new table initialized for `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + initSkipTable(result, sub) + +func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. + rtl, extern: "nsuFindStrA".} = + ## Searches for `sub` in `s` inside range `start..last` using preprocessed ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last ## element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let - last = if last == 0: s.high else: last + last = if last < 0: s.high else: last subLast = sub.len - 1 if subLast == -1: @@ -1822,6 +1935,7 @@ proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int # This is an implementation of the Boyer-Moore Horspool algorithms # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + result = -1 var skip = start while last - skip >= subLast: @@ -1831,130 +1945,162 @@ proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int return skip dec i inc skip, a[s[skip + subLast]] - return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): - proc c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} const hasCStringBuiltin = true else: const hasCStringBuiltin = false -proc find*(s: string, sub: char, start: Natural = 0, last = 0): int {.noSideEffect, - rtl, extern: "nsuFindChar".} = - ## Searches for `sub` in `s` inside range ``start..last`` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). +func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuFindChar".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - ## Otherwise the index returned is relative to ``s[0]``, not ``start``. - ## Use `s[start..last].rfind` for a ``start``-origin index. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind proc<#rfind,string,char,Natural,int>`_ - ## * `replace proc<#replace,string,char,char>`_ - let last = if last == 0: s.high else: last - when nimvm: + ## * `rfind func<#rfind,string,char,Natural,int>`_ + ## * `replace func<#replace,string,char,char>`_ + result = -1 + let last = if last < 0: s.high else: last + + template findImpl = for i in int(start)..last: - if sub == s[i]: return i + if s[i] == sub: + return i + + when nimvm: + findImpl() else: when hasCStringBuiltin: - let L = last-start+1 - if L > 0: - let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + let length = last-start+1 + if length > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + return cast[int](found) -% cast[int](s.cstring) else: - for i in int(start)..last: - if sub == s[i]: return i - return -1 + findImpl() -proc find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {.noSideEffect, - rtl, extern: "nsuFindCharSet".} = - ## Searches for `chars` in `s` inside range ``start..last`` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). +func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. + rtl, extern: "nsuFindCharSet".} = + ## Searches for `chars` in `s` inside range `start..last` (both ends included). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. - ## Otherwise the index returned is relative to ``s[0]``, not ``start``. - ## Use `s[start..last].find` for a ``start``-origin index. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind proc<#rfind,string,set[char],Natural,int>`_ - ## * `multiReplace proc<#multiReplace,string,varargs[]>`_ - let last = if last == 0: s.high else: last + ## * `rfind func<#rfind,string,set[char],Natural,int>`_ + ## * `multiReplace func<#multiReplace,string,varargs[]>`_ + result = -1 + let last = if last < 0: s.high else: last for i in int(start)..last: - if s[i] in chars: return i - return -1 - -proc find*(s, sub: string, start: Natural = 0, last = 0): int {.noSideEffect, - rtl, extern: "nsuFindStr".} = - ## Searches for `sub` in `s` inside range ``start..last`` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + if s[i] in chars: + return i + +when defined(linux): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: """#define _GNU_SOURCE +#include <string.h>""".} +elif defined(bsd) or (defined(macosx) and not defined(ios)): + proc memmem(haystack: pointer, haystacklen: csize_t, + needle: pointer, needlelen: csize_t): pointer {.importc, header: "#include <string.h>".} + +func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuFindStr".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - ## Otherwise the index returned is relative to ``s[0]``, not ``start``. - ## Use `s[start..last].find` for a ``start``-origin index. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `rfind proc<#rfind,string,string,Natural,int>`_ - ## * `replace proc<#replace,string,string,string>`_ - if sub.len > s.len: return -1 + ## * `rfind func<#rfind,string,string,Natural,int>`_ + ## * `replace func<#replace,string,string,string>`_ + if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) - var a {.noinit.}: SkipTable - initSkipTable(a, sub) - result = find(a, s, sub, start, last) -proc rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.noSideEffect, - rtl, extern: "nsuRFindChar".} = - ## Searches for `sub` in `s` inside range ``start..last`` (both ends included) + template useSkipTable = + result = find(initSkipTable(sub), s, sub, start, last) + + when nimvm: + useSkipTable() + else: + when declared(memmem): + let subLen = sub.len + if last < 0 and start < s.len and subLen != 0: + let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) + result = if not found.isNil: + cast[int](found) -% cast[int](s.cstring) + else: + -1 + else: + useSkipTable() + else: + useSkipTable() + +func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuRFindChar".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included) ## in reverse -- starting at high indexes and moving lower to the first - ## character or ``start``. If `last` is unspecified, it defaults to `s.high` + ## character or `start`. If `last` is unspecified, it defaults to `s.high` ## (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - ## Otherwise the index returned is relative to ``s[0]``, not ``start``. - ## Use `s[start..last].find` for a ``start``-origin index. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `find proc<#find,string,char,Natural,int>`_ + ## * `find func<#find,string,char,Natural,int>`_ let last = if last == -1: s.high else: last for i in countdown(last, start): if sub == s[i]: return i return -1 -proc rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {.noSideEffect, - rtl, extern: "nsuRFindCharSet".} = - ## Searches for `chars` in `s` inside range ``start..last`` (both ends +func rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. + rtl, extern: "nsuRFindCharSet".} = + ## Searches for `chars` in `s` inside range `start..last` (both ends ## included) in reverse -- starting at high indexes and moving lower to the - ## first character or ``start``. If `last` is unspecified, it defaults to + ## first character or `start`. If `last` is unspecified, it defaults to ## `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. - ## Otherwise the index returned is relative to ``s[0]``, not ``start``. - ## Use `s[start..last].rfind` for a ``start``-origin index. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `find proc<#find,string,set[char],Natural,int>`_ + ## * `find func<#find,string,set[char],Natural,int>`_ let last = if last == -1: s.high else: last for i in countdown(last, start): if s[i] in chars: return i return -1 -proc rfind*(s, sub: string, start: Natural = 0, last = -1): int {.noSideEffect, - rtl, extern: "nsuRFindStr".} = - ## Searches for `sub` in `s` inside range ``start..last`` (both ends included) +func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, + extern: "nsuRFindStr".} = + ## Searches for `sub` in `s` inside range `start..last` (both ends included) ## included) in reverse -- starting at high indexes and moving lower to the - ## first character or ``start``. If `last` is unspecified, it defaults to + ## first character or `start`. If `last` is unspecified, it defaults to ## `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. - ## Otherwise the index returned is relative to ``s[0]``, not ``start``. - ## Use `s[start..last].rfind` for a ``start``-origin index. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: - ## * `find proc<#find,string,string,Natural,int>`_ + ## * `find func<#find,string,string,Natural,int>`_ if sub.len == 0: + let rightIndex: Natural = if last < 0: s.len else: last + return max(start, rightIndex) + if sub.len > s.len - start: return -1 let last = if last == -1: s.high else: last + result = 0 for i in countdown(last - sub.len + 1, start): for j in 0..sub.len-1: result = i @@ -1965,34 +2111,36 @@ proc rfind*(s, sub: string, start: Natural = 0, last = -1): int {.noSideEffect, return -1 -proc count*(s: string, sub: char): int {.noSideEffect, - rtl, extern: "nsuCountChar".} = - ## Count the occurrences of the character `sub` in the string `s`. +func count*(s: string, sub: char): int {.rtl, extern: "nsuCountChar".} = + ## Counts the occurrences of the character `sub` in the string `s`. ## ## See also: - ## * `countLines proc<#countLines,string>`_ + ## * `countLines func<#countLines,string>`_ + result = 0 for c in s: if c == sub: inc result -proc count*(s: string, subs: set[char]): int {.noSideEffect, - rtl, extern: "nsuCountCharSet".} = - ## Count the occurrences of the group of character `subs` in the string `s`. +func count*(s: string, subs: set[char]): int {.rtl, + extern: "nsuCountCharSet".} = + ## Counts the occurrences of the group of character `subs` in the string `s`. ## ## See also: - ## * `countLines proc<#countLines,string>`_ + ## * `countLines func<#countLines,string>`_ doAssert card(subs) > 0 + result = 0 for c in s: if c in subs: inc result -proc count*(s: string, sub: string, overlapping: bool = false): int {. - noSideEffect, rtl, extern: "nsuCountString".} = - ## Count the occurrences of a substring `sub` in the string `s`. +func count*(s: string, sub: string, overlapping: bool = false): int {.rtl, + extern: "nsuCountString".} = + ## Counts the occurrences of a substring `sub` in the string `s`. ## Overlapping occurrences of `sub` only count when `overlapping` ## is set to true (default: false). ## ## See also: - ## * `countLines proc<#countLines,string>`_ + ## * `countLines func<#countLines,string>`_ doAssert sub.len > 0 + result = 0 var i = 0 while true: i = s.find(sub, i) @@ -2001,12 +2149,11 @@ proc count*(s: string, sub: string, overlapping: bool = false): int {. else: i += sub.len inc result -proc countLines*(s: string): int {.noSideEffect, - rtl, extern: "nsuCountLines".} = +func countLines*(s: string): int {.rtl, extern: "nsuCountLines".} = ## Returns the number of lines in the string `s`. ## - ## This is the same as ``len(splitLines(s))``, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every + ## This is the same as `len(splitLines(s))`, but much more efficient + ## because it doesn't modify the string creating temporary objects. Every ## `character literal <manual.html#lexical-analysis-character-literals>`_ ## newline combination (CR, LF, CR-LF) is supported. ## @@ -2014,7 +2161,7 @@ proc countLines*(s: string): int {.noSideEffect, ## A line can be an empty string. ## ## See also: - ## * `splitLines proc<#splitLines,string>`_ + ## * `splitLines func<#splitLines,string>`_ runnableExamples: doAssert countLines("First line\l and second line.") == 2 result = 1 @@ -2029,30 +2176,30 @@ proc countLines*(s: string): int {.noSideEffect, inc i -proc contains*(s, sub: string): bool {.noSideEffect.} = - ## Same as ``find(s, sub) >= 0``. +func contains*(s, sub: string): bool = + ## Same as `find(s, sub) >= 0`. ## ## See also: - ## * `find proc<#find,string,string,Natural,int>`_ + ## * `find func<#find,string,string,Natural,int>`_ return find(s, sub) >= 0 -proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} = - ## Same as ``find(s, chars) >= 0``. +func contains*(s: string, chars: set[char]): bool = + ## Same as `find(s, chars) >= 0`. ## ## See also: - ## * `find proc<#find,string,set[char],Natural,int>`_ + ## * `find func<#find,string,set[char],Natural,int>`_ return find(s, chars) >= 0 -proc replace*(s, sub: string, by = ""): string {.noSideEffect, - rtl, extern: "nsuReplaceStr".} = - ## Replaces `sub` in `s` by the string `by`. +func replace*(s, sub: string, by = ""): string {.rtl, + extern: "nsuReplaceStr".} = + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## See also: - ## * `find proc<#find,string,string,Natural,int>`_ - ## * `replace proc<#replace,string,char,char>`_ for replacing + ## * `find func<#find,string,string,Natural,int>`_ + ## * `replace func<#replace,string,char,char>`_ for replacing ## single characters - ## * `replaceWord proc<#replaceWord,string,string,string>`_ - ## * `multiReplace proc<#multiReplace,string,varargs[]>`_ + ## * `replaceWord func<#replaceWord,string,string,string>`_ + ## * `multiReplace func<#multiReplace,string,varargs[]>`_ result = "" let subLen = sub.len if subLen == 0: @@ -2072,8 +2219,7 @@ proc replace*(s, sub: string, by = ""): string {.noSideEffect, # copy the rest: add result, substr(s, i) else: - var a {.noinit.}: SkipTable - initSkipTable(a, sub) + var a = initSkipTable(sub) let last = s.high var i = 0 while true: @@ -2085,17 +2231,18 @@ proc replace*(s, sub: string, by = ""): string {.noSideEffect, # copy the rest: add result, substr(s, i) -proc replace*(s: string, sub, by: char): string {.noSideEffect, - rtl, extern: "nsuReplaceChar".} = - ## Replaces `sub` in `s` by the character `by`. +func replace*(s: string, sub, by: char): string {.rtl, + extern: "nsuReplaceChar".} = + ## Replaces every occurrence of the character `sub` in `s` with the character + ## `by`. ## ## Optimized version of `replace <#replace,string,string,string>`_ for ## characters. ## ## See also: - ## * `find proc<#find,string,char,Natural,int>`_ - ## * `replaceWord proc<#replaceWord,string,string,string>`_ - ## * `multiReplace proc<#multiReplace,string,varargs[]>`_ + ## * `find func<#find,string,char,Natural,int>`_ + ## * `replaceWord func<#replaceWord,string,string,string>`_ + ## * `multiReplace func<#multiReplace,string,varargs[]>`_ result = newString(s.len) var i = 0 while i < s.len: @@ -2103,18 +2250,17 @@ proc replace*(s: string, sub, by: char): string {.noSideEffect, else: result[i] = s[i] inc(i) -proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, - rtl, extern: "nsuReplaceWord".} = - ## Replaces `sub` in `s` by the string `by`. +func replaceWord*(s, sub: string, by = ""): string {.rtl, + extern: "nsuReplaceWord".} = + ## Replaces every occurrence of the string `sub` in `s` with the string `by`. ## ## Each occurrence of `sub` has to be surrounded by word boundaries - ## (comparable to ``\b`` in regular expressions), otherwise it is not + ## (comparable to `\b` in regular expressions), otherwise it is not ## replaced. if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) + var a = initSkipTable(sub) var i = 0 let last = s.high let sublen = sub.len @@ -2134,19 +2280,32 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect, # copy the rest: add result, substr(s, i) -proc multiReplace*(s: string, replacements: varargs[(string, string)]): - string {.noSideEffect.} = - ## Same as replace, but specialized for doing multiple replacements in a single - ## pass through the input string. +func multiReplace*(s: string, replacements: varargs[(string, string)]): string = + ## Same as `replace<#replace,string,string,string>`_, but specialized for + ## doing multiple replacements in a single pass through the input string. ## - ## `multiReplace` performs all replacements in a single pass, this means it - ## can be used to swap the occurrences of "a" and "b", for instance. + ## `multiReplace` scans the input string from left to right and replaces the + ## matching substrings in the same order as passed in the argument list. + ## + ## The implications of the order of scanning the string and matching the + ## replacements: + ## - In case of multiple matches at a given position, the earliest + ## replacement is applied. + ## - Overlaps are not handled. After performing a replacement, the scan + ## continues from the character after the matched substring. If the + ## resulting string then contains a possible match starting in a newly + ## placed substring, the additional replacement is not performed. ## ## If the resulting string is not longer than the original input string, ## only a single memory allocation is required. ## - ## The order of the replacements does matter. Earlier replacements are - ## preferred over later replacements in the argument list. + runnableExamples: + # Swapping occurrences of 'a' and 'b': + doAssert multireplace("abba", [("a", "b"), ("b", "a")]) == "baab" + + # The second replacement ("ab") is matched and performed first, the scan then + # continues from 'c', so the "bc" replacement is never matched and thus skipped. + doAssert multireplace("abc", [("bc", "x"), ("ab", "_b")]) == "_bc" result = newStringOfCap(s.len) var i = 0 var fastChk: set[char] = {} @@ -2170,8 +2329,8 @@ proc multiReplace*(s: string, replacements: varargs[(string, string)]): -proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, - rtl, extern: "nsuInsertSep".} = +func insertSep*(s: string, sep = '_', digits = 3): string {.rtl, + extern: "nsuInsertSep".} = ## Inserts the separator `sep` after `digits` characters (default: 3) ## from right to left. ## @@ -2179,31 +2338,50 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, ## if `s` contains a number. runnableExamples: doAssert insertSep("1000000") == "1_000_000" - - var L = (s.len-1) div digits + s.len - result = newString(L) + result = newStringOfCap(s.len) + let hasPrefix = isDigit(s[s.low]) == false + var idx: int + if hasPrefix: + result.add s[s.low] + for i in (s.low + 1)..s.high: + idx = i + if not isDigit(s[i]): + result.add s[i] + else: + break + let partsLen = s.len - idx + var L = (partsLen-1) div digits + partsLen + result.setLen(L + idx) var j = 0 dec(L) - for i in countdown(len(s)-1, 0): + for i in countdown(partsLen-1, 0): if j == digits: - result[L] = sep + result[L + idx] = sep dec(L) j = 0 - result[L] = s[i] + result[L + idx] = s[i + idx] inc(j) dec(L) -proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, - rtl, extern: "nsuEscape".} = - ## Escapes a string `s`. See `system.addEscapedChar - ## <system.html#addEscapedChar,string,char>`_ for the escaping scheme. +func escape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, + extern: "nsuEscape".} = + ## Escapes a string `s`. + ## + ## .. note:: The escaping scheme is different from + ## `system.addEscapedChar`. + ## + ## * replaces `'\0'..'\31'` and `'\127'..'\255'` by `\xHH` where `HH` is its hexadecimal value + ## * replaces ``\`` by `\\` + ## * replaces `'` by `\'` + ## * replaces `"` by `\"` ## ## The resulting string is prefixed with `prefix` and suffixed with `suffix`. ## Both may be empty strings. ## ## See also: - ## * `unescape proc<#unescape,string,string,string>`_ for the opposite - ## operation + ## * `addEscapedChar proc<system.html#addEscapedChar,string,char>`_ + ## * `unescape func<#unescape,string,string,string>`_ for the opposite + ## operation result = newStringOfCap(s.len + s.len shr 2) result.add(prefix) for c in items(s): @@ -2217,14 +2395,14 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, else: add(result, c) add(result, suffix) -proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, - rtl, extern: "nsuUnescape".} = +func unescape*(s: string, prefix = "\"", suffix = "\""): string {.rtl, + extern: "nsuUnescape".} = ## Unescapes a string `s`. ## - ## This complements `escape proc<#escape,string,string,string>`_ + ## This complements `escape func<#escape,string,string,string>`_ ## as it performs the opposite operations. ## - ## If `s` does not begin with ``prefix`` and end with ``suffix`` a + ## If `s` does not begin with `prefix` and end with `suffix` a ## ValueError exception will be raised. result = newStringOfCap(s.len) var i = prefix.len @@ -2240,7 +2418,7 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, case s[i+1]: of 'x': inc i, 2 - var c: int + var c = 0 i += parseutils.parseHex(s, c, i, maxLen = 2) result.add(chr(c)) dec i, 2 @@ -2260,8 +2438,7 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, raise newException(ValueError, "String does not end in: " & suffix) -proc validIdentifier*(s: string): bool {.noSideEffect, - rtl, extern: "nsuValidIdentifier".} = +func validIdentifier*(s: string): bool {.rtl, extern: "nsuValidIdentifier".} = ## Returns true if `s` is a valid identifier. ## ## A valid identifier starts with a character of the set `IdentStartChars` @@ -2277,113 +2454,105 @@ proc validIdentifier*(s: string): bool {.noSideEffect, # floating point formatting: when not defined(js): - proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} + func c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs.} type FloatFormatMode* = enum - ## the different modes of floating point formatting + ## The different modes of floating point formatting. ffDefault, ## use the shorter floating point notation ffDecimal, ## use decimal floating point notation - ffScientific ## use scientific notation (using ``e`` character) + ffScientific ## use scientific notation (using `e` character) -proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, +func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; - decimalSep = '.'): string {. - noSideEffect, rtl, extern: "nsu$1".} = + decimalSep = '.'): string {.rtl, extern: "nsu$1".} = ## Converts a floating point value `f` to a string. ## - ## If ``format == ffDecimal`` then precision is the number of digits to + ## If `format == ffDecimal` then precision is the number of digits to ## be printed after the decimal point. - ## If ``format == ffScientific`` then precision is the maximum number + ## If `format == ffScientific` then precision is the maximum number ## of significant digits to be printed. ## `precision`'s default value is the maximum number of meaningful digits - ## after the decimal point for Nim's ``biggestFloat`` type. + ## after the decimal point for Nim's `biggestFloat` type. ## - ## If ``precision == -1``, it tries to format it nicely. + ## If `precision == -1`, it tries to format it nicely. runnableExamples: let x = 123.456 doAssert x.formatBiggestFloat() == "123.4560000000000" doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" - when defined(js): - var precision = precision - if precision == -1: - # use the same default precision as c_sprintf - precision = 6 - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - if 1.0 / f == -Inf: - # JavaScript removes the "-" from negative Zero, add it back here - result = "-" & $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalSep + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision >= 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - when defined(nimNoArrayToCstringConversion): - L = c_sprintf(addr buf, addr frmtstr, precision, f) - else: - L = c_sprintf(buf, frmtstr, precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_snprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - when defined(nimNoArrayToCstringConversion): - L = c_sprintf(addr buf, addr frmtstr, f) + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), precision, f) else: - L = c_sprintf(buf, frmtstr, f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalSep - else: result[i] = buf[i] - since (1, 1): - # remove trailing dot, compatible with Python's formatter and JS backend - if result[^1] == decimalSep: - result.setLen(len(result)-1) - when defined(windows): - # VS pre 2015 violates the C standard: "The exponent always contains at - # least two digits, and only as many more digits as necessary to - # represent the exponent." [C11 §7.21.6.1] - # The following post-processing fixes this behavior. - if result.len > 4 and result[^4] == '+' and result[^3] == '0': - result[^3] = result[^2] - result[^2] = result[^1] - result.setLen(result.len - 1) - -proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_snprintf(cast[cstring](addr buf), csize_t(2501), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) + +func formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; decimalSep = '.'): string {. - noSideEffect, rtl, extern: "nsu$1".} = + rtl, extern: "nsu$1".} = ## Converts a floating point value `f` to a string. ## - ## If ``format == ffDecimal`` then precision is the number of digits to + ## If `format == ffDecimal` then precision is the number of digits to ## be printed after the decimal point. - ## If ``format == ffScientific`` then precision is the maximum number + ## If `format == ffScientific` then precision is the maximum number ## of significant digits to be printed. ## `precision`'s default value is the maximum number of meaningful digits - ## after the decimal point for Nim's ``float`` type. + ## after the decimal point for Nim's `float` type. ## - ## If ``precision == -1``, it tries to format it nicely. + ## If `precision == -1`, it tries to format it nicely. runnableExamples: let x = 123.456 doAssert x.formatFloat() == "123.4560000000000" @@ -2392,9 +2561,9 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, result = formatBiggestFloat(f, format, precision, decimalSep) -proc trimZeros*(x: var string; decimalSep = '.') {.noSideEffect.} = +func trimZeros*(x: var string; decimalSep = '.') = ## Trim trailing zeros from a formatted floating point - ## value `x` (must be declared as ``var``). + ## value `x` (must be declared as `var`). ## ## This modifies `x` itself, it does not return a copy. runnableExamples: @@ -2409,17 +2578,18 @@ proc trimZeros*(x: var string; decimalSep = '.') {.noSideEffect.} = var pos = last while pos >= 0 and x[pos] == '0': dec(pos) if pos > sPos: inc(pos) - x.delete(pos, last) + if last >= pos: + x.delete(pos..last) type - BinaryPrefixMode* = enum ## the different names for binary prefixes + BinaryPrefixMode* = enum ## The different names for binary prefixes. bpIEC, # use the IEC/ISO standard prefixes such as kibi bpColloquial # use the colloquial kilo, mega etc -proc formatSize*(bytes: int64, +func formatSize*(bytes: int64, decimalSep = '.', prefix = bpIEC, - includeSpace = false): string {.noSideEffect.} = + includeSpace = false): string = ## Rounds and formats `bytes`. ## ## By default, uses the IEC/ISO standard binary prefixes, so 1024 will be @@ -2445,7 +2615,7 @@ proc formatSize*(bytes: int64, xb: int64 = bytes fbytes: float lastXb: int64 = bytes - matchedIndex: int + matchedIndex = 0 prefixes: array[9, string] if prefix == bpColloquial: prefixes = collPrefixes @@ -2472,13 +2642,13 @@ proc formatSize*(bytes: int64, result &= prefixes[matchedIndex] result &= "B" -proc formatEng*(f: BiggestFloat, +func formatEng*(f: BiggestFloat, precision: range[0..32] = 10, trim: bool = true, siPrefix: bool = false, unit: string = "", decimalSep = '.', - useUnitSpace = false): string {.noSideEffect.} = + useUnitSpace = false): string = ## Converts a floating point value `f` to a string using engineering notation. ## ## Numbers in of the range -1000.0<f<1000.0 will be formatted without an @@ -2501,13 +2671,13 @@ proc formatEng*(f: BiggestFloat, ## decimal point or (if `trim` is true) the maximum number of digits to be ## shown. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(0, 2, trim=false) == "0.00" ## formatEng(0, 2) == "0" ## formatEng(0.053, 0) == "53e-3" ## formatEng(52731234, 2) == "52.73e6" ## formatEng(-52731234, 2) == "-52.73e6" + ## ``` ## ## If `siPrefix` is set to true, the number will be displayed with the SI ## prefix corresponding to the exponent. For example 4100 will be displayed @@ -2522,8 +2692,7 @@ proc formatEng*(f: BiggestFloat, ## different to appending the unit to the result as the location of the space ## is altered depending on whether there is an exponent. ## - ## .. code-block:: nim - ## + ## ```nim ## formatEng(4100, siPrefix=true, unit="V") == "4.1 kV" ## formatEng(4.1, siPrefix=true, unit="V") == "4.1 V" ## formatEng(4.1, siPrefix=true) == "4.1" # Note lack of space @@ -2533,6 +2702,7 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100) == "4.1e3" ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="", useUnitSpace=true) == "4.1e3 " # Space with useUnitSpace=true + ## ``` ## ## `decimalSep` is used as the decimal separator. ## @@ -2545,7 +2715,7 @@ proc formatEng*(f: BiggestFloat, exponent: int splitResult: seq[string] suffix: string = "" - proc getPrefix(exp: int): char = + func getPrefix(exp: int): char = ## Get the SI prefix for a given exponent ## ## Assumes exponent is a multiple of 3; returns ' ' if no prefix found @@ -2612,7 +2782,7 @@ proc formatEng*(f: BiggestFloat, result &= "e" & $exponent result &= suffix -proc findNormalized(x: string, inArray: openArray[string]): int = +func findNormalized(x: string, inArray: openArray[string]): int = var i = 0 while i < high(inArray): if cmpIgnoreStyle(x, inArray[i]) == 0: return i @@ -2620,12 +2790,12 @@ proc findNormalized(x: string, inArray: openArray[string]): int = # security hole... return -1 -proc invalidFormatString() {.noinline.} = - raise newException(ValueError, "invalid format string") +func invalidFormatString(formatstr: string) {.noinline.} = + raise newException(ValueError, "invalid format string: " & formatstr) -proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. - noSideEffect, rtl, extern: "nsuAddf".} = - ## The same as ``add(s, formatstr % a)``, but more efficient. +func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, + extern: "nsuAddf".} = + ## The same as `add(s, formatstr % a)`, but more efficient. const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '\128'..'\255', '_'} var i = 0 var num = 0 @@ -2633,7 +2803,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. if formatstr[i] == '$' and i+1 < len(formatstr): case formatstr[i+1] of '#': - if num > a.high: invalidFormatString() + if num > a.high: invalidFormatString(formatstr) add s, a[num] inc i, 2 inc num @@ -2649,7 +2819,7 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) let idx = if not negative: j-1 else: a.len-j - if idx < 0 or idx > a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] of '{': var j = i+2 @@ -2666,28 +2836,28 @@ proc addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {. inc(j) if isNumber == 1: let idx = if not negative: k-1 else: a.len-k - if idx < 0 or idx > a.high: invalidFormatString() + if idx < 0 or idx > a.high: invalidFormatString(formatstr) add s, a[idx] else: var x = findNormalized(substr(formatstr, i+2, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j+1 of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 while j < formatstr.len and formatstr[j] in PatternChars: inc(j) var x = findNormalized(substr(formatstr, i+1, j-1), a) if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + else: invalidFormatString(formatstr) i = j else: - invalidFormatString() + invalidFormatString(formatstr) else: add s, formatstr[i] inc(i) -proc `%` *(formatstr: string, a: openArray[string]): string {.noSideEffect, - rtl, extern: "nsuFormatOpenArray".} = +func `%`*(formatstr: string, a: openArray[string]): string {.rtl, + extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. ## ## The `substitution`:idx: operator performs string substitutions in @@ -2696,35 +2866,40 @@ proc `%` *(formatstr: string, a: openArray[string]): string {.noSideEffect, ## ## This is best explained by an example: ## - ## .. code-block:: nim + ## ```nim ## "$1 eats $2." % ["The cat", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## - ## The substitution variables (the thing after the ``$``) are enumerated - ## from 1 to ``a.len``. - ## To produce a verbatim ``$``, use ``$$``. - ## The notation ``$#`` can be used to refer to the next substitution + ## The substitution variables (the thing after the `$`) are enumerated + ## from 1 to `a.len`. + ## To produce a verbatim `$`, use `$$`. + ## The notation `$#` can be used to refer to the next substitution ## variable: ## - ## .. code-block:: nim + ## ```nim ## "$# eats $#." % ["The cat", "fish"] + ## ``` ## ## Substitution variables can also be words (that is - ## ``[A-Za-z_]+[A-Za-z0-9_]*``) in which case the arguments in `a` with even + ## `[A-Za-z_]+[A-Za-z0-9_]*`) in which case the arguments in `a` with even ## indices are keys and with odd indices are the corresponding values. ## An example: ## - ## .. code-block:: nim + ## ```nim ## "$animal eats $food." % ["animal", "The cat", "food", "fish"] + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## "The cat eats fish." + ## ``` ## ## The variables are compared with `cmpIgnoreStyle`. `ValueError` is ## raised if an ill-formed format string has been passed to the `%` operator. @@ -2734,17 +2909,17 @@ proc `%` *(formatstr: string, a: openArray[string]): string {.noSideEffect, result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) -proc `%` *(formatstr, a: string): string {.noSideEffect, - rtl, extern: "nsuFormatSingleElem".} = - ## This is the same as ``formatstr % [a]`` (see - ## `% proc<#%25,string,openArray[string]>`_). +func `%`*(formatstr, a: string): string {.rtl, + extern: "nsuFormatSingleElem".} = + ## This is the same as `formatstr % [a]` (see + ## `% func<#%25,string,openArray[string]>`_). result = newStringOfCap(formatstr.len + a.len) addf(result, formatstr, [a]) -proc format*(formatstr: string, a: varargs[string, `$`]): string {.noSideEffect, - rtl, extern: "nsuFormatVarargs".} = - ## This is the same as ``formatstr % a`` (see - ## `% proc<#%25,string,openArray[string]>`_) except that it supports +func format*(formatstr: string, a: varargs[string, `$`]): string {.rtl, + extern: "nsuFormatVarargs".} = + ## This is the same as `formatstr % a` (see + ## `% func<#%25,string,openArray[string]>`_) except that it supports ## auto stringification. ## ## See also: @@ -2753,9 +2928,8 @@ proc format*(formatstr: string, a: varargs[string, `$`]): string {.noSideEffect, addf(result, formatstr, a) -proc strip*(s: string, leading = true, trailing = true, - chars: set[char] = Whitespace): string - {.noSideEffect, rtl, extern: "nsuStrip".} = +func strip*(s: string, leading = true, trailing = true, + chars: set[char] = Whitespace): string {.rtl, extern: "nsuStrip".} = ## Strips leading or trailing `chars` (default: whitespace characters) ## from `s` and returns the resulting string. ## @@ -2764,7 +2938,8 @@ proc strip*(s: string, leading = true, trailing = true, ## If both are false, the string is returned unchanged. ## ## See also: - ## * `stripLineEnd proc<#stripLineEnd,string>`_ + ## * `strip proc<strbasics.html#strip,string,set[char]>`_ Inplace version. + ## * `stripLineEnd func<#stripLineEnd,string>`_ runnableExamples: let a = " vhellov " let b = strip(a) @@ -2786,13 +2961,13 @@ proc strip*(s: string, leading = true, trailing = true, if leading: while first <= last and s[first] in chars: inc(first) if trailing: - while last >= 0 and s[last] in chars: dec(last) + while last >= first and s[last] in chars: dec(last) result = substr(s, first, last) -proc stripLineEnd*(s: var string) = - ## Returns ``s`` stripped from one of these suffixes: - ## ``\r, \n, \r\n, \f, \v`` (at most once instance). - ## For example, can be useful in conjunction with ``osproc.execCmdEx``. +func stripLineEnd*(s: var string) = + ## Strips one of these suffixes from `s` in-place: + ## `\r, \n, \r\n, \f, \v` (at most once instance). + ## For example, can be useful in conjunction with `osproc.execCmdEx`. ## aka: `chomp`:idx: runnableExamples: var s = "foo\n\n" @@ -2822,13 +2997,14 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## Substrings are separated by a substring containing only `seps`. ## Example: ## - ## .. code-block:: nim + ## ```nim ## for word in tokenize(" this is an example "): ## writeLine(stdout, word) + ## ``` ## ## Results in: ## - ## .. code-block:: nim + ## ```nim ## (" ", true) ## ("this", false) ## (" ", true) @@ -2838,6 +3014,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ ## (" ", true) ## ("example", false) ## (" ", true) + ## ``` var i = 0 while true: var j = i @@ -2849,228 +3026,7 @@ iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ break i = j -proc isEmptyOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, +func isEmptyOrWhitespace*(s: string): bool {.rtl, extern: "nsuIsEmptyOrWhitespace".} = ## Checks if `s` is empty or consists entirely of whitespace characters. result = s.allCharsInSet(Whitespace) - -proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl, - extern: "nsuIsNilOrWhitespace", - deprecated: "use isEmptyOrWhitespace instead".} = - ## Alias for isEmptyOrWhitespace - result = isEmptyOrWhitespace(s) - -when isMainModule: - proc nonStaticTests = - doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000" - doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235" # bugs 8242, 12586 - doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6" - doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001" - doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in - ["1,0e-11", "1,0e-011"] - # bug #6589 - when not defined(js): - doAssert formatFloat(123.456, ffScientific, precision = -1) == "1.234560e+02" - - doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c" - doAssert "${1}12 ${-1}$2" % ["a", "b"] == "a12 bb" - - block: # formatSize tests - when not defined(js): - doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" # <=== bug #8231 - doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" - doAssert formatSize(4096) == "4KiB" - doAssert formatSize(4096, prefix = bpColloquial, includeSpace = true) == "4 kB" - doAssert formatSize(4096, includeSpace = true) == "4 KiB" - doAssert formatSize(5_378_934, prefix = bpColloquial, decimalSep = ',') == "5,13MB" - - block: # formatEng tests - doAssert formatEng(0, 2, trim = false) == "0.00" - doAssert formatEng(0, 2) == "0" - doAssert formatEng(53, 2, trim = false) == "53.00" - doAssert formatEng(0.053, 2, trim = false) == "53.00e-3" - doAssert formatEng(0.053, 4, trim = false) == "53.0000e-3" - doAssert formatEng(0.053, 4, trim = true) == "53e-3" - doAssert formatEng(0.053, 0) == "53e-3" - doAssert formatEng(52731234) == "52.731234e6" - doAssert formatEng(-52731234) == "-52.731234e6" - doAssert formatEng(52731234, 1) == "52.7e6" - doAssert formatEng(-52731234, 1) == "-52.7e6" - doAssert formatEng(52731234, 1, decimalSep = ',') == "52,7e6" - doAssert formatEng(-52731234, 1, decimalSep = ',') == "-52,7e6" - - doAssert formatEng(4100, siPrefix = true, unit = "V") == "4.1 kV" - doAssert formatEng(4.1, siPrefix = true, unit = "V", - useUnitSpace = true) == "4.1 V" - doAssert formatEng(4.1, siPrefix = true) == "4.1" # Note lack of space - doAssert formatEng(4100, siPrefix = true) == "4.1 k" - doAssert formatEng(4.1, siPrefix = true, unit = "", - useUnitSpace = true) == "4.1 " # Includes space - doAssert formatEng(4100, siPrefix = true, unit = "") == "4.1 k" - doAssert formatEng(4100) == "4.1e3" - doAssert formatEng(4100, unit = "V", useUnitSpace = true) == "4.1e3 V" - doAssert formatEng(4100, unit = "", useUnitSpace = true) == "4.1e3 " - # Don't use SI prefix as number is too big - doAssert formatEng(3.1e22, siPrefix = true, unit = "a", - useUnitSpace = true) == "31e21 a" - # Don't use SI prefix as number is too small - doAssert formatEng(3.1e-25, siPrefix = true, unit = "A", - useUnitSpace = true) == "310e-27 A" - - proc staticTests = - doAssert align("abc", 4) == " abc" - doAssert align("a", 0) == "a" - doAssert align("1232", 6) == " 1232" - doAssert align("1232", 6, '#') == "##1232" - - doAssert alignLeft("abc", 4) == "abc " - doAssert alignLeft("a", 0) == "a" - doAssert alignLeft("1232", 6) == "1232 " - doAssert alignLeft("1232", 6, '#') == "1232##" - - doAssert "$animal eats $food." % ["animal", "The cat", "food", "fish"] == - "The cat eats fish." - - doAssert "-ld a-ldz -ld".replaceWord("-ld") == " a-ldz " - doAssert "-lda-ldz -ld abc".replaceWord("-ld") == "-lda-ldz abc" - - doAssert "-lda-ldz -ld abc".replaceWord("") == "-lda-ldz -ld abc" - doAssert "oo".replace("", "abc") == "oo" - - type MyEnum = enum enA, enB, enC, enuD, enE - doAssert parseEnum[MyEnum]("enu_D") == enuD - - doAssert parseEnum("invalid enum value", enC) == enC - - doAssert center("foo", 13) == " foo " - doAssert center("foo", 0) == "foo" - doAssert center("foo", 3, fillChar = 'a') == "foo" - doAssert center("foo", 10, fillChar = '\t') == "\t\t\tfoo\t\t\t\t" - - doAssert count("foofoofoo", "foofoo") == 1 - doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 - doAssert count("foofoofoo", 'f') == 3 - doAssert count("foofoofoobar", {'f', 'b'}) == 4 - - doAssert strip(" foofoofoo ") == "foofoofoo" - doAssert strip("sfoofoofoos", chars = {'s'}) == "foofoofoo" - doAssert strip("barfoofoofoobar", chars = {'b', 'a', 'r'}) == "foofoofoo" - doAssert strip("stripme but don't strip this stripme", - chars = {'s', 't', 'r', 'i', 'p', 'm', 'e'}) == - " but don't strip this " - doAssert strip("sfoofoofoos", leading = false, chars = {'s'}) == "sfoofoofoo" - doAssert strip("sfoofoofoos", trailing = false, chars = {'s'}) == "foofoofoos" - - doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar" - - doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab" - doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", - "PEOPLE!")) == "HELLO PEOPLE!" - doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa" - - doAssert isAlphaAscii('r') - doAssert isAlphaAscii('A') - doAssert(not isAlphaAscii('$')) - - doAssert isAlphaNumeric('3') - doAssert isAlphaNumeric('R') - doAssert(not isAlphaNumeric('!')) - - doAssert isDigit('3') - doAssert(not isDigit('a')) - doAssert(not isDigit('%')) - - doAssert isSpaceAscii('\t') - doAssert isSpaceAscii('\l') - doAssert(not isSpaceAscii('A')) - - doAssert(isEmptyOrWhitespace("")) - doAssert(isEmptyOrWhitespace(" ")) - doAssert(isEmptyOrWhitespace("\t\l \v\r\f")) - doAssert(not isEmptyOrWhitespace("ABc \td")) - - doAssert isLowerAscii('a') - doAssert isLowerAscii('z') - doAssert(not isLowerAscii('A')) - doAssert(not isLowerAscii('5')) - doAssert(not isLowerAscii('&')) - doAssert(not isLowerAscii(' ')) - - doAssert isUpperAscii('A') - doAssert(not isUpperAscii('b')) - doAssert(not isUpperAscii('5')) - doAssert(not isUpperAscii('%')) - - doAssert rsplit("foo bar", seps = Whitespace) == @["foo", "bar"] - doAssert rsplit(" foo bar", seps = Whitespace, maxsplit = 1) == @[" foo", "bar"] - doAssert rsplit(" foo bar ", seps = Whitespace, maxsplit = 1) == @[ - " foo bar", ""] - doAssert rsplit(":foo:bar", sep = ':') == @["", "foo", "bar"] - doAssert rsplit(":foo:bar", sep = ':', maxsplit = 2) == @["", "foo", "bar"] - doAssert rsplit(":foo:bar", sep = ':', maxsplit = 3) == @["", "foo", "bar"] - doAssert rsplit("foothebar", sep = "the") == @["foo", "bar"] - - doAssert(unescape(r"\x013", "", "") == "\x013") - - doAssert join(["foo", "bar", "baz"]) == "foobarbaz" - doAssert join(@["foo", "bar", "baz"], ", ") == "foo, bar, baz" - doAssert join([1, 2, 3]) == "123" - doAssert join(@[1, 2, 3], ", ") == "1, 2, 3" - - doAssert """~~!!foo -~~!!bar -~~!!baz""".unindent(2, "~~!!") == "foo\nbar\nbaz" - - doAssert """~~!!foo -~~!!bar -~~!!baz""".unindent(2, "~~!!aa") == "~~!!foo\n~~!!bar\n~~!!baz" - doAssert """~~foo -~~ bar -~~ baz""".unindent(4, "~") == "foo\n bar\n baz" - doAssert """foo -bar - baz - """.unindent(4) == "foo\nbar\nbaz\n" - doAssert """foo - bar - baz - """.unindent(2) == "foo\n bar\n baz\n" - doAssert """foo - bar - baz - """.unindent(100) == "foo\nbar\nbaz\n" - - doAssert """foo - foo - bar - """.unindent() == "foo\nfoo\nbar\n" - - let s = " this is an example " - let s2 = ":this;is;an:example;;" - - doAssert s.split() == @["", "this", "is", "an", "example", "", ""] - doAssert s2.split(seps = {':', ';'}) == @["", "this", "is", "an", "example", - "", ""] - doAssert s.split(maxsplit = 4) == @["", "this", "is", "an", "example "] - doAssert s.split(' ', maxsplit = 1) == @["", "this is an example "] - doAssert s.split(" ", maxsplit = 4) == @["", "this", "is", "an", "example "] - - doAssert s.splitWhitespace() == @["this", "is", "an", "example"] - doAssert s.splitWhitespace(maxsplit = 1) == @["this", "is an example "] - doAssert s.splitWhitespace(maxsplit = 2) == @["this", "is", "an example "] - doAssert s.splitWhitespace(maxsplit = 3) == @["this", "is", "an", "example "] - doAssert s.splitWhitespace(maxsplit = 4) == @["this", "is", "an", "example"] - - block: # startsWith / endsWith char tests - var s = "abcdef" - doAssert s.startsWith('a') - doAssert s.startsWith('b') == false - doAssert s.endsWith('f') - doAssert s.endsWith('a') == false - doAssert s.endsWith('\0') == false - - #echo("strutils tests passed") - - nonStaticTests() - staticTests() - static: staticTests() diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index 5719e8dd7..90ba20c13 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -10,18 +10,23 @@ ## This module implements nice syntactic sugar based on Nim's ## macro system. -include system/inclrtl +import std/private/since +import std/macros -import macros -import typetraits +proc checkPragma(ex, prag: var NimNode) = + since (1, 3): + if ex.kind == nnkPragmaExpr: + prag = ex[1] + ex = ex[0] -proc createProcType(p, b: NimNode): NimNode {.compileTime.} = - #echo treeRepr(p) - #echo treeRepr(b) +proc createProcType(p, b: NimNode): NimNode = result = newNimNode(nnkProcTy) - var formalParams = newNimNode(nnkFormalParams) + var + formalParams = newNimNode(nnkFormalParams).add(b) + p = p + prag = newEmptyNode() - formalParams.add b + checkPragma(p, prag) case p.kind of nnkPar, nnkTupleConstr: @@ -45,105 +50,152 @@ proc createProcType(p, b: NimNode): NimNode {.compileTime.} = formalParams.add identDefs result.add formalParams - result.add newEmptyNode() - #echo(treeRepr(result)) - #echo(result.toStrLit()) + result.add prag macro `=>`*(p, b: untyped): untyped = - ## Syntax sugar for anonymous procedures. + ## Syntax sugar for anonymous procedures. It also supports pragmas. ## - ## .. code-block:: nim - ## - ## proc passTwoAndTwo(f: (int, int) -> int): int = - ## f(2, 2) - ## - ## passTwoAndTwo((x, y) => x + y) # 4 + ## .. warning:: Semicolons can not be used to separate procedure arguments. + runnableExamples: + proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) + + assert passTwoAndTwo((x, y) => x + y) == 4 + + type + Bot = object + call: (string {.noSideEffect.} -> string) + + var myBot = Bot() + + myBot.call = (name: string) {.noSideEffect.} => "Hello " & name & ", I'm a bot." + assert myBot.call("John") == "Hello John, I'm a bot." + + let f = () => (discard) # simplest proc that returns void + f() + + var + params = @[ident"auto"] + name = newEmptyNode() + kind = nnkLambda + pragma = newEmptyNode() + p = p + + checkPragma(p, pragma) + + if p.kind == nnkInfix and p[0].kind == nnkIdent and p[0].eqIdent"->": + params[0] = p[2] + p = p[1] - #echo treeRepr(p) - #echo(treeRepr(b)) - var params: seq[NimNode] = @[newIdentNode("auto")] + checkPragma(p, pragma) # check again after -> transform case p.kind of nnkPar, nnkTupleConstr: - for c in children(p): + var untypedBeforeColon = 0 + for i, c in p: var identDefs = newNimNode(nnkIdentDefs) case c.kind of nnkExprColonExpr: + let t = c[1] + since (1, 3): + # + 1 here because of return type in params + for j in (i - untypedBeforeColon + 1) .. i: + params[j][1] = t + untypedBeforeColon = 0 identDefs.add(c[0]) - identDefs.add(c[1]) + identDefs.add(t) identDefs.add(newEmptyNode()) of nnkIdent: identDefs.add(c) identDefs.add(newIdentNode("auto")) identDefs.add(newEmptyNode()) + inc untypedBeforeColon of nnkInfix: - if c[0].kind == nnkIdent and c[0].ident == !"->": + if c[0].kind == nnkIdent and c[0].eqIdent"->": var procTy = createProcType(c[1], c[2]) params[0] = procTy[0][0] for i in 1 ..< procTy[0].len: params.add(procTy[0][i]) else: - error("Expected proc type (->) got (" & $c[0].ident & ").") + error("Expected proc type (->) got (" & c[0].strVal & ").", c) break else: - echo treeRepr c - error("Incorrect procedure parameter list.") + error("Incorrect procedure parameter.", c) params.add(identDefs) - of nnkIdent: + of nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym: var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) - identDefs.add(newIdentNode("auto")) + identDefs.add(ident $p) + identDefs.add(ident"auto") identDefs.add(newEmptyNode()) params.add(identDefs) - of nnkInfix: - if p[0].kind == nnkIdent and p[0].ident == !"->": - var procTy = createProcType(p[1], p[2]) - params[0] = procTy[0][0] - for i in 1 ..< procTy[0].len: - params.add(procTy[0][i]) - else: - error("Expected proc type (->) got (" & $p[0].ident & ").") else: - error("Incorrect procedure parameter list.") - result = newProc(params = params, body = b, procType = nnkLambda) - #echo(result.treeRepr) - #echo(result.toStrLit()) - #return result # TODO: Bug? + error("Incorrect procedure parameter list.", p) + result = newProc(body = b, params = params, + pragmas = pragma, name = name, + procType = kind) macro `->`*(p, b: untyped): untyped = - ## Syntax sugar for procedure types. - ## - ## .. code-block:: nim - ## - ## proc pass2(f: (float, float) -> float): float = - ## f(2, 2) - ## - ## # is the same as: + ## Syntax sugar for procedure types. It also supports pragmas. ## - ## proc pass2(f: proc (x, y: float): float): float = - ## f(2, 2) + ## .. warning:: Semicolons can not be used to separate procedure arguments. + runnableExamples: + proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2) + # is the same as: + # proc passTwoAndTwo(f: proc (x, y: int): int): int = f(2, 2) + + assert passTwoAndTwo((x, y) => x + y) == 4 + + proc passOne(f: (int {.noSideEffect.} -> int)): int = f(1) + # is the same as: + # proc passOne(f: proc (x: int): int {.noSideEffect.}): int = f(1) + + assert passOne(x {.noSideEffect.} => x + 1) == 2 result = createProcType(p, b) -macro dump*(x: typed): untyped = +macro dump*(x: untyped): untyped = ## Dumps the content of an expression, useful for debugging. ## It accepts any expression and prints a textual representation ## of the tree representing the expression - as it would appear in ## source code - together with the value of the expression. ## - ## As an example, - ## - ## .. code-block:: nim - ## let - ## x = 10 - ## y = 20 - ## dump(x + y) - ## - ## will print ``x + y = 30``. + ## See also: `dumpToString` which is more convenient and useful since + ## it expands intermediate templates/macros, returns a string instead of + ## calling `echo`, and works with statements and expressions. + runnableExamples("-r:off"): + let + x = 10 + y = 20 + dump(x + y) # prints: `x + y = 30` + let s = x.toStrLit - let r = quote do: + result = quote do: debugEcho `s`, " = ", `x` - return r + +macro dumpToStringImpl(s: static string, x: typed): string = + let s2 = x.toStrLit + if x.typeKind == ntyVoid: + result = quote do: + `s` & ": " & `s2` + else: + result = quote do: + `s` & ": " & `s2` & " = " & $`x` + +macro dumpToString*(x: untyped): string = + ## Returns the content of a statement or expression `x` after semantic analysis, + ## useful for debugging. + runnableExamples: + const a = 1 + let x = 10 + assert dumpToString(a + 2) == "a + 2: 3 = 3" + assert dumpToString(a + x) == "a + x: 1 + x = 11" + template square(x): untyped = x * x + assert dumpToString(square(x)) == "square(x): x * x = 100" + assert not compiles dumpToString(1 + nonexistent) + import std/strutils + assert "failedAssertImpl" in dumpToString(assert true) # example with a statement + result = newCall(bindSym"dumpToStringImpl") + result.add newLit repr(x) + result.add x # TODO: consider exporting this in macros.nim proc freshIdentNodes(ast: NimNode): NimNode = @@ -151,7 +203,7 @@ proc freshIdentNodes(ast: NimNode): NimNode = # see also https://github.com/nim-lang/Nim/pull/8531#issuecomment-410436458 proc inspect(node: NimNode): NimNode = case node.kind: - of nnkIdent, nnkSym: + of nnkIdent, nnkSym, nnkOpenSymChoice, nnkClosedSymChoice, nnkOpenSym: result = ident($node) of nnkEmpty, nnkLiterals: result = node @@ -161,206 +213,217 @@ proc freshIdentNodes(ast: NimNode): NimNode = result.add inspect(child) result = inspect(ast) -template distinctBase*(T: typedesc): typedesc {.deprecated: "use distinctBase from typetraits instead".} = - ## reverses ``type T = distinct A``; works recursively. - typetraits.distinctBase(T) - -macro capture*(locals: openArray[typed], body: untyped): untyped {.since: (1, 1).} = +macro capture*(locals: varargs[typed], body: untyped): untyped {.since: (1, 1).} = ## Useful when creating a closure in a loop to capture some local loop variables - ## by their current iteration values. Example: - ## - ## .. code-block:: Nim - ## import strformat, sequtils, sugar - ## var myClosure : proc() - ## for i in 5..7: - ## for j in 7..9: - ## if i * j == 42: - ## capture [i, j]: - ## myClosure = proc () = echo fmt"{i} * {j} = 42" - ## myClosure() # output: 6 * 7 == 42 - ## let m = @[proc (s: string): string = "to " & s, proc (s: string): string = "not to " & s] - ## var l = m.mapIt(capture([it], proc (s: string): string = it(s))) - ## let r = l.mapIt(it("be")) - ## echo r[0] & ", or " & r[1] # output: to be, or not to be + ## by their current iteration values. + runnableExamples: + import std/strformat + + var myClosure: () -> string + for i in 5..7: + for j in 7..9: + if i * j == 42: + capture i, j: + myClosure = () => fmt"{i} * {j} = 42" + assert myClosure() == "6 * 7 = 42" + var params = @[newIdentNode("auto")] + let locals = if locals.len == 1 and locals[0].kind == nnkBracket: locals[0] + else: locals for arg in locals: - params.add(newIdentDefs(ident(arg.strVal), freshIdentNodes getTypeImpl arg)) + proc getIdent(n: NimNode): NimNode = + case n.kind + of nnkIdent, nnkSym: + let nStr = n.strVal + if nStr == "result": + error("The variable name cannot be `result`!", n) + result = ident(nStr) + of nnkHiddenDeref: result = n[0].getIdent() + else: + error("The argument to be captured `" & n.repr & "` is not a pure identifier. " & + "It is an unsupported `" & $n.kind & "` node.", n) + let argName = getIdent(arg) + params.add(newIdentDefs(argName, freshIdentNodes getTypeInst arg)) result = newNimNode(nnkCall) - result.add(newProc(newEmptyNode(), params, body, nnkProcDef)) + result.add(newProc(newEmptyNode(), params, body, nnkLambda)) for arg in locals: result.add(arg) -macro outplace*[T](arg: T, call: untyped; inplaceArgPosition: static[int] = 1): T {.since: (1, 1).} = - ## Turns an `in-place`:idx: algorithm into one that works on - ## a copy and returns this copy. The second parameter is the - ## index of the calling expression that is replaced by a copy - ## of this expression. - ## **Since**: Version 1.2. - runnableExamples: - import algorithm - - var a = @[1, 2, 3, 4, 5, 6, 7, 8, 9] - doAssert a.outplace(sort()) == sorted(a) - #Chaining: - var aCopy = a - aCopy.insert(10) - - doAssert a.outplace(insert(10)).outplace(sort()) == sorted(aCopy) - - expectKind call, nnkCallKinds - let tmp = genSym(nskVar, "outplaceResult") - var callsons = call[0..^1] - callsons.insert(tmp, inplaceArgPosition) - result = newTree(nnkStmtListExpr, - newVarStmt(tmp, arg), - copyNimNode(call).add callsons, - tmp) - -proc transLastStmt(n, res, bracketExpr: NimNode): (NimNode, NimNode, NimNode) {.since: (1, 1).} = +since (1, 1): + import std/private/underscored_calls + + macro dup*[T](arg: T, calls: varargs[untyped]): T = + ## Turns an `in-place`:idx: algorithm into one that works on + ## a copy and returns this copy, without modifying its input. + ## + ## This macro also allows for (otherwise in-place) function chaining. + ## + ## **Since:** Version 1.2. + runnableExamples: + import std/algorithm + + let a = @[1, 2, 3, 4, 5, 6, 7, 8, 9] + assert a.dup(sort) == sorted(a) + + # Chaining: + var aCopy = a + aCopy.insert(10) + assert a.dup(insert(10), sort) == sorted(aCopy) + + let s1 = "abc" + let s2 = "xyz" + assert s1 & s2 == s1.dup(&= s2) + + # An underscore (_) can be used to denote the place of the argument you're passing: + assert "".dup(addQuoted(_, "foo")) == "\"foo\"" + # but `_` is optional here since the substitution is in 1st position: + assert "".dup(addQuoted("foo")) == "\"foo\"" + + proc makePalindrome(s: var string) = + for i in countdown(s.len-2, 0): + s.add(s[i]) + + let c = "xyz" + + # chaining: + let d = dup c: + makePalindrome # xyzyx + sort(_, SortOrder.Descending) # zyyxx + makePalindrome # zyyxxxyyz + assert d == "zyyxxxyyz" + + result = newNimNode(nnkStmtListExpr, arg) + let tmp = genSym(nskVar, "dupResult") + result.add newVarStmt(tmp, arg) + underscoredCalls(result, calls, tmp) + result.add tmp + +proc trans(n, res, bracketExpr: NimNode): (NimNode, NimNode, NimNode) {.since: (1, 1).} = # Looks for the last statement of the last statement, etc... case n.kind - of nnkIfExpr, nnkIfStmt, nnkTryStmt, nnkCaseStmt: + of nnkIfExpr, nnkIfStmt, nnkTryStmt, nnkCaseStmt, nnkWhenStmt: result[0] = copyNimTree(n) result[1] = copyNimTree(n) result[2] = copyNimTree(n) - for i in ord(n.kind == nnkCaseStmt)..<n.len: - (result[0][i], result[1][^1], result[2][^1]) = transLastStmt(n[i], res, bracketExpr) + for i in ord(n.kind == nnkCaseStmt) ..< n.len: + (result[0][i], result[1][^1], result[2][^1]) = trans(n[i], res, bracketExpr) of nnkStmtList, nnkStmtListExpr, nnkBlockStmt, nnkBlockExpr, nnkWhileStmt, nnkForStmt, nnkElifBranch, nnkElse, nnkElifExpr, nnkOfBranch, nnkExceptBranch: result[0] = copyNimTree(n) result[1] = copyNimTree(n) result[2] = copyNimTree(n) if n.len >= 1: - (result[0][^1], result[1][^1], result[2][^1]) = transLastStmt(n[^1], res, bracketExpr) + (result[0][^1], result[1][^1], result[2][^1]) = trans(n[^1], + res, bracketExpr) of nnkTableConstr: result[1] = n[0][0] result[2] = n[0][1] + if bracketExpr.len == 0: + bracketExpr.add(ident"initTable") # don't import tables if bracketExpr.len == 1: - bracketExpr.add([newCall(bindSym"typeof", newEmptyNode()), newCall( - bindSym"typeof", newEmptyNode())]) + bracketExpr.add([newCall(bindSym"typeof", + newEmptyNode()), newCall(bindSym"typeof", newEmptyNode())]) template adder(res, k, v) = res[k] = v result[0] = getAst(adder(res, n[0][0], n[0][1])) of nnkCurly: result[2] = n[0] + if bracketExpr.len == 0: + bracketExpr.add(ident"initHashSet") if bracketExpr.len == 1: bracketExpr.add(newCall(bindSym"typeof", newEmptyNode())) template adder(res, v) = res.incl(v) result[0] = getAst(adder(res, n[0])) else: result[2] = n + if bracketExpr.len == 0: + bracketExpr.add(bindSym"newSeq") if bracketExpr.len == 1: bracketExpr.add(newCall(bindSym"typeof", newEmptyNode())) template adder(res, v) = res.add(v) result[0] = getAst(adder(res, n)) +proc collectImpl(init, body: NimNode): NimNode {.since: (1, 1).} = + let res = genSym(nskVar, "collectResult") + var bracketExpr: NimNode + if init != nil: + expectKind init, {nnkCall, nnkIdent, nnkSym, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym} + bracketExpr = newTree(nnkBracketExpr, + if init.kind in {nnkCall, nnkClosedSymChoice, nnkOpenSymChoice, nnkOpenSym}: + freshIdentNodes(init[0]) else: freshIdentNodes(init)) + else: + bracketExpr = newTree(nnkBracketExpr) + let (resBody, keyType, valueType) = trans(body, res, bracketExpr) + if bracketExpr.len == 3: + bracketExpr[1][1] = keyType + bracketExpr[2][1] = valueType + else: + bracketExpr[1][1] = valueType + let call = newTree(nnkCall, bracketExpr) + if init != nil and init.kind == nnkCall: + for i in 1 ..< init.len: + call.add init[i] + result = newTree(nnkStmtListExpr, newVarStmt(res, call), resBody, res) + macro collect*(init, body: untyped): untyped {.since: (1, 1).} = - ## Comprehension for seq/set/table collections. ``init`` is - ## the init call, and so custom collections are supported. - ## - ## The last statement of ``body`` has special syntax that specifies - ## the collection's add operation. Use ``{e}`` for set's ``incl``, - ## ``{k: v}`` for table's ``[]=`` and ``e`` for seq's ``add``. + ## Comprehension for seqs/sets/tables. ## - ## The ``init`` proc can be called with any number of arguments, - ## i.e. ``initTable(initialSize)``. + ## The last expression of `body` has special syntax that specifies + ## the collection's add operation. Use `{e}` for set's `incl`, + ## `{k: v}` for table's `[]=` and `e` for seq's `add`. + # analyse the body, find the deepest expression 'it' and replace it via + # 'result.add it' runnableExamples: - import sets, tables + import std/[sets, tables] + let data = @["bird", "word"] + ## seq: let k = collect(newSeq): for i, d in data.pairs: if i mod 2 == 0: d - assert k == @["bird"] + ## seq with initialSize: let x = collect(newSeqOfCap(4)): for i, d in data.pairs: if i mod 2 == 0: d - assert x == @["bird"] + ## HashSet: - let y = initHashSet.collect: + let y = collect(initHashSet()): for d in data.items: {d} - assert y == data.toHashSet + ## Table: let z = collect(initTable(2)): for i, d in data.pairs: {i: d} + assert z == {0: "bird", 1: "word"}.toTable - assert z == {1: "word", 0: "bird"}.toTable - # analyse the body, find the deepest expression 'it' and replace it via - # 'result.add it' - let res = genSym(nskVar, "collectResult") - expectKind init, {nnkCall, nnkIdent, nnkSym} - let bracketExpr = newTree(nnkBracketExpr, - if init.kind == nnkCall: init[0] else: init) - let (resBody, keyType, valueType) = transLastStmt(body, res, bracketExpr) - if bracketExpr.len == 3: - bracketExpr[1][1] = keyType - bracketExpr[2][1] = valueType - else: - bracketExpr[1][1] = valueType - let call = newTree(nnkCall, bracketExpr) - if init.kind == nnkCall: - for i in 1 ..< init.len: - call.add init[i] - result = newTree(nnkStmtListExpr, newVarStmt(res, call), resBody, res) - -when isMainModule: - since (1, 1): - import algorithm - - var a = @[1, 2, 3, 4, 5, 6, 7, 8, 9] - doAssert outplace(a, sort()) == sorted(a) - doAssert a.outplace(sort()) == sorted(a) - #Chaining: - var aCopy = a - aCopy.insert(10) - doAssert a.outplace(insert(10)).outplace(sort()) == sorted(aCopy) - - import random + result = collectImpl(init, body) - const b = @[0, 1, 2] - let c = b.outplace shuffle() - doAssert c[0] == 1 - doAssert c[1] == 0 +macro collect*(body: untyped): untyped {.since: (1, 5).} = + ## Same as `collect` but without an `init` parameter. + ## + ## **See also:** + ## * `sequtils.toSeq proc<sequtils.html#toSeq.t%2Cuntyped>`_ + ## * `sequtils.mapIt template<sequtils.html#mapIt.t%2Ctyped%2Cuntyped>`_ + runnableExamples: + import std/[sets, tables] + let data = @["bird", "word"] - #test collect - import sets, tables + # seq: + let k = collect: + for i, d in data.pairs: + if i mod 2 == 0: d + assert k == @["bird"] - let data = @["bird", "word"] # if this gets stuck in your head, its not my fault - assert collect(newSeq, for (i, d) in data.pairs: (if i mod 2 == 0: d)) == @["bird"] - assert collect(initTable(2), for (i, d) in data.pairs: {i: d}) == {1: "word", - 0: "bird"}.toTable - assert initHashSet.collect(for d in data.items: {d}) == data.toHashSet + ## HashSet: + let n = collect: + for d in data.items: {d} + assert n == data.toHashSet - let x = collect(newSeqOfCap(4)): - for (i, d) in data.pairs: - if i mod 2 == 0: d - assert x == @["bird"] + ## Table: + let m = collect: + for i, d in data.pairs: {i: d} + assert m == {0: "bird", 1: "word"}.toTable - # bug #12874 - - let bug1 = collect( - newSeq, - for (i, d) in data.pairs:( - block: - if i mod 2 == 0: - d - else: - d & d - ) - ) - assert bug1 == @["bird", "wordword"] - - import strutils - let y = collect(newSeq): - for (i, d) in data.pairs: - try: parseInt(d) except: 0 - assert y == @[0, 0] - - let z = collect(newSeq): - for (i, d) in data.pairs: - case d - of "bird": "word" - else: d - assert z == @["word", "word"] + result = collectImpl(nil, body) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index e798bbaf1..53b3d61da 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -12,17 +12,62 @@ ## sequences and does not depend on any other module, on Windows it uses the ## Windows API. ## Changing the style is permanent even after program termination! Use the -## code ``system.addQuitProc(resetAttributes)`` to restore the defaults. +## code `exitprocs.addExitProc(resetAttributes)` to restore the defaults. ## Similarly, if you hide the cursor, make sure to unhide it with -## ``showCursor`` before quitting. +## `showCursor` before quitting. +## +## Progress bar +## ============ +## +## Basic progress bar example: +runnableExamples("-r:off"): + import std/[os, strutils] + + for i in 0..100: + stdout.styledWriteLine(fgRed, "0% ", fgWhite, '#'.repeat i, if i > 50: fgGreen else: fgYellow, "\t", $i , "%") + sleep 42 + cursorUp 1 + eraseLine() -import macros -import strformat -from strutils import toLowerAscii, `%` -import colors + stdout.resetAttributes() + +##[ +## Playing with colorful and styled text +]## + +## Procs like `styledWriteLine`, `styledEcho` etc. have a temporary effect on +## text parameters. Style parameters only affect the text parameter right after them. +## After being called, these procs will reset the default style of the terminal. +## While `setBackGroundColor`, `setForeGroundColor` etc. have a lasting +## influence on the terminal, you can use `resetAttributes` to +## reset the default style of the terminal. +runnableExamples("-r:off"): + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text without style ") + + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + + # You can specify multiple text parameters. Style parameters + # only affect the text parameter right after them. + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + +import std/macros +import std/strformat +from std/strutils import toLowerAscii, `%`, parseInt +import std/colors when defined(windows): - import winlean + import std/winlean + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] type PTerminal = ref object @@ -37,7 +82,10 @@ type var gTerm {.threadvar.}: owned(PTerminal) -proc newTerminal(): owned(PTerminal) {.gcsafe.} +when defined(windows) and defined(consoleapp): + proc newTerminal(): owned(PTerminal) {.gcsafe, raises: [OSError].} +else: + proc newTerminal(): owned(PTerminal) {.gcsafe, raises: [].} proc getTerminal(): PTerminal {.inline.} = if isNil(gTerm): @@ -45,13 +93,14 @@ proc getTerminal(): PTerminal {.inline.} = result = gTerm const - fgPrefix = "\x1b[38;2;" - bgPrefix = "\x1b[48;2;" + fgPrefix = "\e[38;2;" + bgPrefix = "\e[48;2;" ansiResetCode* = "\e[0m" + getPos = "\e[6n" stylePrefix = "\e[" when defined(windows): - import winlean, os + import std/[winlean, os] const DUPLICATE_SAME_ACCESS = 2 @@ -125,6 +174,7 @@ when defined(windows): return 0 proc terminalWidth*(): int = + ## Returns the terminal width in columns. var w: int = 0 w = terminalWidthIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -133,6 +183,7 @@ when defined(windows): return 80 proc terminalHeight*(): int = + ## Returns the terminal height in rows. var h: int = 0 h = terminalHeightIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -170,6 +221,9 @@ when defined(windows): raiseOSError(osLastError()) return (int(c.dwCursorPosition.x), int(c.dwCursorPosition.y)) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError, OSError].} = + return getCursorPos(getStdHandle(STD_OUTPUT_HANDLE)) + proc setCursorPos(h: Handle, x, y: int) = var c: COORD c.x = int16(x) @@ -203,7 +257,7 @@ when defined(windows): if f == stderr: term.hStderr else: term.hStdout else: - import termios, posix, os, parseutils + import std/[termios, posix, os, parseutils] proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = var mode: Termios @@ -217,6 +271,48 @@ else: mode.c_cc[VTIME] = 0.cuchar discard fd.tcSetAttr(time, addr mode) + proc getCursorPos*(): tuple [x, y: int] {.raises: [ValueError, IOError].} = + ## Returns cursor position (x, y) + ## writes to stdout and expects the terminal to respond via stdin + var + xStr = "" + yStr = "" + ch: char + ct: int + readX = false + + # use raw mode to ask terminal for cursor position + let fd = getFileHandle(stdin) + var oldMode: Termios + discard fd.tcGetAttr(addr oldMode) + fd.setRaw() + stdout.write(getPos) + flushFile(stdout) + + try: + # parse response format: [yyy;xxxR + while true: + let n = readBuffer(stdin, addr ch, 1) + if n == 0 or ch == 'R': + if xStr == "" or yStr == "": + raise newException(ValueError, "Got character position message that was missing data") + break + ct += 1 + if ct > 16: + raise newException(ValueError, "Got unterminated character position message from terminal") + if ch == ';': + readX = true + elif ch in {'0'..'9'}: + if readX: + xStr.add(ch) + else: + yStr.add(ch) + finally: + # restore previous terminal mode + discard fd.tcSetAttr(TCSADRAIN, addr oldMode) + + return (parseInt(xStr), parseInt(yStr)) + proc terminalWidthIoctl*(fds: openArray[int]): int = ## Returns terminal width from first fd that supports the ioctl. @@ -241,25 +337,57 @@ else: ## Returns some reasonable terminal width from either standard file ## descriptors, controlling terminal, environment variables or tradition. - var w = terminalWidthIoctl([0, 1, 2]) #Try standard file descriptors + # POSIX environment variable takes precendence. + # _COLUMNS_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred width in column positions for + # the terminal screen or window. If this variable is unset or null, + # the implementation determines the number of columns, appropriate + # for the terminal or window, in an unspecified manner. + # When COLUMNS is set, any terminal-width information implied by TERM + # is overridden. Users and conforming applications should not set COLUMNS + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var w: int + var s = getEnv("COLUMNS") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, w) > 0 and w > 0: + return w + w = terminalWidthIoctl([0, 1, 2]) # Try standard file descriptors if w > 0: return w - var cterm = newString(L_ctermid) #Try controlling tty + var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) if fd != -1: w = terminalWidthIoctl([int(fd)]) discard close(fd) if w > 0: return w - var s = getEnv("COLUMNS") #Try standard env var - if len(s) > 0 and parseInt(string(s), w) > 0 and w > 0: - return w - return 80 #Finally default to venerable value + return 80 # Finally default to venerable value proc terminalHeight*(): int = ## Returns some reasonable terminal height from either standard file ## descriptors, controlling terminal, environment variables or tradition. ## Zero is returned if the height could not be determined. - var h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors + # POSIX environment variable takes precendence. + # _LINES_: This variable shall represent a decimal integer >0 used + # to indicate the user's preferred number of lines on a page or + # the vertical screen or window size in lines. A line in this case + # is a vertical measure large enough to hold the tallest character + # in the character set being displayed. If this variable is unset or null, + # the implementation determines the number of lines, appropriate + # for the terminal or window (size, terminal baud rate, and so on), + # in an unspecified manner. + # When LINES is set, any terminal-height information implied by TERM + # is overridden. Users and conforming applications should not set LINES + # unless they wish to override the system selection and produce output + # unrelated to the terminal characteristics. + # See POSIX Base Definitions Section 8.1 Environment Variable Definition + + var h: int + var s = getEnv("LINES") # Try standard env var + if len(s) > 0 and parseSaturatedNatural(s, h) > 0 and h > 0: + return h + h = terminalHeightIoctl([0, 1, 2]) # Try standard file descriptors if h > 0: return h var cterm = newString(L_ctermid) # Try controlling tty var fd = open(ctermid(cstring(cterm)), O_RDONLY) @@ -267,9 +395,6 @@ else: h = terminalHeightIoctl([int(fd)]) discard close(fd) if h > 0: return h - var s = getEnv("LINES") # Try standard env var - if len(s) > 0 and parseInt(string(s), h) > 0 and h > 0: - return h return 0 # Could not determine height proc terminalSize*(): tuple[w, h: int] = @@ -329,7 +454,7 @@ when defined(windows): proc setCursorYPos*(f: File, y: int) = ## Sets the terminal's cursor to the y position. ## The x position is not changed. - ## **Warning**: This is not supported on UNIX! + ## .. warning:: This is not supported on UNIX! when defined(windows): let h = conHandle(f) var scrbuf: CONSOLE_SCREEN_BUFFER_INFO @@ -344,6 +469,9 @@ when defined(windows): proc cursorUp*(f: File, count = 1) = ## Moves the cursor up by `count` rows. + runnableExamples("-r:off"): + stdout.cursorUp(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -354,6 +482,9 @@ proc cursorUp*(f: File, count = 1) = proc cursorDown*(f: File, count = 1) = ## Moves the cursor down by `count` rows. + runnableExamples("-r:off"): + stdout.cursorDown(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -364,6 +495,9 @@ proc cursorDown*(f: File, count = 1) = proc cursorForward*(f: File, count = 1) = ## Moves the cursor forward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorForward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -374,6 +508,9 @@ proc cursorForward*(f: File, count = 1) = proc cursorBackward*(f: File, count = 1) = ## Moves the cursor backward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorBackward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -415,6 +552,9 @@ else: proc eraseLine*(f: File) = ## Erases the entire current line. + runnableExamples("-r:off"): + write(stdout, "never mind") + stdout.eraseLine() # nothing will be printed on the screen when defined(windows): let h = conHandle(f) var scrbuf: CONSOLE_SCREEN_BUFFER_INFO @@ -477,7 +617,7 @@ proc resetAttributes*(f: File) = gBG = 0 type - Style* = enum ## different styles for text output + Style* = enum ## Different styles for text output. styleBright = 1, ## bright text styleDim, ## dim text styleItalic, ## italic (or reverse on terminals not supporting) @@ -531,7 +671,7 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = stdout.write(ansiStyleCode(gBG)) type - ForegroundColor* = enum ## terminal's foreground colors + ForegroundColor* = enum ## Terminal's foreground colors. fgBlack = 30, ## black fgRed, ## red fgGreen, ## green @@ -540,10 +680,10 @@ type fgMagenta, ## magenta fgCyan, ## cyan fgWhite, ## white - fg8Bit, ## 256-color (not supported, see ``enableTrueColors`` instead.) + fg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) fgDefault ## default terminal foreground color - BackgroundColor* = enum ## terminal's background colors + BackgroundColor* = enum ## Terminal's background colors. bgBlack = 40, ## black bgRed, ## red bgGreen, ## green @@ -552,7 +692,7 @@ type bgMagenta, ## magenta bgCyan, ## cyan bgWhite, ## white - bg8Bit, ## 256-color (not supported, see ``enableTrueColors`` instead.) + bg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) bgDefault ## default terminal background color when defined(windows): @@ -576,12 +716,12 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright = false) = (FOREGROUND_RED or FOREGROUND_BLUE), (FOREGROUND_BLUE or FOREGROUND_GREEN), (FOREGROUND_BLUE or FOREGROUND_GREEN or FOREGROUND_RED), - 0, # fg8Bit not supported, see ``enableTrueColors`` instead. + 0, # fg8Bit not supported, see `enableTrueColors` instead. 0] # unused if fg == fgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultForegroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultForegroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[fg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[fg]))) else: gFG = ord(fg) if bright: inc(gFG, 60) @@ -605,12 +745,12 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright = false) = (BACKGROUND_RED or BACKGROUND_BLUE), (BACKGROUND_BLUE or BACKGROUND_GREEN), (BACKGROUND_BLUE or BACKGROUND_GREEN or BACKGROUND_RED), - 0, # bg8Bit not supported, see ``enableTrueColors`` instead. + 0, # bg8Bit not supported, see `enableTrueColors` instead. 0] # unused if bg == bgDefault: - discard setConsoleTextAttribute(h, toU16(old or defaultBackgroundColor)) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](defaultBackgroundColor))) else: - discard setConsoleTextAttribute(h, toU16(old or lookup[bg])) + discard setConsoleTextAttribute(h, cast[int16](cast[uint16](old) or cast[uint16](lookup[bg]))) else: gBG = ord(bg) if bright: inc(gBG, 60) @@ -689,24 +829,19 @@ template styledEchoProcessArg(f: File, color: Color) = template styledEchoProcessArg(f: File, cmd: TerminalCmd) = when cmd == resetStyle: resetAttributes(f) - when cmd == fgColor: - fgSetColor = true - when cmd == bgColor: - fgSetColor = false + elif cmd in {fgColor, bgColor}: + let term = getTerminal() + term.fgSetColor = cmd == fgColor macro styledWrite*(f: File, m: varargs[typed]): untyped = - ## Similar to ``write``, but treating terminal style arguments specially. - ## When some argument is ``Style``, ``set[Style]``, ``ForegroundColor``, - ## ``BackgroundColor`` or ``TerminalCmd`` then it is not sent directly to - ## ``f``, but instead corresponding terminal style proc is called. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## stdout.styledWrite(fgRed, "red text ") - ## stdout.styledWrite(fgGreen, "green text") - ## + ## Similar to `write`, but treating terminal style arguments specially. + ## When some argument is `Style`, `set[Style]`, `ForegroundColor`, + ## `BackgroundColor` or `TerminalCmd` then it is not sent directly to + ## `f`, but instead corresponding terminal style proc is called. + runnableExamples("-r:off"): + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgGreen, "green text") + var reset = false result = newNimNode(nnkStmtList) @@ -728,24 +863,20 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = if reset: result.add(newCall(bindSym"resetAttributes", f)) template styledWriteLine*(f: File, args: varargs[untyped]) = - ## Calls ``styledWrite`` and appends a newline at the end. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## proc error(msg: string) = - ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) - ## + ## Calls `styledWrite` and appends a newline at the end. + runnableExamples: + proc error(msg: string) = + styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + styledWrite(f, args) write(f, "\n") template styledEcho*(args: varargs[untyped]) = - ## Echoes styles arguments to stdout using ``styledWriteLine``. + ## Echoes styles arguments to stdout using `styledWriteLine`. stdout.styledWriteLine(args) proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. + ## Reads a single character from the terminal, blocking until it is entered. ## The character is not printed to the terminal. when defined(windows): let fd = getStdHandle(STD_INPUT_HANDLE) @@ -767,42 +898,35 @@ proc getch*(): char = discard fd.tcSetAttr(TCSADRAIN, addr oldMode) when defined(windows): - from unicode import toUTF8, Rune, runeLenAt - - proc readPasswordFromStdin*(prompt: string, password: var TaintedString): + proc readPasswordFromStdin*(prompt: string, password: var string): bool {.tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a `password` from stdin without printing it. `password` must not - ## be ``nil``! Returns ``false`` if the end of the file has been reached, - ## ``true`` otherwise. - password.string.setLen(0) + ## be `nil`! Returns `false` if the end of the file has been reached, + ## `true` otherwise. + password.setLen(0) stdout.write(prompt) - while true: - let c = getch() - case c.char - of '\r', chr(0xA): - break - of '\b': - # ensure we delete the whole UTF-8 character: - var i = 0 - var x = 1 - while i < password.len: - x = runeLenAt(password.string, i) - inc i, x - password.string.setLen(max(password.len - x, 0)) - of chr(0x0): - # modifier key - ignore - for details see - # https://github.com/nim-lang/Nim/issues/7764 - continue - else: - password.string.add(toUTF8(c.Rune)) + let hi = createFileA("CONIN$", + GENERIC_READ or GENERIC_WRITE, 0, nil, OPEN_EXISTING, 0, 0) + var mode = DWORD 0 + discard getConsoleMode(hi, addr mode) + let origMode = mode + const + ENABLE_PROCESSED_INPUT = 1 + ENABLE_ECHO_INPUT = 4 + mode = (mode or ENABLE_PROCESSED_INPUT) and not ENABLE_ECHO_INPUT + + discard setConsoleMode(hi, mode) + result = readLine(stdin, password) + discard setConsoleMode(hi, origMode) + discard closeHandle(hi) stdout.write "\n" else: - import termios + import std/termios - proc readPasswordFromStdin*(prompt: string, password: var TaintedString): + proc readPasswordFromStdin*(prompt: string, password: var string): bool {.tags: [ReadIOEffect, WriteIOEffect].} = - password.string.setLen(0) + password.setLen(0) let fd = stdin.getFileHandle() var cur, old: Termios discard fd.tcGetAttr(cur.addr) @@ -814,9 +938,9 @@ else: stdout.write "\n" discard fd.tcSetAttr(TCSADRAIN, old.addr) -proc readPasswordFromStdin*(prompt = "password: "): TaintedString = +proc readPasswordFromStdin*(prompt = "password: "): string = ## Reads a password from stdin without printing it. - result = TaintedString("") + result = "" discard readPasswordFromStdin(prompt, result) @@ -846,7 +970,7 @@ template setBackgroundColor*(color: Color) = proc resetAttributes*() {.noconv.} = ## Resets all attributes on stdout. ## It is advisable to register this as a quit proc with - ## ``system.addQuitProc(resetAttributes)``. + ## `exitprocs.addExitProc(resetAttributes)`. resetAttributes(stdout) proc isTrueColorSupported*(): bool = @@ -854,10 +978,10 @@ proc isTrueColorSupported*(): bool = return getTerminal().trueColorIsSupported when defined(windows): - import os + import std/os proc enableTrueColors*() = - ## Enable true color. + ## Enables true color. var term = getTerminal() when defined(windows): var @@ -885,12 +1009,12 @@ proc enableTrueColors*() = else: term.trueColorIsEnabled = true else: - term.trueColorIsSupported = string(getEnv("COLORTERM")).toLowerAscii() in [ + term.trueColorIsSupported = getEnv("COLORTERM").toLowerAscii() in [ "truecolor", "24bit"] term.trueColorIsEnabled = term.trueColorIsSupported proc disableTrueColors*() = - ## Disable true color. + ## Disables true color. var term = getTerminal() when defined(windows): if term.trueColorIsSupported: @@ -907,51 +1031,3 @@ proc newTerminal(): owned(PTerminal) = new result when defined(windows): initTerminal(result) - -when not defined(testing) and isMainModule: - assert ansiStyleCode(styleBright) == "\e[1m" - assert ansiStyleCode(styleStrikethrough) == "\e[9m" - #system.addQuitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") - stdout.styledWriteLine("italic text ", {styleItalic}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.write("blue text in cyan background") - stdout.resetAttributes() - echo "" - stdout.writeLine("ordinary text") - echo "more ordinary text" - styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" - echo "ordinary text again" - styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :(" - echo "ordinary text again" - setForeGroundColor(fgGreen) - echo "green text" - echo "more green text" - setForeGroundColor(fgBlue) - echo "blue text" - resetAttributes() - echo "ordinary text" - - stdout.styledWriteLine(fgRed, "red text ") - stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") - stdout.styledWriteLine(" ordinary text ") - stdout.styledWriteLine(fgGreen, "green text") - - writeStyled("underscored text", {styleUnderscore}) - stdout.styledWrite(fgRed, " red text ") - writeStyled("bright text ", {styleBright}) - echo "ordinary text" - - stdout.styledWrite(fgRed, "red text ") - stdout.styledWrite(fgWhite, bgRed, "white text in red background") - stdout.styledWrite(" ordinary text ") - stdout.styledWrite(fgGreen, "green text") - echo "" - echo "ordinary text" - stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, - " bold text in yellow bg", bgDefault, " bold text") - echo "ordinary text" diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 4a33197e2..e59153455 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -8,20 +8,20 @@ # ##[ - The ``times`` module contains routines and types for dealing with time using + The `times` module contains routines and types for dealing with time using the `proleptic Gregorian calendar<https://en.wikipedia.org/wiki/Proleptic_Gregorian_calendar>`_. It's also available for the `JavaScript target <backends.html#backends-the-javascript-target>`_. - Although the ``times`` module supports nanosecond time resolution, the - resolution used by ``getTime()`` depends on the platform and backend + Although the `times` module supports nanosecond time resolution, the + resolution used by `getTime()` depends on the platform and backend (JS is limited to millisecond precision). Examples ======== - .. code-block:: nim - import times, os + ```nim + import std/[times, os] # Simple benchmarking let time = cpuTime() sleep(100) # Replace this with something to be timed @@ -37,138 +37,150 @@ # Arithmetic using TimeInterval echo "One year from now : ", now() + 1.years echo "One month from now : ", now() + 1.months + ``` Parsing and Formatting Dates ============================ - The ``DateTime`` type can be parsed and formatted using the different - ``parse`` and ``format`` procedures. - - .. code-block:: nim + The `DateTime` type can be parsed and formatted using the different + `parse` and `format` procedures. + ```nim let dt = parse("2000-01-01", "yyyy-MM-dd") echo dt.format("yyyy-MM-dd") + ``` The different format patterns that are supported are documented below. - ============= ================================================================================= ================================================ - Pattern Description Example - ============= ================================================================================= ================================================ - ``d`` Numeric value representing the day of the month, | ``1/04/2012 -> 1`` - it will be either one or two digits long. | ``21/04/2012 -> 21`` - ``dd`` Same as above, but is always two digits. | ``1/04/2012 -> 01`` - | ``21/04/2012 -> 21`` - ``ddd`` Three letter string which indicates the day of the week. | ``Saturday -> Sat`` - | ``Monday -> Mon`` - ``dddd`` Full string for the day of the week. | ``Saturday -> Saturday`` - | ``Monday -> Monday`` - ``h`` The hours in one digit if possible. Ranging from 1-12. | ``5pm -> 5`` - | ``2am -> 2`` - ``hh`` The hours in two digits always. If the hour is one digit, 0 is prepended. | ``5pm -> 05`` - | ``11am -> 11`` - ``H`` The hours in one digit if possible, ranging from 0-23. | ``5pm -> 17`` - | ``2am -> 2`` - ``HH`` The hours in two digits always. 0 is prepended if the hour is one digit. | ``5pm -> 17`` - | ``2am -> 02`` - ``m`` The minutes in one digit if possible. | ``5:30 -> 30`` - | ``2:01 -> 1`` - ``mm`` Same as above but always two digits, 0 is prepended if the minute is one digit. | ``5:30 -> 30`` - | ``2:01 -> 01`` - ``M`` The month in one digit if possible. | ``September -> 9`` - | ``December -> 12`` - ``MM`` The month in two digits always. 0 is prepended if the month value is one digit. | ``September -> 09`` - | ``December -> 12`` - ``MMM`` Abbreviated three-letter form of the month. | ``September -> Sep`` - | ``December -> Dec`` - ``MMMM`` Full month string, properly capitalized. | ``September -> September`` - ``s`` Seconds as one digit if possible. | ``00:00:06 -> 6`` - ``ss`` Same as above but always two digits. 0 is prepended if the second is one digit. | ``00:00:06 -> 06`` - ``t`` ``A`` when time is in the AM. ``P`` when time is in the PM. | ``5pm -> P`` - | ``2am -> A`` - ``tt`` Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. | ``5pm -> PM`` - | ``2am -> AM`` - ``yy`` The last two digits of the year. When parsing, the current century is assumed. | ``2012 AD -> 12`` - ``yyyy`` The year, padded to at least four digits. | ``2012 AD -> 2012`` - Is always positive, even when the year is BC. | ``24 AD -> 0024`` - When the year is more than four digits, '+' is prepended. | ``24 BC -> 00024`` - | ``12345 AD -> +12345`` - ``YYYY`` The year without any padding. | ``2012 AD -> 2012`` - Is always positive, even when the year is BC. | ``24 AD -> 24`` - | ``24 BC -> 24`` - | ``12345 AD -> 12345`` - ``uuuu`` The year, padded to at least four digits. Will be negative when the year is BC. | ``2012 AD -> 2012`` - When the year is more than four digits, '+' is prepended unless the year is BC. | ``24 AD -> 0024`` - | ``24 BC -> -0023`` - | ``12345 AD -> +12345`` - ``UUUU`` The year without any padding. Will be negative when the year is BC. | ``2012 AD -> 2012`` - | ``24 AD -> 24`` - | ``24 BC -> -23`` - | ``12345 AD -> 12345`` - ``z`` Displays the timezone offset from UTC. | ``UTC+7 -> +7`` - | ``UTC-5 -> -5`` - ``zz`` Same as above but with leading 0. | ``UTC+7 -> +07`` - | ``UTC-5 -> -05`` - ``zzz`` Same as above but with ``:mm`` where *mm* represents minutes. | ``UTC+7 -> +07:00`` - | ``UTC-5 -> -05:00`` - ``zzzz`` Same as above but with ``:ss`` where *ss* represents seconds. | ``UTC+7 -> +07:00:00`` - | ``UTC-5 -> -05:00:00`` - ``g`` Era: AD or BC | ``300 AD -> AD`` - | ``300 BC -> BC`` - ``fff`` Milliseconds display | ``1000000 nanoseconds -> 1`` - ``ffffff`` Microseconds display | ``1000000 nanoseconds -> 1000`` - ``fffffffff`` Nanoseconds display | ``1000000 nanoseconds -> 1000000`` - ============= ================================================================================= ================================================ - - Other strings can be inserted by putting them in ``''``. For example - ``hh'->'mm`` will give ``01->56``. The following characters can be - inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` - ``,``. A literal ``'`` can be specified with ``''``. + =========== ================================================================================= ============================================== + Pattern Description Example + =========== ================================================================================= ============================================== + `d` Numeric value representing the day of the month, | `1/04/2012 -> 1` + it will be either one or two digits long. | `21/04/2012 -> 21` + `dd` Same as above, but is always two digits. | `1/04/2012 -> 01` + | `21/04/2012 -> 21` + `ddd` Three letter string which indicates the day of the week. | `Saturday -> Sat` + | `Monday -> Mon` + `dddd` Full string for the day of the week. | `Saturday -> Saturday` + | `Monday -> Monday` + `GG` The last two digits of the Iso Week-Year | `30/12/2012 -> 13` + `GGGG` The Iso week-calendar year padded to four digits | `30/12/2012 -> 2013` + `h` The hours in one digit if possible. Ranging from 1-12. | `5pm -> 5` + | `2am -> 2` + `hh` The hours in two digits always. If the hour is one digit, 0 is prepended. | `5pm -> 05` + | `11am -> 11` + `H` The hours in one digit if possible, ranging from 0-23. | `5pm -> 17` + | `2am -> 2` + `HH` The hours in two digits always. 0 is prepended if the hour is one digit. | `5pm -> 17` + | `2am -> 02` + `m` The minutes in one digit if possible. | `5:30 -> 30` + | `2:01 -> 1` + `mm` Same as above but always two digits, 0 is prepended if the minute is one digit. | `5:30 -> 30` + | `2:01 -> 01` + `M` The month in one digit if possible. | `September -> 9` + | `December -> 12` + `MM` The month in two digits always. 0 is prepended if the month value is one digit. | `September -> 09` + | `December -> 12` + `MMM` Abbreviated three-letter form of the month. | `September -> Sep` + | `December -> Dec` + `MMMM` Full month string, properly capitalized. | `September -> September` + `s` Seconds as one digit if possible. | `00:00:06 -> 6` + `ss` Same as above but always two digits. 0 is prepended if the second is one digit. | `00:00:06 -> 06` + `t` `A` when time is in the AM. `P` when time is in the PM. | `5pm -> P` + | `2am -> A` + `tt` Same as above, but `AM` and `PM` instead of `A` and `P` respectively. | `5pm -> PM` + | `2am -> AM` + `yy` The last two digits of the year. When parsing, the current century is assumed. | `2012 AD -> 12` + `yyyy` The year, padded to at least four digits. | `2012 AD -> 2012` + Is always positive, even when the year is BC. | `24 AD -> 0024` + When the year is more than four digits, '+' is prepended. | `24 BC -> 00024` + | `12345 AD -> +12345` + `YYYY` The year without any padding. | `2012 AD -> 2012` + Is always positive, even when the year is BC. | `24 AD -> 24` + | `24 BC -> 24` + | `12345 AD -> 12345` + `uuuu` The year, padded to at least four digits. Will be negative when the year is BC. | `2012 AD -> 2012` + When the year is more than four digits, '+' is prepended unless the year is BC. | `24 AD -> 0024` + | `24 BC -> -0023` + | `12345 AD -> +12345` + `UUUU` The year without any padding. Will be negative when the year is BC. | `2012 AD -> 2012` + | `24 AD -> 24` + | `24 BC -> -23` + | `12345 AD -> 12345` + `V` The Iso Week-Number as one or two digits | `3/2/2012 -> 5` + | `1/4/2012 -> 13` + `VV` The Iso Week-Number as two digits always. 0 is prepended if one digit. | `3/2/2012 -> 05` + | `1/4/2012 -> 13` + `z` Displays the timezone offset from UTC. | `UTC+7 -> +7` + | `UTC-5 -> -5` + `zz` Same as above but with leading 0. | `UTC+7 -> +07` + | `UTC-5 -> -05` + `zzz` Same as above but with `:mm` where *mm* represents minutes. | `UTC+7 -> +07:00` + | `UTC-5 -> -05:00` + `ZZZ` Same as above but with `mm` where *mm* represents minutes. | `UTC+7 -> +0700` + | `UTC-5 -> -0500` + `zzzz` Same as above but with `:ss` where *ss* represents seconds. | `UTC+7 -> +07:00:00` + | `UTC-5 -> -05:00:00` + `ZZZZ` Same as above but with `ss` where *ss* represents seconds. | `UTC+7 -> +070000` + | `UTC-5 -> -050000` + `g` Era: AD or BC | `300 AD -> AD` + | `300 BC -> BC` + `fff` Milliseconds display | `1000000 nanoseconds -> 1` + `ffffff` Microseconds display | `1000000 nanoseconds -> 1000` + `fffffffff` Nanoseconds display | `1000000 nanoseconds -> 1000000` + =========== ================================================================================= ============================================== + + Other strings can be inserted by putting them in `''`. For example + `hh'->'mm` will give `01->56`. In addition to spaces, + the following characters can be inserted without quoting them: + `:` `-` `,` `.` `(` `)` `/` `[` `]`. + A literal `'` can be specified with `''`. However you don't need to necessarily separate format patterns, as an - unambiguous format string like ``yyyyMMddhhmmss`` is also valid (although + unambiguous format string like `yyyyMMddhhmmss` is also valid (although only for years in the range 1..9999). Duration vs TimeInterval ============================ - The ``times`` module exports two similar types that are both used to + The `times` module exports two similar types that are both used to represent some amount of time: `Duration <#Duration>`_ and `TimeInterval <#TimeInterval>`_. This section explains how they differ and when one should be preferred over the - other (short answer: use ``Duration`` unless support for months and years is + other (short answer: use `Duration` unless support for months and years is needed). Duration ---------------------------- - A ``Duration`` represents a duration of time stored as seconds and - nanoseconds. A ``Duration`` is always fully normalized, so -``initDuration(hours = 1)`` and ``initDuration(minutes = 60)`` are equivalent. + A `Duration` represents a duration of time stored as seconds and + nanoseconds. A `Duration` is always fully normalized, so + `initDuration(hours = 1)` and `initDuration(minutes = 60)` are equivalent. - Arithmetic with a ``Duration`` is very fast, especially when used with the - ``Time`` type, since it only involves basic arithmetic. Because ``Duration`` + Arithmetic with a `Duration` is very fast, especially when used with the + `Time` type, since it only involves basic arithmetic. Because `Duration` is more performant and easier to understand it should generally preferred. TimeInterval ---------------------------- - A ``TimeInterval`` represents an amount of time expressed in calendar + A `TimeInterval` represents an amount of time expressed in calendar units, for example "1 year and 2 days". Since some units cannot be normalized (the length of a year is different for leap years for example), - the ``TimeInterval`` type uses separate fields for every unit. The - ``TimeInterval``'s returned from this module generally don't normalize + the `TimeInterval` type uses separate fields for every unit. The + `TimeInterval`'s returned from this module generally don't normalize **anything**, so even units that could be normalized (like seconds, milliseconds and so on) are left untouched. - Arithmetic with a ``TimeInterval`` can be very slow, because it requires + Arithmetic with a `TimeInterval` can be very slow, because it requires timezone information. - Since it's slower and more complex, the ``TimeInterval`` type should be + Since it's slower and more complex, the `TimeInterval` type should be avoided unless the program explicitly needs the features it offers that - ``Duration`` doesn't have. + `Duration` doesn't have. How long is a day? ---------------------------- It should be especially noted that the handling of days differs between - ``TimeInterval`` and ``Duration``. The ``Duration`` type always treats a day - as exactly 86400 seconds. For ``TimeInterval``, it's more complex. + `TimeInterval` and `Duration`. The `Duration` type always treats a day + as exactly 86400 seconds. For `TimeInterval`, it's more complex. As an example, consider the amount of time between these two timestamps, both in the same timezone: @@ -182,8 +194,8 @@ timezones that use daylight savings time. Because of this change, the amount of time that has passed is actually 25 hours. - The ``TimeInterval`` type uses calendar units, and will say that exactly one - day has passed. The ``Duration`` type on the other hand normalizes everything + The `TimeInterval` type uses calendar units, and will say that exactly one + day has passed. The `Duration` type on the other hand normalizes everything to seconds, and will therefore say that 90000 seconds has passed, which is the same as 25 hours. @@ -192,12 +204,17 @@ * `monotimes module <monotimes.html>`_ ]## -import strutils, math, options +import std/[strutils, math, options] +import std/private/since include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js): - import jscore + import std/jscore # This is really bad, but overflow checks are broken badly for # ints on the JS backend. See #6752. @@ -221,28 +238,19 @@ when defined(js): {.pop.} elif defined(posix): - import posix + import std/posix type CTime = posix.Time - when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): - var timezone {.importc, header: "<time.h>".}: int - when not defined(valgrind_workaround_10121): - tzset() - when defined(macosx): proc gettimeofday(tp: var Timeval, unused: pointer = nil) - {.importc: "gettimeofday", header: "<sys/time.h>".} + {.importc: "gettimeofday", header: "<sys/time.h>", sideEffect.} elif defined(windows): - import winlean, std/time_t - - type CTime = time_t.Time - - # visual c's c runtime exposes these under a different name - var timezone {.importc: "_timezone", header: "<time.h>".}: int + import std/winlean, std/time_t type + CTime = time_t.Time Tm {.importc: "struct tm", header: "<time.h>", final, pure.} = object tm_sec*: cint ## Seconds [0,60]. tm_min*: cint ## Minutes [0,59]. @@ -254,12 +262,12 @@ elif defined(windows): tm_yday*: cint ## Day of year [0,365]. tm_isdst*: cint ## Daylight Savings flag. - proc localtime(a1: var CTime): ptr Tm {.importc, header: "<time.h>".} + proc localtime(a1: var CTime): ptr Tm {.importc, header: "<time.h>", sideEffect.} type - Month* = enum ## Represents a month. Note that the enum starts at ``1``, - ## so ``ord(month)`` will give the month number in the - ## range ``1..12``. + Month* = enum ## Represents a month. Note that the enum starts at `1`, + ## so `ord(month)` will give the month number in the + ## range `1..12`. mJan = (1, "January") mFeb = "February" mMar = "March" @@ -282,78 +290,53 @@ type dSat = "Saturday" dSun = "Sunday" -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -type - DateTimeLocale* = object - MMM*: array[mJan..mDec, string] - MMMM*: array[mJan..mDec, string] - ddd*: array[dMon..dSun, string] - dddd*: array[dMon..dSun, string] - -when defined(nimHasStyleChecks): - {.pop.} - type MonthdayRange* = range[1..31] HourRange* = range[0..23] MinuteRange* = range[0..59] - SecondRange* = range[0..60] + SecondRange* = range[0..60] ## \ + ## Includes the value 60 to allow for a leap second. Note however + ## that the `second` of a `DateTime` will never be a leap second. YeardayRange* = range[0..365] NanosecondRange* = range[0..999_999_999] + IsoWeekRange* = range[1 .. 53] + ## An ISO 8601 calendar week number. + IsoYear* = distinct int + ## An ISO 8601 calendar year number. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + Time* = object ## Represents a point in time. seconds: int64 nanosecond: NanosecondRange DateTime* = object of RootObj ## \ - ## Represents a time in different parts. Although this type can represent - ## leap seconds, they are generally not supported in this module. They are - ## not ignored, but the ``DateTime``'s returned by procedures in this - ## module will never have a leap second. - ## - ## **Warning**: even though the fields of ``DateTime`` are exported, - ## they should never be mutated directly. Doing so is unsafe and will - ## result in the ``DateTime`` ending up in an invalid state. - ## - ## Instead of mutating the fields directly, use the `Duration <#Duration>`_ - ## and `TimeInterval <#TimeInterval>`_ types for arithmetic and use the - ## `initDateTime proc <#initDateTime,MonthdayRange,Month,int,HourRange,MinuteRange,SecondRange,NanosecondRange,Timezone>`_ - ## for changing a specific field. - nanosecond*: NanosecondRange ## The number of nanoseconds after the second, - ## in the range 0 to 999_999_999. - second*: SecondRange ## The number of seconds after the minute, - ## normally in the range 0 to 59, but can - ## be up to 60 to allow for a leap second. - minute*: MinuteRange ## The number of minutes after the hour, - ## in the range 0 to 59. - hour*: HourRange ## The number of hours past midnight, - ## in the range 0 to 23. - monthday*: MonthdayRange ## The day of the month, in the range 1 to 31. - month*: Month ## The month. - year*: int ## The year, using astronomical year numbering - ## (meaning that before year 1 is year 0, - ## then year -1 and so on). - weekday*: WeekDay ## The day of the week. - yearday*: YeardayRange ## The number of days since January 1, - ## in the range 0 to 365. - isDst*: bool ## Determines whether DST is in effect. - ## Always false for the JavaScript backend. - timezone*: Timezone ## The timezone represented as an implementation - ## of ``Timezone``. - utcOffset*: int ## The offset in seconds west of UTC, including - ## any offset due to DST. Note that the sign of - ## this number is the opposite of the one in a - ## formatted offset string like ``+01:00`` (which - ## would be equivalent to the UTC offset - ## ``-3600``). + ## Represents a time in different parts. Although this type can represent + ## leap seconds, they are generally not supported in this module. They are + ## not ignored, but the `DateTime`'s returned by procedures in this + ## module will never have a leap second. + nanosecond: NanosecondRange + second: SecondRange + minute: MinuteRange + hour: HourRange + monthdayZero: int + monthZero: int + year: int + weekday: WeekDay + yearday: YeardayRange + isDst: bool + timezone: Timezone + utcOffset: int Duration* = object ## Represents a fixed duration of time, meaning a duration ## that has constant length independent of the context. ## - ## To create a new ``Duration``, use `initDuration proc + ## To create a new `Duration`, use `initDuration ## <#initDuration,int64,int64,int64,int64,int64,int64,int64,int64>`_. + ## Instead of trying to access the private attributes, use + ## `inSeconds <#inSeconds,Duration>`_ for converting to seconds and + ## `inNanoseconds <#inNanoseconds,Duration>`_ for converting to nanoseconds. seconds: int64 nanosecond: NanosecondRange @@ -362,23 +345,23 @@ type Weeks, Months, Years FixedTimeUnit* = range[Nanoseconds..Weeks] ## \ - ## Subrange of ``TimeUnit`` that only includes units of fixed duration. - ## These are the units that can be represented by a ``Duration``. + ## Subrange of `TimeUnit` that only includes units of fixed duration. + ## These are the units that can be represented by a `Duration`. TimeInterval* = object ## \ ## Represents a non-fixed duration of time. Can be used to add and ## subtract non-fixed time units from a `DateTime <#DateTime>`_ or ## `Time <#Time>`_. ## - ## Create a new ``TimeInterval`` with `initTimeInterval proc + ## Create a new `TimeInterval` with `initTimeInterval proc ## <#initTimeInterval,int,int,int,int,int,int,int,int,int,int>`_. ## - ## Note that ``TimeInterval`` doesn't represent a fixed duration of time, + ## Note that `TimeInterval` doesn't represent a fixed duration of time, ## since the duration of some units depend on the context (e.g a year ## can be either 365 or 366 days long). The non-fixed time units are ## years, months, days and week. ## - ## Note that ``TimeInterval``'s returned from the ``times`` module are + ## Note that `TimeInterval`'s returned from the `times` module are ## never normalized. If you want to normalize a time unit, ## `Duration <#Duration>`_ should be used instead. nanoseconds*: int ## The number of nanoseconds @@ -394,8 +377,8 @@ type Timezone* = ref object ## \ ## Timezone interface for supporting `DateTime <#DateTime>`_\s of arbitrary - ## timezones. The ``times`` module only supplies implementations for the - ## systems local time and UTC. + ## timezones. The `times` module only supplies implementations for the + ## system's local time and UTC. zonedTimeFromTimeImpl: proc (x: Time): ZonedTime {.tags: [], raises: [], benign.} zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime @@ -412,7 +395,6 @@ type DurationParts* = array[FixedTimeUnit, int64] # Array of Duration parts starts TimeIntervalParts* = array[TimeUnit, int] # Array of Duration parts starts - TimesMutableTypes = DateTime | Time | Duration | TimeInterval const secondsInMin = 60 @@ -434,15 +416,21 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] -const DefaultLocale* = DateTimeLocale( - MMM: ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", - "Nov", "Dec"], - MMMM: ["January", "February", "March", "April", "May", "June", "July", - "August", "September", "October", "November", "December"], - ddd: ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], - dddd: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", - "Sunday"], -) +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} + + +# +# Helper procs +# + +{.pragma: operator, rtl, noSideEffect, benign.} proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T {.inline.} = @@ -459,8 +447,8 @@ proc convert*[T: SomeInteger](unitFrom, unitTo: FixedTimeUnit, quantity: T): T proc normalize[T: Duration|Time](seconds, nanoseconds: int64): T = ## Normalize a (seconds, nanoseconds) pair and return it as either - ## a ``Duration`` or ``Time``. A normalized ``Duration|Time`` has a - ## positive nanosecond part in the range ``NanosecondRange``. + ## a `Duration` or `Time`. A normalized `Duration|Time` has a + ## positive nanosecond part in the range `NanosecondRange`. result.seconds = seconds + convert(Nanoseconds, Seconds, nanoseconds) var nanosecond = nanoseconds mod convert(Seconds, Nanoseconds, 1) if nanosecond < 0: @@ -468,177 +456,15 @@ proc normalize[T: Duration|Time](seconds, nanoseconds: int64): T = result.seconds -= 1 result.nanosecond = nanosecond.int -# Forward declarations -proc utcTzInfo(time: Time): ZonedTime - {.tags: [], raises: [], benign.} -proc localZonedTimeFromTime(time: Time): ZonedTime - {.tags: [], raises: [], benign.} -proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime - {.tags: [], raises: [], benign.} -proc initTime*(unix: int64, nanosecond: NanosecondRange): Time - {.tags: [], raises: [], benign, noSideEffect.} - -proc nanosecond*(time: Time): NanosecondRange = - ## Get the fractional part of a ``Time`` as the number - ## of nanoseconds of the second. - time.nanosecond - -proc initDuration*(nanoseconds, microseconds, milliseconds, - seconds, minutes, hours, days, weeks: int64 = 0): Duration = - ## Create a new `Duration <#Duration>`_. - runnableExamples: - let dur = initDuration(seconds = 1, milliseconds = 1) - doAssert dur.milliseconds == 1 - doAssert dur.seconds == 1 - - let seconds = convert(Weeks, Seconds, weeks) + - convert(Days, Seconds, days) + - convert(Minutes, Seconds, minutes) + - convert(Hours, Seconds, hours) + - convert(Seconds, Seconds, seconds) + - convert(Milliseconds, Seconds, milliseconds) + - convert(Microseconds, Seconds, microseconds) + - convert(Nanoseconds, Seconds, nanoseconds) - let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + - convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + - nanoseconds mod 1_000_000_000).int - # Nanoseconds might be negative so we must normalize. - result = normalize[Duration](seconds, nanoseconds) - -template convert(dur: Duration, unit: static[FixedTimeUnit]): int64 = - # The correction is required due to how durations are normalized. - # For example,` initDuration(nanoseconds = -1)` is stored as - # { seconds = -1, nanoseconds = 999999999 }. - when unit == Nanoseconds: - dur.seconds * 1_000_000_000 + dur.nanosecond - else: - let correction = dur.seconds < 0 and dur.nanosecond > 0 - when unit >= Seconds: - convert(Seconds, unit, dur.seconds + ord(correction)) - else: - if correction: - convert(Seconds, unit, dur.seconds + 1) - - convert(Nanoseconds, unit, - convert(Seconds, Nanoseconds, 1) - dur.nanosecond) - else: - convert(Seconds, unit, dur.seconds) + - convert(Nanoseconds, unit, dur.nanosecond) - -proc inWeeks*(dur: Duration): int64 = - ## Convert the duration to the number of whole weeks. - runnableExamples: - let dur = initDuration(days = 8) - doAssert dur.inWeeks == 1 - dur.convert(Weeks) - -proc inDays*(dur: Duration): int64 = - ## Convert the duration to the number of whole days. - runnableExamples: - let dur = initDuration(hours = -50) - doAssert dur.inDays == -2 - dur.convert(Days) - -proc inHours*(dur: Duration): int64 = - ## Convert the duration to the number of whole hours. - runnableExamples: - let dur = initDuration(minutes = 60, days = 2) - doAssert dur.inHours == 49 - dur.convert(Hours) - -proc inMinutes*(dur: Duration): int64 = - ## Convert the duration to the number of whole minutes. - runnableExamples: - let dur = initDuration(hours = 2, seconds = 10) - doAssert dur.inMinutes == 120 - dur.convert(Minutes) - -proc inSeconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole seconds. - runnableExamples: - let dur = initDuration(hours = 2, milliseconds = 10) - doAssert dur.inSeconds == 2 * 60 * 60 - dur.convert(Seconds) - -proc inMilliseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole milliseconds. - runnableExamples: - let dur = initDuration(seconds = -2) - doAssert dur.inMilliseconds == -2000 - dur.convert(Milliseconds) - -proc inMicroseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole microseconds. - runnableExamples: - let dur = initDuration(seconds = -2) - doAssert dur.inMicroseconds == -2000000 - dur.convert(Microseconds) - -proc inNanoseconds*(dur: Duration): int64 = - ## Convert the duration to the number of whole nanoseconds. - runnableExamples: - let dur = initDuration(seconds = -2) - doAssert dur.inNanoseconds == -2000000000 - dur.convert(Nanoseconds) - -proc fromUnix*(unix: int64): Time - {.benign, tags: [], raises: [], noSideEffect.} = - ## Convert a unix timestamp (seconds since ``1970-01-01T00:00:00Z``) - ## to a ``Time``. - runnableExamples: - doAssert $fromUnix(0).utc == "1970-01-01T00:00:00Z" - initTime(unix, 0) - -proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = - ## Convert ``t`` to a unix timestamp (seconds since ``1970-01-01T00:00:00Z``). - ## See also `toUnixFloat` for subsecond resolution. - runnableExamples: - doAssert fromUnix(0).toUnix() == 0 - t.seconds - -proc fromUnixFloat(seconds: float): Time {.benign, tags: [], raises: [], noSideEffect.} = - ## Convert a unix timestamp in seconds to a `Time`; same as `fromUnix` - ## but with subsecond resolution. - runnableExamples: - doAssert fromUnixFloat(123456.0) == fromUnixFloat(123456) - doAssert fromUnixFloat(-123456.0) == fromUnixFloat(-123456) - let secs = seconds.floor - let nsecs = (seconds - secs) * 1e9 - initTime(secs.int64, nsecs.NanosecondRange) - -proc toUnixFloat(t: Time): float {.benign, tags: [], raises: [].} = - ## Same as `toUnix` but using subsecond resolution. - runnableExamples: - let t = getTime() - # `<` because of rounding errors - doAssert abs(t.toUnixFloat().fromUnixFloat - t) < initDuration(nanoseconds = 1000) - t.seconds.float + t.nanosecond / convert(Seconds, Nanoseconds, 1) - -since((1, 1)): - export fromUnixFloat - export toUnixFloat - -proc fromWinTime*(win: int64): Time = - ## Convert a Windows file time (100-nanosecond intervals since - ## ``1601-01-01T00:00:00Z``) to a ``Time``. - const hnsecsPerSec = convert(Seconds, Nanoseconds, 1) div 100 - let nanos = floorMod(win, hnsecsPerSec) * 100 - let seconds = floorDiv(win - epochDiff, hnsecsPerSec) - result = initTime(seconds, nanos) - -proc toWinTime*(t: Time): int64 = - ## Convert ``t`` to a Windows file time (100-nanosecond intervals - ## since ``1601-01-01T00:00:00Z``). - result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 - proc isLeapYear*(year: int): bool = - ## Returns true if ``year`` is a leap year. + ## Returns true if `year` is a leap year. runnableExamples: doAssert isLeapYear(2000) doAssert not isLeapYear(1900) year mod 4 == 0 and (year mod 100 != 0 or year mod 400 == 0) proc getDaysInMonth*(month: Month, year: int): int = - ## Get the number of days in ``month`` of ``year``. + ## Get the number of days in `month` of `year`. # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month runnableExamples: doAssert getDaysInMonth(mFeb, 2000) == 29 @@ -648,13 +474,6 @@ proc getDaysInMonth*(month: Month, year: int): int = of mApr, mJun, mSep, mNov: result = 30 else: result = 31 -proc getDaysInYear*(year: int): int = - ## Get the number of days in a ``year`` - runnableExamples: - doAssert getDaysInYear(2000) == 366 - doAssert getDaysInYear(2001) == 365 - result = 365 + (if isLeapYear(year): 1 else: 0) - proc assertValidDate(monthday: MonthdayRange, month: Month, year: int) {.inline.} = assert monthday <= getDaysInMonth(month, year), @@ -698,7 +517,7 @@ proc fromEpochDay(epochday: int64): proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign.} = ## Returns the day of the year. - ## Equivalent with ``initDateTime(monthday, month, year, 0, 0, 0).yearday``. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).yearday`. runnableExamples: doAssert getDayOfYear(1, mJan, 2000) == 0 doAssert getDayOfYear(10, mJan, 2000) == 9 @@ -718,7 +537,7 @@ proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign.} = ## Returns the day of the week enum from day, month and year. - ## Equivalent with ``initDateTime(monthday, month, year, 0, 0, 0).weekday``. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).weekday`. runnableExamples: doAssert getDayOfWeek(13, mJun, 1990) == dWed doAssert $getDayOfWeek(13, mJun, 1990) == "Wednesday" @@ -726,13 +545,91 @@ proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay assertValidDate monthday, month, year # 1970-01-01 is a Thursday, we adjust to the previous Monday let days = toEpochDay(monthday, month, year) - 3 - let weeks = floorDiv(days, 7) + let weeks = floorDiv(days, 7'i64) let wd = days - weeks * 7 # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. # so we must correct for the WeekDay type. result = if wd == 0: dSun else: WeekDay(wd - 1) -{.pragma: operator, rtl, noSideEffect, benign.} +proc getDaysInYear*(year: int): int = + ## Get the number of days in a `year` + runnableExamples: + doAssert getDaysInYear(2000) == 366 + doAssert getDaysInYear(2001) == 365 + result = 365 + (if isLeapYear(year): 1 else: 0) + +proc `==`*(a, b: IsoYear): bool {.borrow.} +proc `$`*(p: IsoYear): string {.borrow.} + +proc getWeeksInIsoYear*(y: IsoYear): IsoWeekRange {.since: (1, 5).} = + ## Returns the number of weeks in the specified ISO 8601 week-based year, which can be + ## either 53 or 52. + runnableExamples: + assert getWeeksInIsoYear(IsoYear(2019)) == 52 + assert getWeeksInIsoYear(IsoYear(2020)) == 53 + + var y = int(y) + + # support negative years + y = if y < 0: 400 + y mod 400 else: y + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let p = (y + (y div 4) - (y div 100) + (y div 400)) mod 7 + let y1 = y - 1 + let p1 = (y1 + (y1 div 4) - (y1 div 100) + (y1 div 400)) mod 7 + if p == 4 or p1 == 3: 53 else: 52 + +proc getIsoWeekAndYear*(dt: DateTime): + tuple[isoweek: IsoWeekRange, isoyear: IsoYear] {.since: (1, 5).} = + ## Returns the ISO 8601 week and year. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert getIsoWeekAndYear(initDateTime(21, mApr, 2018, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2018.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) + assert w == 01.IsoWeekRange + assert y == 2020.IsoYear + assert getIsoWeekAndYear(initDateTime(13, mSep, 2020, 00, 00, 00)) == (isoweek: 37.IsoWeekRange, isoyear: 2020.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(2, mJan, 2021, 00, 00, 00)) + assert w.int > 52 + assert w.int < 54 + assert y.int mod 100 == 20 + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + var w = (dt.yearday.int - dt.weekday.int + 10) div 7 + if w < 1: + (isoweek: getWeeksInIsoYear(IsoYear(dt.year - 1)), isoyear: IsoYear(dt.year - 1)) + elif (w > getWeeksInIsoYear(IsoYear(dt.year))): + (isoweek: IsoWeekRange(1), isoyear: IsoYear(dt.year + 1)) + else: + (isoweek: IsoWeekRange(w), isoyear: IsoYear(dt.year)) + +proc stringifyUnit(value: int | int64, unit: TimeUnit): string = + ## Stringify time unit with it's name, lowercased + let strUnit = $unit + result = "" + result.addInt value + result.add ' ' + if abs(value) != 1: + result.add(strUnit.toLowerAscii()) + else: + result.add(strUnit[0..^2].toLowerAscii()) + +proc humanizeParts(parts: seq[string]): string = + ## Make date string parts human-readable + result = "" + if parts.len == 0: + result.add "0 nanoseconds" + elif parts.len == 1: + result = parts[0] + elif parts.len == 2: + result = parts[0] & " and " & parts[1] + else: + for i in 0..high(parts)-1: + result.add parts[i] & ", " + result.add "and " & parts[high(parts)] template subImpl[T: Duration|Time](a: Duration|Time, b: Duration|Time): T = normalize[T](a.seconds - b.seconds, a.nanosecond - b.nanosecond) @@ -750,21 +647,122 @@ template lqImpl(a: Duration|Time, b: Duration|Time): bool = template eqImpl(a: Duration|Time, b: Duration|Time): bool = a.seconds == b.seconds and a.nanosecond == b.nanosecond -const DurationZero* = initDuration() ## \ + +# +# Duration +# + +const DurationZero* = Duration() ## \ ## Zero value for durations. Useful for comparisons. - ## - ## .. code-block:: nim - ## + ## ```nim ## doAssert initDuration(seconds = 1) > DurationZero ## doAssert initDuration(seconds = 0) == DurationZero + ## ``` + +proc initDuration*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, days, weeks: int64 = 0): Duration = + ## Create a new `Duration <#Duration>`_. + runnableExamples: + let dur = initDuration(seconds = 1, milliseconds = 1) + doAssert dur.inMilliseconds == 1001 + doAssert dur.inSeconds == 1 + + let seconds = convert(Weeks, Seconds, weeks) + + convert(Days, Seconds, days) + + convert(Minutes, Seconds, minutes) + + convert(Hours, Seconds, hours) + + convert(Seconds, Seconds, seconds) + + convert(Milliseconds, Seconds, milliseconds) + + convert(Microseconds, Seconds, microseconds) + + convert(Nanoseconds, Seconds, nanoseconds) + let nanoseconds = (convert(Milliseconds, Nanoseconds, milliseconds mod 1000) + + convert(Microseconds, Nanoseconds, microseconds mod 1_000_000) + + nanoseconds mod 1_000_000_000).int + # Nanoseconds might be negative so we must normalize. + result = normalize[Duration](seconds, nanoseconds) + +template convert(dur: Duration, unit: static[FixedTimeUnit]): int64 = + # The correction is required due to how durations are normalized. + # For example,` initDuration(nanoseconds = -1)` is stored as + # { seconds = -1, nanoseconds = 999999999 }. + when unit == Nanoseconds: + dur.seconds * 1_000_000_000 + dur.nanosecond + else: + let correction = dur.seconds < 0 and dur.nanosecond > 0 + when unit >= Seconds: + convert(Seconds, unit, dur.seconds + ord(correction)) + else: + if correction: + convert(Seconds, unit, dur.seconds + 1) - + convert(Nanoseconds, unit, + convert(Seconds, Nanoseconds, 1) - dur.nanosecond) + else: + convert(Seconds, unit, dur.seconds) + + convert(Nanoseconds, unit, dur.nanosecond) + +proc inWeeks*(dur: Duration): int64 = + ## Converts the duration to the number of whole weeks. + runnableExamples: + let dur = initDuration(days = 8) + doAssert dur.inWeeks == 1 + dur.convert(Weeks) + +proc inDays*(dur: Duration): int64 = + ## Converts the duration to the number of whole days. + runnableExamples: + let dur = initDuration(hours = -50) + doAssert dur.inDays == -2 + dur.convert(Days) + +proc inHours*(dur: Duration): int64 = + ## Converts the duration to the number of whole hours. + runnableExamples: + let dur = initDuration(minutes = 60, days = 2) + doAssert dur.inHours == 49 + dur.convert(Hours) + +proc inMinutes*(dur: Duration): int64 = + ## Converts the duration to the number of whole minutes. + runnableExamples: + let dur = initDuration(hours = 2, seconds = 10) + doAssert dur.inMinutes == 120 + dur.convert(Minutes) + +proc inSeconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole seconds. + runnableExamples: + let dur = initDuration(hours = 2, milliseconds = 10) + doAssert dur.inSeconds == 2 * 60 * 60 + dur.convert(Seconds) + +proc inMilliseconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole milliseconds. + runnableExamples: + let dur = initDuration(seconds = -2) + doAssert dur.inMilliseconds == -2000 + dur.convert(Milliseconds) + +proc inMicroseconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole microseconds. + runnableExamples: + let dur = initDuration(seconds = -2) + doAssert dur.inMicroseconds == -2000000 + dur.convert(Microseconds) + +proc inNanoseconds*(dur: Duration): int64 = + ## Converts the duration to the number of whole nanoseconds. + runnableExamples: + let dur = initDuration(seconds = -2) + doAssert dur.inNanoseconds == -2000000000 + dur.convert(Nanoseconds) proc toParts*(dur: Duration): DurationParts = ## Converts a duration into an array consisting of fixed time units. ## ## Each value in the array gives information about a specific unit of - ## time, for example ``result[Days]`` gives a count of days. + ## time, for example `result[Days]` gives a count of days. ## - ## This procedure is useful for converting ``Duration`` values to strings. + ## This procedure is useful for converting `Duration` values to strings. runnableExamples: var dp = toParts(initDuration(weeks = 2, days = 1)) doAssert dp[Days] == 1 @@ -793,33 +791,8 @@ proc toParts*(dur: Duration): DurationParts = result[unit] = quantity -proc stringifyUnit(value: int | int64, unit: TimeUnit): string = - ## Stringify time unit with it's name, lowercased - let strUnit = $unit - result = "" - result.add($value) - result.add(" ") - if abs(value) != 1: - result.add(strUnit.toLowerAscii()) - else: - result.add(strUnit[0..^2].toLowerAscii()) - -proc humanizeParts(parts: seq[string]): string = - ## Make date string parts human-readable - result = "" - if parts.len == 0: - result.add "0 nanoseconds" - elif parts.len == 1: - result = parts[0] - elif parts.len == 2: - result = parts[0] & " and " & parts[1] - else: - for i in 0..high(parts)-1: - result.add parts[i] & ", " - result.add "and " & parts[high(parts)] - proc `$`*(dur: Duration): string = - ## Human friendly string representation of a ``Duration``. + ## Human friendly string representation of a `Duration`. runnableExamples: doAssert $initDuration(seconds = 2) == "2 seconds" doAssert $initDuration(weeks = 1, days = 2) == "1 week and 2 days" @@ -859,9 +832,9 @@ proc `-`*(a: Duration): Duration {.operator, extern: "ntReverseDuration".} = proc `<`*(a, b: Duration): bool {.operator, extern: "ntLtDuration".} = ## Note that a duration can be negative, - ## so even if ``a < b`` is true ``a`` might + ## so even if `a < b` is true `a` might ## represent a larger absolute duration. - ## Use ``abs(a) < abs(b)`` to compare the absolute + ## Use `abs(a) < abs(b)` to compare the absolute ## duration. runnableExamples: doAssert initDuration(seconds = 1) < initDuration(seconds = 2) @@ -896,6 +869,15 @@ proc `*`*(a: Duration, b: int64): Duration {.operator, doAssert initDuration(minutes = 45) * 3 == initDuration(hours = 2, minutes = 15) b * a +proc `+=`*(d1: var Duration, d2: Duration) = + d1 = d1 + d2 + +proc `-=`*(dt: var Duration, ti: Duration) = + dt = dt - ti + +proc `*=`*(a: var Duration, b: int) = + a = a * b + proc `div`*(a: Duration, b: int64): Duration {.operator, extern: "ntDivDuration".} = ## Integer division for durations. @@ -909,11 +891,113 @@ proc `div`*(a: Duration, b: int64): Duration {.operator, let carryOver = convert(Seconds, Nanoseconds, a.seconds mod b) normalize[Duration](a.seconds div b, (a.nanosecond + carryOver) div b) +proc high*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration. + initDuration(seconds = high(int64), nanoseconds = high(NanosecondRange)) + +proc low*(typ: typedesc[Duration]): Duration = + ## Get the longest representable duration of negative direction. + initDuration(seconds = low(int64)) + +proc abs*(a: Duration): Duration = + runnableExamples: + doAssert initDuration(milliseconds = -1500).abs == + initDuration(milliseconds = 1500) + initDuration(seconds = abs(a.seconds), nanoseconds = -a.nanosecond) + +# +# Time +# + proc initTime*(unix: int64, nanosecond: NanosecondRange): Time = ## Create a `Time <#Time>`_ from a unix timestamp and a nanosecond part. result.seconds = unix result.nanosecond = nanosecond +proc nanosecond*(time: Time): NanosecondRange = + ## Get the fractional part of a `Time` as the number + ## of nanoseconds of the second. + time.nanosecond + +proc fromUnix*(unix: int64): Time + {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp (seconds since `1970-01-01T00:00:00Z`) + ## to a `Time`. + runnableExamples: + doAssert $fromUnix(0).utc == "1970-01-01T00:00:00Z" + initTime(unix, 0) + +proc toUnix*(t: Time): int64 {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert `t` to a unix timestamp (seconds since `1970-01-01T00:00:00Z`). + ## See also `toUnixFloat` for subsecond resolution. + runnableExamples: + doAssert fromUnix(0).toUnix() == 0 + t.seconds + +proc fromUnixFloat(seconds: float): Time {.benign, tags: [], raises: [], noSideEffect.} = + ## Convert a unix timestamp in seconds to a `Time`; same as `fromUnix` + ## but with subsecond resolution. + runnableExamples: + doAssert fromUnixFloat(123456.0) == fromUnixFloat(123456) + doAssert fromUnixFloat(-123456.0) == fromUnixFloat(-123456) + let secs = seconds.floor + let nsecs = (seconds - secs) * 1e9 + initTime(secs.int64, nsecs.NanosecondRange) + +proc toUnixFloat(t: Time): float {.benign, tags: [], raises: [].} = + ## Same as `toUnix` but using subsecond resolution. + runnableExamples: + let t = getTime() + # `<` because of rounding errors + doAssert abs(t.toUnixFloat().fromUnixFloat - t) < initDuration(nanoseconds = 1000) + t.seconds.float + t.nanosecond / convert(Seconds, Nanoseconds, 1) + +since((1, 1)): + export fromUnixFloat + export toUnixFloat + + +proc fromWinTime*(win: int64): Time = + ## Convert a Windows file time (100-nanosecond intervals since + ## `1601-01-01T00:00:00Z`) to a `Time`. + const hnsecsPerSec = convert(Seconds, Nanoseconds, 1) div 100 + let nanos = floorMod(win, hnsecsPerSec) * 100 + let seconds = floorDiv(win - epochDiff, hnsecsPerSec) + result = initTime(seconds, nanos) + +proc toWinTime*(t: Time): int64 = + ## Convert `t` to a Windows file time (100-nanosecond intervals + ## since `1601-01-01T00:00:00Z`). + result = t.seconds * rateDiff + epochDiff + t.nanosecond div 100 + +proc getTimeImpl(typ: typedesc[Time]): Time = + discard "implemented in the vm" + +proc getTime*(): Time {.tags: [TimeEffect], benign.} = + ## Gets the current time as a `Time` with up to nanosecond resolution. + when nimvm: + result = getTimeImpl(Time) + else: + when defined(js): + let millis = newDate().getTime() + let seconds = convert(Milliseconds, Seconds, millis) + let nanos = convert(Milliseconds, Nanoseconds, + millis mod convert(Seconds, Milliseconds, 1).int) + result = initTime(seconds, nanos) + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = initTime(a.tv_sec.int64, + convert(Microseconds, Nanoseconds, a.tv_usec.int)) + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) + elif defined(windows): + var f {.noinit.}: FILETIME + getSystemTimeAsFileTime(f) + result = fromWinTime(rdFileTime(f)) + proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = ## Computes the duration between two points in time. runnableExamples: @@ -922,53 +1006,155 @@ proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = subImpl[Duration](a, b) proc `+`*(a: Time, b: Duration): Time {.operator, extern: "ntAddTime".} = - ## Add a duration of time to a ``Time``. + ## Add a duration of time to a `Time`. runnableExamples: doAssert (fromUnix(0) + initDuration(seconds = 1)) == fromUnix(1) addImpl[Time](a, b) proc `-`*(a: Time, b: Duration): Time {.operator, extern: "ntSubTime".} = - ## Subtracts a duration of time from a ``Time``. + ## Subtracts a duration of time from a `Time`. runnableExamples: doAssert (fromUnix(0) - initDuration(seconds = 1)) == fromUnix(-1) subImpl[Time](a, b) proc `<`*(a, b: Time): bool {.operator, extern: "ntLtTime".} = - ## Returns true if ``a < b``, that is if ``a`` happened before ``b``. + ## Returns true if `a < b`, that is if `a` happened before `b`. runnableExamples: doAssert initTime(50, 0) < initTime(99, 0) ltImpl(a, b) proc `<=`*(a, b: Time): bool {.operator, extern: "ntLeTime".} = - ## Returns true if ``a <= b``. + ## Returns true if `a <= b`. lqImpl(a, b) proc `==`*(a, b: Time): bool {.operator, extern: "ntEqTime".} = - ## Returns true if ``a == b``, that is if both times represent the same point in time. + ## Returns true if `a == b`, that is if both times represent the same point in time. eqImpl(a, b) +proc `+=`*(t: var Time, b: Duration) = + t = t + b + +proc `-=`*(t: var Time, b: Duration) = + t = t - b + proc high*(typ: typedesc[Time]): Time = initTime(high(int64), high(NanosecondRange)) proc low*(typ: typedesc[Time]): Time = - initTime(low(int64), 0) + initTime(0, 0) -proc high*(typ: typedesc[Duration]): Duration = - ## Get the longest representable duration. - initDuration(seconds = high(int64), nanoseconds = high(NanosecondRange)) - -proc low*(typ: typedesc[Duration]): Duration = - ## Get the longest representable duration of negative direction. - initDuration(seconds = low(int64)) +# +# DateTime & Timezone +# -proc abs*(a: Duration): Duration = - runnableExamples: - doAssert initDuration(milliseconds = -1500).abs == - initDuration(milliseconds = 1500) - initDuration(seconds = abs(a.seconds), nanoseconds = -a.nanosecond) +template assertDateTimeInitialized(dt: DateTime) = + assert dt.monthdayZero != 0, "Uninitialized datetime" + +proc nanosecond*(dt: DateTime): NanosecondRange {.inline.} = + ## The number of nanoseconds after the second, + ## in the range 0 to 999_999_999. + assertDateTimeInitialized(dt) + dt.nanosecond + +proc second*(dt: DateTime): SecondRange {.inline.} = + ## The number of seconds after the minute, + ## in the range 0 to 59. + assertDateTimeInitialized(dt) + dt.second + +proc minute*(dt: DateTime): MinuteRange {.inline.} = + ## The number of minutes after the hour, + ## in the range 0 to 59. + assertDateTimeInitialized(dt) + dt.minute + +proc hour*(dt: DateTime): HourRange {.inline.} = + ## The number of hours past midnight, + ## in the range 0 to 23. + assertDateTimeInitialized(dt) + dt.hour + +proc monthday*(dt: DateTime): MonthdayRange {.inline.} = + ## The day of the month, in the range 1 to 31. + assertDateTimeInitialized(dt) + # 'cast' to avoid extra range check + cast[MonthdayRange](dt.monthdayZero) + +proc month*(dt: DateTime): Month = + ## The month as an enum, the ordinal value + ## is in the range 1 to 12. + assertDateTimeInitialized(dt) + # 'cast' to avoid extra range check + cast[Month](dt.monthZero) + +proc year*(dt: DateTime): int {.inline.} = + ## The year, using astronomical year numbering + ## (meaning that before year 1 is year 0, + ## then year -1 and so on). + assertDateTimeInitialized(dt) + dt.year + +proc weekday*(dt: DateTime): WeekDay {.inline.} = + ## The day of the week as an enum, the ordinal + ## value is in the range 0 (monday) to 6 (sunday). + assertDateTimeInitialized(dt) + dt.weekday + +proc yearday*(dt: DateTime): YeardayRange {.inline.} = + ## The number of days since January 1, + ## in the range 0 to 365. + assertDateTimeInitialized(dt) + dt.yearday + +proc isDst*(dt: DateTime): bool {.inline.} = + ## Determines whether DST is in effect. + ## Always false for the JavaScript backend. + assertDateTimeInitialized(dt) + dt.isDst + +proc timezone*(dt: DateTime): Timezone {.inline.} = + ## The timezone represented as an implementation + ## of `Timezone`. + assertDateTimeInitialized(dt) + dt.timezone + +proc utcOffset*(dt: DateTime): int {.inline.} = + ## The offset in seconds west of UTC, including + ## any offset due to DST. Note that the sign of + ## this number is the opposite of the one in a + ## formatted offset string like `+01:00` (which + ## would be equivalent to the UTC offset + ## `-3600`). + assertDateTimeInitialized(dt) + dt.utcOffset + +proc isInitialized(dt: DateTime): bool = + # Returns true if `dt` is not the (invalid) default value for `DateTime`. + runnableExamples: + doAssert now().isInitialized + doAssert not default(DateTime).isInitialized + dt.monthZero != 0 + +since((1, 3)): + export isInitialized + +proc isLeapDay*(dt: DateTime): bool {.since: (1, 1).} = + ## Returns whether `t` is a leap day, i.e. Feb 29 in a leap year. This matters + ## as it affects time offset calculations. + runnableExamples: + let dt = dateTime(2020, mFeb, 29, 00, 00, 00, 00, utc()) + doAssert dt.isLeapDay + doAssert dt+1.years-1.years != dt + let dt2 = dateTime(2020, mFeb, 28, 00, 00, 00, 00, utc()) + doAssert not dt2.isLeapDay + doAssert dt2+1.years-1.years == dt2 + doAssertRaises(Exception): discard dateTime(2021, mFeb, 29, 00, 00, 00, 00, utc()) + assertDateTimeInitialized dt + dt.year.isLeapYear and dt.month == mFeb and dt.monthday == 29 proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = - ## Converts a ``DateTime`` to a ``Time`` representing the same point in time. + ## Converts a `DateTime` to a `Time` representing the same point in time. + assertDateTimeInitialized dt let epochDay = toEpochDay(dt.monthday, dt.month, dt.year) var seconds = epochDay * secondsInDay seconds.inc dt.hour * secondsInHour @@ -978,7 +1164,7 @@ proc toTime*(dt: DateTime): Time {.tags: [], raises: [], benign.} = result = initTime(seconds, dt.nanosecond) proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = - ## Create a new ``DateTime`` using ``ZonedTime`` in the specified timezone. + ## Create a new `DateTime` using `ZonedTime` in the specified timezone. let adjTime = zt.time - initDuration(seconds = zt.utcOffset) let s = adjTime.seconds let epochday = floorDiv(s, secondsInDay) @@ -993,8 +1179,8 @@ proc initDateTime(zt: ZonedTime, zone: Timezone): DateTime = DateTime( year: y, - month: m, - monthday: d, + monthZero: m.int, + monthdayZero: d, hour: hour, minute: minute, second: second, @@ -1013,11 +1199,11 @@ proc newTimezone*( zonedTimeFromAdjTimeImpl: proc (adjTime: Time): ZonedTime {.tags: [], raises: [], benign.} ): owned Timezone = - ## Create a new ``Timezone``. + ## Create a new `Timezone`. ## - ## ``zonedTimeFromTimeImpl`` and ``zonedTimeFromAdjTimeImpl`` is used - ## as the underlying implementations for ``zonedTimeFromTime`` and - ## ``zonedTimeFromAdjTime``. + ## `zonedTimeFromTimeImpl` and `zonedTimeFromAdjTimeImpl` is used + ## as the underlying implementations for `zonedTimeFromTime` and + ## `zonedTimeFromAdjTime`. ## ## If possible, the name parameter should match the name used in the ## tz database. If the timezone doesn't exist in the tz database, or if the @@ -1041,48 +1227,48 @@ proc name*(zone: Timezone): string = ## If the timezone doesn't exist in the tz database, or if the timezone ## name is unknown, then any string that describes the timezone ## unambiguously might be used. For example, the string "LOCAL" is used - ## for the systems local timezone. + ## for the system's local timezone. ## ## See also: https://en.wikipedia.org/wiki/Tz_database zone.name proc zonedTimeFromTime*(zone: Timezone, time: Time): ZonedTime = - ## Returns the ``ZonedTime`` for some point in time. + ## Returns the `ZonedTime` for some point in time. zone.zonedTimeFromTimeImpl(time) proc zonedTimeFromAdjTime*(zone: Timezone, adjTime: Time): ZonedTime = - ## Returns the ``ZonedTime`` for some local time. + ## Returns the `ZonedTime` for some local time. ## - ## Note that the ``Time`` argument does not represent a point in time, it - ## represent a local time! E.g if ``adjTime`` is ``fromUnix(0)``, it should be - ## interpreted as 1970-01-01T00:00:00 in the ``zone`` timezone, not in UTC. + ## Note that the `Time` argument does not represent a point in time, it + ## represent a local time! E.g if `adjTime` is `fromUnix(0)`, it should be + ## interpreted as 1970-01-01T00:00:00 in the `zone` timezone, not in UTC. zone.zonedTimeFromAdjTimeImpl(adjTime) proc `$`*(zone: Timezone): string = ## Returns the name of the timezone. - zone.name + if zone != nil: result = zone.name proc `==`*(zone1, zone2: Timezone): bool = - ## Two ``Timezone``'s are considered equal if their name is equal. + ## Two `Timezone`'s are considered equal if their name is equal. + runnableExamples: + doAssert local() == local() + doAssert local() != utc() if system.`==`(zone1, zone2): return true if zone1.isNil or zone2.isNil: return false - - runnableExamples: - doAssert local() == local() - doAssert local() != utc() zone1.name == zone2.name proc inZone*(time: Time, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = - ## Convert ``time`` into a ``DateTime`` using ``zone`` as the timezone. + ## Convert `time` into a `DateTime` using `zone` as the timezone. result = initDateTime(zone.zonedTimeFromTime(time), zone) proc inZone*(dt: DateTime, zone: Timezone): DateTime {.tags: [], raises: [], benign.} = - ## Returns a ``DateTime`` representing the same point in time as ``dt`` but - ## using ``zone`` as the timezone. + ## Returns a `DateTime` representing the same point in time as `dt` but + ## using `zone` as the timezone. + assertDateTimeInitialized dt dt.toTime.inZone(zone) proc toAdjTime(dt: DateTime): Time = @@ -1094,14 +1280,14 @@ proc toAdjTime(dt: DateTime): Time = result = initTime(seconds, dt.nanosecond) when defined(js): - proc localZonedTimeFromTime(time: Time): ZonedTime = + proc localZonedTimeFromTime(time: Time): ZonedTime {.benign.} = let jsDate = newDate(time.seconds * 1000) let offset = jsDate.getTimezoneOffset() * secondsInMin result.time = time result.utcOffset = offset result.isDst = false - proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime {.benign.} = let utcDate = newDate(adjTime.seconds * 1000) let localDate = newDate(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate(), utcDate.getUTCHours(), utcDate.getUTCMinutes(), @@ -1148,13 +1334,13 @@ else: return ((a.int64 - tm.toAdjUnix).int, tm.tm_isdst > 0) return (0, false) - proc localZonedTimeFromTime(time: Time): ZonedTime = + proc localZonedTimeFromTime(time: Time): ZonedTime {.benign.} = let (offset, dst) = getLocalOffsetAndDst(time.seconds) result.time = time result.utcOffset = offset result.isDst = dst - proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime = + proc localZonedTimeFromAdjTime(adjTime: Time): ZonedTime {.benign.} = var adjUnix = adjTime.seconds let past = adjUnix - secondsInDay let (pastOffset, _) = getLocalOffsetAndDst(past) @@ -1187,7 +1373,7 @@ var utcInstance {.threadvar.}: Timezone var localInstance {.threadvar.}: Timezone proc utc*(): Timezone = - ## Get the ``Timezone`` implementation for the UTC timezone. + ## Get the `Timezone` implementation for the UTC timezone. runnableExamples: doAssert now().utc.timezone == utc() doAssert utc().name == "Etc/UTC" @@ -1196,7 +1382,7 @@ proc utc*(): Timezone = result = utcInstance proc local*(): Timezone = - ## Get the ``Timezone`` implementation for the local timezone. + ## Get the `Timezone` implementation for the local timezone. runnableExamples: doAssert now().timezone == local() doAssert local().name == "LOCAL" @@ -1206,272 +1392,42 @@ proc local*(): Timezone = result = localInstance proc utc*(dt: DateTime): DateTime = - ## Shorthand for ``dt.inZone(utc())``. + ## Shorthand for `dt.inZone(utc())`. dt.inZone(utc()) proc local*(dt: DateTime): DateTime = - ## Shorthand for ``dt.inZone(local())``. + ## Shorthand for `dt.inZone(local())`. dt.inZone(local()) proc utc*(t: Time): DateTime = - ## Shorthand for ``t.inZone(utc())``. + ## Shorthand for `t.inZone(utc())`. t.inZone(utc()) proc local*(t: Time): DateTime = - ## Shorthand for ``t.inZone(local())``. + ## Shorthand for `t.inZone(local())`. t.inZone(local()) -proc getTime*(): Time {.tags: [TimeEffect], benign.} = - ## Gets the current time as a ``Time`` with up to nanosecond resolution. - when defined(js): - let millis = newDate().getTime() - let seconds = convert(Milliseconds, Seconds, millis) - let nanos = convert(Milliseconds, Nanoseconds, - millis mod convert(Seconds, Milliseconds, 1).int) - result = initTime(seconds, nanos) - elif defined(macosx): - var a: Timeval - gettimeofday(a) - result = initTime(a.tv_sec.int64, - convert(Microseconds, Nanoseconds, a.tv_usec.int)) - elif defined(posix): - var ts: Timespec - discard clock_gettime(CLOCK_REALTIME, ts) - result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) - elif defined(windows): - var f: FILETIME - getSystemTimeAsFileTime(f) - result = fromWinTime(rdFileTime(f)) - proc now*(): DateTime {.tags: [TimeEffect], benign.} = - ## Get the current time as a ``DateTime`` in the local timezone. + ## Get the current time as a `DateTime` in the local timezone. + ## Shorthand for `getTime().local`. ## - ## Shorthand for ``getTime().local``. + ## .. warning:: Unsuitable for benchmarking, use `monotimes.getMonoTime` or + ## `cpuTime` instead, depending on the use case. getTime().local -proc initTimeInterval*(nanoseconds, microseconds, milliseconds, - seconds, minutes, hours, - days, weeks, months, years: int = 0): TimeInterval = - ## Creates a new `TimeInterval <#TimeInterval>`_. - ## - ## This proc doesn't perform any normalization! For example, - ## ``initTimeInterval(hours = 24)`` and ``initTimeInterval(days = 1)`` are - ## not equal. - ## - ## You can also use the convenience procedures called ``milliseconds``, - ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. - runnableExamples: - let day = initTimeInterval(hours = 24) - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) - doAssert $(dt + day) == "2000-01-02T12:00:00Z" - doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) - result.nanoseconds = nanoseconds - result.microseconds = microseconds - result.milliseconds = milliseconds - result.seconds = seconds - result.minutes = minutes - result.hours = hours - result.days = days - result.weeks = weeks - result.months = months - result.years = years - -proc `+`*(ti1, ti2: TimeInterval): TimeInterval = - ## Adds two ``TimeInterval`` objects together. - result.nanoseconds = ti1.nanoseconds + ti2.nanoseconds - result.microseconds = ti1.microseconds + ti2.microseconds - result.milliseconds = ti1.milliseconds + ti2.milliseconds - result.seconds = ti1.seconds + ti2.seconds - result.minutes = ti1.minutes + ti2.minutes - result.hours = ti1.hours + ti2.hours - result.days = ti1.days + ti2.days - result.weeks = ti1.weeks + ti2.weeks - result.months = ti1.months + ti2.months - result.years = ti1.years + ti2.years - -proc `-`*(ti: TimeInterval): TimeInterval = - ## Reverses a time interval - runnableExamples: - let day = -initTimeInterval(hours = 24) - doAssert day.hours == -24 - - result = TimeInterval( - nanoseconds: -ti.nanoseconds, - microseconds: -ti.microseconds, - milliseconds: -ti.milliseconds, - seconds: -ti.seconds, - minutes: -ti.minutes, - hours: -ti.hours, - days: -ti.days, - weeks: -ti.weeks, - months: -ti.months, - years: -ti.years - ) - -proc `-`*(ti1, ti2: TimeInterval): TimeInterval = - ## Subtracts TimeInterval ``ti1`` from ``ti2``. - ## - ## Time components are subtracted one-by-one, see output: - runnableExamples: - let ti1 = initTimeInterval(hours = 24) - let ti2 = initTimeInterval(hours = 4) - doAssert (ti1 - ti2) == initTimeInterval(hours = 20) - - result = ti1 + (-ti2) - -proc getDateStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## Gets the current local date as a string of the format ``YYYY-MM-DD``. - runnableExamples: - echo getDateStr(now() - 1.months) - result = $dt.year & '-' & intToStr(ord(dt.month), 2) & - '-' & intToStr(dt.monthday, 2) - -proc getClockStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = - ## Gets the current local clock time as a string of the format ``HH:mm:ss``. - runnableExamples: - echo getClockStr(now() - 1.hours) - result = intToStr(dt.hour, 2) & ':' & intToStr(dt.minute, 2) & - ':' & intToStr(dt.second, 2) - -proc toParts*(ti: TimeInterval): TimeIntervalParts = - ## Converts a ``TimeInterval`` into an array consisting of its time units, - ## starting with nanoseconds and ending with years. - ## - ## This procedure is useful for converting ``TimeInterval`` values to strings. - ## E.g. then you need to implement custom interval printing - runnableExamples: - var tp = toParts(initTimeInterval(years = 1, nanoseconds = 123)) - doAssert tp[Years] == 1 - doAssert tp[Nanoseconds] == 123 - - var index = 0 - for name, value in fieldPairs(ti): - result[index.TimeUnit()] = value - index += 1 - -proc `$`*(ti: TimeInterval): string = - ## Get string representation of ``TimeInterval``. - runnableExamples: - doAssert $initTimeInterval(years = 1, nanoseconds = 123) == - "1 year and 123 nanoseconds" - doAssert $initTimeInterval() == "0 nanoseconds" - - var parts: seq[string] = @[] - var tiParts = toParts(ti) - for unit in countdown(Years, Nanoseconds): - if tiParts[unit] != 0: - parts.add(stringifyUnit(tiParts[unit], unit)) - - result = humanizeParts(parts) - -proc nanoseconds*(nanos: int): TimeInterval {.inline.} = - ## TimeInterval of ``nanos`` nanoseconds. - initTimeInterval(nanoseconds = nanos) - -proc microseconds*(micros: int): TimeInterval {.inline.} = - ## TimeInterval of ``micros`` microseconds. - initTimeInterval(microseconds = micros) - -proc milliseconds*(ms: int): TimeInterval {.inline.} = - ## TimeInterval of ``ms`` milliseconds. - initTimeInterval(milliseconds = ms) - -proc seconds*(s: int): TimeInterval {.inline.} = - ## TimeInterval of ``s`` seconds. - ## - ## ``echo getTime() + 5.seconds`` - initTimeInterval(seconds = s) - -proc minutes*(m: int): TimeInterval {.inline.} = - ## TimeInterval of ``m`` minutes. - ## - ## ``echo getTime() + 5.minutes`` - initTimeInterval(minutes = m) - -proc hours*(h: int): TimeInterval {.inline.} = - ## TimeInterval of ``h`` hours. - ## - ## ``echo getTime() + 2.hours`` - initTimeInterval(hours = h) - -proc days*(d: int): TimeInterval {.inline.} = - ## TimeInterval of ``d`` days. - ## - ## ``echo getTime() + 2.days`` - initTimeInterval(days = d) - -proc weeks*(w: int): TimeInterval {.inline.} = - ## TimeInterval of ``w`` weeks. - ## - ## ``echo getTime() + 2.weeks`` - initTimeInterval(weeks = w) - -proc months*(m: int): TimeInterval {.inline.} = - ## TimeInterval of ``m`` months. - ## - ## ``echo getTime() + 2.months`` - initTimeInterval(months = m) - -proc years*(y: int): TimeInterval {.inline.} = - ## TimeInterval of ``y`` years. - ## - ## ``echo getTime() + 2.years`` - initTimeInterval(years = y) - -proc evaluateInterval(dt: DateTime, interval: TimeInterval): - tuple[adjDur, absDur: Duration] = - ## Evaluates how many nanoseconds the interval is worth - ## in the context of ``dt``. - ## The result in split into an adjusted diff and an absolute diff. - var months = interval.years * 12 + interval.months - var curYear = dt.year - var curMonth = dt.month - # Subtracting - if months < 0: - for mth in countdown(-1 * months, 1): - if curMonth == mJan: - curMonth = mDec - curYear.dec - else: - curMonth.dec() - let days = getDaysInMonth(curMonth, curYear) - result.adjDur = result.adjDur - initDuration(days = days) - # Adding - else: - for mth in 1 .. months: - let days = getDaysInMonth(curMonth, curYear) - result.adjDur = result.adjDur + initDuration(days = days) - if curMonth == mDec: - curMonth = mJan - curYear.inc - else: - curMonth.inc() - - result.adjDur = result.adjDur + initDuration( - days = interval.days, - weeks = interval.weeks) - result.absDur = initDuration( - nanoseconds = interval.nanoseconds, - microseconds = interval.microseconds, - milliseconds = interval.milliseconds, - seconds = interval.seconds, - minutes = interval.minutes, - hours = interval.hours) - -proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, - hour: HourRange, minute: MinuteRange, second: SecondRange, - nanosecond: NanosecondRange, - zone: Timezone = local()): DateTime = +proc dateTime*(year: int, month: Month, monthday: MonthdayRange, + hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0, + nanosecond: NanosecondRange = 0, + zone: Timezone = local()): DateTime = ## Create a new `DateTime <#DateTime>`_ in the specified timezone. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) - doAssert $dt1 == "2017-03-30T00:00:00Z" + assert $dateTime(2017, mMar, 30, zone = utc()) == "2017-03-30T00:00:00Z" assertValidDate monthday, month, year let dt = DateTime( - monthday: monthday, + monthdayZero: monthday, year: year, - month: month, + monthZero: month.int, hour: hour, minute: minute, second: second, @@ -1481,57 +1437,24 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, hour: HourRange, minute: MinuteRange, second: SecondRange, - zone: Timezone = local()): DateTime = + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = ## Create a new `DateTime <#DateTime>`_ in the specified timezone. - runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - doAssert $dt1 == "2017-03-30T00:00:00Z" - initDateTime(monthday, month, year, hour, minute, second, 0, zone) - - -proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = - ## Adds ``interval`` to ``dt``. Components from ``interval`` are added - ## in the order of their size, i.e. first the ``years`` component, then the - ## ``months`` component and so on. The returned ``DateTime`` will have the - ## same timezone as the input. - ## - ## Note that when adding months, monthday overflow is allowed. This means that - ## if the resulting month doesn't have enough days it, the month will be - ## incremented and the monthday will be set to the number of days overflowed. - ## So adding one month to `31 October` will result in `31 November`, which - ## will overflow and result in `1 December`. - runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" - # This is correct and happens due to monthday overflow. - doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" - let (adjDur, absDur) = evaluateInterval(dt, interval) - - if adjDur != DurationZero: - var zt = dt.timezone.zonedTimeFromAdjTime(dt.toAdjTime + adjDur) - if absDur != DurationZero: - zt = dt.timezone.zonedTimeFromTime(zt.time + absDur) - result = initDateTime(zt, dt.timezone) - else: - result = initDateTime(zt, dt.timezone) - else: - var zt = dt.timezone.zonedTimeFromTime(dt.toTime + absDur) - result = initDateTime(zt, dt.timezone) - -proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = - ## Subtract ``interval`` from ``dt``. Components from ``interval`` are - ## subtracted in the order of their size, i.e. first the ``years`` component, - ## then the ``months`` component and so on. The returned ``DateTime`` will - ## have the same timezone as the input. - runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) - dt + (-interval) +proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} = + ## Create a new `DateTime <#DateTime>`_ in the specified timezone. + runnableExamples("--warning:deprecated:off"): + assert $initDateTime(30, mMar, 2017, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z" + dateTime(year, month, monthday, hour, minute, second, 0, zone) proc `+`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(hours = 5) doAssert $(dt + dur) == "2017-03-30T05:00:00Z" @@ -1539,206 +1462,97 @@ proc `+`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(days = 5) doAssert $(dt - dur) == "2017-03-25T00:00:00Z" (dt.toTime - dur).inZone(dt.timezone) proc `-`*(dt1, dt2: DateTime): Duration = - ## Compute the duration between ``dt1`` and ``dt2``. + ## Compute the duration between `dt1` and `dt2`. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) + let dt1 = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dt2 = dateTime(2017, mMar, 25, 00, 00, 00, 00, utc()) doAssert dt1 - dt2 == initDuration(days = 5) dt1.toTime - dt2.toTime proc `<`*(a, b: DateTime): bool = - ## Returns true if ``a`` happened before ``b``. + ## Returns true if `a` happened before `b`. return a.toTime < b.toTime proc `<=`*(a, b: DateTime): bool = - ## Returns true if ``a`` happened before or at the same time as ``b``. + ## Returns true if `a` happened before or at the same time as `b`. return a.toTime <= b.toTime proc `==`*(a, b: DateTime): bool = - ## Returns true if ``a`` and ``b`` represent the same point in time. - return a.toTime == b.toTime - -proc isStaticInterval(interval: TimeInterval): bool = - interval.years == 0 and interval.months == 0 and - interval.days == 0 and interval.weeks == 0 - -proc evaluateStaticInterval(interval: TimeInterval): Duration = - assert interval.isStaticInterval - initDuration(nanoseconds = interval.nanoseconds, - microseconds = interval.microseconds, - milliseconds = interval.milliseconds, - seconds = interval.seconds, - minutes = interval.minutes, - hours = interval.hours) + ## Returns true if `a` and `b` represent the same point in time. + if not a.isInitialized: not b.isInitialized + elif not b.isInitialized: false + else: a.toTime == b.toTime -proc between*(startDt, endDt: DateTime): TimeInterval = - ## Gives the difference between ``startDt`` and ``endDt`` as a - ## ``TimeInterval``. The following guarantees about the result is given: - ## - ## - All fields will have the same sign. - ## - If `startDt.timezone == endDt.timezone`, it is guaranteed that - ## `startDt + between(startDt, endDt) == endDt`. - ## - If `startDt.timezone != endDt.timezone`, then the result will be - ## equivalent to `between(startDt.utc, endDt.utc)`. - runnableExamples: - var a = initDateTime(25, mMar, 2015, 12, 0, 0, utc()) - var b = initDateTime(1, mApr, 2017, 15, 0, 15, utc()) - var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) - doAssert between(a, b) == ti - doAssert between(a, b) == -between(b, a) - - if startDt.timezone != endDt.timezone: - return between(startDt.utc, endDt.utc) - elif endDt < startDt: - return -between(endDt, startDt) - - type Date = tuple[year, month, monthday: int] - var startDate: Date = (startDt.year, startDt.month.ord, startDt.monthday) - var endDate: Date = (endDt.year, endDt.month.ord, endDt.monthday) - - # Subtract one day from endDate if time of day is earlier than startDay - # The subtracted day will be counted by fixed units (hour and lower) - # at the end of this proc - if (endDt.hour, endDt.minute, endDt.second, endDt.nanosecond) < - (startDt.hour, startDt.minute, startDt.second, startDt.nanosecond): - if endDate.month == 1 and endDate.monthday == 1: - endDate.year.dec - endDate.monthday = 31 - endDate.month = 12 - elif endDate.monthday == 1: - endDate.month.dec - endDate.monthday = getDaysInMonth(endDate.month.Month, endDate.year) - else: - endDate.monthday.dec - - # Years - result.years.inc endDate.year - startDate.year - 1 - if (startDate.month, startDate.monthday) <= (endDate.month, endDate.monthday): - result.years.inc - startDate.year.inc result.years - - # Months - if startDate.year < endDate.year: - result.months.inc 12 - startDate.month # Move to dec - if endDate.month != 1 or (startDate.monthday <= endDate.monthday): - result.months.inc - startDate.year = endDate.year - startDate.month = 1 - else: - startDate.month = 12 - if startDate.year == endDate.year: - if (startDate.monthday <= endDate.monthday): - result.months.inc endDate.month - startDate.month - startDate.month = endDate.month - elif endDate.month != 1: - let month = endDate.month - 1 - let daysInMonth = getDaysInMonth(month.Month, startDate.year) - if daysInMonth < startDate.monthday: - if startDate.monthday - daysInMonth < endDate.monthday: - result.months.inc endDate.month - startDate.month - 1 - startDate.month = endDate.month - startDate.monthday = startDate.monthday - daysInMonth - else: - result.months.inc endDate.month - startDate.month - 2 - startDate.month = endDate.month - 2 - else: - result.months.inc endDate.month - startDate.month - 1 - startDate.month = endDate.month - 1 - - # Days - # This means that start = dec and end = jan - if startDate.year < endDate.year: - result.days.inc 31 - startDate.monthday + endDate.monthday - startDate = endDate - else: - while startDate.month < endDate.month: - let daysInMonth = getDaysInMonth(startDate.month.Month, startDate.year) - result.days.inc daysInMonth - startDate.monthday + 1 - startDate.month.inc - startDate.monthday = 1 - result.days.inc endDate.monthday - startDate.monthday - result.weeks = result.days div 7 - result.days = result.days mod 7 - startDate = endDate +proc `+=`*(a: var DateTime, b: Duration) = + a = a + b - # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds - let newStartDt = initDateTime(startDate.monthday, startDate.month.Month, - startDate.year, startDt.hour, startDt.minute, startDt.second, - startDt.nanosecond, startDt.timezone) - let dur = endDt - newStartDt - let parts = toParts(dur) - # There can still be a full day in `parts` since `Duration` and `TimeInterval` - # models days differently. - result.hours = parts[Hours].int + parts[Days].int * 24 - result.minutes = parts[Minutes].int - result.seconds = parts[Seconds].int - result.milliseconds = parts[Milliseconds].int - result.microseconds = parts[Microseconds].int - result.nanoseconds = parts[Nanoseconds].int +proc `-=`*(a: var DateTime, b: Duration) = + a = a - b -proc `+`*(time: Time, interval: TimeInterval): Time = - ## Adds `interval` to `time`. - ## If `interval` contains any years, months, weeks or days the operation - ## is performed in the local timezone. +proc getDateStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## Gets the current local date as a string of the format `YYYY-MM-dd`. runnableExamples: - let tm = fromUnix(0) - doAssert tm + 5.seconds == fromUnix(5) - - if interval.isStaticInterval: - time + evaluateStaticInterval(interval) - else: - toTime(time.local + interval) + echo getDateStr(now() - 1.months) + assertDateTimeInitialized dt + result = newStringOfCap(10) # len("YYYY-MM-dd") == 10 + result.addInt dt.year + result.add '-' + result.add intToStr(dt.monthZero, 2) + result.add '-' + result.add intToStr(dt.monthday, 2) -proc `-`*(time: Time, interval: TimeInterval): Time = - ## Subtracts `interval` from Time `time`. - ## If `interval` contains any years, months, weeks or days the operation - ## is performed in the local timezone. +proc getClockStr*(dt = now()): string {.rtl, extern: "nt$1", tags: [TimeEffect].} = + ## Gets the current local clock time as a string of the format `HH:mm:ss`. runnableExamples: - let tm = fromUnix(5) - doAssert tm - 5.seconds == fromUnix(0) + echo getClockStr(now() - 1.hours) + assertDateTimeInitialized dt + result = newStringOfCap(8) # len("HH:mm:ss") == 8 + result.add intToStr(dt.hour, 2) + result.add ':' + result.add intToStr(dt.minute, 2) + result.add ':' + result.add intToStr(dt.second, 2) - if interval.isStaticInterval: - time - evaluateStaticInterval(interval) - else: - toTime(time.local - interval) -proc `+=`*[T, U: TimesMutableTypes](a: var T, b: U) = - ## Modify ``a`` in place by adding ``b``. - runnableExamples: - var tm = fromUnix(0) - tm += initDuration(seconds = 1) - doAssert tm == fromUnix(1) - a = a + b +# +# Iso week forward declarations +# -proc `-=`*[T, U: TimesMutableTypes](a: var T, b: U) = - ## Modify ``a`` in place by subtracting ``b``. - runnableExamples: - var tm = fromUnix(5) - tm -= initDuration(seconds = 5) - doAssert tm == fromUnix(0) - a = a - b +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} -proc `*=`*[T: TimesMutableTypes, U](a: var T, b: U) = - # Mutable type is often multiplied by number - runnableExamples: - var dur = initDuration(seconds = 1) - dur *= 5 - doAssert dur == initDuration(seconds = 5) - a = a * b +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.gcsafe, raises: [], tags: [], since: (1, 5).} # -# Parse & format implementation +# TimeFormat # +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +type + DateTimeLocale* = object + MMM*: array[mJan..mDec, string] + MMMM*: array[mJan..mDec, string] + ddd*: array[dMon..dSun, string] + dddd*: array[dMon..dSun, string] + +when defined(nimHasStyleChecks): + {.pop.} + type AmPm = enum apUnknown, apAm, apPm @@ -1752,6 +1566,9 @@ type year: Option[int] month: Option[int] monthday: Option[int] + isoyear: Option[int] + yearweek: Option[int] + weekday: Option[WeekDay] utcOffset: Option[int] # '0' as default for these work fine @@ -1766,45 +1583,59 @@ type FormatPattern {.pure.} = enum d, dd, ddd, dddd + GG, GGGG h, hh, H, HH m, mm, M, MM, MMM, MMMM s, ss fff, ffffff, fffffffff t, tt - y, yy, yyy, yyyy, yyyyy + yy, yyyy YYYY uuuu UUUU + V, VV z, zz, zzz, zzzz + ZZZ, ZZZZ g # This is a special value used to mark literal format values. - # See the doc comment for ``TimeFormat.patterns``. + # See the doc comment for `TimeFormat.patterns`. Lit TimeFormat* = object ## Represents a format for parsing and printing ## time types. ## - ## To create a new ``TimeFormat`` use `initTimeFormat proc + ## To create a new `TimeFormat` use `initTimeFormat proc ## <#initTimeFormat,string>`_. patterns: seq[byte] ## \ ## Contains the patterns encoded as bytes. ## Literal values are encoded in a special way. - ## They start with ``Lit.byte``, then the length of the literal, then the + ## They start with `Lit.byte`, then the length of the literal, then the ## raw char values of the literal. For example, the literal `foo` would - ## be encoded as ``@[Lit.byte, 3.byte, 'f'.byte, 'o'.byte, 'o'.byte]``. + ## be encoded as `@[Lit.byte, 3.byte, 'f'.byte, 'o'.byte, 'o'.byte]`. formatStr: string TimeParseError* = object of ValueError ## \ - ## Raised when parsing input using a ``TimeFormat`` fails. + ## Raised when parsing input using a `TimeFormat` fails. TimeFormatParseError* = object of ValueError ## \ - ## Raised when parsing a ``TimeFormat`` string fails. + ## Raised when parsing a `TimeFormat` string fails. -const FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ','} +const + DefaultLocale* = DateTimeLocale( + MMM: ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", + "Nov", "Dec"], + MMMM: ["January", "February", "March", "April", "May", "June", "July", + "August", "September", "October", "November", "December"], + ddd: ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], + dddd: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", + "Sunday"], + ) + + FormatLiterals = {' ', '-', '/', ':', '(', ')', '[', ']', ',', '.'} proc `$`*(f: TimeFormat): string = - ## Returns the format string that was used to construct ``f``. + ## Returns the format string that was used to construct `f`. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") doAssert $f == "yyyy-MM-dd" @@ -1891,6 +1722,8 @@ proc stringToPattern(str: string): FormatPattern = of "dd": result = dd of "ddd": result = ddd of "dddd": result = dddd + of "GG": result = GG + of "GGGG": result = GGGG of "h": result = h of "hh": result = hh of "H": result = H @@ -1908,18 +1741,19 @@ proc stringToPattern(str: string): FormatPattern = of "fffffffff": result = fffffffff of "t": result = t of "tt": result = tt - of "y": result = y of "yy": result = yy - of "yyy": result = yyy of "yyyy": result = yyyy - of "yyyyy": result = yyyyy of "YYYY": result = YYYY of "uuuu": result = uuuu of "UUUU": result = UUUU + of "V": result = V + of "VV": result = VV of "z": result = z of "zz": result = zz of "zzz": result = zzz of "zzzz": result = zzzz + of "ZZZ": result = ZZZ + of "ZZZZ": result = ZZZZ of "g": result = g else: raise newException(TimeFormatParseError, "'" & str & "' is not a valid pattern") @@ -1928,7 +1762,7 @@ proc initTimeFormat*(format: string): TimeFormat = ## Construct a new time format for parsing & formatting time types. ## ## See `Parsing and formatting dates`_ for documentation of the - ## ``format`` argument. + ## `format` argument. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") doAssert "2000-01-01" == "2000-01-01".parse(f).format(f) @@ -1963,6 +1797,10 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add loc.ddd[dt.weekday] of dddd: result.add loc.dddd[dt.weekday] + of GG: + result.add (dt.getIsoWeekAndYear.isoyear.int mod 100).intToStr(2) + of GGGG: + result.add $dt.getIsoWeekAndYear.isoyear of h: result.add( if dt.hour == 0: "12" @@ -2005,20 +1843,14 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add if dt.hour >= 12: "P" else: "A" of tt: result.add if dt.hour >= 12: "PM" else: "AM" - of y: # Deprecated - result.add $(dt.yearOfEra mod 10) of yy: result.add (dt.yearOfEra mod 100).intToStr(2) - of yyy: # Deprecated - result.add (dt.yearOfEra mod 1000).intToStr(3) of yyyy: let year = dt.yearOfEra if year < 10000: result.add year.intToStr(4) else: result.add '+' & $year - of yyyyy: # Deprecated - result.add (dt.yearOfEra mod 100_000).intToStr(5) of YYYY: if dt.year < 1: result.add $(abs(dt.year) + 1) @@ -2032,7 +1864,11 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add '+' & $year of UUUU: result.add $dt.year - of z, zz, zzz, zzzz: + of V: + result.add $dt.getIsoWeekAndYear.isoweek + of VV: + result.add dt.getIsoWeekAndYear.isoweek.intToStr(2) + of z, zz, zzz, zzzz, ZZZ, ZZZZ: if dt.timezone != nil and dt.timezone.name == "Etc/UTC": result.add 'Z' else: @@ -2043,16 +1879,18 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, result.add $(absOffset div 3600) of zz: result.add (absOffset div 3600).intToStr(2) - of zzz: + of zzz, ZZZ: let h = (absOffset div 3600).intToStr(2) let m = ((absOffset div 60) mod 60).intToStr(2) - result.add h & ":" & m - of zzzz: + let sep = if pattern == zzz: ":" else: "" + result.add h & sep & m + of zzzz, ZZZZ: let absOffset = abs(dt.utcOffset) let h = (absOffset div 3600).intToStr(2) let m = ((absOffset div 60) mod 60).intToStr(2) let s = (absOffset mod 60).intToStr(2) - result.add h & ":" & m & ":" & s + let sep = if pattern == zzzz: ":" else: "" + result.add h & sep & m & sep & s else: assert false of g: result.add if dt.year < 1: "BC" else: "AD" @@ -2061,7 +1899,7 @@ proc formatPattern(dt: DateTime, pattern: FormatPattern, result: var string, proc parsePattern(input: string, pattern: FormatPattern, i: var int, parsed: var ParsedTime, loc: DateTimeLocale): bool = template takeInt(allowedWidth: Slice[int], allowSign = false): int = - var sv: int + var sv = 0 var pd = parseInt(input, sv, i, allowedWidth.b, allowSign) if pd < allowedWidth.a: return false @@ -2084,18 +1922,30 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, result = monthday in MonthdayRange of ddd: result = false - for v in loc.ddd: + for d, v in loc.ddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break of dddd: result = false - for v in loc.dddd: + for d, v in loc.dddd: if input.substr(i, i+v.len-1).cmpIgnoreCase(v) == 0: + parsed.weekday = some(d.WeekDay) result = true i.inc v.len break + of GG: + # Assumes current century + var isoyear = takeInt(2..2) + var thisCen = now().year div 100 + parsed.isoyear = some(thisCen*100 + isoyear) + result = isoyear > 0 + of GGGG: + let isoyear = takeInt(1..high(int)) + parsed.isoyear = some(isoyear) + result = isoyear > 0 of h, H: parsed.hour = takeInt(1..2) result = parsed.hour in HourRange @@ -2186,7 +2036,15 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, parsed.year = some(year) of UUUU: parsed.year = some(takeInt(1..high(int), allowSign = true)) - of z, zz, zzz, zzzz: + of V: + let yearweek = takeInt(1..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of VV: + let yearweek = takeInt(2..2) + parsed.yearweek = some(yearweek) + result = yearweek in IsoWeekRange + of z, zz, zzz, zzzz, ZZZ, ZZZZ: case input[i] of '+', '-': let sign = if input[i] == '-': 1 else: -1 @@ -2197,21 +2055,24 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, offset = takeInt(1..2) * 3600 of zz: offset = takeInt(2..2) * 3600 - of zzz: + of zzz, ZZZ: offset.inc takeInt(2..2) * 3600 - if input[i] != ':': - return false - i.inc + if pattern == zzz: + if input[i] != ':': + return false + i.inc offset.inc takeInt(2..2) * 60 - of zzzz: + of zzzz, ZZZZ: offset.inc takeInt(2..2) * 3600 - if input[i] != ':': - return false - i.inc + if pattern == zzzz: + if input[i] != ':': + return false + i.inc offset.inc takeInt(2..2) * 60 - if input[i] != ':': - return false - i.inc + if pattern == zzzz: + if input[i] != ':': + return false + i.inc offset.inc takeInt(2..2) else: assert false parsed.utcOffset = some(offset * sign) @@ -2229,25 +2090,13 @@ proc parsePattern(input: string, pattern: FormatPattern, i: var int, i.inc 2 else: result = false - of y, yyy, yyyyy: - raiseAssert "Pattern is invalid for parsing: " & $pattern - of Lit: doAssert false, "Can't happen" + of Lit: raiseAssert "Can't happen" proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, input: string): DateTime = - var month = mJan - var year: int - var monthday: int - # `now()` is an expensive call, so we avoid it when possible - (year, month, monthday) = - if p.year.isNone or p.month.isNone or p.monthday.isNone: - let n = now() - (p.year.get(n.year), - p.month.get(n.month.int).Month, - p.monthday.get(n.monthday)) - else: - (p.year.get(), p.month.get().Month, p.monthday.get()) - + var year = p.year.get(0) + var month = p.month.get(1).Month + var monthday = p.monthday.get(1) year = case p.era of eraUnknown: @@ -2288,26 +2137,55 @@ proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, $year & "-" & ord(month).intToStr(2) & "-" & $monthday & " is not a valid date") - result = DateTime( - year: year, month: month, monthday: monthday, - hour: hour, minute: minute, second: second, nanosecond: nanosecond - ) - if p.utcOffset.isNone: # No timezone parsed - assume timezone is `zone` - result = initDateTime(zone.zonedTimeFromAdjTime(result.toAdjTime), zone) + result = dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) else: # Otherwise convert to `zone` - result.utcOffset = p.utcOffset.get() - result = result.toTime.inZone(zone) + result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) + +proc toDateTimeByWeek(p: ParsedTime, zone: Timezone, f: TimeFormat, + input: string): DateTime = + var isoyear = p.isoyear.get(0) + var yearweek = p.yearweek.get(1) + var weekday = p.weekday.get(dMon) + + if p.amPm != apUnknown: + raiseParseException(f, input, "Parsing iso weekyear dates does not support am/pm") + + if p.year.isSome: + raiseParseException(f, input, "Use iso-year GG or GGGG as year with iso week number") + + if p.month.isSome: + raiseParseException(f, input, "Use either iso week number V or VV or month") + + if p.monthday.isSome: + raiseParseException(f, input, "Use weekday ddd or dddd as day with with iso week number") + + if p.isoyear.isNone: + raiseParseException(f, input, "Need iso-year with week number") + + let hour = p.hour + let minute = p.minute + let second = p.second + let nanosecond = p.nanosecond + + if p.utcOffset.isNone: + result = initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone) + else: + result = (initDateTime(weekday, yearweek.IsoWeekRange, isoyear.IsoYear, hour, minute, second, nanosecond, zone).toTime + + initDuration(seconds = p.utcOffset.get())).inZone(zone) proc format*(dt: DateTime, f: TimeFormat, loc: DateTimeLocale = DefaultLocale): string {.raises: [].} = - ## Format ``dt`` using the format specified by ``f``. + ## Format `dt` using the format specified by `f`. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == dt.format(f) + assertDateTimeInitialized dt + result = "" var idx = 0 while idx <= f.patterns.high: case f.patterns[idx].FormatPattern @@ -2324,61 +2202,56 @@ proc format*(dt: DateTime, f: TimeFormat, proc format*(dt: DateTime, f: string, loc: DateTimeLocale = DefaultLocale): string {.raises: [TimeFormatParseError].} = - ## Shorthand for constructing a ``TimeFormat`` and using it to format ``dt``. + ## Shorthand for constructing a `TimeFormat` and using it to format `dt`. ## ## See `Parsing and formatting dates`_ for documentation of the - ## ``format`` argument. + ## `format` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") let dtFormat = initTimeFormat(f) result = dt.format(dtFormat, loc) proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = - ## Overload that validates ``format`` at compile time. + ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = dt.format(f2) -proc formatValue*(result: var string; value: DateTime, specifier: string) = +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = ## adapter for strformat. Not intended to be called directly. result.add format(value, if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) proc format*(time: Time, f: string, zone: Timezone = local()): string {.raises: [TimeFormatParseError].} = - ## Shorthand for constructing a ``TimeFormat`` and using it to format - ## ``time``. Will use the timezone specified by ``zone``. + ## Shorthand for constructing a `TimeFormat` and using it to format + ## `time`. Will use the timezone specified by `zone`. ## ## See `Parsing and formatting dates`_ for documentation of the - ## ``f`` argument. + ## `f` argument. runnableExamples: - var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + var dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, utc()) var tm = dt.toTime() doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" time.inZone(zone).format(f) proc format*(time: Time, f: static[string], zone: Timezone = local()): string {.raises: [].} = - ## Overload that validates ``f`` at compile time. + ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = time.inZone(zone).format(f2) -template formatValue*(result: var string; value: Time, specifier: string) = - ## adapter for ``strformat``. Not intended to be called directly. - result.add format(value, specifier) - proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, Defect].} = - ## Parses ``input`` as a ``DateTime`` using the format specified by ``f``. - ## If no UTC offset was parsed, then ``input`` is assumed to be specified in - ## the ``zone`` timezone. If a UTC offset was parsed, the result will be - ## converted to the ``zone`` timezone. + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = + ## Parses `input` as a `DateTime` using the format specified by `f`. + ## If no UTC offset was parsed, then `input` is assumed to be specified in + ## the `zone` timezone. If a UTC offset was parsed, the result will be + ## converted to the `zone` timezone. ## - ## Month and day names from the passed in ``loc`` are used. + ## Month and day names from the passed in `loc` are used. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == "2000-01-01".parse(f, utc()) var inpIdx = 0 # Input index var patIdx = 0 # Pattern index @@ -2409,136 +2282,551 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), raiseParseException(f, input, "Parsing ended but there was still patterns remaining") - result = toDateTime(parsed, zone, f, input) + if parsed.yearweek.isSome: + result = toDateTimeByWeek(parsed, zone, f, input) + elif parsed.isoyear.isSome: + raiseParseException(f, input, "Iso year GG or GGGG require iso week V or VV") + else: + result = toDateTime(parsed, zone, f, input) proc parse*(input, f: string, tz: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = - ## Shorthand for constructing a ``TimeFormat`` and using it to parse - ## ``input`` as a ``DateTime``. + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = + ## Shorthand for constructing a `TimeFormat` and using it to parse + ## `input` as a `DateTime`. ## ## See `Parsing and formatting dates`_ for documentation of the - ## ``f`` argument. + ## `f` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) let dtFormat = initTimeFormat(f) result = input.parse(dtFormat, tz, loc = loc) proc parse*(input: string, f: static[string], zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): - DateTime {.raises: [TimeParseError, Defect].} = - ## Overload that validates ``f`` at compile time. + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = + ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone, loc = loc) -proc parseTime*(input, f: string, zone: Timezone): Time - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = - ## Shorthand for constructing a ``TimeFormat`` and using it to parse - ## ``input`` as a ``DateTime``, then converting it a ``Time``. +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = + ## Shorthand for constructing a `TimeFormat` and using it to parse + ## `input` as a `DateTime`, then converting it a `Time`. ## ## See `Parsing and formatting dates`_ for documentation of the - ## ``format`` argument. + ## `format` argument. runnableExamples: let tStr = "1970-01-01T00:00:00+00:00" doAssert parseTime(tStr, "yyyy-MM-dd'T'HH:mm:sszzz", utc()) == fromUnix(0) parse(input, f, zone).toTime() proc parseTime*(input: string, f: static[string], zone: Timezone): Time - {.raises: [TimeParseError, Defect].} = - ## Overload that validates ``format`` at compile time. + {.parseRaises.} = + ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone).toTime() -# -# End of parse & format implementation -# - proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = ## Converts a `DateTime` object to a string representation. - ## It uses the format ``yyyy-MM-dd'T'HH:mm:sszzz``. + ## It uses the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $dt == "2000-01-01T12:00:00Z" - result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") + doAssert $default(DateTime) == "Uninitialized DateTime" + if not dt.isInitialized: + result = "Uninitialized DateTime" + else: + result = format(dt, "yyyy-MM-dd'T'HH:mm:sszzz") proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## Converts a `Time` value to a string representation. It will use the local - ## time zone and use the format ``yyyy-MM-dd'T'HH:mm:sszzz``. + ## time zone and use the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, local()) let tm = dt.toTime() doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local -proc countLeapYears*(yearSpan: int): int - {.deprecated.} = - ## Returns the number of leap years spanned by a given number of years. +# +# TimeInterval +# + +proc initTimeInterval*(nanoseconds, microseconds, milliseconds, + seconds, minutes, hours, + days, weeks, months, years: int = 0): TimeInterval = + ## Creates a new `TimeInterval <#TimeInterval>`_. + ## + ## This proc doesn't perform any normalization! For example, + ## `initTimeInterval(hours = 24)` and `initTimeInterval(days = 1)` are + ## not equal. + ## + ## You can also use the convenience procedures called `milliseconds`, + ## `seconds`, `minutes`, `hours`, `days`, `months`, and `years`. + runnableExamples: + let day = initTimeInterval(hours = 24) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) + doAssert $(dt + day) == "2000-01-02T12:00:00Z" + doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) + result.nanoseconds = nanoseconds + result.microseconds = microseconds + result.milliseconds = milliseconds + result.seconds = seconds + result.minutes = minutes + result.hours = hours + result.days = days + result.weeks = weeks + result.months = months + result.years = years + +proc `+`*(ti1, ti2: TimeInterval): TimeInterval = + ## Adds two `TimeInterval` objects together. + result.nanoseconds = ti1.nanoseconds + ti2.nanoseconds + result.microseconds = ti1.microseconds + ti2.microseconds + result.milliseconds = ti1.milliseconds + ti2.milliseconds + result.seconds = ti1.seconds + ti2.seconds + result.minutes = ti1.minutes + ti2.minutes + result.hours = ti1.hours + ti2.hours + result.days = ti1.days + ti2.days + result.weeks = ti1.weeks + ti2.weeks + result.months = ti1.months + ti2.months + result.years = ti1.years + ti2.years + +proc `-`*(ti: TimeInterval): TimeInterval = + ## Reverses a time interval + runnableExamples: + let day = -initTimeInterval(hours = 24) + doAssert day.hours == -24 + + result = TimeInterval( + nanoseconds: -ti.nanoseconds, + microseconds: -ti.microseconds, + milliseconds: -ti.milliseconds, + seconds: -ti.seconds, + minutes: -ti.minutes, + hours: -ti.hours, + days: -ti.days, + weeks: -ti.weeks, + months: -ti.months, + years: -ti.years + ) + +proc `-`*(ti1, ti2: TimeInterval): TimeInterval = + ## Subtracts TimeInterval `ti1` from `ti2`. + ## + ## Time components are subtracted one-by-one, see output: + runnableExamples: + let ti1 = initTimeInterval(hours = 24) + let ti2 = initTimeInterval(hours = 4) + doAssert (ti1 - ti2) == initTimeInterval(hours = 20) + + result = ti1 + (-ti2) + +proc `+=`*(a: var TimeInterval, b: TimeInterval) = + a = a + b + +proc `-=`*(a: var TimeInterval, b: TimeInterval) = + a = a - b + +proc isStaticInterval(interval: TimeInterval): bool = + interval.years == 0 and interval.months == 0 and + interval.days == 0 and interval.weeks == 0 + +proc evaluateStaticInterval(interval: TimeInterval): Duration = + assert interval.isStaticInterval + initDuration(nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) + +proc between*(startDt, endDt: DateTime): TimeInterval = + ## Gives the difference between `startDt` and `endDt` as a + ## `TimeInterval`. The following guarantees about the result is given: + ## + ## - All fields will have the same sign. + ## - If `startDt.timezone == endDt.timezone`, it is guaranteed that + ## `startDt + between(startDt, endDt) == endDt`. + ## - If `startDt.timezone != endDt.timezone`, then the result will be + ## equivalent to `between(startDt.utc, endDt.utc)`. + runnableExamples: + var a = dateTime(2015, mMar, 25, 12, 0, 0, 00, utc()) + var b = dateTime(2017, mApr, 1, 15, 0, 15, 00, utc()) + var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) + doAssert between(a, b) == ti + doAssert between(a, b) == -between(b, a) + + if startDt.timezone != endDt.timezone: + return between(startDt.utc, endDt.utc) + elif endDt < startDt: + return -between(endDt, startDt) + + type Date = tuple[year, month, monthday: int] + var startDate: Date = (startDt.year, startDt.month.ord, startDt.monthday) + var endDate: Date = (endDt.year, endDt.month.ord, endDt.monthday) + + # Subtract one day from endDate if time of day is earlier than startDay + # The subtracted day will be counted by fixed units (hour and lower) + # at the end of this proc + if (endDt.hour, endDt.minute, endDt.second, endDt.nanosecond) < + (startDt.hour, startDt.minute, startDt.second, startDt.nanosecond): + if endDate.month == 1 and endDate.monthday == 1: + endDate.year.dec + endDate.monthday = 31 + endDate.month = 12 + elif endDate.monthday == 1: + endDate.month.dec + endDate.monthday = getDaysInMonth(endDate.month.Month, endDate.year) + else: + endDate.monthday.dec + + # Years + result.years = endDate.year - startDate.year - 1 + if (startDate.month, startDate.monthday) <= (endDate.month, endDate.monthday): + result.years.inc + startDate.year.inc result.years + + # Months + if startDate.year < endDate.year: + result.months.inc 12 - startDate.month # Move to dec + if endDate.month != 1 or (startDate.monthday <= endDate.monthday): + result.months.inc + startDate.year = endDate.year + startDate.month = 1 + else: + startDate.month = 12 + if startDate.year == endDate.year: + if (startDate.monthday <= endDate.monthday): + result.months.inc endDate.month - startDate.month + startDate.month = endDate.month + elif endDate.month != 1: + let month = endDate.month - 1 + let daysInMonth = getDaysInMonth(month.Month, startDate.year) + if daysInMonth < startDate.monthday: + if startDate.monthday - daysInMonth < endDate.monthday: + result.months.inc endDate.month - startDate.month - 1 + startDate.month = endDate.month + startDate.monthday = startDate.monthday - daysInMonth + else: + result.months.inc endDate.month - startDate.month - 2 + startDate.month = endDate.month - 2 + else: + result.months.inc endDate.month - startDate.month - 1 + startDate.month = endDate.month - 1 + + # Days + # This means that start = dec and end = jan + if startDate.year < endDate.year: + result.days.inc 31 - startDate.monthday + endDate.monthday + startDate = endDate + else: + while startDate.month < endDate.month: + let daysInMonth = getDaysInMonth(startDate.month.Month, startDate.year) + result.days.inc daysInMonth - startDate.monthday + 1 + startDate.month.inc + startDate.monthday = 1 + result.days.inc endDate.monthday - startDate.monthday + result.weeks = result.days div 7 + result.days = result.days mod 7 + startDate = endDate + + # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds + let newStartDt = dateTime(startDate.year, startDate.month.Month, + startDate.monthday, startDt.hour, startDt.minute, startDt.second, + startDt.nanosecond, startDt.timezone) + let dur = endDt - newStartDt + let parts = toParts(dur) + # There can still be a full day in `parts` since `Duration` and `TimeInterval` + # models days differently. + result.hours = parts[Hours].int + parts[Days].int * 24 + result.minutes = parts[Minutes].int + result.seconds = parts[Seconds].int + result.milliseconds = parts[Milliseconds].int + result.microseconds = parts[Microseconds].int + result.nanoseconds = parts[Nanoseconds].int + +proc toParts*(ti: TimeInterval): TimeIntervalParts = + ## Converts a `TimeInterval` into an array consisting of its time units, + ## starting with nanoseconds and ending with years. + ## + ## This procedure is useful for converting `TimeInterval` values to strings. + ## E.g. then you need to implement custom interval printing + runnableExamples: + var tp = toParts(initTimeInterval(years = 1, nanoseconds = 123)) + doAssert tp[Years] == 1 + doAssert tp[Nanoseconds] == 123 + + var index = 0 + for name, value in fieldPairs(ti): + result[index.TimeUnit()] = value + index += 1 + +proc `$`*(ti: TimeInterval): string = + ## Get string representation of `TimeInterval`. + runnableExamples: + doAssert $initTimeInterval(years = 1, nanoseconds = 123) == + "1 year and 123 nanoseconds" + doAssert $initTimeInterval() == "0 nanoseconds" + + var parts: seq[string] = @[] + var tiParts = toParts(ti) + for unit in countdown(Years, Nanoseconds): + if tiParts[unit] != 0: + parts.add(stringifyUnit(tiParts[unit], unit)) + + result = humanizeParts(parts) + +proc nanoseconds*(nanos: int): TimeInterval {.inline.} = + ## TimeInterval of `nanos` nanoseconds. + initTimeInterval(nanoseconds = nanos) + +proc microseconds*(micros: int): TimeInterval {.inline.} = + ## TimeInterval of `micros` microseconds. + initTimeInterval(microseconds = micros) + +proc milliseconds*(ms: int): TimeInterval {.inline.} = + ## TimeInterval of `ms` milliseconds. + initTimeInterval(milliseconds = ms) + +proc seconds*(s: int): TimeInterval {.inline.} = + ## TimeInterval of `s` seconds. ## - ## **Note:** For leap years, start date is assumed to be 1 AD. - ## counts the number of leap years up to January 1st of a given year. - ## Keep in mind that if specified year is a leap year, the leap day - ## has not happened before January 1st of that year. + ## `echo getTime() + 5.seconds` + initTimeInterval(seconds = s) + +proc minutes*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` minutes. ## - ## **Deprecated since v0.20.0**. - (yearSpan - 1) div 4 - (yearSpan - 1) div 100 + (yearSpan - 1) div 400 + ## `echo getTime() + 5.minutes` + initTimeInterval(minutes = m) -proc countDays*(yearSpan: int): int - {.deprecated.} = - ## Returns the number of days spanned by a given number of years. +proc hours*(h: int): TimeInterval {.inline.} = + ## TimeInterval of `h` hours. ## - ## **Deprecated since v0.20.0**. - (yearSpan - 1) * 365 + countLeapYears(yearSpan) + ## `echo getTime() + 2.hours` + initTimeInterval(hours = h) -proc countYears*(daySpan: int): int - {.deprecated.} = - ## Returns the number of years spanned by a given number of days. +proc days*(d: int): TimeInterval {.inline.} = + ## TimeInterval of `d` days. ## - ## **Deprecated since v0.20.0**. - ((daySpan - countLeapYears(daySpan div 365)) div 365) + ## `echo getTime() + 2.days` + initTimeInterval(days = d) + +proc weeks*(w: int): TimeInterval {.inline.} = + ## TimeInterval of `w` weeks. + ## + ## `echo getTime() + 2.weeks` + initTimeInterval(weeks = w) + +proc months*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` months. + ## + ## `echo getTime() + 2.months` + initTimeInterval(months = m) + +proc years*(y: int): TimeInterval {.inline.} = + ## TimeInterval of `y` years. + ## + ## `echo getTime() + 2.years` + initTimeInterval(years = y) + +proc evaluateInterval(dt: DateTime, interval: TimeInterval): + tuple[adjDur, absDur: Duration] = + ## Evaluates how many nanoseconds the interval is worth + ## in the context of `dt`. + ## The result in split into an adjusted diff and an absolute diff. + var months = interval.years * 12 + interval.months + var curYear = dt.year + var curMonth = dt.month + result = default(tuple[adjDur, absDur: Duration]) + # Subtracting + if months < 0: + for mth in countdown(-1 * months, 1): + if curMonth == mJan: + curMonth = mDec + curYear.dec + else: + curMonth.dec() + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur - initDuration(days = days) + # Adding + else: + for mth in 1 .. months: + let days = getDaysInMonth(curMonth, curYear) + result.adjDur = result.adjDur + initDuration(days = days) + if curMonth == mDec: + curMonth = mJan + curYear.inc + else: + curMonth.inc() + + result.adjDur = result.adjDur + initDuration( + days = interval.days, + weeks = interval.weeks) + result.absDur = initDuration( + nanoseconds = interval.nanoseconds, + microseconds = interval.microseconds, + milliseconds = interval.milliseconds, + seconds = interval.seconds, + minutes = interval.minutes, + hours = interval.hours) -proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] - {.deprecated.} = - ## Returns the number of years spanned by a given number of days and the - ## remainder as days. +proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Adds `interval` to `dt`. Components from `interval` are added + ## in the order of their size, i.e. first the `years` component, then the + ## `months` component and so on. The returned `DateTime` will have the + ## same timezone as the input. ## - ## **Deprecated since v0.20.0**. - let days = daySpan - countLeapYears(daySpan div 365) - result.years = days div 365 - result.days = days mod 365 - -proc toTimeInterval*(time: Time): TimeInterval - {.deprecated: "Use `between` instead".} = - ## Converts a Time to a TimeInterval. To be used when diffing times. + ## Note that when adding months, monthday overflow is allowed. This means that + ## if the resulting month doesn't have enough days it, the month will be + ## incremented and the monthday will be set to the number of days overflowed. + ## So adding one month to `31 October` will result in `31 November`, which + ## will overflow and result in `1 December`. + runnableExamples: + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" + # This is correct and happens due to monthday overflow. + doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" + let (adjDur, absDur) = evaluateInterval(dt, interval) + + if adjDur != DurationZero: + var zt = dt.timezone.zonedTimeFromAdjTime(dt.toAdjTime + adjDur) + if absDur != DurationZero: + zt = dt.timezone.zonedTimeFromTime(zt.time + absDur) + result = initDateTime(zt, dt.timezone) + else: + result = initDateTime(zt, dt.timezone) + else: + var zt = dt.timezone.zonedTimeFromTime(dt.toTime + absDur) + result = initDateTime(zt, dt.timezone) + +proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = + ## Subtract `interval` from `dt`. Components from `interval` are + ## subtracted in the order of their size, i.e. first the `years` component, + ## then the `months` component and so on. The returned `DateTime` will + ## have the same timezone as the input. + runnableExamples: + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" + + dt + (-interval) + +proc `+`*(time: Time, interval: TimeInterval): Time = + ## Adds `interval` to `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(0) + doAssert tm + 5.seconds == fromUnix(5) + + if interval.isStaticInterval: + time + evaluateStaticInterval(interval) + else: + toTime(time.local + interval) + +proc `-`*(time: Time, interval: TimeInterval): Time = + ## Subtracts `interval` from Time `time`. + ## If `interval` contains any years, months, weeks or days the operation + ## is performed in the local timezone. + runnableExamples: + let tm = fromUnix(5) + doAssert tm - 5.seconds == fromUnix(0) + + if interval.isStaticInterval: + time - evaluateStaticInterval(interval) + else: + toTime(time.local - interval) + +proc `+=`*(a: var DateTime, b: TimeInterval) = + a = a + b + +proc `-=`*(a: var DateTime, b: TimeInterval) = + a = a - b + +proc `+=`*(t: var Time, b: TimeInterval) = + t = t + b + +proc `-=`*(t: var Time, b: TimeInterval) = + t = t - b + +# +# Iso week +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year + ## in the specified timezone. ## - ## **Deprecated since version 0.20.0:** Use the `between proc - ## <#between,DateTime,DateTime>`_ instead. + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. runnableExamples: - let a = fromUnix(10) - let b = fromUnix(1_500_000_000) - let ti = b.toTimeInterval() - a.toTimeInterval() - doAssert a + ti == b - var dt = time.local - initTimeInterval(dt.nanosecond, 0, 0, dt.second, dt.minute, dt.hour, - dt.monthday, 0, dt.month.ord - 1, dt.year) + assert initDateTime(21, mApr, 2018, 00, 00, 00) == initDateTime(dSat, 16, 2018.IsoYear, 00, 00, 00) + assert initDateTime(30, mDec, 2019, 00, 00, 00) == initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) + assert initDateTime(13, mSep, 2020, 00, 00, 00) == initDateTime(dSun, 37, 2020.IsoYear, 00, 00, 00) + assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) + + # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00, zone).weekday.int - 4 + initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.raises: [], tags: [], since: (1, 5).} = + initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) + +# +# Other +# + +proc epochTime*(): float {.tags: [TimeEffect].} = + ## Gets time after the UNIX epoch (1970) in seconds. It is a float + ## because sub-second resolution is likely to be supported (depending + ## on the hardware/OS). + ## + ## `getTime` should generally be preferred over this proc. + ## + ## .. warning:: Unsuitable for benchmarking (but still better than `now`), + ## use `monotimes.getMonoTime` or `cpuTime` instead, depending on the use case. + when defined(js): + result = newDate().getTime() / 1000 + elif defined(macosx): + var a {.noinit.}: Timeval + gettimeofday(a) + result = toBiggestFloat(a.tv_sec.int64) + toBiggestFloat( + a.tv_usec)*0.00_0001 + elif defined(posix): + var ts {.noinit.}: Timespec + discard clock_gettime(CLOCK_REALTIME, ts) + result = toBiggestFloat(ts.tv_sec.int64) + + toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 + elif defined(windows): + var f {.noinit.}: winlean.FILETIME + getSystemTimeAsFileTime(f) + var i64 = rdFileTime(f) - epochDiff + var secs = i64 div rateDiff + var subsecs = i64 mod rateDiff + result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 + else: + {.error: "unknown OS".} when not defined(js): type Clock {.importc: "clock_t".} = distinct int proc getClock(): Clock - {.importc: "clock", header: "<time.h>", tags: [TimeEffect], used.} + {.importc: "clock", header: "<time.h>", tags: [TimeEffect], used, sideEffect.} var clocksPerSec {.importc: "CLOCKS_PER_SEC", nodecl, used.}: int proc cpuTime*(): float {.tags: [TimeEffect].} = - ## gets time spent that the CPU spent to run the current process in - ## seconds. This may be more useful for benchmarking than ``epochTime``. + ## Gets time spent that the CPU spent to run the current process in + ## seconds. This may be more useful for benchmarking than `epochTime`. ## However, it may measure the real time instead (depending on the OS). ## The value of the result has no meaning. ## To generate useful timing values, take the difference between - ## the results of two ``cpuTime`` calls: + ## the results of two `cpuTime` calls: runnableExamples: var t0 = cpuTime() # some useless work here (calculate fibonacci) @@ -2547,6 +2835,8 @@ when not defined(js): fib.add(fib[^1] + fib[^2]) echo "CPU time [s] ", cpuTime() - t0 echo "Fib is [s] ", fib + ## When the flag `--benchmarkVM` is passed to the compiler, this proc is + ## also available at compile time when defined(posix) and not defined(osx) and declared(CLOCK_THREAD_CPUTIME_ID): # 'clocksPerSec' is a compile-time constant, possibly a # rather awful one, so use clock_gettime instead @@ -2557,243 +2847,43 @@ when not defined(js): else: result = toFloat(int(getClock())) / toFloat(clocksPerSec) - proc epochTime*(): float {.tags: [TimeEffect].} = - ## gets time after the UNIX epoch (1970) in seconds. It is a float - ## because sub-second resolution is likely to be supported (depending - ## on the hardware/OS). - ## - ## ``getTime`` should generally be preferred over this proc. - when defined(macosx): - var a: Timeval - gettimeofday(a) - result = toBiggestFloat(a.tv_sec.int64) + toBiggestFloat( - a.tv_usec)*0.00_0001 - elif defined(posix): - var ts: Timespec - discard clock_gettime(CLOCK_REALTIME, ts) - result = toBiggestFloat(ts.tv_sec.int64) + - toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 - elif defined(windows): - var f: winlean.FILETIME - getSystemTimeAsFileTime(f) - var i64 = rdFileTime(f) - epochDiff - var secs = i64 div rateDiff - var subsecs = i64 mod rateDiff - result = toFloat(int(secs)) + toFloat(int(subsecs)) * 0.0000001 - else: - {.error: "unknown OS".} - -when defined(js): - proc epochTime*(): float {.tags: [TimeEffect].} = - newDate().getTime() / 1000 -# Deprecated procs +# +# Deprecations +# -proc weeks*(dur: Duration): int64 - {.inline, deprecated: "Use `inWeeks` instead".} = - ## Number of whole weeks represented by the duration. - ## - ## **Deprecated since version v0.20.0**: Use the `inWeeks proc - ## <#inWeeks,Duration>`_ instead. - runnableExamples: - let dur = initDuration(weeks = 1, days = 2, hours = 3, minutes = 4) - doAssert dur.weeks == 1 - dur.inWeeks +proc `nanosecond=`*(dt: var DateTime, value: NanosecondRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.nanosecond = value -proc days*(dur: Duration): int64 - {.inline, deprecated: "Use `inDays` instead".} = - ## Number of whole days represented by the duration. - ## - ## **Deprecated since version v0.20.0**: Use the `inDays proc - ## <#inDays,Duration>`_ instead. - runnableExamples: - let dur = initDuration(weeks = 1, days = 2, hours = 3, minutes = 4) - doAssert dur.days == 9 - dur.inDays +proc `second=`*(dt: var DateTime, value: SecondRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.second = value -proc hours*(dur: Duration): int64 - {.inline, deprecated: "Use `inHours` instead".} = - ## Number of whole hours represented by the duration. - ## - ## **Deprecated since version v0.20.0**: Use the `inHours proc - ## <#inHours,Duration>`_ instead. - runnableExamples: - let dur = initDuration(days = 1, hours = 2, minutes = 3) - doAssert dur.hours == 26 - dur.inHours +proc `minute=`*(dt: var DateTime, value: MinuteRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.minute = value -proc minutes*(dur: Duration): int64 - {.inline, deprecated: "Use `inMinutes` instead".} = - ## Number of whole minutes represented by the duration. - ## - ## **Deprecated since version v0.20.0**: Use the `inMinutes proc - ## <#inMinutes,Duration>`_ instead. - runnableExamples: - let dur = initDuration(days = 1, hours = 2, minutes = 3) - doAssert dur.minutes == 1563 - dur.inMinutes +proc `hour=`*(dt: var DateTime, value: HourRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.hour = value -proc seconds*(dur: Duration): int64 - {.inline, deprecated: "Use `inSeconds` instead".} = - ## Number of whole seconds represented by the duration. - ## - ## **Deprecated since version v0.20.0**: Use the `inSeconds proc - ## <#inSeconds,Duration>`_ instead. - runnableExamples: - let dur = initDuration(minutes = 10, seconds = 30) - doAssert dur.seconds == 630 - dur.inSeconds +proc `monthdayZero=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.monthdayZero = value -proc milliseconds*(dur: Duration): int {.inline, deprecated.} = - ## Number of whole milliseconds represented by the **fractional** - ## part of the duration. - ## - ## **Deprecated since version v0.20.0**. - runnableExamples: - let dur = initDuration(minutes = 5, seconds = 6, milliseconds = 7, - microseconds = 8, nanoseconds = 9) - doAssert dur.milliseconds == 7 - result = convert(Nanoseconds, Milliseconds, dur.nanosecond) +proc `monthZero=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.monthZero = value -proc microseconds*(dur: Duration): int {.inline, deprecated.} = - ## Number of whole microseconds represented by the **fractional** - ## part of the duration. - ## - ## **Deprecated since version v0.20.0**. - runnableExamples: - let dur = initDuration(minutes = 5, seconds = 6, milliseconds = 7, - microseconds = 8, nanoseconds = 9) - doAssert dur.microseconds == 7008 - result = convert(Nanoseconds, Microseconds, dur.nanosecond) +proc `year=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.year = value -proc nanoseconds*(dur: Duration): NanosecondRange {.inline, deprecated.} = - ## Number of whole microseconds represented by the **fractional** - ## part of the duration. - ## - ## **Deprecated since version v0.20.0**. - runnableExamples: - let dur = initDuration(minutes = 5, seconds = 6, milliseconds = 7, - microseconds = 8, nanoseconds = 9) - doAssert dur.nanoseconds == 7008009 - dur.nanosecond +proc `weekday=`*(dt: var DateTime, value: WeekDay) {.deprecated: "Deprecated since v1.3.1".} = + dt.weekday = value -proc fractional*(dur: Duration): Duration {.inline, deprecated.} = - ## The fractional part of `dur`, as a duration. - ## - ## **Deprecated since version v0.20.0**. - runnableExamples: - let dur = initDuration(minutes = 5, seconds = 6, milliseconds = 7, - microseconds = 8, nanoseconds = 9) - doAssert dur.fractional == initDuration(milliseconds = 7, microseconds = 8, - nanoseconds = 9) - initDuration(nanoseconds = dur.nanosecond) +proc `yearday=`*(dt: var DateTime, value: YeardayRange) {.deprecated: "Deprecated since v1.3.1".} = + dt.yearday = value -when not defined(js): - proc unixTimeToWinTime*(time: CTime): int64 - {.deprecated: "Use toWinTime instead".} = - ## Converts a UNIX `Time` (``time_t``) to a Windows file time - ## - ## **Deprecated:** use ``toWinTime`` instead. - result = int64(time) * rateDiff + epochDiff +proc `isDst=`*(dt: var DateTime, value: bool) {.deprecated: "Deprecated since v1.3.1".} = + dt.isDst = value - proc winTimeToUnixTime*(time: int64): CTime - {.deprecated: "Use fromWinTime instead".} = - ## Converts a Windows time to a UNIX `Time` (``time_t``) - ## - ## **Deprecated:** use ``fromWinTime`` instead. - result = CTime((time - epochDiff) div rateDiff) - -proc initInterval*(seconds, minutes, hours, days, months, years: int = 0): - TimeInterval {.deprecated.} = - ## **Deprecated since v0.18.0:** use ``initTimeInterval`` instead. - initTimeInterval(0, 0, 0, seconds, minutes, hours, days, 0, months, years) - -proc fromSeconds*(since1970: float): Time - {.tags: [], raises: [], benign, deprecated: "Use fromUnixFloat or fromUnix".} = - ## Takes a float which contains the number of seconds since the unix epoch and - ## returns a time object. - ## - ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - fromUnixFloat(since1970) +proc `timezone=`*(dt: var DateTime, value: Timezone) {.deprecated: "Deprecated since v1.3.1".} = + dt.timezone = value -proc fromSeconds*(since1970: int64): Time - {.tags: [], raises: [], benign, deprecated.} = - ## Takes an int which contains the number of seconds since the unix epoch and - ## returns a time object. - ## - ## **Deprecated since v0.18.0:** use ``fromUnix`` instead - fromUnix(since1970) - -proc toSeconds*(time: Time): float - {.tags: [], raises: [], benign, deprecated: "Use toUnixFloat or toUnix".} = - ## Returns the time in seconds since the unix epoch, with subsecond resolution. - toUnixFloat(time) - -proc getLocalTime*(time: Time): DateTime - {.tags: [], raises: [], benign, deprecated.} = - ## Converts the calendar time `time` to broken-time representation, - ## expressed relative to the user's specified time zone. - ## - ## **Deprecated since v0.18.0:** use ``local`` instead - time.local - -proc getGMTime*(time: Time): DateTime - {.tags: [], raises: [], benign, deprecated.} = - ## Converts the calendar time `time` to broken-down time representation, - ## expressed in Coordinated Universal Time (UTC). - ## - ## **Deprecated since v0.18.0:** use ``utc`` instead - time.utc - -proc getTimezone*(): int - {.tags: [TimeEffect], raises: [], benign, deprecated.} = - ## Returns the offset of the local (non-DST) timezone in seconds west of UTC. - ## - ## **Deprecated since v0.18.0:** use ``now().utcOffset`` to get the current - ## utc offset (including DST). - when defined(js): - return newDate().getTimezoneOffset() * 60 - elif defined(freebsd) or defined(netbsd) or defined(openbsd): - # This is wrong since it will include DST offsets, but the behavior has - # always been wrong for bsd and the proc is deprecated so lets ignore it. - return now().utcOffset - else: - return timezone - -proc getDayOfWeek*(day, month, year: int): WeekDay - {.tags: [], raises: [], benign, deprecated.} = - ## **Deprecated since v0.18.0:** use - ## ``getDayOfWeek(monthday: MonthdayRange; month: Month; year: int)`` instead. - getDayOfWeek(day, month.Month, year) - -proc getDayOfWeekJulian*(day, month, year: int): WeekDay {.deprecated.} = - ## Returns the day of the week enum from day, month and year, - ## according to the Julian calendar. - ## **Deprecated since v0.18.0** - # Day & month start from one. - let - a = (14 - month) div 12 - y = year - a - m = month + (12*a) - 2 - d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 - result = d.WeekDay - -proc adjTime*(zt: ZonedTime): Time - {.deprecated: "Use zt.time instead".} = - ## **Deprecated since v0.19.0:** use the ``time`` field instead. - zt.time - initDuration(seconds = zt.utcOffset) - -proc `adjTime=`*(zt: var ZonedTime, adjTime: Time) - {.deprecated: "Use zt.time instead".} = - ## **Deprecated since v0.19.0:** use the ``time`` field instead. - zt.time = adjTime + initDuration(seconds = zt.utcOffset) - -proc zoneInfoFromUtc*(zone: Timezone, time: Time): ZonedTime - {.deprecated: "Use zonedTimeFromTime instead".} = - ## **Deprecated since v0.19.0:** use ``zonedTimeFromTime`` instead. - zone.zonedTimeFromTime(time) - -proc zoneInfoFromTz*(zone: Timezone, adjTime: Time): ZonedTime - {.deprecated: "Use zonedTimeFromAdjTime instead".} = - ## **Deprecated since v0.19.0:** use the ``zonedTimeFromAdjTime`` instead. - zone.zonedTimeFromAdjTime(adjTime) +proc `utcOffset=`*(dt: var DateTime, value: int) {.deprecated: "Deprecated since v1.3.1".} = + dt.utcOffset = value diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index fd729b59e..78af84fdd 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -12,122 +12,365 @@ ## ## Unstable API. +import std/private/since export system.`$` # for backward compatibility -include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/assertions -proc name*(t: typedesc): string {.magic: "TypeTrait".} - ## Returns the name of the given type. + +type HoleyEnum* = (not Ordinal) and enum ## Enum with holes. +type OrdinalEnum* = Ordinal and enum ## Enum without holes. + +runnableExamples: + type A = enum a0 = 2, a1 = 4, a2 + type B = enum b0 = 2, b1, b2 + assert A is enum + assert A is HoleyEnum + assert A isnot OrdinalEnum + assert B isnot HoleyEnum + assert B is OrdinalEnum + assert int isnot HoleyEnum + type C[T] = enum h0 = 2, h1 = 4 + assert C[float] is HoleyEnum + +proc name*(t: typedesc): string {.magic: "TypeTrait".} = + ## Returns the name of `t`. ## - ## Alias for system.`$`(t) since Nim v0.20. + ## Alias for `system.\`$\`(t) <dollars.html#$,typedesc>`_ since Nim v0.20. + runnableExamples: + doAssert name(int) == "int" + doAssert name(seq[string]) == "seq[string]" proc arity*(t: typedesc): int {.magic: "TypeTrait".} = - ## Returns the arity of the given type. This is the number of "type" - ## components or the number of generic parameters a given type ``t`` has. + ## Returns the arity of `t`. This is the number of "type" + ## components or the number of generic parameters a given type `t` has. runnableExamples: - assert arity(seq[string]) == 1 - assert arity(array[3, int]) == 2 - assert arity((int, int, float, string)) == 4 + doAssert arity(int) == 0 + doAssert arity(seq[string]) == 1 + doAssert arity(array[3, int]) == 2 + doAssert arity((int, int, float, string)) == 4 -proc genericHead*(t: typedesc): typedesc {.magic: "TypeTrait".} +proc genericHead*(t: typedesc): typedesc {.magic: "TypeTrait".} = ## Accepts an instantiated generic type and returns its ## uninstantiated form. - ## - ## For example: - ## * `seq[int].genericHead` will be just `seq` - ## * `seq[int].genericHead[float]` will be `seq[float]` - ## ## A compile-time error will be produced if the supplied type ## is not generic. ## - ## See also: - ## * `stripGenericParams <#stripGenericParams,typedesc>`_ - ## - ## Example: - ## - ## .. code-block:: nim - ## type - ## Functor[A] = concept f - ## type MatchedGenericType = genericHead(f.type) - ## # `f` will be a value of a type such as `Option[T]` - ## # `MatchedGenericType` will become the `Option` type + ## **See also:** + ## * `stripGenericParams proc <#stripGenericParams,typedesc>`_ + runnableExamples: + type + Foo[T] = object + FooInst = Foo[int] + Foo2 = genericHead(FooInst) + + doAssert Foo2 is Foo and Foo is Foo2 + doAssert genericHead(Foo[seq[string]]) is Foo + doAssert not compiles(genericHead(int)) + + type Generic = concept f + type _ = genericHead(typeof(f)) + proc bar(a: Generic): typeof(a) = a -proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} + doAssert bar(Foo[string].default) == Foo[string]() + doAssert not compiles bar(string.default) + + when false: # these don't work yet + doAssert genericHead(Foo[int])[float] is Foo[float] + doAssert seq[int].genericHead is seq + +proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} = ## This trait is similar to `genericHead <#genericHead,typedesc>`_, but - ## instead of producing error for non-generic types, it will just return + ## instead of producing an error for non-generic types, it will just return ## them unmodified. + runnableExamples: + type Foo[T] = object + + doAssert stripGenericParams(Foo[string]) is Foo + doAssert stripGenericParams(int) is int proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} - ## This trait returns true iff the type ``t`` is safe to use for - ## `copyMem`:idx:. + ## Returns true if `t` is safe to use for `copyMem`:idx:. ## ## Other languages name a type like these `blob`:idx:. -proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} - ## Return true for named tuples, false for any other type. +proc hasDefaultValue*(t: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true if `t` has a valid default value. + runnableExamples: + {.experimental: "strictNotNil".} + type + NilableObject = ref object + a: int + Object = NilableObject not nil + RequiresInit[T] = object + a {.requiresInit.}: T -proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} - ## Returns base type for distinct types, works only for distinct types. - ## compile time error otherwise + assert hasDefaultValue(NilableObject) + assert not hasDefaultValue(Object) + assert hasDefaultValue(string) + assert not hasDefaultValue(var string) + assert not hasDefaultValue(RequiresInit[int]) -import std/macros +proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = + ## Returns true for named tuples, false for any other type. + runnableExamples: + doAssert not isNamedTuple(int) + doAssert not isNamedTuple((string, int)) + doAssert isNamedTuple(tuple[name: string, age: int]) -macro lenTuple*(t: tuple): int {.since: (1, 1).} = - ## Return number of elements of `t` - newLit t.len +template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = + ## Returns `T` for `ref T | ptr T`. + runnableExamples: + assert (ref int).pointerBase is int + type A = ptr seq[float] + assert A.pointerBase is seq[float] + assert (ref A).pointerBase is A # not seq[float] + assert (var s = "abc"; s[0].addr).typeof.pointerBase is char + T -macro lenTuple*(t: typedesc[tuple]): int {.since: (1, 1).} = - ## Return number of elements of `T` - newLit t.len +proc rangeBase*(T: typedesc[range]): typedesc {.magic: "TypeTrait".} = + ## Returns the base type for range types, or the type itself otherwise. + ## + ## **See also:** + ## * `rangeBase template <#rangeBase.t,T>`_ + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + doAssert rangeBase(MyRange) is int + doAssert rangeBase(MyEnumRange) is MyEnum + doAssert rangeBase(range['a'..'z']) is char + +template rangeBase*[T: range](a: T): untyped = + ## Overload of `rangeBase <#rangeBase,typedesc,static[bool]>`_ for values. + runnableExamples: + type MyRange = range[0..5] + type MyEnum = enum a, b, c + type MyEnumRange = range[b..c] + let x = MyRange(3) + doAssert rangeBase(x) is int + doAssert $typeof(rangeBase(x)) == "int" + doAssert rangeBase(x) == 3 + let y: set[MyEnumRange] = {c} + for e in y: + doAssert rangeBase(e) is MyEnum + doAssert $typeof(rangeBase(e)) == "MyEnum" + doAssert rangeBase(e) == c + let z: seq[range['a'..'z']] = @['c'] + doAssert rangeBase(z[0]) is char + doAssert $typeof(rangeBase(z[0])) == "char" + doAssert rangeBase(z[0]) == 'c' + rangeBase(typeof(T))(a) + +proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = + ## Returns the base type for distinct types, or the type itself otherwise. + ## If `recursive` is false, only the immediate distinct base will be returned. + ## + ## **See also:** + ## * `distinctBase template <#distinctBase.t,T,static[bool]>`_ + runnableExamples: + type MyInt = distinct int + type MyOtherInt = distinct MyInt + doAssert distinctBase(MyInt) is int + doAssert distinctBase(MyOtherInt) is int + doAssert distinctBase(MyOtherInt, false) is MyInt + doAssert distinctBase(int) is int since (1, 1): + template distinctBase*[T](a: T, recursive: static bool = true): untyped = + ## Overload of `distinctBase <#distinctBase,typedesc,static[bool]>`_ for values. + runnableExamples: + type MyInt = distinct int + type MyOtherInt = distinct MyInt + doAssert 12.MyInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase(false) is MyInt + doAssert 12.distinctBase == 12 + when T is distinct: + distinctBase(typeof(a), recursive)(a) + else: # avoids hint ConvFromXtoItselfNotNeeded + a + + proc tupleLen*(T: typedesc[tuple]): int {.magic: "TypeTrait".} = + ## Returns the number of elements of the tuple type `T`. + ## + ## **See also:** + ## * `tupleLen template <#tupleLen.t>`_ + runnableExamples: + doAssert tupleLen((int, int, float, string)) == 4 + doAssert tupleLen(tuple[name: string, age: int]) == 2 + + template tupleLen*(t: tuple): int = + ## Returns the number of elements of the tuple `t`. + ## + ## **See also:** + ## * `tupleLen proc <#tupleLen,typedesc>`_ + runnableExamples: + doAssert tupleLen((1, 2)) == 2 + + tupleLen(typeof(t)) + template get*(T: typedesc[tuple], i: static int): untyped = - ## Return `i`th element of `T` + ## Returns the `i`-th element of `T`. # Note: `[]` currently gives: `Error: no generic parameters allowed for ...` - type(default(T)[i]) + runnableExamples: + doAssert get((int, int, float, string), 2) is float + + typeof(default(T)[i]) + + type StaticParam*[value: static type] = object + ## Used to wrap a static value in `genericParams <#genericParams.t,typedesc>`_. + +since (1, 3, 5): + template elementType*(a: untyped): typedesc = + ## Returns the element type of `a`, which can be any iterable (over which you + ## can iterate). + runnableExamples: + iterator myiter(n: int): auto = + for i in 0 ..< n: + yield i + + doAssert elementType(@[1,2]) is int + doAssert elementType("asdf") is char + doAssert elementType(myiter(3)) is int -macro genericParams*(T: typedesc): untyped {.since: (1, 1).} = - ## return tuple of generic params for generic `T` + typeof(block: (for ai in a: ai)) + +import std/macros + +macro enumLen*(T: typedesc[enum]): int = + ## Returns the number of items in the enum `T`. runnableExamples: - type Foo[T1, T2]=object - doAssert genericParams(Foo[float, string]) is (float, string) + type Foo = enum + fooItem1 + fooItem2 + + doAssert Foo.enumLen == 2 + + let bracketExpr = getType(T) + expectKind(bracketExpr, nnkBracketExpr) + let enumTy = bracketExpr[1] + expectKind(enumTy, nnkEnumTy) + result = newLit(enumTy.len - 1) + +macro genericParamsImpl(T: typedesc): untyped = + # auxiliary macro needed, can't do it directly in `genericParams` result = newNimNode(nnkTupleConstr) var impl = getTypeImpl(T) expectKind(impl, nnkBracketExpr) impl = impl[1] while true: case impl.kind - of nnkSym: - impl = impl.getImpl - continue - of nnkTypeDef: - impl = impl[2] - continue - of nnkBracketExpr: - for i in 1..<impl.len: - result.add impl[i] - break - else: - error "wrong kind: " & $impl.kind - -when isMainModule: - static: - doAssert $type(42) == "int" - doAssert int.name == "int" - - const a1 = name(int) - const a2 = $(int) - const a3 = $int - doAssert a1 == "int" - doAssert a2 == "int" - doAssert a3 == "int" - - proc fun[T: typedesc](t: T) = - const a1 = name(t) - const a2 = $(t) - const a3 = $t - doAssert a1 == "int" - doAssert a2 == "int" - doAssert a3 == "int" - fun(int) + of nnkSym: + impl = impl.getImpl + of nnkTypeDef: + impl = impl[2] + of nnkTypeOfExpr: + impl = getTypeInst(impl[0]) + of nnkBracketExpr: + for i in 1..<impl.len: + let ai = impl[i] + var ret: NimNode = nil + case ai.typeKind + of ntyTypeDesc: + ret = ai + of ntyStatic: raiseAssert "unreachable" + else: + # getType from a resolved symbol might return a typedesc symbol. + # If so, use it directly instead of wrapping it in StaticParam. + if (ai.kind == nnkSym and ai.symKind == nskType) or + (ai.kind == nnkBracketExpr and ai[0].kind == nnkSym and + ai[0].symKind == nskType) or ai.kind in {nnkRefTy, nnkVarTy, nnkPtrTy, nnkProcTy}: + ret = ai + elif ai.kind == nnkInfix and ai[0].kind == nnkIdent and + ai[0].strVal == "..": + # For built-in array types, the "2" is translated to "0..1" then + # automagically translated to "range[0..1]". However this is not + # reflected in the AST, thus requiring manual transformation here. + # + # We will also be losing some context here: + # var a: array[10, int] + # will be translated to: + # var a: array[0..9, int] + # after typecheck. This means that we can't get the exact + # definition as typed by the user, which will cause confusion for + # users expecting: + # genericParams(typeof(a)) is (StaticParam(10), int) + # to be true while in fact the result will be: + # genericParams(typeof(a)) is (range[0..9], int) + ret = newTree(nnkBracketExpr, @[bindSym"range", ai]) + else: + since (1, 1): + ret = newTree(nnkBracketExpr, @[bindSym"StaticParam", ai]) + result.add ret + break + else: + error "wrong kind: " & $impl.kind, impl + +since (1, 1): + template genericParams*(T: typedesc): untyped = + ## Returns the tuple of generic parameters for the generic type `T`. + ## + ## **Note:** For the builtin array type, the index generic parameter will + ## **always** become a range type after it's bound to a variable. + runnableExamples: + type Foo[T1, T2] = object + + doAssert genericParams(Foo[float, string]) is (float, string) + + type Bar[N: static float, T] = object + + doAssert genericParams(Bar[1.0, string]) is (StaticParam[1.0], string) + doAssert genericParams(Bar[1.0, string]).get(0).value == 1.0 + doAssert genericParams(seq[Bar[2.0, string]]).get(0) is Bar[2.0, string] + var s: seq[Bar[3.0, string]] + doAssert genericParams(typeof(s)) is (Bar[3.0, string],) + + doAssert genericParams(array[10, int]) is (StaticParam[10], int) + var a: array[10, int] + doAssert genericParams(typeof(a)) is (range[0..9], int) + + type T2 = T + genericParamsImpl(T2) + + +proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = + ## Returns true if the func/proc/etc `fn` has `closure`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasClosureImpl(fn) + +template toUnsigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns an unsigned type with same bit size as `T`. + runnableExamples: + assert int8.toUnsigned is uint8 + assert uint.toUnsigned is uint + assert int.toUnsigned is uint + # range types are currently unsupported: + assert not compiles(toUnsigned(range[0..7])) + when T is int8: uint8 + elif T is int16: uint16 + elif T is int32: uint32 + elif T is int64: uint64 + elif T is int: uint + else: T + +template toSigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns a signed type with same bit size as `T`. + runnableExamples: + assert int8.toSigned is int8 + assert uint16.toSigned is int16 + # range types are currently unsupported: + assert not compiles(toSigned(range[0..7])) + when T is uint8: int8 + elif T is uint16: int16 + elif T is uint32: int32 + elif T is uint64: int64 + elif T is uint: int + else: T diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 8d76cc787..8cbe117bb 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -20,10 +20,17 @@ ## * `unidecode module <unidecode.html>`_ ## * `encodings module <encodings.html>`_ - -{.deadCodeElim: on.} # dce option deprecated - include "system/inclrtl" +import std/strbasics +template toOa(s: string): auto = s.toOpenArray(0, s.high) + +proc substr(s: openArray[char] , first, last: int): string = + # Copied substr from system + let first = max(first, 0) + let L = max(min(last, high(s)) - first + 1, 0) + result = newString(L) + for i in 0 .. L-1: + result[i] = s[i+first] type RuneImpl = int32 # underlying type of Rune @@ -31,16 +38,18 @@ type ## Type that can hold a single Unicode code point. ## ## A Rune may be composed with other Runes to a character on the screen. + ## `RuneImpl` is the underlying type used to store Runes, currently `int32`. template ones(n: untyped): untyped = ((1 shl n)-1) -proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = +proc runeLen*(s: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Returns the number of runes of the string ``s``. runnableExamples: let a = "añyóng" doAssert a.runeLen == 6 ## note: a.len == 8 + result = 0 var i = 0 while i < len(s): if uint(s[i]) <= 127: inc(i) @@ -52,7 +61,7 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = else: inc i inc(result) -proc runeLenAt*(s: string, i: Natural): int = +proc runeLenAt*(s: openArray[char], i: Natural): int = ## Returns the number of bytes the rune starting at ``s[i]`` takes. ## ## See also: @@ -72,7 +81,7 @@ proc runeLenAt*(s: string, i: Natural): int = const replRune = Rune(0xFFFD) -template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = +template fastRuneAt*(s: openArray[char] or string, i: int, result: untyped, doInc = true) = ## Returns the rune ``s[i]`` in ``result``. ## ## If ``doInc == true`` (default), ``i`` is incremented by the number @@ -150,7 +159,7 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = result = Rune(uint(s[i])) when doInc: inc(i) -proc runeAt*(s: string, i: Natural): Rune = +proc runeAt*(s: openArray[char], i: Natural): Rune = ## Returns the rune in ``s`` at **byte index** ``i``. ## ## See also: @@ -164,7 +173,7 @@ proc runeAt*(s: string, i: Natural): Rune = doAssert a.runeAt(3) == "y".runeAt(0) fastRuneAt(s, i, result, false) -proc validateUtf8*(s: string): int = +proc validateUtf8*(s: openArray[char]): int = ## Returns the position of the invalid byte in ``s`` if the string ``s`` does ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. ## @@ -301,7 +310,7 @@ proc `$`*(runes: seq[Rune]): string = for rune in runes: result.add rune -proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = +proc runeOffset*(s: openArray[char], pos: Natural, start: Natural = 0): int = ## Returns the byte position of rune ## at position ``pos`` in ``s`` with an optional start byte position. ## Returns the special value -1 if it runs out of the string. @@ -328,13 +337,13 @@ proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int = inc i return o -proc runeReverseOffset*(s: string, rev: Positive): (int, int) = +proc runeReverseOffset*(s: openArray[char], rev: Positive): (int, int) = ## Returns a tuple with the byte offset of the ## rune at position ``rev`` in ``s``, counting ## from the end (starting with 1) and the total ## number of runes in the string. ## - ## Returns a negative value for offset if there are to few runes in + ## Returns a negative value for offset if there are too few runes in ## the string to satisfy the request. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -347,18 +356,16 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = a = rev.int o = 0 x = 0 + let times = 2*rev.int-s.runeLen # transformed from rev.int - a < s.runeLen - rev.int while o < s.len: let r = runeLenAt(s, o) o += r - if a < 0: + if a > times: x += r dec a + result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) - if a > 0: - return (-a, rev.int-a) - return (x, -a+rev.int) - -proc runeAtPos*(s: string, pos: int): Rune = +proc runeAtPos*(s: openArray[char], pos: int): Rune = ## Returns the rune at position ``pos``. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -371,7 +378,7 @@ proc runeAtPos*(s: string, pos: int): Rune = ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ fastRuneAt(s, runeOffset(s, pos), result, false) -proc runeStrAtPos*(s: string, pos: Natural): string = +proc runeStrAtPos*(s: openArray[char], pos: Natural): string = ## Returns the rune at position ``pos`` as UTF8 String. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -383,9 +390,9 @@ proc runeStrAtPos*(s: string, pos: Natural): string = ## * `runeAtPos proc <#runeAtPos,string,int>`_ ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ let o = runeOffset(s, pos) - s[o .. (o+runeLenAt(s, o)-1)] + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) -proc runeSubStr*(s: string, pos: int, len: int = int.high): string = +proc runeSubStr*(s: openArray[char], pos: int, len: int = int.high): string = ## Returns the UTF-8 substring starting at code point ``pos`` ## with ``len`` code points. ## @@ -404,7 +411,7 @@ proc runeSubStr*(s: string, pos: int, len: int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s.substr(o, s.len-1) + result = s.substr(o, s.high) elif len < 0: let e = rl + len if e < 0: @@ -457,7 +464,7 @@ proc `==`*(a, b: Rune): bool = include "includes/unicode_ranges" -proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = +proc binarySearch(c: RuneImpl, tab: openArray[int32], len, stride: int): int = var n = len var t = 0 while n > 1: @@ -472,7 +479,7 @@ proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int = return t return -1 -proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = +proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1".} = ## Converts ``c`` into lower case. This works for any rune. ## ## If possible, prefer ``toLower`` over ``toUpper``. @@ -490,7 +497,7 @@ proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c + toLowerSinglets[p+1] - 500) return Rune(c) -proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = +proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1".} = ## Converts ``c`` into upper case. This works for any rune. ## ## If possible, prefer ``toLower`` over ``toUpper``. @@ -508,7 +515,7 @@ proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c + toUpperSinglets[p+1] - 500) return Rune(c) -proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = +proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1".} = ## Converts ``c`` to title case. ## ## See also: @@ -521,7 +528,7 @@ proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = return Rune(c + toTitleSinglets[p+1] - 500) return Rune(c) -proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1".} = ## Returns true if ``c`` is a lower case rune. ## ## If possible, prefer ``isLower`` over ``isUpper``. @@ -539,7 +546,7 @@ proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c == toUpperSinglets[p]: return true -proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1".} = ## Returns true if ``c`` is a upper case rune. ## ## If possible, prefer ``isLower`` over ``isUpper``. @@ -559,7 +566,7 @@ proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c == toLowerSinglets[p]: return true -proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1".} = ## Returns true if ``c`` is an *alpha* rune (i.e., a letter). ## ## See also: @@ -578,7 +585,7 @@ proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c == alphaSinglets[p]: return true -proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1".} = ## Returns true if ``c`` is a Unicode titlecase code point. ## ## See also: @@ -589,7 +596,7 @@ proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = ## * `isWhiteSpace proc <#isWhiteSpace,Rune>`_ return isUpper(c) and isLower(c) -proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1".} = ## Returns true if ``c`` is a Unicode whitespace code point. ## ## See also: @@ -602,7 +609,7 @@ proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = if p >= 0 and c >= spaceRanges[p] and c <= spaceRanges[p+1]: return true -proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = +proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1".} = ## Returns true if ``c`` is a Unicode combining code unit. ## ## See also: @@ -629,7 +636,7 @@ template runeCheck(s, runeProc) = fastRuneAt(s, i, rune, doInc = true) result = runeProc(rune) and result -proc isAlpha*(s: string): bool {.noSideEffect, procvar, +proc isAlpha*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all alphabetic runes. runnableExamples: @@ -637,7 +644,7 @@ proc isAlpha*(s: string): bool {.noSideEffect, procvar, doAssert a.isAlpha runeCheck(s, isAlpha) -proc isSpace*(s: string): bool {.noSideEffect, procvar, +proc isSpace*(s: openArray[char]): bool {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Returns true if ``s`` contains all whitespace runes. runnableExamples: @@ -658,21 +665,21 @@ template convertRune(s, runeProc) = rune = runeProc(rune) fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc toUpper*(s: string): string {.noSideEffect, procvar, +proc toUpper*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into upper-case runes. runnableExamples: doAssert toUpper("abγ") == "ABΓ" convertRune(s, toUpper) -proc toLower*(s: string): string {.noSideEffect, procvar, +proc toLower*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1Str".} = ## Converts ``s`` into lower-case runes. runnableExamples: doAssert toLower("ABΓ") == "abγ" convertRune(s, toLower) -proc swapCase*(s: string): string {.noSideEffect, procvar, +proc swapCase*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Swaps the case of runes in ``s``. ## @@ -694,22 +701,25 @@ proc swapCase*(s: string): string {.noSideEffect, procvar, rune = rune.toUpper() fastToUTF8Copy(rune, result, resultIndex, doInc = true) -proc capitalize*(s: string): string {.noSideEffect, procvar, +proc capitalize*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts the first character of ``s`` into an upper-case rune. runnableExamples: doAssert capitalize("βeta") == "Βeta" if len(s) == 0: - return s + return "" var rune: Rune i = 0 fastRuneAt(s, i, rune, doInc = true) - result = $toUpper(rune) & substr(s, i) + result = $toUpper(rune) & substr(s.toOpenArray(i, s.high)) -proc translate*(s: string, replacements: proc(key: string): string): string {. - rtl, extern: "nuc$1".} = +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + +proc translate*(s: openArray[char], replacements: proc(key: string): string): string {. + rtl, extern: "nuc$1", effectsOf: replacements.} = ## Translates words in a string using the ``replacements`` proc to substitute ## words inside ``s`` with their replacements. ## @@ -743,7 +753,7 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if whiteSpace and inWord: # If we've reached the end of a word - let word = s[wordStart ..< lastIndex] + let word = substr(s.toOpenArray(wordStart, lastIndex - 1)) result.add(replacements(word)) result.add($rune) inWord = false @@ -758,10 +768,10 @@ proc translate*(s: string, replacements: proc(key: string): string): string {. if wordStart < len(s) and inWord: # Get the trailing word at the end - let word = s[wordStart .. ^1] + let word = substr(s.toOpenArray(wordStart, s.high)) result.add(replacements(word)) -proc title*(s: string): string {.noSideEffect, procvar, +proc title*(s: openArray[char]): string {.noSideEffect, rtl, extern: "nuc$1".} = ## Converts ``s`` to a unicode title. ## @@ -787,7 +797,7 @@ proc title*(s: string): string {.noSideEffect, procvar, fastToUTF8Copy(rune, result, resultIndex, doInc = true) -iterator runes*(s: string): Rune = +iterator runes*(s: openArray[char]): Rune = ## Iterates over any rune of the string ``s`` returning runes. var i = 0 @@ -796,7 +806,7 @@ iterator runes*(s: string): Rune = fastRuneAt(s, i, result, true) yield result -iterator utf8*(s: string): string = +iterator utf8*(s: openArray[char]): string = ## Iterates over any rune of the string ``s`` returning utf8 values. ## ## See also: @@ -807,14 +817,14 @@ iterator utf8*(s: string): string = var o = 0 while o < s.len: let n = runeLenAt(s, o) - yield s[o .. (o+n-1)] + yield substr(s.toOpenArray(o, (o+n-1))) o += n -proc toRunes*(s: string): seq[Rune] = +proc toRunes*(s: openArray[char]): seq[Rune] = ## Obtains a sequence containing the Runes in ``s``. ## ## See also: - ## * `$ proc <#$,seq[T][Rune]>`_ for a reverse operation + ## * `$ proc <#$,Rune>`_ for a reverse operation runnableExamples: let a = toRunes("aáä") doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] @@ -823,12 +833,12 @@ proc toRunes*(s: string): seq[Rune] = for r in s.runes: result.add(r) -proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = +proc cmpRunesIgnoreCase*(a, b: openArray[char]): int {.rtl, extern: "nuc$1".} = ## Compares two UTF-8 strings and ignores the case. Returns: ## - ## | 0 if a == b - ## | < 0 if a < b - ## | > 0 if a > b + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b var i = 0 var j = 0 var ar, br: Rune @@ -836,11 +846,16 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} = # slow path: fastRuneAt(a, i, ar) fastRuneAt(b, j, br) - result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) + when sizeof(int) < 4: + const lo = low(int).int32 + const hi = high(int).int32 + result = clamp(RuneImpl(toLower(ar)) - RuneImpl(toLower(br)), lo, hi).int + else: + result = RuneImpl(toLower(ar)) - RuneImpl(toLower(br)) if result != 0: return result = a.len - b.len -proc reversed*(s: string): string = +proc reversed*(s: openArray[char]): string = ## Returns the reverse of ``s``, interpreting it as runes. ## ## Unicode combining characters are correctly interpreted as well. @@ -875,9 +890,9 @@ proc reversed*(s: string): string = reverseUntil(len(s)) -proc graphemeLen*(s: string; i: Natural): Natural = +proc graphemeLen*(s: openArray[char]; i: Natural): Natural = ## The number of bytes belonging to byte index ``s[i]``, - ## including following combining code unit. + ## including following combining code units. runnableExamples: let a = "añyóng" doAssert a.graphemeLen(1) == 2 ## ñ @@ -894,7 +909,7 @@ proc graphemeLen*(s: string; i: Natural): Natural = if not isCombining(r2): break result = j-i -proc lastRune*(s: string; last: int): (Rune, int) = +proc lastRune*(s: openArray[char]; last: int): (Rune, int) = ## Length of the last rune in ``s[0..last]``. Returns the rune and its length ## in bytes. if s[last] <= chr(127): @@ -923,83 +938,61 @@ proc size*(r: Rune): int {.noSideEffect.} = else: result = 1 # --------- Private templates for different split separators ----------- -proc stringHasSep(s: string, index: int, seps: openArray[Rune]): bool = +proc stringHasSep(s: openArray[char], index: int, seps: openArray[Rune]): bool = var rune: Rune fastRuneAt(s, index, rune, false) return seps.contains(rune) -proc stringHasSep(s: string, index: int, sep: Rune): bool = +proc stringHasSep(s: openArray[char], index: int, sep: Rune): bool = var rune: Rune fastRuneAt(s, index, rune, false) return sep == rune -template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) = +template splitCommon(s, sep, maxsplit: untyped) = ## Common code for split procedures. + let + sLen = len(s) var last = 0 splits = maxsplit - if len(s) > 0: - while last <= len(s): + if sLen > 0: + while last <= sLen: var first = last - while last < len(s) and not stringHasSep(s, last, sep): - when sep is Rune: - inc(last, sepLen) - else: - inc(last, runeLenAt(s, last)) - if splits == 0: last = len(s) - yield s[first .. (last - 1)] + while last < sLen and not stringHasSep(s, last, sep): + inc(last, runeLenAt(s, last)) + if splits == 0: last = sLen + yield substr(s.toOpenArray(first, (last - 1))) if splits == 0: break dec(splits) - when sep is Rune: - inc(last, sepLen) - else: - inc(last, if last < len(s): runeLenAt(s, last) else: 1) + inc(last, if last < sLen: runeLenAt(s, last) else: 1) -iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, +iterator split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a group of separators. ## ## Substrings are separated by a substring containing only ``seps``. - ## - ## .. code-block:: nim - ## for word in split("this\lis an\texample"): - ## writeLine(stdout, word) - ## - ## ...generates this output: - ## - ## .. code-block:: - ## "this" - ## "is" - ## "an" - ## "example" - ## - ## And the following code: - ## - ## .. code-block:: nim - ## for word in split("this:is;an$example", {';', ':', '$'}): - ## writeLine(stdout, word) - ## - ## ...produces the same output as the first example. The code: - ## - ## .. code-block:: nim - ## let date = "2012-11-20T22:08:08.398990" - ## let separators = {' ', '-', ':', 'T'} - ## for number in split(date, separators): - ## writeLine(stdout, number) - ## - ## ...results in: - ## - ## .. code-block:: - ## "2012" - ## "11" - ## "20" - ## "22" - ## "08" - ## "08.398990" - ## + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + splitCommon(s, seps, maxsplit) -iterator splitWhitespace*(s: string): string = +iterator splitWhitespace*(s: openArray[char]): string = ## Splits a unicode string at whitespace runes. splitCommon(s, unicodeSpaces, -1) @@ -1007,51 +1000,36 @@ template accResult(iter: untyped) = result = @[] for x in iter: add(result, x) -proc splitWhitespace*(s: string): seq[string] {.noSideEffect, +proc splitWhitespace*(s: openArray[char]): seq[string] {.noSideEffect, rtl, extern: "ncuSplitWhitespace".} = ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ ## iterator, but is a proc that returns a sequence of substrings. accResult(splitWhitespace(s)) -iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = +iterator split*(s: openArray[char], sep: Rune, maxsplit: int = -1): string = ## Splits the unicode string ``s`` into substrings using a single separator. - ## ## Substrings are separated by the rune ``sep``. - ## The code: - ## - ## .. code-block:: nim - ## for word in split(";;this;is;an;;example;;;", ';'): - ## writeLine(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: - ## "" - ## "" - ## "this" - ## "is" - ## "an" - ## "" - ## "example" - ## "" - ## "" - ## "" - ## - splitCommon(s, sep, maxsplit, sep.size) + runnableExamples: + import std/sequtils -proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(s, sep, maxsplit) + +proc split*(s: openArray[char], seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRunes".} = ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, ## but is a proc that returns a sequence of substrings. accResult(split(s, seps, maxsplit)) -proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, +proc split*(s: openArray[char], sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nucSplitRune".} = ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc ## that returns a sequence of substrings. accResult(split(s, sep, maxsplit)) -proc strip*(s: string, leading = true, trailing = true, +proc strip*(s: openArray[char], leading = true, trailing = true, runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, rtl, extern: "nucStrip".} = ## Strips leading or trailing ``runes`` from ``s`` and returns @@ -1106,7 +1084,7 @@ proc strip*(s: string, leading = true, trailing = true, let newLen = eI - sI + 1 result = newStringOfCap(newLen) if newLen > 0: - result.add s[sI .. eI] + result.add substr(s.toOpenArray(sI, eI)) proc repeat*(c: Rune, count: Natural): string {.noSideEffect, rtl, extern: "nucRepeatRune".} = @@ -1122,7 +1100,7 @@ proc repeat*(c: Rune, count: Natural): string {.noSideEffect, for i in 0 ..< count: result.add s -proc align*(s: string, count: Natural, padding = ' '.Rune): string {. +proc align*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect, rtl, extern: "nucAlignString".} = ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length ## of ``count``. @@ -1147,9 +1125,9 @@ proc align*(s: string, count: Natural, padding = ' '.Rune): string {. for i in 0 ..< spaces: result.add padStr result.add s else: - result = s + result = s.substr -proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. +proc alignLeft*(s: openArray[char], count: Natural, padding = ' '.Rune): string {. noSideEffect.} = ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a ## rune-length of ``count``. @@ -1173,301 +1151,365 @@ proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. for i in sLen ..< count: result.add padStr else: - result = s + result = s.substr -# ----------------------------------------------------------------------------- -# deprecated -template runeCaseCheck(s, runeProc, skipNonAlpha) = - ## Common code for rune.isLower and rune.isUpper. - if len(s) == 0: return false - var - i = 0 - rune: Rune - hasAtleastOneAlphaRune = false - while i < len(s): - fastRuneAt(s, i, rune, doInc = true) - if skipNonAlpha: - var runeIsAlpha = isAlpha(rune) - if not hasAtleastOneAlphaRune: - hasAtleastOneAlphaRune = runeIsAlpha - if runeIsAlpha and (not runeProc(rune)): - return false - else: - if not runeProc(rune): - return false - return if skipNonAlpha: hasAtleastOneAlphaRune else: true +proc runeLen*(s: string): int {.inline.} = + ## Returns the number of runes of the string ``s``. + runnableExamples: + let a = "añyóng" + doAssert a.runeLen == 6 + ## note: a.len == 8 + runeLen(toOa(s)) + +proc runeLenAt*(s: string, i: Natural): int {.inline.} = + ## Returns the number of bytes the rune starting at ``s[i]`` takes. + ## + ## See also: + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeLenAt(0) == 1 + doAssert a.runeLenAt(1) == 2 + runeLenAt(toOa(s), i) -proc isLower*(s: string, skipNonAlpha: bool): bool {. - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## **Deprecated since version 0.20 since its semantics are unclear** +proc runeAt*(s: string, i: Natural): Rune {.inline.} = + ## Returns the rune in ``s`` at **byte index** ``i``. ## - ## Checks whether ``s`` is lower case. + ## See also: + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeAt(1) == "ñ".runeAt(0) + doAssert a.runeAt(2) == "ñ".runeAt(1) + doAssert a.runeAt(3) == "y".runeAt(0) + fastRuneAt(s, i, result, false) + +proc validateUtf8*(s: string): int {.inline.} = + ## Returns the position of the invalid byte in ``s`` if the string ``s`` does + ## not hold valid UTF-8 data. Otherwise ``-1`` is returned. ## - ## If ``skipNonAlpha`` is true, returns true if all alphabetical - ## runes in ``s`` are lower case. Returns false if none of the - ## runes in ``s`` are alphabetical. + ## See also: + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + validateUtf8(toOa(s)) + +proc runeOffset*(s: string, pos: Natural, start: Natural = 0): int {.inline.} = + ## Returns the byte position of rune + ## at position ``pos`` in ``s`` with an optional start byte position. + ## Returns the special value -1 if it runs out of the string. ## - ## If ``skipNonAlpha`` is false, returns true only if all runes in - ## ``s`` are alphabetical and lower case. + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. ## - ## For either value of ``skipNonAlpha``, returns false if ``s`` is - ## an empty string. - runeCaseCheck(s, isLower, skipNonAlpha) + ## See also: + ## * `runeReverseOffset proc <#runeReverseOffset,string,Positive>`_ + runnableExamples: + let a = "añyóng" + doAssert a.runeOffset(1) == 1 + doAssert a.runeOffset(3) == 4 + doAssert a.runeOffset(4) == 6 + runeOffset(toOa(s), pos, start) -proc isUpper*(s: string, skipNonAlpha: bool): bool {. - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## **Deprecated since version 0.20 since its semantics are unclear** +proc runeReverseOffset*(s: string, rev: Positive): (int, int) {.inline.} = + ## Returns a tuple with the byte offset of the + ## rune at position ``rev`` in ``s``, counting + ## from the end (starting with 1) and the total + ## number of runes in the string. + ## + ## Returns a negative value for offset if there are too few runes in + ## the string to satisfy the request. ## - ## Checks whether ``s`` is upper case. + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. ## - ## If ``skipNonAlpha`` is true, returns true if all alphabetical - ## runes in ``s`` are upper case. Returns false if none of the - ## runes in ``s`` are alphabetical. + ## See also: + ## * `runeOffset proc <#runeOffset,string,Natural,Natural>`_ + runeReverseOffset(toOa(s), rev) + +proc runeAtPos*(s: string, pos: int): Rune {.inline.} = + ## Returns the rune at position ``pos``. ## - ## If ``skipNonAlpha`` is false, returns true only if all runes in - ## ``s`` are alphabetical and upper case. + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. ## - ## For either value of ``skipNonAlpha``, returns false if ``s`` is - ## an empty string. - runeCaseCheck(s, isUpper, skipNonAlpha) + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeStrAtPos proc <#runeStrAtPos,string,Natural>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + fastRuneAt(toOa(s), runeOffset(s, pos), result, false) -proc isTitle*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nuc$1Str", - deprecated: "Deprecated since version 0.20 since its semantics are unclear".} = - ## **Deprecated since version 0.20 since its semantics are unclear** +proc runeStrAtPos*(s: string, pos: Natural): string {.inline.} = + ## Returns the rune at position ``pos`` as UTF8 String. ## - ## Checks whether or not ``s`` is a unicode title. + ## **Beware:** This can lead to unoptimized code and slow execution! + ## Most problems can be solved more efficiently by using an iterator + ## or conversion to a seq of Rune. ## - ## Returns true if the first character in each word inside ``s`` - ## are upper case and there is at least one character in ``s``. - if s.len == 0: - return false - result = true - var - i = 0 - rune: Rune - var firstRune = true + ## See also: + ## * `runeAt proc <#runeAt,string,Natural>`_ + ## * `runeAtPos proc <#runeAtPos,string,int>`_ + ## * `fastRuneAt template <#fastRuneAt.t,string,int,untyped>`_ + let o = runeOffset(s, pos) + substr(s.toOpenArray(o, (o+runeLenAt(s, o)-1))) - while i < len(s) and result: - fastRuneAt(s, i, rune, doInc = true) - if not rune.isWhiteSpace() and firstRune: - result = rune.isUpper() and result - firstRune = false - elif rune.isWhiteSpace(): - firstRune = true +proc runeSubStr*(s: string, pos: int, len: int = int.high): string {.inline.} = + ## Returns the UTF-8 substring starting at code point ``pos`` + ## with ``len`` code points. + ## + ## If ``pos`` or ``len`` is negative they count from + ## the end of the string. If ``len`` is not given it means the longest + ## possible string. + runnableExamples: + let s = "Hänsel ««: 10,00€" + doAssert(runeSubStr(s, 0, 2) == "Hä") + doAssert(runeSubStr(s, 10, 1) == ":") + doAssert(runeSubStr(s, -6) == "10,00€") + doAssert(runeSubStr(s, 10) == ": 10,00€") + doAssert(runeSubStr(s, 12, 5) == "10,00") + doAssert(runeSubStr(s, -6, 3) == "10,") + runeSubStr(toOa(s), pos, len) +proc isAlpha*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all alphabetic runes. + runnableExamples: + let a = "añyóng" + doAssert a.isAlpha + isAlpha(toOa(s)) -when isMainModule: +proc isSpace*(s: string): bool {.noSideEffect, inline.} = + ## Returns true if ``s`` contains all whitespace runes. + runnableExamples: + let a = "\t\l \v\r\f" + doAssert a.isSpace + isSpace(toOa(s)) - proc asRune(s: static[string]): Rune = - ## Compile-time conversion proc for converting string literals to a Rune - ## value. Returns the first Rune of the specified string. - ## - ## Shortcuts code like ``"å".runeAt(0)`` to ``"å".asRune`` and returns a - ## compile-time constant. - if s.len == 0: Rune(0) - else: s.runeAt(0) - let - someString = "öÑ" - someRunes = toRunes(someString) - compared = (someString == $someRunes) - doAssert compared == true - - proc testReplacements(word: string): string = - case word - of "two": - return "2" - of "foo": - return "BAR" - of "βeta": - return "beta" - of "alpha": - return "αlpha" - else: - return "12345" - - doAssert translate("two not alpha foo βeta", testReplacements) == "2 12345 αlpha BAR beta" - doAssert translate(" two not foo βeta ", testReplacements) == " 2 12345 BAR beta " - - doAssert title("foo bar") == "Foo Bar" - doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" - doAssert title("") == "" - - doAssert capitalize("βeta") == "Βeta" - doAssert capitalize("foo") == "Foo" - doAssert capitalize("") == "" - - doAssert swapCase("FooBar") == "fOObAR" - doAssert swapCase(" ") == " " - doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" - doAssert swapCase("a✓B") == "A✓b" - doAssert swapCase("Јамогујестистаклоитоминештети") == "јАМОГУЈЕСТИСТАКЛОИТОМИНЕШТЕТИ" - doAssert swapCase("ὕαλονϕαγεῖνδύναμαιτοῦτοοὔμεβλάπτει") == "ὝΑΛΟΝΦΑΓΕῖΝΔΎΝΑΜΑΙΤΟῦΤΟΟὔΜΕΒΛΆΠΤΕΙ" - doAssert swapCase("Կրնամապակիուտեևինծիանհանգիստչըներ") == "կՐՆԱՄԱՊԱԿԻՈՒՏԵևԻՆԾԻԱՆՀԱՆԳԻՍՏՉԸՆԵՐ" - doAssert swapCase("") == "" - - doAssert isAlpha("r") - doAssert isAlpha("α") - doAssert isAlpha("ϙ") - doAssert isAlpha("ஶ") - doAssert(not isAlpha("$")) - doAssert(not isAlpha("")) - - doAssert isAlpha("Βeta") - doAssert isAlpha("Args") - doAssert isAlpha("𐌼𐌰𐌲𐌲𐌻𐌴𐍃𐍄𐌰𐌽") - doAssert isAlpha("ὕαλονϕαγεῖνδύναμαιτοῦτοοὔμεβλάπτει") - doAssert isAlpha("Јамогујестистаклоитоминештети") - doAssert isAlpha("Կրնամապակիուտեևինծիանհանգիստչըներ") - doAssert(not isAlpha("$Foo✓")) - doAssert(not isAlpha("⠙⠕⠑⠎⠝⠞")) - - doAssert isSpace("\t") - doAssert isSpace("\l") - doAssert(not isSpace("Β")) - doAssert(not isSpace("Βeta")) - - doAssert isSpace("\t\l \v\r\f") - doAssert isSpace(" ") - doAssert(not isSpace("")) - doAssert(not isSpace("ΑΓc \td")) - - doAssert(not isLower(' '.Rune)) - - doAssert(not isUpper(' '.Rune)) - - doAssert toUpper("Γ") == "Γ" - doAssert toUpper("b") == "B" - doAssert toUpper("α") == "Α" - doAssert toUpper("✓") == "✓" - doAssert toUpper("ϙ") == "Ϙ" - doAssert toUpper("") == "" - - doAssert toUpper("ΑΒΓ") == "ΑΒΓ" - doAssert toUpper("AAccβ") == "AACCΒ" - doAssert toUpper("A✓$β") == "A✓$Β" - - doAssert toLower("a") == "a" - doAssert toLower("γ") == "γ" - doAssert toLower("Γ") == "γ" - doAssert toLower("4") == "4" - doAssert toLower("Ϙ") == "ϙ" - doAssert toLower("") == "" - - doAssert toLower("abcdγ") == "abcdγ" - doAssert toLower("abCDΓ") == "abcdγ" - doAssert toLower("33aaΓ") == "33aaγ" - - doAssert reversed("Reverse this!") == "!siht esreveR" - doAssert reversed("先秦兩漢") == "漢兩秦先" - doAssert reversed("as⃝df̅") == "f̅ds⃝a" - doAssert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" - doAssert reversed("ὕαλονϕαγεῖνδύναμαιτοῦτοοὔμεβλάπτει") == "ιετπάλβεμὔοοτῦοτιαμανύδνῖεγαϕνολαὕ" - doAssert reversed("Јамогујестистаклоитоминештети") == "итетшенимотиолкатситсејугомаЈ" - doAssert reversed("Կրնամապակիուտեևինծիանհանգիստչըներ") == "րենըչտսիգնահնաիծնիևետւոիկապամանրԿ" - doAssert len(toRunes("as⃝df̅")) == runeLen("as⃝df̅") - const test = "as⃝" - doAssert lastRune(test, test.len-1)[1] == 3 - doAssert graphemeLen("è", 0) == 2 - - # test for rune positioning and runeSubStr() - let s = "Hänsel ««: 10,00€" - - var t = "" - for c in s.utf8: - t.add c - - doAssert(s == t) - - doAssert(runeReverseOffset(s, 1) == (20, 18)) - doAssert(runeReverseOffset(s, 19) == (-1, 18)) - - doAssert(runeStrAtPos(s, 0) == "H") - doAssert(runeSubStr(s, 0, 1) == "H") - doAssert(runeStrAtPos(s, 10) == ":") - doAssert(runeSubStr(s, 10, 1) == ":") - doAssert(runeStrAtPos(s, 9) == "«") - doAssert(runeSubStr(s, 9, 1) == "«") - doAssert(runeStrAtPos(s, 17) == "€") - doAssert(runeSubStr(s, 17, 1) == "€") - # echo runeStrAtPos(s, 18) # index error - - doAssert(runeSubStr(s, 0) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, -18) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, 10) == ": 10,00€") - doAssert(runeSubStr(s, 18) == "") - doAssert(runeSubStr(s, 0, 10) == "Hänsel ««") - - doAssert(runeSubStr(s, 12) == "10,00€") - doAssert(runeSubStr(s, -6) == "10,00€") - - doAssert(runeSubStr(s, 12, 5) == "10,00") - doAssert(runeSubStr(s, 12, -1) == "10,00") - doAssert(runeSubStr(s, -6, 5) == "10,00") - doAssert(runeSubStr(s, -6, -1) == "10,00") - - doAssert(runeSubStr(s, 0, 100) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, -100, 100) == "Hänsel ««: 10,00€") - doAssert(runeSubStr(s, 0, -100) == "") - doAssert(runeSubStr(s, 100, -100) == "") - - block splitTests: - let s = " this is an example " - let s2 = ":this;is;an:example;;" - let s3 = ":this×is×an:example××" - doAssert s.split() == @["", "this", "is", "an", "example", "", ""] - doAssert s2.split(seps = [':'.Rune, ';'.Rune]) == @["", "this", "is", "an", - "example", "", ""] - doAssert s3.split(seps = [':'.Rune, "×".asRune]) == @["", "this", "is", - "an", "example", "", ""] - doAssert s.split(maxsplit = 4) == @["", "this", "is", "an", "example "] - doAssert s.split(' '.Rune, maxsplit = 1) == @["", "this is an example "] - - block stripTests: - doAssert(strip("") == "") - doAssert(strip(" ") == "") - doAssert(strip("y") == "y") - doAssert(strip(" foofoofoo ") == "foofoofoo") - doAssert(strip("sfoofoofoos", runes = ['s'.Rune]) == "foofoofoo") - - block: - let stripTestRunes = ['b'.Rune, 'a'.Rune, 'r'.Rune] - doAssert(strip("barfoofoofoobar", runes = stripTestRunes) == "foofoofoo") - doAssert(strip("sfoofoofoos", leading = false, runes = ['s'.Rune]) == "sfoofoofoo") - doAssert(strip("sfoofoofoos", trailing = false, runes = ['s'.Rune]) == "foofoofoos") - - block: - let stripTestRunes = ["«".asRune, "»".asRune] - doAssert(strip("«TEXT»", runes = stripTestRunes) == "TEXT") - doAssert(strip("copyright©", leading = false, runes = ["©".asRune]) == "copyright") - doAssert(strip("¿Question?", trailing = false, runes = ["¿".asRune]) == "Question?") - doAssert(strip("×text×", leading = false, runes = ["×".asRune]) == "×text") - doAssert(strip("×text×", trailing = false, runes = ["×".asRune]) == "text×") - - block repeatTests: - doAssert repeat('c'.Rune, 5) == "ccccc" - doAssert repeat("×".asRune, 5) == "×××××" - - block alignTests: - doAssert align("abc", 4) == " abc" - doAssert align("a", 0) == "a" - doAssert align("1232", 6) == " 1232" - doAssert align("1232", 6, '#'.Rune) == "##1232" - doAssert align("1232", 6, "×".asRune) == "××1232" - doAssert alignLeft("abc", 4) == "abc " - doAssert alignLeft("a", 0) == "a" - doAssert alignLeft("1232", 6) == "1232 " - doAssert alignLeft("1232", 6, '#'.Rune) == "1232##" - doAssert alignLeft("1232", 6, "×".asRune) == "1232××" - - block differentSizes: - # upper and lower variants have different number of bytes - doAssert toLower("AẞC") == "aßc" - doAssert toLower("ȺẞCD") == "ⱥßcd" - doAssert toUpper("ⱥbc") == "ȺBC" - doAssert toUpper("rsⱦuv") == "RSȾUV" - doAssert swapCase("ⱥbCd") == "ȺBcD" - doAssert swapCase("XyꟆaB") == "xYᶎAb" - doAssert swapCase("aᵹcᲈd") == "AꝽCꙊD" +proc toUpper*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into upper-case runes. + runnableExamples: + doAssert toUpper("abγ") == "ABΓ" + toUpper(toOa(s)) + +proc toLower*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` into lower-case runes. + runnableExamples: + doAssert toLower("ABΓ") == "abγ" + toLower(toOa(s)) + +proc swapCase*(s: string): string {.noSideEffect, inline.} = + ## Swaps the case of runes in ``s``. + ## + ## Returns a new string such that the cases of all runes + ## are swapped if possible. + runnableExamples: + doAssert swapCase("Αlpha Βeta Γamma") == "αLPHA βETA γAMMA" + swapCase(toOa(s)) + +proc capitalize*(s: string): string {.noSideEffect.} = + ## Converts the first character of ``s`` into an upper-case rune. + runnableExamples: + doAssert capitalize("βeta") == "Βeta" + capitalize(toOa(s)) + + +proc translate*(s: string, replacements: proc(key: string): string): string {.effectsOf: replacements, inline.} = + ## Translates words in a string using the ``replacements`` proc to substitute + ## words inside ``s`` with their replacements. + ## + ## ``replacements`` is any proc that takes a word and returns + ## a new word to fill it's place. + runnableExamples: + proc wordToNumber(s: string): string = + case s + of "one": "1" + of "two": "2" + else: s + let a = "one two three four" + doAssert a.translate(wordToNumber) == "1 2 three four" + translate(toOa(s), replacements) + +proc title*(s: string): string {.noSideEffect, inline.} = + ## Converts ``s`` to a unicode title. + ## + ## Returns a new string such that the first character + ## in each word inside ``s`` is capitalized. + runnableExamples: + doAssert title("αlpha βeta γamma") == "Αlpha Βeta Γamma" + title(toOa(s)) + + +iterator runes*(s: string): Rune = + ## Iterates over any rune of the string ``s`` returning runes. + for rune in runes(toOa(s)): + yield rune + +iterator utf8*(s: string): string = + ## Iterates over any rune of the string ``s`` returning utf8 values. + ## + ## See also: + ## * `validateUtf8 proc <#validateUtf8,string>`_ + ## * `toUTF8 proc <#toUTF8,Rune>`_ + ## * `$ proc <#$,Rune>`_ alias for `toUTF8` + ## * `fastToUTF8Copy template <#fastToUTF8Copy.t,Rune,string,int>`_ + for str in utf8(toOa(s)): + yield str + +proc toRunes*(s: string): seq[Rune] {.inline.} = + ## Obtains a sequence containing the Runes in ``s``. + ## + ## See also: + ## * `$ proc <#$,Rune>`_ for a reverse operation + runnableExamples: + let a = toRunes("aáä") + doAssert a == @["a".runeAt(0), "á".runeAt(0), "ä".runeAt(0)] + toRunes(toOa(s)) + +proc cmpRunesIgnoreCase*(a, b: string): int {.inline.} = + ## Compares two UTF-8 strings and ignores the case. Returns: + ## + ## | `0` if a == b + ## | `< 0` if a < b + ## | `> 0` if a > b + cmpRunesIgnoreCase(a.toOa(), b.toOa()) + +proc reversed*(s: string): string {.inline.} = + ## Returns the reverse of ``s``, interpreting it as runes. + ## + ## Unicode combining characters are correctly interpreted as well. + runnableExamples: + assert reversed("Reverse this!") == "!siht esreveR" + assert reversed("先秦兩漢") == "漢兩秦先" + assert reversed("as⃝df̅") == "f̅ds⃝a" + assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞" + reversed(toOa(s)) + +proc graphemeLen*(s: string; i: Natural): Natural {.inline.} = + ## The number of bytes belonging to byte index ``s[i]``, + ## including following combining code unit. + runnableExamples: + let a = "añyóng" + doAssert a.graphemeLen(1) == 2 ## ñ + doAssert a.graphemeLen(2) == 1 + doAssert a.graphemeLen(4) == 2 ## ó + graphemeLen(toOa(s), i) + +proc lastRune*(s: string; last: int): (Rune, int) {.inline.} = + ## Length of the last rune in ``s[0..last]``. Returns the rune and its length + ## in bytes. + lastRune(toOa(s), last) + +iterator split*(s: string, seps: openArray[Rune] = unicodeSpaces, + maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a group of separators. + ## + ## Substrings are separated by a substring containing only ``seps``. + runnableExamples: + import std/sequtils + + assert toSeq("hÃllo\lthis\lis an\texample\l是".split) == + @["hÃllo", "this", "is", "an", "example", "是"] + + # And the following code splits the same string using a sequence of Runes. + assert toSeq(split("añyóng:hÃllo;是$example", ";:$".toRunes)) == + @["añyóng", "hÃllo", "是", "example"] + + # example with a `Rune` separator and unused one `;`: + assert toSeq(split("ab是de:f:", ";:是".toRunes)) == @["ab", "de", "f", ""] + + # Another example that splits a string containing a date. + let date = "2012-11-20T22:08:08.398990" + + assert toSeq(split(date, " -:T".toRunes)) == + @["2012", "11", "20", "22", "08", "08.398990"] + + splitCommon(toOa(s), seps, maxsplit) + +iterator splitWhitespace*(s: string): string = + ## Splits a unicode string at whitespace runes. + splitCommon(s.toOa(), unicodeSpaces, -1) + + +proc splitWhitespace*(s: string): seq[string] {.noSideEffect, inline.}= + ## The same as the `splitWhitespace <#splitWhitespace.i,string>`_ + ## iterator, but is a proc that returns a sequence of substrings. + accResult(splitWhitespace(toOa(s))) + +iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = + ## Splits the unicode string ``s`` into substrings using a single separator. + ## Substrings are separated by the rune ``sep``. + runnableExamples: + import std/sequtils + + assert toSeq(split(";;hÃllo;this;is;an;;example;;;是", ";".runeAt(0))) == + @["", "", "hÃllo", "this", "is", "an", "", "example", "", "", "是"] + + splitCommon(toOa(s), sep, maxsplit) + +proc split*(s: string, seps: openArray[Rune] = unicodeSpaces, maxsplit: int = -1): + seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,openArray[Rune],int>`_, + ## but is a proc that returns a sequence of substrings. + accResult(split(toOa(s), seps, maxsplit)) + +proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffect, inline.} = + ## The same as the `split iterator <#split.i,string,Rune,int>`_, but is a proc + ## that returns a sequence of substrings. + accResult(split(toOa(s), sep, maxsplit)) + +proc strip*(s: string, leading = true, trailing = true, + runes: openArray[Rune] = unicodeSpaces): string {.noSideEffect, inline.} = + ## Strips leading or trailing ``runes`` from ``s`` and returns + ## the resulting string. + ## + ## If ``leading`` is true (default), leading ``runes`` are stripped. + ## If ``trailing`` is true (default), trailing ``runes`` are stripped. + ## If both are false, the string is returned unchanged. + runnableExamples: + let a = "\táñyóng " + doAssert a.strip == "áñyóng" + doAssert a.strip(leading = false) == "\táñyóng" + doAssert a.strip(trailing = false) == "áñyóng " + strip(toOa(s), leading, trailing, runes) + + +proc align*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length + ## of ``count``. + ## + ## ``padding`` characters (by default spaces) are added before ``s`` resulting in + ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to left align a string use the `alignLeft + ## proc <#alignLeft,string,Natural>`_. + runnableExamples: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + assert align("1232", 6, '#'.Rune) == "##1232" + assert align("Åge", 5) == " Åge" + assert align("×", 4, '_'.Rune) == "___×" + align(toOa(s), count, padding) + +proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {.noSideEffect, inline.} = + ## Left-aligns a unicode string ``s`` with ``padding``, so that it has a + ## rune-length of ``count``. + ## + ## ``padding`` characters (by default spaces) are added after ``s`` resulting in + ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is + ## returned unchanged. If you need to right align a string use the `align + ## proc <#align,string,Natural>`_. + runnableExamples: + assert alignLeft("abc", 4) == "abc " + assert alignLeft("a", 0) == "a" + assert alignLeft("1232", 6) == "1232 " + assert alignLeft("1232", 6, '#'.Rune) == "1232##" + assert alignLeft("Åge", 5) == "Åge " + assert alignLeft("×", 4, '_'.Rune) == "×___" + alignLeft(toOa(s), count, padding) diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index f0647ea6c..2fb69f7b2 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,4 +1,4 @@ -#! usr/bin/env python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module @@ -11,7 +11,7 @@ try: except ImportError: pass -def main2(): +def main(): f = open("unidecode.dat", "wb+") for x in range(128, 0xffff + 1): u = eval("u'\\u%04x'" % x) @@ -19,12 +19,12 @@ def main2(): val = unidecode(u) # f.write("%x | " % x) - if x==0x2028: # U+2028 = LINE SEPARATOR + if x == 0x2028: # U+2028 = LINE SEPARATOR val = "" - elif x==0x2029: # U+2028 = PARAGRAPH SEPARATOR + elif x == 0x2029: # U+2029 = PARAGRAPH SEPARATOR val = "" f.write("%s\n" % val) f.close() -main2() \ No newline at end of file +main() diff --git a/lib/pure/unidecode/unidecode.nim b/lib/pure/unidecode/unidecode.nim index 30aaa7cd8..9affc53f6 100644 --- a/lib/pure/unidecode/unidecode.nim +++ b/lib/pure/unidecode/unidecode.nim @@ -7,30 +7,31 @@ # distribution, for details about the copyright. # -## This module is based on Python's Unidecode module by Tomaz Solc, -## which in turn is based on the ``Text::Unidecode`` Perl module by -## Sean M. Burke -## (http://search.cpan.org/~sburke/Text-Unidecode-0.04/lib/Text/Unidecode.pm ). +## This module is based on Python's [Unidecode](https://pypi.org/project/Unidecode/) +## module by Tomaz Solc, which in turn is based on the +## [Text::Unidecode](https://metacpan.org/pod/Text::Unidecode) +## Perl module by Sean M. Burke. ## -## It provides a single proc that does Unicode to ASCII transliterations: -## It finds the sequence of ASCII characters that is the closest approximation -## to the Unicode string. +## It provides a `unidecode proc <#unidecode,string>`_ that does +## Unicode to ASCII transliterations: It finds the sequence of ASCII characters +## that is the closest approximation to the Unicode string. ## ## For example, the closest to string "Äußerst" in ASCII is "Ausserst". Some ## information is lost in this transformation, of course, since several Unicode -## strings can be transformed in the same ASCII representation. So this is a -## strictly one-way transformation. However a human reader will probably -## still be able to guess what original string was meant from the context. +## strings can be transformed to the same ASCII representation. So this is a +## strictly one-way transformation. However, a human reader will probably +## still be able to guess from the context, what the original string was. ## -## This module needs the data file "unidecode.dat" to work: This file is -## embedded as a resource into your application by default. But you an also -## define the symbol ``--define:noUnidecodeTable`` during compile time and -## use the `loadUnidecodeTable` proc to initialize this module. +## This module needs the data file `unidecode.dat` to work: This file is +## embedded as a resource into your application by default. You can also +## define the symbol `--define:noUnidecodeTable` during compile time and +## use the `loadUnidecodeTable proc <#loadUnidecodeTable>`_ to initialize +## this module. -import unicode +import std/unicode when not defined(noUnidecodeTable): - import strutils + import std/strutils const translationTable = splitLines(slurp"unidecode/unidecode.dat") else: @@ -38,36 +39,26 @@ else: var translationTable: seq[string] proc loadUnidecodeTable*(datafile = "unidecode.dat") = - ## loads the datafile that `unidecode` to work. This is only required if - ## the module was compiled with the ``--define:noUnidecodeTable`` switch. - ## This needs to be called by the main thread before any thread can make a - ## call to `unidecode`. + ## Loads the datafile that `unidecode <#unidecode,string>`_ needs to work. + ## This is only required if the module was compiled with the + ## `--define:noUnidecodeTable` switch. This needs to be called by the + ## main thread before any thread can make a call to `unidecode`. when defined(noUnidecodeTable): newSeq(translationTable, 0xffff) var i = 0 for line in lines(datafile): - translationTable[i] = line.string + translationTable[i] = line inc(i) proc unidecode*(s: string): string = ## Finds the sequence of ASCII characters that is the closest approximation ## to the UTF-8 string `s`. - ## - ## Example: - ## - ## ..code-block:: nim - ## - ## unidecode("北京") - ## - ## Results in: "Bei Jing" - ## + runnableExamples: + doAssert unidecode("北京") == "Bei Jing " + doAssert unidecode("Äußerst") == "Ausserst" + result = "" for r in runes(s): var c = int(r) if c <=% 127: add(result, chr(c)) - elif c <% translationTable.len: add(result, translationTable[c-128]) - -when isMainModule: - #loadUnidecodeTable("lib/pure/unidecode/unidecode.dat") - doAssert unidecode("Äußerst") == "Ausserst" - doAssert unidecode("北京") == "Bei Jing " + elif c <% translationTable.len: add(result, translationTable[c - 128]) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index c50ef0b1b..cfb762258 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -9,11 +9,6 @@ ## :Author: Zahary Karadjov ## -## **Note**: Instead of ``unittest.nim``, please consider to use -## the ``testament`` tool which offers process isolation for your tests. -## Also ``when isMainModule: doAssert conditionHere`` is usually a -## much simpler solution for testing purposes. -## ## This module implements boilerplate to make unit testing easy. ## ## The test status and name is printed after any output or traceback. @@ -22,51 +17,59 @@ ## parent test as failed. Setup and teardown are inherited. Setup can be ## overridden locally. ## -## Compiled test files return the number of failed test as exit code, while -## ``nim c -r <testfile.nim>`` exits with 0 or 1 +## Compiled test files as well as `nim c -r <testfile.nim>` +## exit with 0 for success (no failed tests) or 1 for failure. +## +## Testament +## ========= +## +## Instead of `unittest`, please consider using +## `the Testament tool <testament.html>`_ which offers process isolation for your tests. +## +## Alternatively using `when isMainModule: doAssert conditionHere` is usually a +## much simpler solution for testing purposes. ## ## Running a single test ## ===================== ## ## Specify the test name as a command line argument. ## -## .. code:: -## +## ```cmd ## nim c -r test "my test name" "another test" +## ``` ## ## Multiple arguments can be used. ## ## Running a single test suite ## =========================== ## -## Specify the suite name delimited by ``"::"``. -## -## .. code:: +## Specify the suite name delimited by `"::"`. ## +## ```cmd ## nim c -r test "my test name::" +## ``` ## ## Selecting tests by pattern ## ========================== ## ## A single ``"*"`` can be used for globbing. ## -## Delimit the end of a suite name with ``"::"``. +## Delimit the end of a suite name with `"::"`. ## ## Tests matching **any** of the arguments are executed. ## -## .. code:: -## +## ```cmd ## nim c -r test fast_suite::mytest1 fast_suite::mytest2 ## nim c -r test "fast_suite::mytest*" ## nim c -r test "auth*::" "crypto::hashing*" ## # Run suites starting with 'bug #' and standalone tests starting with '#' ## nim c -r test 'bug #*::' '::#*' +## ``` ## ## Examples ## ======== ## -## .. code:: nim -## +## ```nim ## suite "description for this stuff": ## echo "suite setup: run once before the tests" ## @@ -88,21 +91,35 @@ ## ## test "out of bounds error is thrown on bad access": ## let v = @[1, 2, 3] # you can do initialization here -## expect(IndexError): +## expect(IndexDefect): ## discard v[4] ## ## echo "suite teardown: run once after the tests" +## ``` +## +## Limitations/Bugs +## ================ +## Since `check` will rewrite some expressions for supporting checkpoints +## (namely assigns expressions to variables), some type conversions are not supported. +## For example `check 4.0 == 2 + 2` won't work. But `doAssert 4.0 == 2 + 2` works. +## Make sure both sides of the operator (such as `==`, `>=` and so on) have the same type. +## + +import std/private/since +import std/exitprocs -import - macros, strutils, streams, times, sets, sequtils +when defined(nimPreviewSlimSystem): + import std/assertions -include "system/inclrtl" +import std/[macros, strutils, streams, times, sets, sequtils] when declared(stdout): - import os + import std/os + +const useTerminal = not defined(js) -when not defined(ECMAScript): - import terminal +when useTerminal: + import std/terminal type TestStatus* = enum ## The status of a test when it is done. @@ -128,21 +145,19 @@ type ConsoleOutputFormatter* = ref object of OutputFormatter colorOutput: bool ## Have test results printed in color. - ## Default is true for the non-js target, - ## for which ``stdout`` is a tty. - ## Setting the environment variable - ## ``NIMTEST_COLOR`` to ``always`` or - ## ``never`` changes the default for the - ## non-js target to true or false respectively. - ## The deprecated environment variable - ## ``NIMTEST_NO_COLOR``, when set, - ## changes the default to true, if - ## ``NIMTEST_COLOR`` is undefined. + ## Default is `auto` depending on `isatty(stdout)`, or override it with + ## `-d:nimUnittestColor:auto|on|off`. + ## + ## Deprecated: Setting the environment variable `NIMTEST_COLOR` to `always` + ## or `never` changes the default for the non-js target to true or false respectively. + ## Deprecated: the environment variable `NIMTEST_NO_COLOR`, when set, changes the + ## default to true, if `NIMTEST_COLOR` is undefined. outputLevel: OutputLevel ## Set the verbosity of test results. - ## Default is ``PRINT_ALL``, unless - ## the ``NIMTEST_OUTPUT_LVL`` environment - ## variable is set for the non-js target. + ## Default is `PRINT_ALL`, or override with: + ## `-d:nimUnittestOutputLevel:PRINT_ALL|PRINT_FAILURES|PRINT_NONE`. + ## + ## Deprecated: the `NIMTEST_OUTPUT_LVL` environment variable is set for the non-js target. isInSuite: bool isInTest: bool @@ -155,17 +170,31 @@ type var abortOnError* {.threadvar.}: bool ## Set to true in order to quit ## immediately on fail. Default is false, - ## unless the ``NIMTEST_ABORT_ON_ERROR`` - ## environment variable is set for - ## the non-js target. + ## or override with `-d:nimUnittestAbortOnError:on|off`. + ## + ## Deprecated: can also override depending on whether + ## `NIMTEST_ABORT_ON_ERROR` environment variable is set. checkpoints {.threadvar.}: seq[string] formatters {.threadvar.}: seq[OutputFormatter] testsFilters {.threadvar.}: HashSet[string] disabledParamFiltering {.threadvar.}: bool +const + outputLevelDefault = PRINT_ALL + nimUnittestOutputLevel {.strdefine.} = $outputLevelDefault + nimUnittestColor {.strdefine.} = "auto" ## auto|on|off + nimUnittestAbortOnError {.booldefine.} = false + +template deprecateEnvVarHere() = + # xxx issue a runtime warning to deprecate this envvar. + discard + +abortOnError = nimUnittestAbortOnError when declared(stdout): - abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR") + if existsEnv("NIMTEST_ABORT_ON_ERROR"): + deprecateEnvVarHere() + abortOnError = true method suiteStarted*(formatter: OutputFormatter, suiteName: string) {.base, gcsafe.} = discard @@ -190,41 +219,49 @@ proc delOutputFormatter*(formatter: OutputFormatter) = proc resetOutputFormatters* {.since: (1, 1).} = formatters = @[] - -proc newConsoleOutputFormatter*(outputLevel: OutputLevel = OutputLevel.PRINT_ALL, - colorOutput = true): <//>ConsoleOutputFormatter = + +proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault, + colorOutput = true): ConsoleOutputFormatter = ConsoleOutputFormatter( outputLevel: outputLevel, colorOutput: colorOutput ) -proc defaultConsoleFormatter*(): <//>ConsoleOutputFormatter = +proc colorOutput(): bool = + let color = nimUnittestColor + case color + of "auto": + when declared(stdout): result = isatty(stdout) + else: result = false + of "on": result = true + of "off": result = false + else: raiseAssert $color + when declared(stdout): - # Reading settings - # On a terminal this branch is executed - var envOutLvl = os.getEnv("NIMTEST_OUTPUT_LVL").string - var colorOutput = isatty(stdout) if existsEnv("NIMTEST_COLOR"): + deprecateEnvVarHere() let colorEnv = getEnv("NIMTEST_COLOR") if colorEnv == "never": - colorOutput = false + result = false elif colorEnv == "always": - colorOutput = true + result = true elif existsEnv("NIMTEST_NO_COLOR"): - colorOutput = false - var outputLevel = OutputLevel.PRINT_ALL - if envOutLvl.len > 0: - for opt in countup(low(OutputLevel), high(OutputLevel)): - if $opt == envOutLvl: - outputLevel = opt - break - result = newConsoleOutputFormatter(outputLevel, colorOutput) - else: - result = newConsoleOutputFormatter() + deprecateEnvVarHere() + result = false + +proc defaultConsoleFormatter*(): ConsoleOutputFormatter = + var colorOutput = colorOutput() + var outputLevel = nimUnittestOutputLevel.parseEnum[:OutputLevel] + when declared(stdout): + const a = "NIMTEST_OUTPUT_LVL" + if existsEnv(a): + # xxx issue a warning to deprecate this envvar. + outputLevel = getEnv(a).parseEnum[:OutputLevel] + result = newConsoleOutputFormatter(outputLevel, colorOutput) method suiteStarted*(formatter: ConsoleOutputFormatter, suiteName: string) = template rawPrint() = echo("\n[Suite] ", suiteName) - when not defined(ECMAScript): + when useTerminal: if formatter.colorOutput: styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, suiteName else: rawPrint() @@ -250,8 +287,8 @@ method testEnded*(formatter: ConsoleOutputFormatter, testResult: TestResult) = let prefix = if testResult.suiteName.len > 0: " " else: "" template rawPrint() = echo(prefix, "[", $testResult.status, "] ", testResult.testName) - when not defined(ECMAScript): - if formatter.colorOutput and not defined(ECMAScript): + when useTerminal: + if formatter.colorOutput: var color = case testResult.status of TestStatus.OK: fgGreen of TestStatus.FAILED: fgRed @@ -281,7 +318,7 @@ proc xmlEscape(s: string): string = else: result.add(c) -proc newJUnitOutputFormatter*(stream: Stream): <//>JUnitOutputFormatter = +proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = ## Creates a formatter that writes report to the specified stream in ## JUnit format. ## The ``stream`` is NOT closed automatically when the test are finished, @@ -396,8 +433,6 @@ proc matchFilter(suiteName, testName, filter: string): bool = return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1]) -when defined(testing): export matchFilter - proc shouldRun(currentSuiteName, testName: string): bool = ## Check if a test should be run by matching suiteName and testName against ## test filters. @@ -414,8 +449,7 @@ proc ensureInitialized() = if formatters.len == 0: formatters = @[OutputFormatter(defaultConsoleFormatter())] - if not disabledParamFiltering and not testsFilters.isValid: - testsFilters.init() + if not disabledParamFiltering: when declared(paramCount): # Read tests to run from the command line. for i in 1 .. paramCount(): @@ -439,27 +473,26 @@ template suite*(name, body) {.dirty.} = ## common fixture (``setup``, ``teardown``). The fixture is executed ## for EACH test. ## - ## .. code-block:: nim - ## suite "test suite for addition": - ## setup: - ## let result = 4 + ## ```nim + ## suite "test suite for addition": + ## setup: + ## let result = 4 ## - ## test "2 + 2 = 4": - ## check(2+2 == result) + ## test "2 + 2 = 4": + ## check(2+2 == result) ## - ## test "(2 + -2) != 4": - ## check(2 + -2 != result) + ## test "(2 + -2) != 4": + ## check(2 + -2 != result) ## - ## # No teardown needed + ## # No teardown needed + ## ``` ## ## The suite will run the individual test cases in the order in which ## they were listed. With default global settings the above code prints: ## - ## .. code-block:: - ## - ## [Suite] test suite for addition - ## [OK] 2 + 2 = 4 - ## [OK] (2 + -2) != 4 + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 bind formatters, ensureInitialized, suiteEnded block: @@ -481,23 +514,29 @@ template suite*(name, body) {.dirty.} = finally: suiteEnded() -template exceptionTypeName(e: typed): string = $e.name +proc exceptionTypeName(e: ref Exception): string {.inline.} = + if e == nil: "<foreign exception>" + else: $e.name + +when not declared(setProgramResult): + {.warning: "setProgramResult not available on platform, unittest will not" & + " give failing exit code on test failure".} + template setProgramResult(a: int) = + discard template test*(name, body) {.dirty.} = ## Define a single test case identified by `name`. ## - ## .. code-block:: nim - ## - ## test "roses are red": - ## let roses = "red" - ## check(roses == "red") + ## ```nim + ## test "roses are red": + ## let roses = "red" + ## check(roses == "red") + ## ``` ## ## The above code outputs: ## - ## .. code-block:: - ## - ## [OK] roses are red - bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName + ## [OK] roses are red + bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName, setProgramResult ensureInitialized() @@ -508,23 +547,28 @@ template test*(name, body) {.dirty.} = for formatter in formatters: formatter.testStarted(name) + {.push warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.push warning[BareExcept]:on.} body + {.pop.} except: - when not defined(js): - let e = getCurrentException() - let eTypeDesc = "[" & exceptionTypeName(e) & "]" - checkpoint("Unhandled exception: " & getCurrentExceptionMsg() & " " & eTypeDesc) + let e = getCurrentException() + let eTypeDesc = "[" & exceptionTypeName(e) & "]" + checkpoint("Unhandled exception: " & getCurrentExceptionMsg() & " " & eTypeDesc) + if e == nil: # foreign + fail() + else: var stackTrace {.inject.} = e.getStackTrace() - fail() + fail() finally: if testStatusIMPL == TestStatus.FAILED: - programResult = 1 + setProgramResult 1 let testResult = TestResult( suiteName: when declared(testSuiteName): testSuiteName else: "", testName: name, @@ -532,16 +576,17 @@ template test*(name, body) {.dirty.} = ) testEnded(testResult) checkpoints = @[] + {.pop.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all ## checkpoints encountered so far are printed out. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## check((42, "the Answer to life and everything") == (1, "a")) - ## checkpoint("Checkpoint B") + ## ```nim + ## checkpoint("Checkpoint A") + ## check((42, "the Answer to life and everything") == (1, "a")) + ## checkpoint("Checkpoint B") + ## ``` ## ## outputs "Checkpoint A" once it fails. checkpoints.add(msg) @@ -553,19 +598,18 @@ template fail* = ## failed (change exit code and test status). This template is useful ## for debugging, but is otherwise mostly used internally. Example: ## - ## .. code-block:: nim - ## - ## checkpoint("Checkpoint A") - ## complicatedProcInThread() - ## fail() + ## ```nim + ## checkpoint("Checkpoint A") + ## complicatedProcInThread() + ## fail() + ## ``` ## ## outputs "Checkpoint A" before quitting. - bind ensureInitialized - + bind ensureInitialized, setProgramResult when declared(testStatusIMPL): testStatusIMPL = TestStatus.FAILED else: - programResult = 1 + setProgramResult 1 ensureInitialized() @@ -576,8 +620,7 @@ template fail* = else: formatter.failureOccurred(checkpoints, "") - when declared(programResult): - if abortOnError: quit(programResult) + if abortOnError: quit(1) checkpoints = @[] @@ -587,11 +630,10 @@ template skip* = ## for reasons depending on outer environment, ## or certain application logic conditions or configurations. ## The test code is still executed. - ## - ## .. code-block:: nim - ## - ## if not isGLContextCreated(): - ## skip() + ## ```nim + ## if not isGLContextCreated(): + ## skip() + ## ``` bind checkpoints testStatusIMPL = TestStatus.SKIPPED @@ -601,19 +643,17 @@ macro check*(conditions: untyped): untyped = ## Verify if a statement or a list of statements is true. ## A helpful error message and set checkpoints are printed out on ## failure (if ``outputLevel`` is not ``PRINT_NONE``). - ## Example: - ## - ## .. code-block:: nim - ## - ## import strutils - ## - ## check("AKB48".toLowerAscii() == "akb48") - ## - ## let teams = {'A', 'K', 'B', '4', '8'} - ## - ## check: - ## "AKB48".toLowerAscii() == "akb48" - ## 'C' in teams + runnableExamples: + import std/strutils + + check("AKB48".toLowerAscii() == "akb48") + + let teams = {'A', 'K', 'B', '4', '8'} + + check: + "AKB48".toLowerAscii() == "akb48" + 'C' notin teams + let checked = callsite()[1] template asgn(a: untyped, value: typed) = @@ -631,7 +671,7 @@ macro check*(conditions: untyped): untyped = var counter = 0 - if exp[0].kind == nnkIdent and + if exp[0].kind in {nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym} and $exp[0] in ["not", "in", "notin", "==", "<=", ">=", "<", ">", "!=", "is", "isnot"]: @@ -642,14 +682,15 @@ macro check*(conditions: untyped): untyped = let paramAst = exp[i] if exp[i].kind == nnkIdent: result.printOuts.add getAst(print(argStr, paramAst)) - if exp[i].kind in nnkCallKinds + {nnkDotExpr, nnkBracketExpr}: + if exp[i].kind in nnkCallKinds + {nnkDotExpr, nnkBracketExpr, nnkPar} and + (exp[i].typeKind notin {ntyTypeDesc} or $exp[0] notin ["is", "isnot"]): let callVar = newIdentNode(":c" & $counter) result.assigns.add getAst(asgn(callVar, paramAst)) result.check[i] = callVar result.printOuts.add getAst(print(argStr, callVar)) if exp[i].kind == nnkExprEqExpr: # ExprEqExpr - # Ident !"v" + # Ident "v" # IntLit 2 result.check[i] = exp[i][1] if exp[i].typeKind notin {ntyTypeDesc}: @@ -670,7 +711,9 @@ macro check*(conditions: untyped): untyped = result = quote do: block: `assigns` - if not `check`: + if `check`: + discard + else: checkpoint(`lineinfo` & ": Check failed: " & `callLit`) `printOuts` fail() @@ -679,14 +722,16 @@ macro check*(conditions: untyped): untyped = result = newNimNode(nnkStmtList) for node in checked: if node.kind != nnkCommentStmt: - result.add(newCall(!"check", node)) + result.add(newCall(newIdentNode("check"), node)) else: let lineinfo = newStrLitNode(checked.lineInfo) let callLit = checked.toStrLit result = quote do: - if not `checked`: + if `checked`: + discard + else: checkpoint(`lineinfo` & ": Check failed: " & `callLit`) fail() @@ -704,40 +749,40 @@ macro expect*(exceptions: varargs[typed], body: untyped): untyped = ## Test if `body` raises an exception found in the passed `exceptions`. ## The test passes if the raised exception is part of the acceptable ## exceptions. Otherwise, it fails. - ## Example: - ## - ## .. code-block:: nim - ## - ## import math, random - ## proc defectiveRobot() = - ## randomize() - ## case random(1..4) - ## of 1: raise newException(OSError, "CANNOT COMPUTE!") - ## of 2: discard parseInt("Hello World!") - ## of 3: raise newException(IOError, "I can't do that Dave.") - ## else: assert 2 + 2 == 5 - ## - ## expect IOError, OSError, ValueError, AssertionError: - ## defectiveRobot() - let exp = callsite() + runnableExamples: + import std/[math, random, strutils] + proc defectiveRobot() = + randomize() + case rand(1..4) + of 1: raise newException(OSError, "CANNOT COMPUTE!") + of 2: discard parseInt("Hello World!") + of 3: raise newException(IOError, "I can't do that Dave.") + else: assert 2 + 2 == 5 + + expect IOError, OSError, ValueError, AssertionDefect: + defectiveRobot() + template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} = + {.push warning[BareExcept]:off.} try: + {.push warning[BareExcept]:on.} body + {.pop.} checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") fail() except errorTypes: discard except: - checkpoint(lineInfoLit & ": Expect Failed, unexpected exception was thrown.") + let err = getCurrentException() + checkpoint(lineInfoLit & ": Expect Failed, " & $err.name & " was thrown.") fail() - - var body = exp[exp.len - 1] + {.pop.} var errorTypes = newNimNode(nnkBracket) - for i in countup(1, exp.len - 2): - errorTypes.add(exp[i]) + for exp in exceptions: + errorTypes.add(exp) - result = getAst(expectBody(errorTypes, exp.lineInfo, body)) + result = getAst(expectBody(errorTypes, errorTypes.lineInfo, body)) proc disableParamFiltering* = ## disables filtering tests with the command line params diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 9112671c7..725d5bbd9 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -11,33 +11,40 @@ ## ## A Uniform Resource Identifier (URI) provides a simple and extensible ## means for identifying a resource. A URI can be further classified -## as a locator, a name, or both. The term “Uniform Resource Locator” +## as a locator, a name, or both. The term "Uniform Resource Locator" ## (URL) refers to the subset of URIs. ## -## Basic usage -## =========== +## .. warning:: URI parsers in this module do not perform security validation. ## -## Combine URIs -## ------------- -## .. code-block:: -## import uri -## let host = parseUri("https://nim-lang.org") -## let blog = "/blog.html" -## let bloguri = host / blog -## assert $host == "https://nim-lang.org" -## assert $bloguri == "https://nim-lang.org/blog.html" -## -## Access URI item -## --------------- -## .. code-block:: -## import uri -## let res = parseUri("sftp://127.0.0.1:4343") -## if isAbsolute(res): -## assert res.port == "4343" -## else: -## echo "Wrong format" - -import strutils, parseutils +## # Basic usage + + +## ## Combine URIs +runnableExamples: + let host = parseUri("https://nim-lang.org") + assert $host == "https://nim-lang.org" + assert $(host / "/blog.html") == "https://nim-lang.org/blog.html" + assert $(host / "blog2.html") == "https://nim-lang.org/blog2.html" + +## ## Access URI item +runnableExamples: + let res = parseUri("sftp://127.0.0.1:4343") + assert isAbsolute(res) + assert res.port == "4343" + +## ## Data URI Base64 +runnableExamples: + assert getDataUri("Hello World", "text/plain") == "data:text/plain;charset=utf-8;base64,SGVsbG8gV29ybGQ=" + assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + + +import std/[strutils, parseutils, base64] +import std/private/[since, decode_helpers] + +when defined(nimPreviewSlimSystem): + import std/assertions + + type Url* = distinct string @@ -45,21 +52,29 @@ type scheme*, username*, password*: string hostname*, port*, path*, query*, anchor*: string opaque*: bool + isIpv6*: bool + + UriParseError* = object of ValueError + + +proc uriParseError*(msg: string) {.noreturn.} = + ## Raises a `UriParseError` exception with message `msg`. + raise newException(UriParseError, msg) -proc encodeUrl*(s: string, usePlus = true): string = +func encodeUrl*(s: string, usePlus = true): string = ## Encodes a URL according to RFC3986. ## ## This means that characters in the set - ## ``{'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~'}`` are + ## `{'a'..'z', 'A'..'Z', '0'..'9', '-', '.', '_', '~'}` are ## carried over to the result. - ## All other characters are encoded as ``%xx`` where ``xx`` + ## All other characters are encoded as `%xx` where `xx` ## denotes its hexadecimal value. ## - ## As a special rule, when the value of ``usePlus`` is true, - ## spaces are encoded as ``+`` instead of ``%20``. + ## As a special rule, when the value of `usePlus` is true, + ## spaces are encoded as `+` instead of `%20`. ## ## **See also:** - ## * `decodeUrl proc<#decodeUrl,string>`_ + ## * `decodeUrl func<#decodeUrl,string>`_ runnableExamples: assert encodeUrl("https://nim-lang.org") == "https%3A%2F%2Fnim-lang.org" assert encodeUrl("https://nim-lang.org/this is a test") == "https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test" @@ -75,29 +90,25 @@ proc encodeUrl*(s: string, usePlus = true): string = add(result, '%') add(result, toHex(ord(c), 2)) -proc decodeUrl*(s: string, decodePlus = true): string = +func decodeUrl*(s: string, decodePlus = true): string = ## Decodes a URL according to RFC3986. ## - ## This means that any ``%xx`` (where ``xx`` denotes a hexadecimal - ## value) are converted to the character with ordinal number ``xx``, + ## This means that any `%xx` (where `xx` denotes a hexadecimal + ## value) are converted to the character with ordinal number `xx`, ## and every other character is carried over. + ## If `xx` is not a valid hexadecimal value, it is left intact. ## - ## As a special rule, when the value of ``decodePlus`` is true, ``+`` + ## As a special rule, when the value of `decodePlus` is true, `+` ## characters are converted to a space. ## ## **See also:** - ## * `encodeUrl proc<#encodeUrl,string>`_ + ## * `encodeUrl func<#encodeUrl,string>`_ runnableExamples: assert decodeUrl("https%3A%2F%2Fnim-lang.org") == "https://nim-lang.org" assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test") == "https://nim-lang.org/this is a test" assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test", false) == "https://nim-lang.org/this is a test" - proc handleHexChar(c: char, x: var int) {.inline.} = - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: assert(false) + assert decodeUrl("abc%xyz") == "abc%xyz" result = newString(s.len) var i = 0 @@ -105,11 +116,7 @@ proc decodeUrl*(s: string, decodePlus = true): string = while i < s.len: case s[i] of '%': - var x = 0 - handleHexChar(s[i+1], x) - handleHexChar(s[i+2], x) - inc(i, 2) - result[j] = chr(x) + result[j] = decodePercent(s, i) of '+': if decodePlus: result[j] = ' ' @@ -120,27 +127,28 @@ proc decodeUrl*(s: string, decodePlus = true): string = inc(j) setLen(result, j) -proc encodeQuery*(query: openArray[(string, string)], usePlus = true, - omitEq = true): string = +func encodeQuery*(query: openArray[(string, string)], usePlus = true, + omitEq = true, sep = '&'): string = ## Encodes a set of (key, value) parameters into a URL query string. ## - ## Every (key, value) pair is URL-encoded and written as ``key=value``. If the - ## value is an empty string then the ``=`` is omitted, unless ``omitEq`` is + ## Every (key, value) pair is URL-encoded and written as `key=value`. If the + ## value is an empty string then the `=` is omitted, unless `omitEq` is ## false. - ## The pairs are joined together by a ``&`` character. + ## The pairs are joined together by the `sep` character. ## - ## The ``usePlus`` parameter is passed down to the `encodeUrl` function that + ## The `usePlus` parameter is passed down to the `encodeUrl` function that ## is used for the URL encoding of the string values. ## ## **See also:** - ## * `encodeUrl proc<#encodeUrl,string>`_ + ## * `encodeUrl func<#encodeUrl,string>`_ runnableExamples: assert encodeQuery({: }) == "" assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + assert encodeQuery({"a": "1", "b": ""}, omitEq = false, sep = ';') == "a=1;b=" for elem in query: - # Encode the `key = value` pairs and separate them with a '&' - if result.len > 0: result.add('&') + # Encode the `key = value` pairs and separate them with 'sep' + if result.len > 0: result.add(sep) let (key, val) = elem result.add(encodeUrl(key, usePlus)) # Omit the '=' if the value string is empty @@ -148,7 +156,52 @@ proc encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add('=') result.add(encodeUrl(val, usePlus)) -proc parseAuthority(authority: string, result: var Uri) = +iterator decodeQuery*(data: string, sep = '&'): tuple[key, value: string] = + ## Reads and decodes the query string `data` and yields the `(key, value)` pairs + ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, + ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded + ## value, which was the behavior in Nim < 1.5.1. + runnableExamples: + import std/sequtils + assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("foo=1;bar=2=3", ';')) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] + + proc parseData(data: string, i: int, field: var string, sep: char): int = + result = i + while result < data.len: + let c = data[result] + case c + of '%': add(field, decodePercent(data, result)) + of '+': add(field, ' ') + of '&': break + else: + if c == sep: break + else: add(field, data[result]) + inc(result) + + var i = 0 + var name = "" + var value = "" + # decode everything in one pass: + while i < data.len: + setLen(name, 0) # reuse memory + i = parseData(data, i, name, '=') + setLen(value, 0) # reuse memory + if i < data.len and data[i] == '=': + inc(i) # skip '=' + when defined(nimLegacyParseQueryStrict): + i = parseData(data, i, value, '=') + else: + i = parseData(data, i, value, sep) + yield (name, value) + if i < data.len: + when defined(nimLegacyParseQueryStrict): + if data[i] != '&': + uriParseError("'&' expected at index '$#' for '$#'" % [$i, data]) + inc(i) + +func parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false var inIPv6 = false @@ -167,6 +220,7 @@ proc parseAuthority(authority: string, result: var Uri) = inPort = true of '[': inIPv6 = true + result.isIpv6 = true of ']': inIPv6 = false else: @@ -176,8 +230,7 @@ proc parseAuthority(authority: string, result: var Uri) = result.hostname.add(authority[i]) i.inc -proc parsePath(uri: string, i: var int, result: var Uri) = - +func parsePath(uri: string, i: var int, result: var Uri) = i.inc parseUntil(uri, result.path, {'?', '#'}, i) # The 'mailto' scheme's PATH actually contains the hostname/username @@ -193,31 +246,34 @@ proc parsePath(uri: string, i: var int, result: var Uri) = i.inc # Skip '#' i.inc parseUntil(uri, result.anchor, {}, i) -proc initUri*(): Uri = - ## Initializes a URI with ``scheme``, ``username``, ``password``, - ## ``hostname``, ``port``, ``path``, ``query`` and ``anchor``. +func initUri*(isIpv6 = false): Uri = + ## Initializes a URI with `scheme`, `username`, `password`, + ## `hostname`, `port`, `path`, `query`, `anchor` and `isIpv6`. ## ## **See also:** ## * `Uri type <#Uri>`_ for available fields in the URI type runnableExamples: - var uri2: Uri - assert initUri() == uri2 + var uri2 = initUri(isIpv6 = true) + uri2.scheme = "tcp" + uri2.hostname = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + uri2.port = "8080" + assert $uri2 == "tcp://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8080" result = Uri(scheme: "", username: "", password: "", hostname: "", port: "", - path: "", query: "", anchor: "") + path: "", query: "", anchor: "", isIpv6: isIpv6) -proc resetUri(uri: var Uri) = +func resetUri(uri: var Uri) = for f in uri.fields: when f is string: f.setLen(0) else: f = false -proc parseUri*(uri: string, result: var Uri) = +func parseUri*(uri: string, result: var Uri) = ## Parses a URI. The `result` variable will be cleared before. ## ## **See also:** ## * `Uri type <#Uri>`_ for available fields in the URI type - ## * `initUri proc <#initUri>`_ for initializing a URI + ## * `initUri func <#initUri>`_ for initializing a URI runnableExamples: var res = initUri() parseUri("https://nim-lang.org/docs/manual.html", res) @@ -229,9 +285,9 @@ proc parseUri*(uri: string, result: var Uri) = var i = 0 # Check if this is a reference URI (relative URI) - let doubleSlash = uri.len > 1 and uri[1] == '/' + let doubleSlash = uri.len > 1 and uri[0] == '/' and uri[1] == '/' if i < uri.len and uri[i] == '/': - # Make sure ``uri`` doesn't begin with '//'. + # Make sure `uri` doesn't begin with '//'. if not doubleSlash: parsePath(uri, i, result) return @@ -260,7 +316,7 @@ proc parseUri*(uri: string, result: var Uri) = # Path parsePath(uri, i, result) -proc parseUri*(uri: string): Uri = +func parseUri*(uri: string): Uri = ## Parses a URI and returns it. ## ## **See also:** @@ -273,10 +329,18 @@ proc parseUri*(uri: string): Uri = result = initUri() parseUri(uri, result) -proc removeDotSegments(path: string): string = +func removeDotSegments(path: string): string = + ## Collapses `..` and `.` in `path` in a similar way as done in `os.normalizedPath` + ## Caution: this is buggy. + runnableExamples: + assert removeDotSegments("a1/a2/../a3/a4/a5/./a6/a7/.//./") == "a1/a3/a4/a5/a6/a7/" + assert removeDotSegments("http://www.ai.") == "http://www.ai." + # xxx adapt or reuse `pathnorm.normalizePath(path, '/')` to make this more reliable, but + # taking into account url specificities such as not collapsing leading `//` in scheme + # `https://`. see `turi` for failing tests. if path.len == 0: return "" var collection: seq[string] = @[] - let endsWithSlash = path[path.len-1] == '/' + let endsWithSlash = path.endsWith '/' var i = 0 var currentSegment = "" while i < path.len: @@ -290,7 +354,7 @@ proc removeDotSegments(path: string): string = discard collection.pop() i.inc 3 continue - elif path[i+1] == '/': + elif i + 1 < path.len and path[i+1] == '/': i.inc 2 continue currentSegment.add path[i] @@ -303,7 +367,7 @@ proc removeDotSegments(path: string): string = result = collection.join("/") if endsWithSlash: result.add '/' -proc merge(base, reference: Uri): string = +func merge(base, reference: Uri): string = # http://tools.ietf.org/html/rfc3986#section-5.2.3 if base.hostname != "" and base.path == "": '/' & reference.path @@ -314,7 +378,7 @@ proc merge(base, reference: Uri): string = else: base.path[0 .. lastSegment] & reference.path -proc combine*(base: Uri, reference: Uri): Uri = +func combine*(base: Uri, reference: Uri): Uri = ## Combines a base URI with a reference URI. ## ## This uses the algorithm specified in @@ -324,7 +388,7 @@ proc combine*(base: Uri, reference: Uri): Uri = ## URIs path affect the resulting URI. ## ## **See also:** - ## * `/ proc <#/,Uri,string>`_ for building URIs + ## * `/ func <#/,Uri,string>`_ for building URIs runnableExamples: let foo = combine(parseUri("https://nim-lang.org/foo/bar"), parseUri("/baz")) assert foo.path == "/baz" @@ -365,11 +429,11 @@ proc combine*(base: Uri, reference: Uri): Uri = result.scheme = base.scheme result.anchor = reference.anchor -proc combine*(uris: varargs[Uri]): Uri = +func combine*(uris: varargs[Uri]): Uri = ## Combines multiple URIs together. ## ## **See also:** - ## * `/ proc <#/,Uri,string>`_ for building URIs + ## * `/ func <#/,Uri,string>`_ for building URIs runnableExamples: let foo = combine(parseUri("https://nim-lang.org/"), parseUri("docs/"), parseUri("manual.html")) @@ -379,24 +443,22 @@ proc combine*(uris: varargs[Uri]): Uri = for i in 1 ..< uris.len: result = combine(result, uris[i]) -proc isAbsolute*(uri: Uri): bool = +func isAbsolute*(uri: Uri): bool = ## Returns true if URI is absolute, false otherwise. runnableExamples: - let foo = parseUri("https://nim-lang.org") - assert isAbsolute(foo) == true - let bar = parseUri("nim-lang") - assert isAbsolute(bar) == false + assert parseUri("https://nim-lang.org").isAbsolute + assert not parseUri("nim-lang").isAbsolute return uri.scheme != "" and (uri.hostname != "" or uri.path != "") -proc `/`*(x: Uri, path: string): Uri = +func `/`*(x: Uri, path: string): Uri = ## Concatenates the path specified to the specified URIs path. ## - ## Contrary to the `combine proc <#combine,Uri,Uri>`_ you do not have to worry about + ## Contrary to the `combine func <#combine,Uri,Uri>`_ you do not have to worry about ## the slashes at the beginning and end of the path and URIs path ## respectively. ## ## **See also:** - ## * `combine proc <#combine,Uri,Uri>`_ + ## * `combine func <#combine,Uri,Uri>`_ runnableExamples: let foo = parseUri("https://nim-lang.org/foo/bar") / "/baz" assert foo.path == "/foo/bar/baz" @@ -422,7 +484,7 @@ proc `/`*(x: Uri, path: string): Uri = result.path.add '/' result.path.add(path) -proc `?`*(u: Uri, query: openArray[(string, string)]): Uri = +func `?`*(u: Uri, query: openArray[(string, string)]): Uri = ## Concatenates the query parameters to the specified URI object. runnableExamples: let foo = parseUri("https://example.com") / "foo" ? {"bar": "qux"} @@ -430,309 +492,81 @@ proc `?`*(u: Uri, query: openArray[(string, string)]): Uri = result = u result.query = encodeQuery(query) -proc `$`*(u: Uri): string = +func `$`*(u: Uri): string = ## Returns the string representation of the specified URI object. runnableExamples: - let foo = parseUri("https://nim-lang.org") - assert $foo == "https://nim-lang.org" - result = "" - if u.scheme.len > 0: - result.add(u.scheme) - if u.opaque: - result.add(":") - else: - result.add("://") - if u.username.len > 0: - result.add(u.username) - if u.password.len > 0: - result.add(":") - result.add(u.password) - result.add("@") + assert $parseUri("https://nim-lang.org") == "https://nim-lang.org" + # Get the len of all the parts. + let schemeLen = u.scheme.len + let usernameLen = u.username.len + let passwordLen = u.password.len + let hostnameLen = u.hostname.len + let portLen = u.port.len + let pathLen = u.path.len + let queryLen = u.query.len + let anchorLen = u.anchor.len + # Prepare a string that fits all the parts and all punctuation chars. + # 12 is the max len required by all possible punctuation chars. + result = newStringOfCap( + schemeLen + usernameLen + passwordLen + hostnameLen + portLen + pathLen + queryLen + anchorLen + 12 + ) + # Insert to result. + if schemeLen > 0: + result.add u.scheme + result.add ':' + if not u.opaque: + result.add '/' + result.add '/' + if usernameLen > 0: + result.add u.username + if passwordLen > 0: + result.add ':' + result.add u.password + result.add '@' if u.hostname.endsWith('/'): - result.add(u.hostname[0..^2]) + if u.isIpv6: + result.add '[' + result.add u.hostname[0 .. ^2] + result.add ']' + else: + result.add u.hostname[0 .. ^2] else: - result.add(u.hostname) - if u.port.len > 0: - result.add(":") - result.add(u.port) - if u.path.len > 0: - if u.hostname.len > 0 and u.path[0] != '/': - result.add('/') - result.add(u.path) - if u.query.len > 0: - result.add("?") - result.add(u.query) - if u.anchor.len > 0: - result.add("#") - result.add(u.anchor) - -when isMainModule: - block: - const test1 = "abc\L+def xyz" - doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz" - doAssert decodeUrl(encodeUrl(test1)) == test1 - doAssert encodeUrl(test1, false) == "abc%0A%2Bdef%20xyz" - doAssert decodeUrl(encodeUrl(test1, false), false) == test1 - doAssert decodeUrl(encodeUrl(test1)) == test1 - - block: - let str = "http://localhost" - let test = parseUri(str) - doAssert test.path == "" - - block: - let str = "http://localhost/" - let test = parseUri(str) - doAssert test.path == "/" - - block: - let str = "http://localhost:8080/test" - let test = parseUri(str) - doAssert test.scheme == "http" - doAssert test.port == "8080" - doAssert test.path == "/test" - doAssert test.hostname == "localhost" - doAssert($test == str) - - block: - let str = "foo://username:password@example.com:8042/over/there" & - "/index.dtb?type=animal&name=narwhal#nose" - let test = parseUri(str) - doAssert test.scheme == "foo" - doAssert test.username == "username" - doAssert test.password == "password" - doAssert test.hostname == "example.com" - doAssert test.port == "8042" - doAssert test.path == "/over/there/index.dtb" - doAssert test.query == "type=animal&name=narwhal" - doAssert test.anchor == "nose" - doAssert($test == str) - - block: - # IPv6 address - let str = "foo://[::1]:1234/bar?baz=true&qux#quux" - let uri = parseUri(str) - doAssert uri.scheme == "foo" - doAssert uri.hostname == "::1" - doAssert uri.port == "1234" - doAssert uri.path == "/bar" - doAssert uri.query == "baz=true&qux" - doAssert uri.anchor == "quux" - - block: - let str = "urn:example:animal:ferret:nose" - let test = parseUri(str) - doAssert test.scheme == "urn" - doAssert test.path == "example:animal:ferret:nose" - doAssert($test == str) - - block: - let str = "mailto:username@example.com?subject=Topic" - let test = parseUri(str) - doAssert test.scheme == "mailto" - doAssert test.username == "username" - doAssert test.hostname == "example.com" - doAssert test.query == "subject=Topic" - doAssert($test == str) - - block: - let str = "magnet:?xt=urn:sha1:72hsga62ba515sbd62&dn=foobar" - let test = parseUri(str) - doAssert test.scheme == "magnet" - doAssert test.query == "xt=urn:sha1:72hsga62ba515sbd62&dn=foobar" - doAssert($test == str) - - block: - let str = "/test/foo/bar?q=2#asdf" - let test = parseUri(str) - doAssert test.scheme == "" - doAssert test.path == "/test/foo/bar" - doAssert test.query == "q=2" - doAssert test.anchor == "asdf" - doAssert($test == str) - - block: - let str = "test/no/slash" - let test = parseUri(str) - doAssert test.path == "test/no/slash" - doAssert($test == str) - - block: - let str = "//git@github.com:dom96/packages" - let test = parseUri(str) - doAssert test.scheme == "" - doAssert test.username == "git" - doAssert test.hostname == "github.com" - doAssert test.port == "dom96" - doAssert test.path == "/packages" - - block: - let str = "file:///foo/bar/baz.txt" - let test = parseUri(str) - doAssert test.scheme == "file" - doAssert test.username == "" - doAssert test.hostname == "" - doAssert test.port == "" - doAssert test.path == "/foo/bar/baz.txt" - - # Remove dot segments tests - block: - doAssert removeDotSegments("/foo/bar/baz") == "/foo/bar/baz" - - # Combine tests - block: - let concat = combine(parseUri("http://google.com/foo/bar/"), parseUri("baz")) - doAssert concat.path == "/foo/bar/baz" - doAssert concat.hostname == "google.com" - doAssert concat.scheme == "http" - - block: - let concat = combine(parseUri("http://google.com/foo"), parseUri("/baz")) - doAssert concat.path == "/baz" - doAssert concat.hostname == "google.com" - doAssert concat.scheme == "http" - - block: - let concat = combine(parseUri("http://google.com/foo/test"), parseUri("bar")) - doAssert concat.path == "/foo/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test"), parseUri("/bar")) - doAssert concat.path == "/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test"), parseUri("bar")) - doAssert concat.path == "/foo/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test/"), parseUri("bar")) - doAssert concat.path == "/foo/test/bar" - - block: - let concat = combine(parseUri("http://google.com/foo/test/"), parseUri("bar/")) - doAssert concat.path == "/foo/test/bar/" - - block: - let concat = combine(parseUri("http://google.com/foo/test/"), parseUri("bar/"), - parseUri("baz")) - doAssert concat.path == "/foo/test/bar/baz" - - # `/` tests - block: - let test = parseUri("http://example.com/foo") / "bar/asd" - doAssert test.path == "/foo/bar/asd" - - block: - let test = parseUri("http://example.com/foo/") / "/bar/asd" - doAssert test.path == "/foo/bar/asd" - - # removeDotSegments tests - block: - # empty test - doAssert removeDotSegments("") == "" - - # bug #3207 - block: - doAssert parseUri("http://qq/1").combine(parseUri("https://qqq")).`$` == "https://qqq" - - # bug #4959 - block: - let foo = parseUri("http://example.com") / "/baz" - doAssert foo.path == "/baz" - - # bug found on stream 13/10/17 - block: - let foo = parseUri("http://localhost:9515") / "status" - doAssert $foo == "http://localhost:9515/status" - - # bug #6649 #6652 - block: - var foo = parseUri("http://example.com") - foo.hostname = "example.com" - foo.path = "baz" - doAssert $foo == "http://example.com/baz" - - foo.hostname = "example.com/" - foo.path = "baz" - doAssert $foo == "http://example.com/baz" - - foo.hostname = "example.com" - foo.path = "/baz" - doAssert $foo == "http://example.com/baz" - - foo.hostname = "example.com/" - foo.path = "/baz" - doAssert $foo == "http://example.com/baz" - - foo.hostname = "example.com/" - foo.port = "8000" - foo.path = "baz" - doAssert $foo == "http://example.com:8000/baz" - - foo = parseUri("file:/dir/file") - foo.path = "relative" - doAssert $foo == "file:relative" - - # isAbsolute tests - block: - doAssert "www.google.com".parseUri().isAbsolute() == false - doAssert "http://www.google.com".parseUri().isAbsolute() == true - doAssert "file:/dir/file".parseUri().isAbsolute() == true - doAssert "file://localhost/dir/file".parseUri().isAbsolute() == true - doAssert "urn:ISSN:1535-3613".parseUri().isAbsolute() == true - - # path-relative URL *relative - doAssert "about".parseUri().isAbsolute == false - doAssert "about/staff.html".parseUri().isAbsolute == false - doAssert "about/staff.html?".parseUri().isAbsolute == false - doAssert "about/staff.html?parameters".parseUri().isAbsolute == false - - # absolute-path-relative URL *relative - doAssert "/".parseUri().isAbsolute == false - doAssert "/about".parseUri().isAbsolute == false - doAssert "/about/staff.html".parseUri().isAbsolute == false - doAssert "/about/staff.html?".parseUri().isAbsolute == false - doAssert "/about/staff.html?parameters".parseUri().isAbsolute == false - - # scheme-relative URL *relative - doAssert "//username:password@example.com:8888".parseUri().isAbsolute == false - doAssert "//username@example.com".parseUri().isAbsolute == false - doAssert "//example.com".parseUri().isAbsolute == false - doAssert "//example.com/".parseUri().isAbsolute == false - doAssert "//example.com/about".parseUri().isAbsolute == false - doAssert "//example.com/about/staff.html".parseUri().isAbsolute == false - doAssert "//example.com/about/staff.html?".parseUri().isAbsolute == false - doAssert "//example.com/about/staff.html?parameters".parseUri().isAbsolute == false - - # absolute URL *absolute - doAssert "https://username:password@example.com:8888".parseUri().isAbsolute == true - doAssert "https://username@example.com".parseUri().isAbsolute == true - doAssert "https://example.com".parseUri().isAbsolute == true - doAssert "https://example.com/".parseUri().isAbsolute == true - doAssert "https://example.com/about".parseUri().isAbsolute == true - doAssert "https://example.com/about/staff.html".parseUri().isAbsolute == true - doAssert "https://example.com/about/staff.html?".parseUri().isAbsolute == true - doAssert "https://example.com/about/staff.html?parameters".parseUri().isAbsolute == true - - # encodeQuery tests - block: - doAssert encodeQuery({:}) == "" - doAssert encodeQuery({"foo": "bar"}) == "foo=bar" - doAssert encodeQuery({"foo": "bar & baz"}) == "foo=bar+%26+baz" - doAssert encodeQuery({"foo": "bar & baz"}, usePlus = false) == "foo=bar%20%26%20baz" - doAssert encodeQuery({"foo": ""}) == "foo" - doAssert encodeQuery({"foo": ""}, omitEq = false) == "foo=" - doAssert encodeQuery({"a": "1", "b": "", "c": "3"}) == "a=1&b&c=3" - doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, omitEq = false) == "a=1&b=&c=3" - - block: - var foo = parseUri("http://example.com") / "foo" ? {"bar": "1", "baz": "qux"} - var foo1 = parseUri("http://example.com/foo?bar=1&baz=qux") - doAssert foo == foo1 - - block: - var foo = parseUri("http://example.com") / "foo" ? {"do": "do", "bar": ""} - var foo1 = parseUri("http://example.com/foo?do=do&bar") - doAssert foo == foo1 - - echo("All good!") + if u.isIpv6: + result.add '[' + result.add u.hostname + result.add ']' + else: + result.add u.hostname + if portLen > 0: + result.add ':' + result.add u.port + if pathLen > 0: + if hostnameLen > 0 and u.path[0] != '/': + result.add '/' + result.add u.path + if queryLen > 0: + result.add '?' + result.add u.query + if anchorLen > 0: + result.add '#' + result.add u.anchor + + +proc getDataUri*(data, mime: string, encoding = "utf-8"): string {.since: (1, 3).} = + ## Convenience proc for `base64.encode` returns a standard Base64 Data URI (RFC-2397) + ## + ## **See also:** + ## * `mimetypes <mimetypes.html>`_ for `mime` argument + ## * https://tools.ietf.org/html/rfc2397 + ## * https://en.wikipedia.org/wiki/Data_URI_scheme + runnableExamples: static: assert getDataUri("Nim", "text/plain") == "data:text/plain;charset=utf-8;base64,Tmlt" + assert encoding.len > 0 and mime.len > 0 # Must *not* be URL-Safe, see RFC-2397 + let base64encoded: string = base64.encode(data) + # ("data:".len + ";charset=".len + ";base64,".len) == 22 + result = newStringOfCap(22 + mime.len + encoding.len + base64encoded.len) + result.add "data:" + result.add mime + result.add ";charset=" + result.add encoding + result.add ";base64," + result.add base64encoded diff --git a/lib/pure/volatile.nim b/lib/pure/volatile.nim index b3705a199..a38247c7d 100644 --- a/lib/pure/volatile.nim +++ b/lib/pure/volatile.nim @@ -10,21 +10,25 @@ ## This module contains code for generating volatile loads and stores, ## which are useful in embedded and systems programming. -template volatileLoad*[T](src: ptr T): T = +proc volatileLoad*[T](src: ptr T): T {.inline, noinit.} = ## Generates a volatile load of the value stored in the container `src`. - ## Note that this only effects code generation on `C` like backends - when defined(js): - src[] + ## Note that this only effects code generation on `C` like backends. + when nimvm: + result = src[] else: - var res: T - {.emit: [res, " = (*(", type(src[]), " volatile*)", src, ");"].} - res + when defined(js): + result = src[] + else: + {.emit: [result, " = (*(", typeof(src[]), " volatile*)", src, ");"].} -template volatileStore*[T](dest: ptr T, val: T) = +proc volatileStore*[T](dest: ptr T, val: T) {.inline.} = ## Generates a volatile store into the container `dest` of the value ## `val`. Note that this only effects code generation on `C` like - ## backends - when defined(js): + ## backends. + when nimvm: dest[] = val else: - {.emit: ["*((", type(dest[]), " volatile*)(", dest, ")) = ", val, ";"].} + when defined(js): + dest[] = val + else: + {.emit: ["*((", typeof(dest[]), " volatile*)(", dest, ")) = ", val, ";"].} diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 2a2d19dca..2c1e4e37c 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -9,7 +9,10 @@ ## This module parses an XML document and creates its XML tree representation. -import streams, parsexml, strtabs, xmltree +import std/[streams, parsexml, strtabs, xmltree] + +when defined(nimPreviewSlimSystem): + import std/syncio type XmlError* = object of ValueError ## Exception that is raised @@ -99,11 +102,11 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = of xmlEof: discard proc parseXml*(s: Stream, filename: string, - errors: var seq[string]): XmlNode = + errors: var seq[string], options: set[XmlParseOption] = {reportComments}): XmlNode = ## Parses the XML from stream ``s`` and returns a ``XmlNode``. Every ## occurred parsing error is added to the ``errors`` sequence. var x: XmlParser - open(x, s, filename, {reportComments}) + open(x, s, filename, options) while true: x.next() case x.kind @@ -118,37 +121,37 @@ proc parseXml*(s: Stream, filename: string, break close(x) -proc parseXml*(s: Stream): XmlNode = +proc parseXml*(s: Stream, options: set[XmlParseOption] = {reportComments}): XmlNode = ## Parses the XML from stream ``s`` and returns a ``XmlNode``. All parsing ## errors are turned into an ``XmlError`` exception. var errors: seq[string] = @[] - result = parseXml(s, "unknown_xml_doc", errors) + result = parseXml(s, "unknown_xml_doc", errors, options) if errors.len > 0: raiseInvalidXml(errors) -proc parseXml*(str: string): XmlNode = +proc parseXml*(str: string, options: set[XmlParseOption] = {reportComments}): XmlNode = ## Parses the XML from string ``str`` and returns a ``XmlNode``. All parsing ## errors are turned into an ``XmlError`` exception. - parseXml(newStringStream(str)) + parseXml(newStringStream(str), options) -proc loadXml*(path: string, errors: var seq[string]): XmlNode = +proc loadXml*(path: string, errors: var seq[string], options: set[XmlParseOption] = {reportComments}): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns ## a ``XmlNode``. Every occurred parsing error is added to the ``errors`` ## sequence. var s = newFileStream(path, fmRead) if s == nil: raise newException(IOError, "Unable to read file: " & path) - result = parseXml(s, path, errors) + result = parseXml(s, path, errors, options) -proc loadXml*(path: string): XmlNode = +proc loadXml*(path: string, options: set[XmlParseOption] = {reportComments}): XmlNode = ## Loads and parses XML from file specified by ``path``, and returns ## a ``XmlNode``. All parsing errors are turned into an ``XmlError`` ## exception. var errors: seq[string] = @[] - result = loadXml(path, errors) + result = loadXml(path, errors, options) if errors.len > 0: raiseInvalidXml(errors) when isMainModule: when not defined(testing): - import os + import std/os var errors: seq[string] = @[] var x = loadXml(paramStr(1), errors) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index b23e00fdb..5c0cbc5e4 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -9,32 +9,33 @@ ## A simple XML tree generator. ## -## .. code-block:: -## import xmltree -## -## var g = newElement("myTag") -## g.add newText("some text") -## g.add newComment("this is comment") -## -## var h = newElement("secondTag") -## h.add newEntity("some entity") -## -## let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes -## let k = newXmlTree("treeTag", [g, h], att) -## -## echo k -## # <treeTag key2="second value" key1="first value"> -## # <myTag>some text<!-- this is comment --></myTag> -## # <secondTag>&some entity;</secondTag> -## # </treeTag> -## -## +runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + + var h = newElement("secondTag") + h.add newEntity("some entity") + + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + let k = newXmlTree("treeTag", [g, h], att) + + doAssert $k == """<treeTag key1="first value" key2="second value"> + <myTag>some text<!-- this is comment --></myTag> + <secondTag>&some entity;</secondTag> +</treeTag>""" + ## **See also:** ## * `xmlparser module <xmlparser.html>`_ for high-level XML parsing ## * `parsexml module <parsexml.html>`_ for low-level XML parsing ## * `htmlgen module <htmlgen.html>`_ for html code generator -import macros, strtabs, strutils +import std/private/since +import std/[macros, strtabs, strutils, sequtils] + +when defined(nimPreviewSlimSystem): + import std/assertions + type XmlNode* = ref XmlNodeObj ## An XML tree consisting of XML nodes. @@ -44,6 +45,7 @@ type XmlNodeKind* = enum ## Different kinds of XML nodes. xnText, ## a text element + xnVerbatimText, ## xnElement, ## an element with 0 or more children xnCData, ## a CDATA node xnEntity, ## an entity (like ``&thing;``) @@ -56,7 +58,7 @@ type XmlNodeObj {.acyclic.} = object case k: XmlNodeKind # private, use the kind() proc to read this field. - of xnText, xnComment, xnCData, xnEntity: + of xnText, xnVerbatimText, xnComment, xnCData, xnEntity: fText: string of xnElement: fTag: string @@ -68,11 +70,19 @@ const xmlHeader* = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" ## Header to use for complete XML output. +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k + proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) -proc newElement*(tag: string): XmlNode = +proc newElement*(tag: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`. ## ## See also: @@ -82,14 +92,16 @@ proc newElement*(tag: string): XmlNode = var a = newElement("firstTag") a.add newElement("childTag") assert a.kind == xnElement - assert $a == "<firstTag><childTag /></firstTag>" + assert $a == """<firstTag> + <childTag /> +</firstTag>""" result = newXmlNode(xnElement) result.fTag = tag result.s = @[] # init attributes lazily to save memory -proc newText*(text: string): XmlNode = +proc newText*(text: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`. runnableExamples: var b = newText("my text") @@ -99,7 +111,13 @@ proc newText*(text: string): XmlNode = result = newXmlNode(xnText) result.fText = text -proc newComment*(comment: string): XmlNode = +proc newVerbatimText*(text: sink string): XmlNode {.since: (1, 3).} = + ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`. + ## **Since**: Version 1.3. + result = newXmlNode(xnVerbatimText) + result.fText = text + +proc newComment*(comment: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`. runnableExamples: var c = newComment("my comment") @@ -109,7 +127,7 @@ proc newComment*(comment: string): XmlNode = result = newXmlNode(xnComment) result.fText = comment -proc newCData*(cdata: string): XmlNode = +proc newCData*(cdata: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`. runnableExamples: var d = newCData("my cdata") @@ -129,28 +147,27 @@ proc newEntity*(entity: string): XmlNode = result = newXmlNode(xnEntity) result.fText = entity -proc newXmlTree*(tag: string, children: openArray[XmlNode], +proc newXmlTree*(tag: sink string, children: openArray[XmlNode], attributes: XmlAttributes = nil): XmlNode = ## Creates a new XML tree with `tag`, `children` and `attributes`. ## ## See also: ## * `newElement proc <#newElement,string>`_ ## * [<> macro](#<>.m,untyped) - ## - ## .. code-block:: - ## var g = newElement("myTag") - ## g.add newText("some text") - ## g.add newComment("this is comment") - ## var h = newElement("secondTag") - ## h.add newEntity("some entity") - ## let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes - ## let k = newXmlTree("treeTag", [g, h], att) - ## - ## echo k - ## ## <treeTag key2="second value" key1="first value"> - ## ## <myTag>some text<!-- this is comment --></myTag> - ## ## <secondTag>&some entity;</secondTag> - ## ## </treeTag> + + runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + var h = newElement("secondTag") + h.add newEntity("some entity") + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + let k = newXmlTree("treeTag", [g, h], att) + + doAssert $k == """<treeTag key1="first value" key2="second value"> + <myTag>some text<!-- this is comment --></myTag> + <secondTag>&some entity;</secondTag> +</treeTag>""" result = newXmlNode(xnElement) result.fTag = tag @@ -158,7 +175,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes -proc text*(n: XmlNode): string {.inline.} = +proc text*(n: XmlNode): lent string {.inline.} = ## Gets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -173,10 +190,10 @@ proc text*(n: XmlNode): string {.inline.} = assert $c == "<!-- my comment -->" assert c.text == "my comment" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} result = n.fText -proc `text=`*(n: XmlNode, text: string){.inline.} = +proc `text=`*(n: XmlNode, text: sink string) {.inline.} = ## Sets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -191,10 +208,10 @@ proc `text=`*(n: XmlNode, text: string){.inline.} = e.text = "a new entity text" assert $e == "&a new entity text;" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnVerbatimText, xnComment, xnCData, xnEntity} n.fText = text -proc tag*(n: XmlNode): string {.inline.} = +proc tag*(n: XmlNode): lent string {.inline.} = ## Gets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -207,13 +224,15 @@ proc tag*(n: XmlNode): string {.inline.} = runnableExamples: var a = newElement("firstTag") a.add newElement("childTag") - assert $a == "<firstTag><childTag /></firstTag>" + assert $a == """<firstTag> + <childTag /> +</firstTag>""" assert a.tag == "firstTag" - assert n.k == xnElement + n.expect xnElement result = n.fTag -proc `tag=`*(n: XmlNode, tag: string) {.inline.} = +proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = ## Sets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -225,11 +244,15 @@ proc `tag=`*(n: XmlNode, tag: string) {.inline.} = runnableExamples: var a = newElement("firstTag") a.add newElement("childTag") - assert $a == "<firstTag><childTag /></firstTag>" + assert $a == """<firstTag> + <childTag /> +</firstTag>""" a.tag = "newTag" - assert $a == "<newTag><childTag /></newTag>" + assert $a == """<newTag> + <childTag /> +</newTag>""" - assert n.k == xnElement + n.expect xnElement n.fTag = tag proc rawText*(n: XmlNode): string {.inline.} = @@ -283,55 +306,206 @@ proc innerText*(n: XmlNode): string = proc add*(father, son: XmlNode) {.inline.} = ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type ## ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newText("my text") f.add newElement("sonTag") f.add newEntity("my entity") assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "<myTag>my text<sonTag />&my entity;</myTag>" + + father.expect xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = - ## Insert the child `son` to a given position in `father`. + ## Inserts the child `son` to a given position in `father`. ## - ## `father` and `son` must be of `xnElement` kind. + ## `father` must be of `xnElement` kind. ## ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: - from strutils import unindent var f = newElement("myTag") f.add newElement("first") f.insert(newElement("second"), 0) - assert ($f).unindent == "<myTag>\n<second />\n<first />\n</myTag>" + assert $f == """<myTag> + <second /> + <first /> +</myTag>""" - assert father.k == xnElement and son.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) -proc delete*(n: XmlNode, i: Natural) {.noSideEffect.} = - ## Delete the `i`'th child of `n`. +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """<myTag> + <second /> + <third /> + <first /> +</myTag>""" + + father.expect xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + +proc delete*(n: XmlNode, i: Natural) = + ## Deletes the `i`'th child of `n`. ## ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") f.insert(newElement("second"), 0) f.delete(0) - assert $f == "<myTag><first /></myTag>" + assert $f == """<myTag> + <first /> +</myTag>""" - assert n.k == xnElement + n.expect xnElement n.s.delete(i) +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """<myTag> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """<myTag> + <third /> + <fourth /> + <first /> +</myTag>""" + + n.expect xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) + proc len*(n: XmlNode): int {.inline.} = ## Returns the number of `n`'s children. runnableExamples: @@ -359,37 +533,35 @@ proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = assert $f[1] == "<first />" assert $f[0] == "<second />" - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = ## Returns the `i`'th child of `n` so that it can be modified. - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc clear*(n: var XmlNode) = - ## Recursively clear all children of an XmlNode. - ## - ## .. code-block:: - ## var g = newElement("myTag") - ## g.add newText("some text") - ## g.add newComment("this is comment") - ## - ## var h = newElement("secondTag") - ## h.add newEntity("some entity") - ## - ## let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes - ## var k = newXmlTree("treeTag", [g, h], att) + ## Recursively clears all children of an XmlNode. ## - ## echo k - ## ## <treeTag key2="second value" key1="first value"> - ## ## <myTag>some text<!-- this is comment --></myTag> - ## ## <secondTag>&some entity;</secondTag> - ## ## </treeTag> - ## - ## clear(k) - ## echo k - ## ## <treeTag key2="second value" key1="first value" /> + runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + + var h = newElement("secondTag") + h.add newEntity("some entity") + + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + var k = newXmlTree("treeTag", [g, h], att) + + doAssert $k == """<treeTag key1="first value" key2="second value"> + <myTag>some text<!-- this is comment --></myTag> + <secondTag>&some entity;</secondTag> +</treeTag>""" + + clear(k) + doAssert $k == """<treeTag key1="first value" key2="second value" />""" for i in 0 ..< n.len: clear(n[i]) @@ -399,45 +571,43 @@ proc clear*(n: var XmlNode) = iterator items*(n: XmlNode): XmlNode {.inline.} = ## Iterates over all direct children of `n`. - ## - ## **Examples:** - ## - ## .. code-block:: - ## var g = newElement("myTag") - ## g.add newText("some text") - ## g.add newComment("this is comment") - ## - ## var h = newElement("secondTag") - ## h.add newEntity("some entity") - ## g.add h - ## - ## assert $g == "<myTag>some text<!-- this is comment --><secondTag>&some entity;</secondTag></myTag>" - ## for x in g: # the same as `for x in items(g):` - ## echo x - ## - ## # some text - ## # <!-- this is comment --> - ## # <secondTag>&some entity;<![CDATA[some cdata]]></secondTag> - assert n.k == xnElement + runnableExamples: + var g = newElement("myTag") + g.add newText("some text") + g.add newComment("this is comment") + + var h = newElement("secondTag") + h.add newEntity("some entity") + g.add h + + assert $g == "<myTag>some text<!-- this is comment --><secondTag>&some entity;</secondTag></myTag>" + + # for x in g: # the same as `for x in items(g):` + # echo x + + # some text + # <!-- this is comment --> + # <secondTag>&some entity;<![CDATA[some cdata]]></secondTag> + + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = ## Iterates over all direct children of `n` so that they can be modified. - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, val: string]]): XmlAttributes = ## Converts `{key: value}` pairs into `XmlAttributes`. ## - ## .. code-block:: - ## let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes - ## var j = newElement("myTag") - ## j.attrs = att - ## - ## echo j - ## ## <myTag key2="second value" key1="first value" /> + runnableExamples: + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes + var j = newElement("myTag") + j.attrs = att + + doAssert $j == """<myTag key1="first value" key2="second value" />""" newStringTable(keyValuePairs) @@ -457,7 +627,7 @@ proc attrs*(n: XmlNode): XmlAttributes {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = @@ -474,7 +644,7 @@ proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = @@ -491,7 +661,7 @@ proc attrsLen*(n: XmlNode): int {.inline.} = j.attrs = att assert j.attrsLen == 2 - assert n.k == xnElement + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) proc attr*(n: XmlNode, name: string): string = @@ -509,7 +679,7 @@ proc attr*(n: XmlNode, name: string): string = assert j.attr("key1") == "first value" assert j.attr("key2") == "second value" - assert n.kind == xnElement + n.expect xnElement if n.attrs == nil: return "" return n.attrs.getOrDefault(name) @@ -541,15 +711,15 @@ proc escape*(s: string): string = ## ## Escapes these characters: ## - ## ------------ ------------------- + ## ============ =================== ## char is converted to - ## ------------ ------------------- + ## ============ =================== ## ``<`` ``<`` ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` ## ``'`` ``'`` - ## ------------ ------------------- + ## ============ =================== ## ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) @@ -558,25 +728,14 @@ proc escape*(s: string): string = proc addIndent(result: var string, indent: int, addNewLines: bool) = if addNewLines: result.add("\n") - for i in 1..indent: result.add(' ') + for i in 1 .. indent: + result.add(' ') -proc noWhitespace(n: XmlNode): bool = - for i in 0..n.len-1: - if n[i].kind in {xnText, xnEntity}: return true - -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, - addNewLines = true) = - ## Adds the textual representation of `n` to string `result`. - runnableExamples: - var - a = newElement("firstTag") - b = newText("my text") - c = newComment("my comment") - s = "" - s.add(c) - s.add(a) - s.add(b) - assert s == "<!-- my comment --><firstTag />my text" +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = + proc noWhitespace(n: XmlNode): bool = + for i in 0 ..< n.len: + if n[i].kind in {xnText, xnVerbatimText, xnEntity}: return true proc addEscapedAttr(result: var string, s: string) = # `addEscaped` alternative with less escaped characters. @@ -590,8 +749,18 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, else: result.add(c) if n == nil: return + case n.k of xnElement: + if indent > 0 and not lastNodeIsText: + result.addIndent(indent, addNewLines) + + let + addNewLines = if n.noWhitespace(): + false + else: + addNewLines + result.add('<') result.add(n.fTag) if not isNil(n.fAttr): @@ -601,29 +770,32 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add("=\"") result.addEscapedAttr(val) result.add('"') - if n.len > 0: - result.add('>') - if n.len > 1: - if noWhitespace(n): - # for mixed leaves, we cannot output whitespace for readability, - # because this would be wrong. For example: ``a<b>b</b>`` is - # different from ``a <b>b</b>``. - for i in 0..n.len-1: - result.add(n[i], indent+indWidth, indWidth, addNewLines) - else: - for i in 0..n.len-1: - result.addIndent(indent+indWidth, addNewLines) - result.add(n[i], indent+indWidth, indWidth, addNewLines) - result.addIndent(indent, addNewLines) - else: - result.add(n[0], indent+indWidth, indWidth, addNewLines) - result.add("</") - result.add(n.fTag) - result.add(">") - else: + + if n.len == 0: result.add(" />") + return + + let + indentNext = if n.noWhitespace(): + indent + else: + indent+indWidth + result.add('>') + var lastNodeIsText = false + for i in 0 ..< n.len: + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = (n[i].kind == xnText) or (n[i].kind == xnVerbatimText) + + if not n.noWhitespace(): + result.addIndent(indent, addNewLines) + + result.add("</") + result.add(n.fTag) + result.add(">") of xnText: result.addEscaped(n.fText) + of xnVerbatimText: + result.add(n.fText) of xnComment: result.add("<!-- ") result.addEscaped(n.fText) @@ -637,6 +809,21 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "<!-- my comment --><firstTag />my text" + result.addImpl(n, indent, indWidth, addNewLines) + proc `$`*(n: XmlNode): string = ## Converts `n` into its string representation. ## @@ -655,7 +842,7 @@ proc child*(n: XmlNode, name: string): XmlNode = f.add newElement("thirdSon") assert $(f.child("secondSon")) == "<secondSon />" - assert n.kind == xnElement + n.expect xnElement for i in items(n): if i.kind == xnElement: if i.tag == name: @@ -691,7 +878,7 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], a.findAll("BAD", s, caseInsensitive = true) assert $s == "@[<bad>c text</bad>, <BAD>d text</BAD>]" - assert n.k == xnElement + n.expect xnElement for child in n.items(): if child.k != xnElement: continue @@ -721,7 +908,7 @@ proc findAll*(n: XmlNode, tag: string, caseInsensitive = false): seq[XmlNode] = newSeq(result, 0) findAll(n, tag, result, caseInsensitive) -proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = +proc xmlConstructor(a: NimNode): NimNode = if a.kind == nnkCall: result = newCall("newXmlTree", toStrLit(a[0])) var attrs = newNimNode(nnkBracket, a) @@ -748,16 +935,12 @@ proc xmlConstructor(a: NimNode): NimNode {.compileTime.} = macro `<>`*(x: untyped): untyped = ## Constructor macro for XML. Example usage: ## - ## .. code-block:: nim + ## ```nim ## <>a(href="http://nim-lang.org", newText("Nim rules.")) + ## ``` ## - ## Produces an XML tree for:: + ## Produces an XML tree for: ## - ## <a href="http://nim-lang.org">Nim rules.</a> + ## <a href="http://nim-lang.org">Nim rules.</a> ## result = xmlConstructor(x) - - -when isMainModule: - assert """<a href="http://nim-lang.org">Nim rules.</a>""" == - $(<>a(href = "http://nim-lang.org", newText("Nim rules."))) diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim new file mode 100644 index 000000000..963451efe --- /dev/null +++ b/lib/std/appdirs.nim @@ -0,0 +1,94 @@ +## This module implements helpers for determining special directories used by apps. + +## .. importdoc:: paths.nim + +from std/private/osappdirs import nil +import std/paths +import std/envvars + +proc getHomeDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getHomeDir()) + +proc getDataDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + result = Path(osappdirs.getDataDir()) + +proc getConfigDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + result = Path(osappdirs.getConfigDir()) + +proc getCacheDir*(): Path {.inline.} = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + # follows https://crates.io/crates/platform-dirs + result = Path(osappdirs.getCacheDir()) + +proc getCacheDir*(app: Path): Path {.inline.} = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + result = Path(osappdirs.getCacheDir(app.string)) + +proc getTempDir*(): Path {.inline, tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## .. Note:: This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + result = Path(osappdirs.getTempDir()) diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim new file mode 100644 index 000000000..56c37d205 --- /dev/null +++ b/lib/std/assertions.nim @@ -0,0 +1,122 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" + include "system/fatal" + +## This module implements assertion handling. + +import std/private/miscdollars +# --------------------------------------------------------------------------- +# helpers + +type InstantiationInfo = tuple[filename: string, line: int, column: int] + +proc `$`(info: InstantiationInfo): string = + # The +1 is needed here + # instead of overriding `$` (and changing its meaning), consider explicit name. + result = "" + result.toLocation(info.filename, info.line, info.column + 1) + +# --------------------------------------------------------------------------- + + +proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = + ## Raises an `AssertionDefect` with `msg`. + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) + +proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = + ## Raises an `AssertionDefect` with `msg`, but this is hidden + ## from the effect system. Called when an assertion failed. + raiseAssert(msg) + +template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = + when enabled: + const + loc = instantiationInfo(fullPaths = compileOption("excessiveStackTrace")) + ploc = $loc + bind instantiationInfo + mixin failedAssertImpl + {.line: loc.}: + if not cond: + failedAssertImpl(ploc & " `" & expr & "` " & msg) + +template assert*(cond: untyped, msg = "") = + ## Raises `AssertionDefect` with `msg` if `cond` is false. Note + ## that `AssertionDefect` is hidden from the effect system, so it doesn't + ## produce `{.raises: [AssertionDefect].}`. This exception is only supposed + ## to be caught by unit testing frameworks. + ## + ## No code will be generated for `assert` when passing `-d:danger` (implied by `--assertions:off`). + ## See `command line switches <nimc.html#compiler-usage-commandminusline-switches>`_. + runnableExamples: assert 1 == 1 + runnableExamples("--assertions:off"): + assert 1 == 2 # no code generated, no failure here + runnableExamples("-d:danger"): assert 1 == 2 # ditto + assertImpl(cond, msg, astToStr(cond), compileOption("assertions")) + +template doAssert*(cond: untyped, msg = "") = + ## Similar to `assert <#assert.t,untyped,string>`_ but is always turned on regardless of `--assertions`. + runnableExamples: + doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off` + assertImpl(cond, msg, astToStr(cond), true) + +template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = + ## Sets an assertion failure handler that will intercept any assert + ## statements following `onFailedAssert` in the current scope. + runnableExamples: + type MyError = object of CatchableError + lineinfo: tuple[filename: string, line: int, column: int] + # block-wide policy to change the failed assert exception type in order to + # include a lineinfo + onFailedAssert(msg): + raise (ref MyError)(msg: msg, lineinfo: instantiationInfo(-2)) + doAssertRaises(MyError): doAssert false + when not defined(nimHasTemplateRedefinitionPragma): + {.pragma: redefine.} + template failedAssertImpl(msgIMPL: string): untyped {.dirty, redefine.} = + let msg = msgIMPL + code + +template doAssertRaises*(exception: typedesc, code: untyped) = + ## Raises `AssertionDefect` if specified `code` does not raise `exception`. + runnableExamples: + doAssertRaises(ValueError): raise newException(ValueError, "Hello World") + doAssertRaises(CatchableError): raise newException(ValueError, "Hello World") + doAssertRaises(AssertionDefect): doAssert false + var wrong = false + const begin = "expected raising '" & astToStr(exception) & "', instead" + const msgEnd = " by: " & astToStr(code) + template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.push warning[BareExcept]:off.} + when Exception is exception: + try: + if true: + code + wrong = true + except Exception as e: discard + except: raisedForeign() + else: + try: + if true: + code + wrong = true + except exception: + discard + except Exception as e: + mixin `$` # alternatively, we could define $cstring in this module + raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) + except: raisedForeign() + {.pop.} + if wrong: + raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim new file mode 100644 index 000000000..0ba4619e5 --- /dev/null +++ b/lib/std/cmdline.nim @@ -0,0 +1,313 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module contains system facilities for reading command +## line parameters. + +## **See also:** +## * `parseopt module <parseopt.html>`_ for command-line parser beyond +## `parseCmdLine proc`_ + + +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/widestrs + +when defined(nodejs): + from std/private/oscommon import ReadDirEffect + + +const weirdTarget = defined(nimscript) or defined(js) + + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix +else: + {.error: "The cmdline module has not been implemented for the target platform.".} + + +# Needed by windows in order to obtain the command line for targets +# other than command line targets +when defined(windows) and not weirdTarget: + template getCommandLine*(): untyped = getCommandLineW() + + +proc parseCmdLine*(c: string): seq[string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a `command line`:idx: into several components. + ## + ## **Note**: This proc is only occasionally useful, better use the + ## `parseopt module <parseopt.html>`_. + ## + ## On Windows, it uses the `following parsing rules + ## <http://msdn.microsoft.com/en-us/library/17w5ykft.aspx>`_: + ## + ## * Arguments are delimited by white space, which is either a space or a tab. + ## * The caret character (^) is not recognized as an escape character or + ## delimiter. The character is handled completely by the command-line parser + ## in the operating system before being passed to the argv array in the + ## program. + ## * A string surrounded by double quotation marks ("string") is interpreted + ## as a single argument, regardless of white space contained within. A + ## quoted string can be embedded in an argument. + ## * A double quotation mark preceded by a backslash (\") is interpreted as a + ## literal double quotation mark character ("). + ## * Backslashes are interpreted literally, unless they immediately precede + ## a double quotation mark. + ## * If an even number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is interpreted as a string delimiter. + ## * If an odd number of backslashes is followed by a double quotation mark, + ## one backslash is placed in the argv array for every pair of backslashes, + ## and the double quotation mark is "escaped" by the remaining backslash, + ## causing a literal double quotation mark (") to be placed in argv. + ## + ## On Posix systems, it uses the following parsing rules: + ## Components are separated by whitespace unless the whitespace + ## occurs within ``"`` or ``'`` quotes. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while i < c.len and c[i] in {' ', '\t', '\l', '\r'}: inc(i) + if i >= c.len: break + when defined(windows): + # parse a single argument according to the above rules: + var inQuote = false + while i < c.len: + case c[i] + of '\\': + var j = i + while j < c.len and c[j] == '\\': inc(j) + if j < c.len and c[j] == '"': + for k in 1..(j-i) div 2: a.add('\\') + if (j-i) mod 2 == 0: + i = j + else: + a.add('"') + i = j+1 + else: + a.add(c[i]) + inc(i) + of '"': + inc(i) + if not inQuote: inQuote = true + elif i < c.len and c[i] == '"': + a.add(c[i]) + inc(i) + else: + inQuote = false + break + of ' ', '\t': + if not inQuote: break + a.add(c[i]) + inc(i) + else: + a.add(c[i]) + inc(i) + else: + case c[i] + of '\'', '\"': + var delim = c[i] + inc(i) # skip ' or " + while i < c.len and c[i] != delim: + add a, c[i] + inc(i) + if i < c.len: inc(i) + else: + while i < c.len and c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, move a) + +when defined(nimdoc): + # Common forward declaration docstring block for parameter retrieval procs. + proc paramCount*(): int {.tags: [ReadIOEffect].} = + ## Returns the number of `command line arguments`:idx: given to the + ## application. + ## + ## Unlike `argc`:idx: in C, if your binary was called without parameters this + ## will return zero. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramCount): + ## # Use paramCount() here + ## else: + ## # Do something else! + ## ``` + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + ## Returns the `i`-th `command line argument`:idx: given to the application. + ## + ## `i` should be in the range `1..paramCount()`, the `IndexDefect` + ## exception will be raised for invalid values. Instead of iterating + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. + ## + ## Similarly to `argv`:idx: in C, + ## it is possible to call `paramStr(0)` but this will return OS specific + ## contents (usually the name of the invoked executable). You should avoid + ## this and call `getAppFilename() <os.html#getAppFilename>`_ instead. + ## + ## **Availability**: When generating a dynamic library (see `--app:lib`) on + ## Posix this proc is not defined. + ## Test for availability using `declared() <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(paramStr): + ## # Use paramStr() here + ## else: + ## # Do something else! + ## ``` + +elif defined(nimscript): discard +elif defined(nodejs): + type Argv = object of JsRoot + let argv {.importjs: "process.argv".} : Argv + proc len(argv: Argv): int {.importjs: "#.length".} + proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} + + proc paramCount*(): int {.tags: [ReadDirEffect].} = + result = argv.len - 2 + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + let i = i + 1 + if i < argv.len and i >= 0: + result = $argv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) +elif defined(windows): + # Since we support GUI applications with Nim, we sometimes generate + # a WinMain entry proc. But a WinMain proc has no access to the parsed + # command line arguments. The way to get them differs. Thus we parse them + # ourselves. This has the additional benefit that the program's behaviour + # is always the same -- independent of the used C compiler. + var + ownArgv {.threadvar.}: seq[string] + ownParsedArgv {.threadvar.}: bool + + proc paramCount*(): int {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + result = ownArgv.len-1 + + proc paramStr*(i: int): string {.rtl, extern: "nos$1", + tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if not ownParsedArgv: + ownArgv = parseCmdLine($getCommandLine()) + ownParsedArgv = true + if i < ownArgv.len and i >= 0: + result = ownArgv[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, ownArgv.len-1)) + +elif defined(genode): + proc paramStr*(i: int): string = + raise newException(OSError, "paramStr is not implemented on Genode") + + proc paramCount*(): int = + raise newException(OSError, "paramCount is not implemented on Genode") +elif weirdTarget or (defined(posix) and appType == "lib"): + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramStr is not implemented on current platform") + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + raise newException(OSError, "paramCount is not implemented on current platform") +elif not defined(createNimRtl) and + not(defined(posix) and appType == "lib"): + # On Posix, there is no portable way to get the command line from a DLL. + var + cmdCount {.importc: "cmdCount".}: cint + cmdLine {.importc: "cmdLine".}: cstringArray + + proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + if i < cmdCount and i >= 0: + result = $cmdLine[i] + else: + raise newException(IndexDefect, formatErrorIndexBound(i, cmdCount-1)) + + proc paramCount*(): int {.tags: [ReadIOEffect].} = + # Docstring in nimdoc block. + result = cmdCount-1 + +when declared(paramCount) or defined(nimdoc): + proc commandLineParams*(): seq[string] = + ## Convenience proc which returns the command line parameters. + ## + ## This returns **only** the parameters. If you want to get the application + ## executable filename, call `getAppFilename() <os.html#getAppFilename>`_. + ## + ## **Availability**: On Posix there is no portable way to get the command + ## line from a DLL and thus the proc isn't defined in this environment. You + ## can test for its availability with `declared() + ## <system.html#declared,untyped>`_. + ## + ## See also: + ## * `parseopt module <parseopt.html>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc <os.html#getAppFilename>`_ + ## + ## **Examples:** + ## + ## ```nim + ## when declared(commandLineParams): + ## # Use commandLineParams() here + ## else: + ## # Do something else! + ## ``` + result = @[] + for i in 1..paramCount(): + result.add(paramStr(i)) +else: + proc commandLineParams*(): seq[string] {.error: + "commandLineParams() unsupported by dynamic libraries".} = + discard diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim new file mode 100644 index 000000000..6d8bd22f4 --- /dev/null +++ b/lib/std/compilesettings.nim @@ -0,0 +1,66 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows querying the compiler about +## diverse configuration settings. See also `compileOption`. + +# Note: Only add new enum values at the end to ensure binary compatibility with +# other Nim compiler versions! + +type + SingleValueSetting* {.pure.} = enum ## \ + ## settings resulting in a single string value + arguments, ## experimental: the arguments passed after '-r' + outFile, ## experimental: the output file + outDir, ## the output directory + nimcacheDir, ## the location of the 'nimcache' directory + projectName, ## the project's name that is being compiled + projectPath, ## experimental: some path to the project that is being compiled + projectFull, ## the full path to the project that is being compiled + command, ## experimental: the command (e.g. 'c', 'cpp', 'doc') passed to + ## the Nim compiler + commandLine, ## experimental: the command line passed to Nim + linkOptions, ## additional options passed to the linker + compileOptions, ## additional options passed to the C/C++ compiler + ccompilerPath ## the path to the C/C++ compiler + backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` + ## and `nim js` would imply backend=js + libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 + gc {.deprecated.} ## gc selected + mm ## memory management selected + + MultipleValueSetting* {.pure.} = enum ## \ + ## settings resulting in a seq of string values + nimblePaths, ## the nimble path(s) + searchPaths, ## the search path for modules + lazyPaths, ## experimental: even more paths + commandArgs, ## the arguments passed to the Nim compiler + cincludes, ## the #include paths passed to the C/C++ compiler + clibs ## libraries passed to the C/C++ compiler + +proc querySetting*(setting: SingleValueSetting): string {. + compileTime, noSideEffect.} = + ## Can be used to get a string compile-time option. + ## + ## See also: + ## * `compileOption <system.html#compileOption,string>`_ for `on|off` options + ## * `compileOption <system.html#compileOption,string,string>`_ for enum options + ## + runnableExamples: + const nimcache = querySetting(SingleValueSetting.nimcacheDir) + +proc querySettingSeq*(setting: MultipleValueSetting): seq[string] {. + compileTime, noSideEffect.} = + ## Can be used to get a multi-string compile-time option. + ## + ## See also: + ## * `compileOption <system.html#compileOption,string>`_ for `on|off` options + ## * `compileOption <system.html#compileOption,string,string>`_ for enum options + runnableExamples: + const nimblePaths = querySettingSeq(MultipleValueSetting.nimblePaths) diff --git a/lib/std/decls.nim b/lib/std/decls.nim new file mode 100644 index 000000000..bb7ec3593 --- /dev/null +++ b/lib/std/decls.nim @@ -0,0 +1,31 @@ +## This module implements syntax sugar for some declarations. + +import std/macros + +macro byaddr*(sect) = + ## Allows a syntax for l-value references, being an exact analog to + ## `auto& a = ex;` in C++. + ## + ## .. warning:: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. + runnableExamples: + var s = @[10, 11, 12] + var a {.byaddr.} = s[0] + a += 100 + assert s == @[110, 11, 12] + assert a is int + var b {.byaddr.}: int = s[0] + assert a.addr == b.addr + expectLen sect, 1 + let def = sect[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = addr(`ex`) + template `lhs`: untyped = tmp[] + result.copyLineInfo(def) diff --git a/lib/std/dirs.nim b/lib/std/dirs.nim new file mode 100644 index 000000000..380d6d08f --- /dev/null +++ b/lib/std/dirs.nim @@ -0,0 +1,135 @@ +## This module implements directory handling. + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir, + moveDir, walkDir, setCurrentDir, + walkDirRec, PathComponent + +export PathComponent + +proc dirExists*(dir: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + result = dirExists(dir.string) + +proc createDir*(dir: Path) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `moveDir proc`_ + createDir(dir.string) + +proc existsOrCreateDir*(dir: Path): bool {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + result = existsOrCreateDir(dir.string) + +proc removeDir*(dir: Path, checkDir = false + ) {.inline, tags: [WriteDirEffect, ReadDirEffect].} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `removeFile proc <files.html#removeFile>`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + removeDir(dir.string, checkDir) + +proc moveDir*(source, dest: Path) {.inline, tags: [ReadIOEffect, WriteIOEffect].} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc <files.html#moveFile>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + moveDir(source.string, dest.string) + +iterator walkDir*(dir: Path; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + for (k, p) in walkDir(dir.string, relative, checkDir, skipSpecial): + yield (k, Path(p)) + +iterator walkDirRec*(dir: Path, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + Path {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkDir iterator`_ + for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, + checkDir, skipSpecial): + yield Path(p) + +proc setCurrentDir*(newDir: Path) {.inline, tags: [].} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getCurrentDir proc <paths.html#getCurrentDir>`_ + osdirs.setCurrentDir(newDir.string) diff --git a/lib/std/editdistance.nim b/lib/std/editdistance.nim index 0cb5edc9b..40c0017ae 100644 --- a/lib/std/editdistance.nim +++ b/lib/std/editdistance.nim @@ -10,26 +10,27 @@ ## This module implements an algorithm to compute the ## `edit distance`:idx: between two Unicode strings. -import unicode +import std/unicode proc editDistance*(a, b: string): int {.noSideEffect.} = - ## Returns the unicode-rune edit distance between ``a`` and ``b``. + ## Returns the **unicode-rune** edit distance between `a` and `b`. ## ## This uses the `Levenshtein`:idx: distance algorithm with only a linear ## memory overhead. - if len(a) > len(b): - # make ``b`` the longer string + runnableExamples: static: doAssert editdistance("Kitten", "Bitten") == 1 + if runeLen(a) > runeLen(b): + # make `b` the longer string return editDistance(b, a) # strip common prefix var - iStart = 0 ## The character starting index of the first rune in both strings ``a`` and ``b`` + iStart = 0 ## The character starting index of the first rune in both strings `a` and `b` iNextA = 0 iNextB = 0 runeA, runeB: Rune - lenRunesA = 0 ## The number of relevant runes in string ``a``. - lenRunesB = 0 ## The number of relevant runes in string ``b``. + lenRunesA = 0 ## The number of relevant runes in string `a`. + lenRunesB = 0 ## The number of relevant runes in string `b`. block commonPrefix: - # ``a`` is the shorter string + # `a` is the shorter string while iStart < len(a): iNextA = iStart a.fastRuneAt(iNextA, runeA, doInc = true) @@ -43,9 +44,9 @@ proc editDistance*(a, b: string): int {.noSideEffect.} = var # we know that we are either at the start of the strings # or that the current value of runeA is not equal to runeB - # => start search for common suffix after the current rune (``i_next_*``) - iEndA = iNextA ## The exclusive upper index bound of string ``a``. - iEndB = iNextB ## The exclusive upper index bound of string ``b``. + # => start search for common suffix after the current rune (`i_next_*`) + iEndA = iNextA ## The exclusive upper index bound of string `a`. + iEndB = iNextB ## The exclusive upper index bound of string `b`. iCurrentA = iNextA iCurrentB = iNextB block commonSuffix: @@ -68,8 +69,8 @@ proc editDistance*(a, b: string): int {.noSideEffect.} = addRunesB = 0 iCurrentA = iNextA iCurrentB = iNextB - if iCurrentA >= len(a): # ``a`` exhausted - if iCurrentB < len(b): # ``b`` not exhausted + if iCurrentA >= len(a): # `a` exhausted + if iCurrentB < len(b): # `b` not exhausted iEndA = iCurrentA iEndB = iCurrentB inc(lenRunesA, addRunesA) @@ -78,7 +79,7 @@ proc editDistance*(a, b: string): int {.noSideEffect.} = b.fastRuneAt(iEndB, runeB) inc(lenRunesB) if iEndB >= len(b): break - elif iCurrentB >= len(b): # ``b`` exhausted and ``a`` not exhausted + elif iCurrentB >= len(b): # `b` exhausted and `a` not exhausted iEndA = iCurrentA iEndB = iCurrentB inc(lenRunesA, addRunesA) @@ -181,6 +182,7 @@ proc editDistanceAscii*(a, b: string): int {.noSideEffect.} = ## ## This uses the `Levenshtein`:idx: distance algorithm with only a linear ## memory overhead. + runnableExamples: static: doAssert editDistanceAscii("Kitten", "Bitten") == 1 var len1 = a.len var len2 = b.len if len1 > len2: @@ -262,44 +264,3 @@ proc editDistanceAscii*(a, b: string): int {.noSideEffect.} = if x > c3: x = c3 row[p] = x result = row[e] - - -when isMainModule: - doAssert editDistance("", "") == 0 - doAssert editDistance("kitten", "sitting") == 3 # from Wikipedia - doAssert editDistance("flaw", "lawn") == 2 # from Wikipedia - - doAssert editDistance("привет", "превет") == 1 - doAssert editDistance("Åge", "Age") == 1 - # editDistance, one string is longer in bytes, but shorter in rune length - # first string: 4 bytes, second: 6 bytes, but only 3 runes - doAssert editDistance("aaaa", "×××") == 4 - - block veryLongStringEditDistanceTest: - const cap = 256 - var - s1 = newStringOfCap(cap) - s2 = newStringOfCap(cap) - while len(s1) < cap: - s1.add 'a' - while len(s2) < cap: - s2.add 'b' - doAssert editDistance(s1, s2) == cap - - block combiningCodePointsEditDistanceTest: - const s = "A\xCC\x8Age" - doAssert editDistance(s, "Age") == 1 - - doAssert editDistanceAscii("", "") == 0 - doAssert editDistanceAscii("kitten", "sitting") == 3 # from Wikipedia - doAssert editDistanceAscii("flaw", "lawn") == 2 # from Wikipedia - - - assert(editDistance("prefix__hallo_suffix", "prefix__hallo_suffix") == 0) - assert(editDistance("prefix__hallo_suffix", "prefix__hallo_suffi1") == 1) - assert(editDistance("prefix__hallo_suffix", "prefix__HALLO_suffix") == 5) - assert(editDistance("prefix__hallo_suffix", "prefix__ha_suffix") == 3) - assert(editDistance("prefix__hallo_suffix", "prefix") == 14) - assert(editDistance("prefix__hallo_suffix", "suffix") == 14) - assert(editDistance("prefix__hallo_suffix", "prefix__hao_suffix") == 2) - assert(editDistance("main", "malign") == 2) diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim new file mode 100644 index 000000000..3d1b4ffd3 --- /dev/null +++ b/lib/std/effecttraits.nim @@ -0,0 +1,63 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides access to the inferred .raises effects +## for Nim's macro system. +## **Since**: Version 1.4. +## +## One can test for the existence of this standard module +## via `defined(nimHasEffectTraitsModule)`. + +import std/macros + +proc getRaisesListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc getTagsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc getForbidsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim" +proc isGcSafeImpl(n: NimNode): bool = discard "see compiler/vmops.nim" +proc hasNoSideEffectsImpl(n: NimNode): bool = discard "see compiler/vmops.nim" + +proc getRaisesList*(fn: NimNode): NimNode = + ## Extracts the `.raises` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getRaisesListImpl(fn) + +proc getTagsList*(fn: NimNode): NimNode = + ## Extracts the `.tags` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getTagsListImpl(fn) + +proc getForbidsList*(fn: NimNode): NimNode = + ## Extracts the `.forbids` list of the func/proc/etc `fn`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = getForbidsListImpl(fn) + +proc isGcSafe*(fn: NimNode): bool = + ## Return true if the func/proc/etc `fn` is `gcsafe`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = isGcSafeImpl(fn) + +proc hasNoSideEffects*(fn: NimNode): bool = + ## Return true if the func/proc/etc `fn` has `noSideEffect`. + ## `fn` has to be a resolved symbol of kind `nnkSym`. This + ## implies that the macro that calls this proc should accept `typed` + ## arguments and not `untyped` arguments. + expectKind fn, nnkSym + result = hasNoSideEffectsImpl(fn) diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim new file mode 100644 index 000000000..beb65ed30 --- /dev/null +++ b/lib/std/enumerate.nim @@ -0,0 +1,70 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements `enumerate` syntactic sugar based on Nim's +## macro system. + +import std/private/since +import std/macros + + +macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = + ## Enumerating iterator for collections. + ## + ## It yields `(count, value)` tuples (which must be immediately unpacked). + ## The default starting count `0` can be manually overridden if needed. + runnableExamples: + let a = [10, 20, 30] + var b: seq[(int, int)] = @[] + for i, x in enumerate(a): + b.add((i, x)) + assert b == @[(0, 10), (1, 20), (2, 30)] + + let c = "abcd" + var d: seq[(int, char)] + for (i, x) in enumerate(97, c): + d.add((i, x)) + assert d == @[(97, 'a'), (98, 'b'), (99, 'c'), (100, 'd')] + + template genCounter(x): untyped = + # We strip off the first for loop variable and use it as an integer counter. + # We must immediately decrement it by one, because it gets incremented before + # the loop body - to be able to use the final expression in other macros. + newVarStmt(x, infix(countStart, "-", newLit(1))) + + template genInc(x): untyped = + newCall(bindSym"inc", x) + + expectKind x, nnkForStmt + # check if the starting count is specified: + var countStart = if x[^2].len == 2: newLit(0) else: x[^2][1] + result = newStmtList() + var body = x[^1] + if body.kind != nnkStmtList: + body = newTree(nnkStmtList, body) + var newFor = newTree(nnkForStmt) + if x.len == 3: # single iteration variable + if x[0].kind == nnkVarTuple: # for (x, y, ...) in iter + result.add genCounter(x[0][0]) + body.insert(0, genInc(x[0][0])) + for i in 1 .. x[0].len-2: + newFor.add x[0][i] + else: + error("Missing second for loop variable") # for x in iter + else: # for x, y, ... in iter + result.add genCounter(x[0]) + body.insert(0, genInc(x[0])) + for i in 1 .. x.len-3: + newFor.add x[i] + # transform enumerate(X) to 'X' + newFor.add x[^2][^1] + newFor.add body + result.add newFor + # now wrap the whole macro in a block to create a new scope + result = newBlockStmt(result) diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim new file mode 100644 index 000000000..9c338817d --- /dev/null +++ b/lib/std/enumutils.nim @@ -0,0 +1,202 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import std/macros +from std/typetraits import OrdinalEnum, HoleyEnum + +when defined(nimPreviewSlimSystem): + import std/assertions + + +# xxx `genEnumCaseStmt` needs tests and runnableExamples + +macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, + userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = + # Generates a case stmt, which assigns the correct enum field given + # a normalized string comparison to the `argSym` input. + # string normalization is done using passed normalizer. + let typ = typ.getTypeInst[1] + let typSym = typ.getTypeImpl.getTypeInst # skip aliases etc to get type sym + let impl = typSym.getImpl[2] + expectKind impl, nnkEnumTy + let normalizerNode = quote: `normalizer` + expectKind normalizerNode, nnkSym + result = nnkCaseStmt.newTree(newCall(normalizerNode, argSym)) + # stores all processed field strings to give error msg for ambiguous enums + var foundFields: seq[string] = @[] + var fVal = "" + var fStr = "" # string of current field + var fNum = BiggestInt(0) # int value of current field + for f in impl: + case f.kind + of nnkEmpty: continue # skip first node of `enumTy` + of nnkSym, nnkIdent: + fVal = f.strVal + fStr = fVal + of nnkAccQuoted: + fVal = "" + for ch in f: + fVal.add ch.strVal + fStr = fVal + of nnkEnumFieldDef: + fVal = f[0].strVal + case f[1].kind + of nnkStrLit: + fStr = f[1].strVal + of nnkTupleConstr: + fStr = f[1][1].strVal + fNum = f[1][0].intVal + of nnkIntLit: + fStr = f[0].strVal + fNum = f[1].intVal + else: + let fAst = f[0].getImpl + if fAst.kind == nnkStrLit: + fStr = fAst.strVal + else: + error("Invalid tuple syntax!", f[1]) + else: error("Invalid node for enum type `" & $f.kind & "`!", f) + # add field if string not already added + if fNum >= userMin and fNum <= userMax: + fStr = normalizer(fStr) + if fStr notin foundFields: + result.add nnkOfBranch.newTree(newLit fStr, newDotExpr(typ, ident fVal)) + foundFields.add fStr + else: + error("Ambiguous enums cannot be parsed, field " & $fStr & + " appears multiple times!", f) + inc fNum + # finally add else branch to raise or use default + if default == nil: + let raiseStmt = quote do: + raise newException(ValueError, "Invalid enum value: " & $`argSym`) + result.add nnkElse.newTree(raiseStmt) + else: + expectKind(default, nnkSym) + result.add nnkElse.newTree(default) + +macro enumFullRange(a: typed): untyped = + newNimNode(nnkBracket).add(a.getType[1][1..^1]) + +macro enumNames(a: typed): untyped = + # this could be exported too; in particular this could be useful for enum with holes. + result = newNimNode(nnkBracket) + for ai in a.getType[1][1..^1]: + assert ai.kind == nnkSym + result.add newLit ai.strVal + +iterator items*[T: HoleyEnum](E: typedesc[T]): T = + ## Iterates over an enum with holes. + runnableExamples: + type + A = enum + a0 = 2 + a1 = 4 + a2 + B[T] = enum + b0 = 2 + b1 = 4 + from std/sequtils import toSeq + assert A.toSeq == [a0, a1, a2] + assert B[float].toSeq == [B[float].b0, B[float].b1] + for a in enumFullRange(E): yield a + +func span(T: typedesc[HoleyEnum]): int = + (T.high.ord - T.low.ord) + 1 + +const invalidSlot = uint8.high + +proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = + const n = span(T) + var i = 0 + assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] + for ai in mitems(ret): ai = invalidSlot + for ai in items(T): + ret[ai.ord - T.low.ord] = uint8(i) + inc(i) + return ret + +func symbolRankImpl[T](a: T): int {.inline.} = + const n = T.span + const thres = 255 # must be <= `invalidSlot`, but this should be tuned. + when n <= thres: + const lookup = genLookup(T) + let lookup2 {.global.} = lookup # xxx improve pending https://github.com/timotheecour/Nim/issues/553 + #[ + This could be optimized using a hash adapted to `T` (possible since it's known at CT) + to get better key distribution before indexing into the lookup table table. + ]# + {.noSideEffect.}: # because it's immutable + let ret = lookup2[ord(a) - T.low.ord] + if ret != invalidSlot: return ret.int + else: + var i = 0 + # we could also generate a case statement as optimization + for ai in items(T): + if ai == a: return i + inc(i) + raise newException(IndexDefect, $ord(a) & " invalid for " & $T) + +template symbolRank*[T: enum](a: T): int = + ## Returns the index in which `a` is listed in `T`. + ## + ## The cost for a `HoleyEnum` is implementation defined, currently optimized + ## for small enums, otherwise is `O(T.enumLen)`. + runnableExamples: + type + A = enum # HoleyEnum + a0 = -3 + a1 = 10 + a2 + a3 = (20, "f3Alt") + B = enum # OrdinalEnum + b0 + b1 + b2 + C = enum # OrdinalEnum + c0 = 10 + c1 + c2 + assert a2.symbolRank == 2 + assert b2.symbolRank == 2 + assert c2.symbolRank == 2 + assert c2.ord == 12 + assert a2.ord == 11 + var invalid = 7.A + doAssertRaises(IndexDefect): discard invalid.symbolRank + when T is Ordinal: ord(a) - T.low.ord.static + else: symbolRankImpl(a) + +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + +func symbolName*[T: enum](a: T): string = + ## Returns the symbol name of an enum. + ## + ## This uses `symbolRank`. + runnableExamples: + type B = enum + b0 = (10, "kb0") + b1 = "kb1" + b2 + let b = B.low + assert b.symbolName == "b0" + assert $b == "kb0" + static: assert B.high.symbolName == "b2" + type C = enum # HoleyEnum + c0 = -3 + c1 = 4 + c2 = 20 + assert c1.symbolName == "c1" + when T is range: + const names = enumNames(rangeBase T) + else: + const names = enumNames(T) + names[a.symbolRank] diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim new file mode 100644 index 000000000..a955077ea --- /dev/null +++ b/lib/std/envvars.nim @@ -0,0 +1,221 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/envvars` module implements environment variable handling. +import std/oserrors + +type + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. + + +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + var num: int + var keys: RootObj + {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} + for i in 0..<num: + var key, value: cstring + {.emit: "`key` = `keys`[`i`]; `value` = process.env[`key`];".} + yield ($key, $value) + + # commented because it must keep working with js+VM + # elif defined(js): + # {.error: "requires -d:nodejs".} + + else: + + when defined(windows): + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "<stdlib.h>".} + from std/private/win_setenv import setEnvImpl + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t", header: "<stdlib.h>".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", + header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) + else: + proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "<stdlib.h>".} + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "<stdlib.h>".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "<stdlib.h>".} + proc getEnvImpl(env: cstring): cstring = c_getenv(env) + + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc`_. + ## + ## See also: + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + + let env = getEnvImpl(key) + if env == nil: + result = default + else: + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + result = getEnvImpl(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + else: + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "<string.h>".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "<string.h>".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # <crt_externs.h>, can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "<crt_externs.h>".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "<stdlib.h>".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim new file mode 100644 index 000000000..f26368f42 --- /dev/null +++ b/lib/std/exitprocs.nim @@ -0,0 +1,87 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module allows adding hooks to program exit. + +import std/locks +when defined(js) and not defined(nodejs): + import std/assertions + +type + FunKind = enum kClosure, kNoconv # extend as needed + Fun = object + case kind: FunKind + of kClosure: fun1: proc () {.closure.} + of kNoconv: fun2: proc () {.noconv.} + +var + gFunsLock: Lock + gFuns {.cursor.}: seq[Fun] #Intentionally use the cursor to break up the lifetime trace and make it compatible with JS. + +initLock(gFunsLock) + +when defined(js): + proc addAtExit(quitProc: proc() {.noconv.}) = + when defined(nodejs): + {.emit: """ + process.on('exit', `quitProc`); + """.} + elif defined(js): + {.emit: """ + window.onbeforeunload = `quitProc`; + """.} +else: + proc addAtExit(quitProc: proc() {.noconv.}) {. + importc: "atexit", header: "<stdlib.h>".} + +proc callClosures() {.noconv.} = + withLock gFunsLock: + for i in countdown(gFuns.len-1, 0): + let fun = gFuns[i] + case fun.kind + of kClosure: fun.fun1() + of kNoconv: fun.fun2() + gFuns.setLen(0) + +template fun() = + if gFuns.len == 0: + addAtExit(callClosures) + +proc addExitProc*(cl: proc () {.closure.}) = + ## Adds/registers a quit procedure. Each call to `addExitProc` registers + ## another quit procedure. They are executed on a last-in, first-out basis. + # Support for `addExitProc` is done by Ansi C's facilities here. + # In case of an unhandled exception the exit handlers should + # not be called explicitly! The user may decide to do this manually though. + withLock gFunsLock: + fun() + gFuns.add Fun(kind: kClosure, fun1: cl) + +proc addExitProc*(cl: proc() {.noconv.}) = + ## overload for `noconv` procs. + withLock gFunsLock: + fun() + gFuns.add Fun(kind: kNoconv, fun2: cl) + +when not defined(nimscript) and (not defined(js) or defined(nodejs)): + proc getProgramResult*(): int = + when defined(js) and defined(nodejs): + {.emit: """ +`result` = process.exitCode; +""".} + else: + result = programResult + + proc setProgramResult*(a: int) = + when defined(js) and defined(nodejs): + {.emit: """ +process.exitCode = `a`; +""".} + else: + programResult = a diff --git a/lib/std/files.nim b/lib/std/files.nim new file mode 100644 index 000000000..c4e0491c9 --- /dev/null +++ b/lib/std/files.nim @@ -0,0 +1,46 @@ +## This module implements file handling. +## +## **See also:** +## * `paths module <paths.html>`_ for path manipulation + +from std/paths import Path, ReadDirEffect, WriteDirEffect + +from std/private/osfiles import fileExists, removeFile, + moveFile + + +proc fileExists*(filename: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + result = fileExists(filename.string) + +proc removeFile*(file: Path) {.inline, tags: [WriteDirEffect].} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc <dirs.html#removeDir>`_ + ## * `moveFile proc`_ + removeFile(file.string) + +proc moveFile*(source, dest: Path) {.inline, + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect].} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc <dirs.html#moveDir>`_ + ## * `removeFile proc`_ + moveFile(source.string, dest.string) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim new file mode 100644 index 000000000..9258245f6 --- /dev/null +++ b/lib/std/formatfloat.nim @@ -0,0 +1,143 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements formatting floats as strings. + +when defined(nimPreviewSlimSystem): + import std/assertions + +proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "<string.h>", discardable.} + +proc addCstringN(result: var string, buf: cstring; buflen: int) = + # no nimvm support needed, so it doesn't need to be fast here either + let oldLen = result.len + let newLen = oldLen + buflen + result.setLen newLen + c_memcpy(result[oldLen].addr, buf, buflen.csize_t) + +import std/private/[dragonbox, schubfach] + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + result = toChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int = + result = float32ToChars(buf, value, forceTrailingDotZero=true).int + buf[result] = '\0' + +proc c_snprintf(buf: cstring, n: csize_t, frmt: cstring): cint {.header: "<stdio.h>", + importc: "snprintf", varargs, noSideEffect.} + +proc writeToBuffer(buf: var array[65, char]; value: cstring) = + var i = 0 + while value[i] != '\0': + buf[i] = value[i] + inc i + +proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + var n = c_snprintf(cast[cstring](addr buf), 65, "%.16g", value).int + var hasDot = false + for i in 0..n-1: + if buf[i] == ',': + buf[i] = '.' + hasDot = true + elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: + hasDot = true + if not hasDot: + buf[n] = '.' + buf[n+1] = '0' + buf[n+2] = '\0' + result = n + 2 + else: + result = n + # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' + # of '-1.#IND' are produced. + # We want to get rid of these here: + if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: + writeToBuffer(buf, "nan") + result = 3 + elif buf[n-1] == 'F': + if buf[0] == '-': + writeToBuffer(buf, "-inf") + result = 4 + else: + writeToBuffer(buf, "inf") + result = 3 + +proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + writeFloatToBufferRoundtrip(buf, value) + else: + writeFloatToBufferSprintf(buf, value) + +proc addFloatRoundtrip*(result: var string; x: float | float32) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferRoundtrip(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +proc addFloatSprintf*(result: var string; x: float) = + when nimvm: + raiseAssert "unreachable" + else: + var buffer {.noinit.}: array[65, char] + let n = writeFloatToBufferSprintf(buffer, x) + result.addCstringN(cast[cstring](buffer[0].addr), n) + +when defined(js): + proc nimFloatToString(a: float): cstring = + ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2 + # print `-0.0` properly + {.emit: """ + function nimOnlyDigitsOrMinus(n) { + return n.toString().match(/^-?\d+$/); + } + if (Number.isSafeInteger(`a`)) + `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"; + else { + `result` = `a`+""; + if(nimOnlyDigitsOrMinus(`result`)){ + `result` = `a`+".0"; + } + } + """.} + +proc addFloat*(result: var string; x: float | float32) {.inline.} = + ## Converts float to its string representation and appends it to `result`. + runnableExamples: + var + s = "foo:" + b = 45.67 + s.addFloat(45.67) + assert s == "foo:45.67" + template impl = + when defined(nimPreviewFloatRoundtrip) or defined(nimPreviewSlimSystem): + addFloatRoundtrip(result, x) + else: + addFloatSprintf(result, x) + when defined(js): + when nimvm: impl() + else: + result.add nimFloatToString(x) + else: impl() + +when defined(nimPreviewSlimSystem): + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim new file mode 100644 index 000000000..d0f07c527 --- /dev/null +++ b/lib/std/genasts.nim @@ -0,0 +1,89 @@ +## This module implements AST generation using captured variables for macros. + +import std/macros + +type GenAstOpt* = enum + kDirtyTemplate, + # When set, uses a dirty template in implementation of `genAst`. This + # is occasionally useful as workaround for issues such as #8220, see + # `strformat limitations <strformat.html#limitations>`_ for details. + # Default is unset, to avoid hijacking of uncaptured local symbols by + # symbols in caller scope. + kNoNewLit, + # don't call call newLit automatically in `genAst` capture parameters + +macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped = + ## Accepts a list of captured variables `a=b` or `a` and a block and returns the + ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured + ## unless `kDirtyTemplate in options`. + runnableExamples: + # This example shows how one could write a simplified version of `unittest.check`. + import std/[macros, strutils] + macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + raiseAssert "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] + let a = 3 + check2 a*2 == a+3 + if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' + + runnableExamples: + # This example goes in more details about the capture semantics. + macro fun(a: string, b: static bool): untyped = + let c = 'z' + var d = 11 # implicitly {.gensym.} and needs to be captured for use in `genAst`. + proc localFun(): auto = 12 # implicitly {.inject.}, doesn't need to be captured. + genAst(a, b, c = true): + # `a`, `b` are captured explicitly, `c` is a local definition masking `c = 'z'`. + const b2 = b # macro static param `b` is forwarded here as a static param. + # `echo d` would give: `var not init` because `d` is not captured. + (a & a, b, c, localFun()) # localFun can be called without capture. + assert fun("ab", false) == ("abab", false, true, 12) + + let params = newTree(nnkFormalParams, newEmptyNode()) + let pragmas = + if kDirtyTemplate in options: + nnkPragma.newTree(ident"dirty") + else: + newEmptyNode() + + template newLitMaybe(a): untyped = + when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)): + a # `proc` actually also covers template, macro + else: newLit(a) + + # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669 + let name = genSym(nskTemplate, "_fun") + let call = newCall(name) + for a in args[0..^2]: + var varName: NimNode + var varVal: NimNode + case a.kind + of nnkExprEqExpr: + varName = a[0] + varVal = a[1] + of nnkIdent: + varName = a + varVal = a + else: error("invalid argument kind: " & $a.kind, a) + if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal) + + params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode()) + call.add varVal + + result = newStmtList() + result.add nnkTemplateDef.newTree( + name, + newEmptyNode(), + newEmptyNode(), + params, + pragmas, + newEmptyNode(), + args[^1]) + result.add newCall(bindSym"getAst", call) + +template genAst*(args: varargs[untyped]): untyped = + ## Convenience wrapper around `genAstOpt`. + genAstOpt({}, args) diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim new file mode 100644 index 000000000..d2da76ea8 --- /dev/null +++ b/lib/std/importutils.nim @@ -0,0 +1,44 @@ +##[ +Utilities related to import and symbol resolution. + +Experimental API, subject to change. +]## + +#[ +Possible future APIs: +* module symbols (https://github.com/nim-lang/Nim/pull/9560) +* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376) +* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530) +* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527) +]# + +when defined(nimImportutilsExample): + type + Foo = object + f0: int # private + Goo*[T] = object + g0: int # private + proc initFoo*(): auto = Foo() + +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = + ## Enables access to private fields of `t` in current scope. + runnableExamples("-d:nimImportutilsExample"): + # here we're importing a module containing: + # type + # Foo = object + # f0: int # private + # Goo*[T] = object + # g0: int # private + # proc initFoo*(): auto = Foo() + var f = initFoo() + block: + assert not compiles(f.f0) + privateAccess(f.type) + f.f0 = 1 # accessible in this scope + block: + assert f.f0 == 1 # still in scope + assert not compiles(f.f0) + + # this also works with generics + privateAccess(Goo) + assert Goo[float](g0: 1).g0 == 1 diff --git a/lib/std/isolation.nim b/lib/std/isolation.nim new file mode 100644 index 000000000..b03e00651 --- /dev/null +++ b/lib/std/isolation.nim @@ -0,0 +1,49 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the `Isolated[T]` type for +## safe construction of isolated subgraphs that can be +## passed efficiently to different channels and threads. +## +## .. warning:: This module is experimental and its interface may change. +## + +type + Isolated*[T] {.sendable.} = object ## Isolated data can only be moved, not copied. + value: T + +proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.} + +proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} = + # delegate to value's sink operation + `=sink`(dest.value, src.value) + +proc `=destroy`*[T](dest: var Isolated[T]) {.inline.} = + # delegate to value's destroy operation + `=destroy`(dest.value) + +func isolate*[T](value: sink T): Isolated[T] {.magic: "Isolate".} = + ## Creates an isolated subgraph from the expression `value`. + ## Isolation is checked at compile time. + ## + ## Please read https://github.com/nim-lang/RFCs/issues/244 + ## for more details. + Isolated[T](value: value) + +func unsafeIsolate*[T](value: sink T): Isolated[T] = + ## Creates an isolated subgraph from the expression `value`. + ## + ## .. warning:: The proc doesn't check whether `value` is isolated. + ## + Isolated[T](value: value) + +func extract*[T](src: var Isolated[T]): T = + ## Returns the internal value of `src`. + ## The value is moved from `src`. + result = move(src.value) diff --git a/lib/std/jsbigints.nim b/lib/std/jsbigints.nim new file mode 100644 index 000000000..4e996ea7b --- /dev/null +++ b/lib/std/jsbigints.nim @@ -0,0 +1,228 @@ +## Arbitrary precision integers. +## * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt +when not defined(js): + {.fatal: "Module jsbigints is designed to be used with the JavaScript backend.".} + +when defined(nimPreviewSlimSystem): + import std/assertions + +type JsBigIntImpl {.importjs: "bigint".} = int # https://github.com/nim-lang/Nim/pull/16606 +type JsBigInt* = distinct JsBigIntImpl ## Arbitrary precision integer for JavaScript target. + +func big*(integer: SomeInteger): JsBigInt {.importjs: "BigInt(#)".} = + ## Constructor for `JsBigInt`. + runnableExamples: + doAssert big(1234567890) == big"1234567890" + doAssert 0b1111100111.big == 0o1747.big and 0o1747.big == 999.big + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func `'big`*(num: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Constructor for `JsBigInt`. + runnableExamples: + doAssert -1'big == 1'big - 2'big + # supports decimal, binary, octal, hex: + doAssert -12'big == big"-12" + doAssert 12'big == 12.big + doAssert 0b101'big == 0b101.big + doAssert 0o701'big == 0o701.big + doAssert 0xdeadbeaf'big == 0xdeadbeaf.big + doAssert 0xffffffffffffffff'big == (1'big shl 64'big) - 1'big + doAssert not compiles(static(12'big)) + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func big*(integer: cstring): JsBigInt {.importjs: "BigInt(#)".} = + ## Alias for `'big` + when nimvm: raiseAssert "JsBigInt can not be used at compile-time nor static context" else: discard + +func toCstring*(this: JsBigInt; radix: 2..36): cstring {.importjs: "#.toString(#)".} = + ## Converts from `JsBigInt` to `cstring` representation. + ## * `radix` Base to use for representing numeric values. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/toString + runnableExamples: + doAssert big"2147483647".toCstring(2) == "1111111111111111111111111111111".cstring + +func toCstring*(this: JsBigInt): cstring {.importjs: "#.toString()".} + ## Converts from `JsBigInt` to `cstring` representation. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/toString + +func `$`*(this: JsBigInt): string = + ## Returns a `string` representation of `JsBigInt`. + runnableExamples: doAssert $big"1024" == "1024n" + $toCstring(this) & 'n' + +func wrapToInt*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: + "(() => { const i = #, b = #; return BigInt.asIntN(b, i) })()".} = + ## Wraps `this` to a signed `JsBigInt` of `bits` bits in `-2 ^ (bits - 1)` .. `2 ^ (bits - 1) - 1`. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/asIntN + runnableExamples: + doAssert (big("3") + big("2") ** big("66")).wrapToInt(13) == big("3") + +func wrapToUint*(this: JsBigInt; bits: Natural): JsBigInt {.importjs: + "(() => { const i = #, b = #; return BigInt.asUintN(b, i) })()".} = + ## Wraps `this` to an unsigned `JsBigInt` of `bits` bits in 0 .. `2 ^ bits - 1`. + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt/asUintN + runnableExamples: + doAssert (big("3") + big("2") ** big("66")).wrapToUint(66) == big("3") + +func toNumber*(this: JsBigInt): int {.importjs: "Number(#)".} = + ## Does not do any bounds check and may or may not return an inexact representation. + runnableExamples: + doAssert toNumber(big"2147483647") == 2147483647.int + +func `+`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert (big"9" + big"1") == big"10" + +func `-`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert (big"9" - big"1") == big"8" + +func `*`*(x, y: JsBigInt): JsBigInt {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert (big"42" * big"9") == big"378" + +func `div`*(x, y: JsBigInt): JsBigInt {.importjs: "(# / #)".} = + ## Same as `div` but for `JsBigInt`(uses JavaScript `BigInt() / BigInt()`). + runnableExamples: + doAssert big"13" div big"3" == big"4" + doAssert big"-13" div big"3" == big"-4" + doAssert big"13" div big"-3" == big"-4" + doAssert big"-13" div big"-3" == big"4" + +func `mod`*(x, y: JsBigInt): JsBigInt {.importjs: "(# % #)".} = + ## Same as `mod` but for `JsBigInt` (uses JavaScript `BigInt() % BigInt()`). + runnableExamples: + doAssert big"13" mod big"3" == big"1" + doAssert big"-13" mod big"3" == big"-1" + doAssert big"13" mod big"-3" == big"1" + doAssert big"-13" mod big"-3" == big"-1" + +func `<`*(x, y: JsBigInt): bool {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert big"2" < big"9" + +func `<=`*(x, y: JsBigInt): bool {.importjs: "(# $1 #)".} = + runnableExamples: + doAssert big"1" <= big"5" + +func `==`*(x, y: JsBigInt): bool {.importjs: "(# == #)".} = + runnableExamples: + doAssert big"42" == big"42" + +func `**`*(x, y: JsBigInt): JsBigInt {.importjs: "((#) $1 #)".} = + # (#) needed due to unary minus + runnableExamples: + doAssert big"2" ** big"64" == big"18446744073709551616" + doAssert big"-2" ** big"3" == big"-8" + doAssert -big"2" ** big"2" == big"4" # parsed as: (-2n) ** 2n + doAssert big"0" ** big"0" == big"1" # edge case + var ok = false + try: discard big"2" ** big"-1" # raises foreign `RangeError` + except: ok = true + doAssert ok + +func `and`*(x, y: JsBigInt): JsBigInt {.importjs: "(# & #)".} = + runnableExamples: + doAssert (big"555" and big"2") == big"2" + +func `or`*(x, y: JsBigInt): JsBigInt {.importjs: "(# | #)".} = + runnableExamples: + doAssert (big"555" or big"2") == big"555" + +func `xor`*(x, y: JsBigInt): JsBigInt {.importjs: "(# ^ #)".} = + runnableExamples: + doAssert (big"555" xor big"2") == big"553" + +func `shl`*(a, b: JsBigInt): JsBigInt {.importjs: "(# << #)".} = + runnableExamples: + doAssert (big"999" shl big"2") == big"3996" + +func `shr`*(a, b: JsBigInt): JsBigInt {.importjs: "(# >> #)".} = + runnableExamples: + doAssert (big"999" shr big"2") == big"249" + +func `-`*(this: JsBigInt): JsBigInt {.importjs: "($1#)".} = + runnableExamples: + doAssert -(big"10101010101") == big"-10101010101" + +func inc*(this: var JsBigInt) {.importjs: "(++[#][0][0])".} = + runnableExamples: + var big1: JsBigInt = big"1" + inc big1 + doAssert big1 == big"2" + +func dec*(this: var JsBigInt) {.importjs: "(--[#][0][0])".} = + runnableExamples: + var big1: JsBigInt = big"2" + dec big1 + doAssert big1 == big"1" + +func inc*(this: var JsBigInt; amount: JsBigInt) {.importjs: "([#][0][0] += #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + inc big1, big"2" + doAssert big1 == big"3" + +func dec*(this: var JsBigInt; amount: JsBigInt) {.importjs: "([#][0][0] -= #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + dec big1, big"2" + doAssert big1 == big"-1" + +func `+=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + big1 += big"2" + doAssert big1 == big"3" + +func `-=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + runnableExamples: + var big1: JsBigInt = big"1" + big1 -= big"2" + doAssert big1 == big"-1" + +func `*=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + runnableExamples: + var big1: JsBigInt = big"2" + big1 *= big"4" + doAssert big1 == big"8" + +func `/=`*(x: var JsBigInt; y: JsBigInt) {.importjs: "([#][0][0] $1 #)".} = + ## Same as `x = x div y`. + runnableExamples: + var big1: JsBigInt = big"11" + big1 /= big"2" + doAssert big1 == big"5" + +proc `+`*(_: JsBigInt): JsBigInt {.error: + "See https://github.com/tc39/proposal-bigint/blob/master/ADVANCED.md#dont-break-asmjs".} # Can not be used by design + ## **Do NOT use.** https://github.com/tc39/proposal-bigint/blob/master/ADVANCED.md#dont-break-asmjs + +proc low*(_: typedesc[JsBigInt]): JsBigInt {.error: + "Arbitrary precision integers do not have a known low.".} ## **Do NOT use.** + +proc high*(_: typedesc[JsBigInt]): JsBigInt {.error: + "Arbitrary precision integers do not have a known high.".} ## **Do NOT use.** + + +runnableExamples: + block: + let big1: JsBigInt = big"2147483647" + let big2: JsBigInt = big"666" + doAssert JsBigInt isnot int + doAssert big1 != big2 + doAssert big1 > big2 + doAssert big1 >= big2 + doAssert big2 < big1 + doAssert big2 <= big1 + doAssert not(big1 == big2) + let z = JsBigInt.default + doAssert $z == "0n" + block: + var a: seq[JsBigInt] + a.setLen 2 + doAssert a == @[big"0", big"0"] + doAssert a[^1] == big"0" + var b: JsBigInt + doAssert b == big"0" + doAssert b == JsBigInt.default diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim new file mode 100644 index 000000000..219594619 --- /dev/null +++ b/lib/std/jsfetch.nim @@ -0,0 +1,202 @@ +## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API +when not defined(js): + {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} + +import std/[asyncjs, jsformdata, jsheaders] +export jsformdata, jsheaders +from std/httpcore import HttpMethod +from std/jsffi import JsObject + +type + FetchOptions* = ref object of JsRoot ## Options for Fetch API. + keepalive*: bool + metod* {.importjs: "method".}: cstring + body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring + headers*: Headers + + FetchModes* = enum ## Mode options. + fmCors = "cors" + fmNoCors = "no-cors" + fmSameOrigin = "same-origin" + + FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials + fcInclude = "include" + fcSameOrigin = "same-origin" + fcOmit = "omit" + + FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache + fchDefault = "default" + fchNoStore = "no-store" + fchReload = "reload" + fchNoCache = "no-cache" + fchForceCache = "force-cache" + + FetchRedirects* = enum ## Redirects options. + frFollow = "follow" + frError = "error" + frManual = "manual" + + FetchReferrerPolicies* = enum ## Referrer Policy options. + frpNoReferrer = "no-referrer" + frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" + frpOrigin = "origin" + frpOriginWhenCrossOrigin = "origin-when-cross-origin" + frpUnsafeUrl = "unsafe-url" + + Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring + + Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: cstring + +func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} + ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. + +func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} + ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. + +func newRequest*(url: cstring; fetchOptions: FetchOptions): Request {.importjs: "(new Request(#, #))".} + ## Constructor for `Request` with `fetchOptions`. Same as `fetch(url, fetchOptions)`. + +func clone*(self: Response | Request): Response {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone + +proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/text + +proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/json + +proc formData*(self: Response): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/formData + +proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring; headers: Headers = newHeaders()): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} + ## .. warning:: Unsafe `newfetchOptions`. + +func newfetchOptions*(metod = HttpGet; body: cstring = nil; + mode = fmCors; credentials = fcSameOrigin; cache = fchDefault; referrerPolicy = frpNoReferrerWhenDowngrade; + keepalive = false; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, + headers: Headers = newHeaders()): FetchOptions = + ## Constructor for `FetchOptions`. + result = FetchOptions( + body: if metod notin {HttpHead, HttpGet}: body else: nil, + mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, headers: headers, + metod: (case metod + of HttpHead: "HEAD".cstring + of HttpGet: "GET".cstring + of HttpPost: "POST".cstring + of HttpPut: "PUT".cstring + of HttpDelete: "DELETE".cstring + of HttpPatch: "PATCH".cstring + else: "GET".cstring + ) + ) + +proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} + ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. + +proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} + ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. + +func toCstring*(self: Request | Response | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} + +func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) + + +runnableExamples("-r:off"): + import std/[asyncjs, jsconsole, jsformdata, jsheaders] + from std/httpcore import HttpMethod + from std/jsffi import JsObject + from std/sugar import `=>` + + block: + let options0: FetchOptions = unsafeNewFetchOptions( + metod = "POST".cstring, + body = """{"key": "value"}""".cstring, + mode = "no-cors".cstring, + credentials = "omit".cstring, + cache = "no-cache".cstring, + referrerPolicy = "no-referrer".cstring, + keepalive = false, + redirect = "follow".cstring, + referrer = "client".cstring, + integrity = "".cstring, + headers = newHeaders() + ) + assert options0.keepalive == false + assert options0.metod == "POST".cstring + assert options0.body == """{"key": "value"}""".cstring + assert options0.mode == "no-cors".cstring + assert options0.credentials == "omit".cstring + assert options0.cache == "no-cache".cstring + assert options0.referrerPolicy == "no-referrer".cstring + assert options0.redirect == "follow".cstring + assert options0.referrer == "client".cstring + assert options0.integrity == "".cstring + assert options0.headers.len == 0 + + block: + let options1: FetchOptions = newFetchOptions( + metod = HttpPost, + body = """{"key": "value"}""".cstring, + mode = fmNoCors, + credentials = fcOmit, + cache = fchNoCache, + referrerPolicy = frpNoReferrer, + keepalive = false, + redirect = frFollow, + referrer = "client".cstring, + integrity = "".cstring, + headers = newHeaders() + ) + assert options1.keepalive == false + assert options1.metod == $HttpPost + assert options1.body == """{"key": "value"}""".cstring + assert options1.mode == $fmNoCors + assert options1.credentials == $fcOmit + assert options1.cache == $fchNoCache + assert options1.referrerPolicy == $frpNoReferrer + assert options1.redirect == $frFollow + assert options1.referrer == "client".cstring + assert options1.integrity == "".cstring + assert options1.headers.len == 0 + + block: + let response: Response = newResponse(body = "-. .. --".cstring) + let request: Request = newRequest(url = "http://nim-lang.org".cstring) + + if not defined(nodejs): + block: + proc doFetch(): Future[Response] {.async.} = + fetch "https://httpbin.org/get".cstring + + proc example() {.async.} = + let response: Response = await doFetch() + assert response.ok + assert response.status == 200.cint + assert response.headers is Headers + assert response.body is cstring + + discard example() + + block: + proc example2 {.async.} = + await fetch("https://api.github.com/users/torvalds".cstring) + .then((response: Response) => response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) + + discard example2() diff --git a/lib/std/jsformdata.nim b/lib/std/jsformdata.nim new file mode 100644 index 000000000..61dcc39a3 --- /dev/null +++ b/lib/std/jsformdata.nim @@ -0,0 +1,69 @@ +## - `FormData` for the JavaScript target: https://developer.mozilla.org/en-US/docs/Web/API/FormData +when not defined(js): + {.fatal: "Module jsformdata is designed to be used with the JavaScript backend.".} + +from std/dom import Blob + +type FormData* = ref object of JsRoot ## FormData API. + +func newFormData*(): FormData {.importjs: "new FormData()".} + +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.append(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. + +func add*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.append(#, #, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/append + ## + ## .. hint:: Duplicate keys are allowed and order is preserved. + +func delete*(self: FormData; name: cstring) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/delete + ## + ## .. warning:: Deletes *all items* with the same key name. + +func getAll*(self: FormData; name: cstring): seq[cstring] {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/getAll + +func hasKey*(self: FormData; name: cstring): bool {.importjs: "#.has(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/has + +func keys*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/keys + +func values*(self: FormData): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/values + +func pairs*(self: FormData): seq[tuple[key, val: cstring]] {.importjs: "Array.from(#.entries())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/entries + +func put*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob; filename: cstring) {.importjs: "#.set(#, #, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set + +func `[]=`*(self: FormData; name: cstring; value: SomeNumber | bool | cstring | Blob) {.importjs: "#.set(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/set + +func `[]`*(self: FormData; name: cstring): cstring {.importjs: "#.get(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FormData/get + +func clear*(self: FormData) {.importjs: + "(() => { const frmdt = #; Array.from(frmdt.keys()).forEach((key) => frmdt.delete(key)) })()".} + ## Convenience func to delete all items from `FormData`. + +func toCstring*(self: FormData): cstring {.importjs: "JSON.stringify(#)".} + +func `$`*(self: FormData): string = $toCstring(self) + +func len*(self: FormData): int {.importjs: "Array.from(#.entries()).length".} + + +runnableExamples("-r:off"): + let data: FormData = newFormData() + data["key0"] = "value0".cstring + data.add("key1".cstring, "value1".cstring) + data.delete("key1") + assert data.hasKey("key0") + assert data["key0"] == "value0".cstring + data.clear() + assert data.len == 0 diff --git a/lib/std/jsheaders.nim b/lib/std/jsheaders.nim new file mode 100644 index 000000000..6fd3b3468 --- /dev/null +++ b/lib/std/jsheaders.nim @@ -0,0 +1,83 @@ +## - HTTP Headers for the JavaScript target: https://developer.mozilla.org/en-US/docs/Web/API/Headers +when not defined(js): + {.fatal: "Module jsheaders is designed to be used with the JavaScript backend.".} + +type Headers* = ref object of JsRoot ## HTTP Headers API. + +func newHeaders*(): Headers {.importjs: "new Headers()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers + +func add*(self: Headers; key: cstring; value: cstring) {.importjs: "#.append(#, #)".} + ## Allows duplicated keys. + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/append + +func delete*(self: Headers; key: cstring) {.importjs: "#.$1(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/delete + ## + ## .. warning:: Delete *all* items with `key` from the headers, including duplicated keys. + +func hasKey*(self: Headers; key: cstring): bool {.importjs: "#.has(#)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/has + +func keys*(self: Headers): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/keys + +func values*(self: Headers): seq[cstring] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/values + +func entries*(self: Headers): seq[tuple[key, value: cstring]] {.importjs: "Array.from(#.$1())".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/entries + +func `[]`*(self: Headers; key: cstring): cstring {.importjs: "#.get(#)".} + ## Get *all* items with `key` from the headers, including duplicated values. + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/get + +func `[]=`*(self: Headers; key: cstring; value: cstring) {.importjs: "#.set(#, #)".} + ## Do *not* allow duplicated keys, overwrites duplicated keys. + ## https://developer.mozilla.org/en-US/docs/Web/API/Headers/set + +func clear*(self: Headers) {.importjs: + "(() => { const header = #; Array.from(header.keys()).forEach((key) => header.delete(key)) })()".} + ## Convenience func to delete all items from `Headers`. + +func toCstring*(self: Headers): cstring {.importjs: "JSON.stringify(Array.from(#.entries()))".} + ## Returns a `cstring` representation of `Headers`. + +func `$`*(self: Headers): string = $toCstring(self) + +func len*(self: Headers): int {.importjs: "Array.from(#.entries()).length".} + + +runnableExamples("-r:off"): + + block: + let header: Headers = newHeaders() + header.add("key", "value") + assert header.hasKey("key") + assert header.keys() == @["key".cstring] + assert header.values() == @["value".cstring] + assert header["key"] == "value".cstring + header["other"] = "another".cstring + assert header["other"] == "another".cstring + assert header.entries() == @[("key".cstring, "value".cstring), ("other".cstring, "another".cstring)] + assert header.toCstring() == """[["key","value"],["other","another"]]""".cstring + header.delete("other") + assert header.entries() == @[("key".cstring, "value".cstring)] + header.clear() + assert header.entries() == @[] + assert header.len == 0 + + block: + let header: Headers = newHeaders() + header.add("key", "a") + header.add("key", "b") ## Duplicated. + header.add("key", "c") ## Duplicated. + assert header["key"] == "a, b, c".cstring + header["key"] = "value".cstring + assert header["key"] == "value".cstring + + block: + let header: Headers = newHeaders() + header["key"] = "a" + header["key"] = "b" ## Overwrites. + assert header["key"] == "b".cstring diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim new file mode 100644 index 000000000..2d28748ce --- /dev/null +++ b/lib/std/jsonutils.nim @@ -0,0 +1,493 @@ +##[ +This module implements a hookable (de)serialization for arbitrary types using JSON. +Design goal: avoid importing modules where a custom serialization is needed; +see strtabs.fromJsonHook,toJsonHook for an example. +]## + +runnableExamples: + import std/[strtabs,json] + type Foo = ref object + t: bool + z1: int8 + let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable]) + let j = a.toJson + assert j.jsonTo(typeof(a)).toJson == j + assert $[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson == """["nan","inf","-inf",0.0,-0.0,1.0,0.01]""" + assert 0.0.toJson.kind == JFloat + assert Inf.toJson.kind == JString + +import std/[json, strutils, tables, sets, strtabs, options, strformat] + +#[ +Future directions: +add a way to customize serialization, for e.g.: +* field renaming +* allow serializing `enum` and `char` as `string` instead of `int` + (enum is more compact/efficient, and robust to enum renamings, but string + is more human readable) +* handle cyclic references, using a cache of already visited addresses +* implement support for serialization and de-serialization of nested variant + objects. +]# + +import std/macros +from std/enumutils import symbolName +from std/typetraits import OrdinalEnum, tupleLen + +when defined(nimPreviewSlimSystem): + import std/assertions + + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +type + Joptions* = object # xxx rename FromJsonOptions + ## Options controlling the behavior of `fromJson`. + allowExtraKeys*: bool + ## If `true` Nim's object to which the JSON is parsed is not required to + ## have a field for every JSON key. + allowMissingKeys*: bool + ## If `true` Nim's object to which JSON is parsed is allowed to have + ## fields without corresponding JSON keys. + # in future work: a key rename could be added + EnumMode* = enum + joptEnumOrd + joptEnumSymbol + joptEnumString + JsonNodeMode* = enum ## controls `toJson` for JsonNode types + joptJsonNodeAsRef ## returns the ref as is + joptJsonNodeAsCopy ## returns a deep copy of the JsonNode + joptJsonNodeAsObject ## treats JsonNode as a regular ref object + ToJsonOptions* = object + enumMode*: EnumMode + jsonNodeMode*: JsonNodeMode + # xxx charMode, etc + +proc initToJsonOptions*(): ToJsonOptions = + ## initializes `ToJsonOptions` with sane options. + ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) + +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a) + +macro getDiscriminants(a: typedesc): seq[string] = + ## return the discriminant keys + # candidate for std/typetraits + var a = a.getTypeImpl + doAssert a.kind == nnkBracketExpr + let sym = a[1] + let t = sym.getTypeImpl + let t2 = t[2] + case t2.kind + of nnkEmpty: # allow empty objects + result = quote do: + seq[string].default + of nnkRecList: + result = newTree(nnkBracket) + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + result.add newLit key.strVal + if result.len > 0: + result = quote do: + @`result` + else: + result = quote do: + seq[string].default + else: + raiseAssert "unexpected kind: " & $t2.kind + +macro initCaseObject(T: typedesc, fun: untyped): untyped = + ## does the minimum to construct a valid case object, only initializing + ## the discriminant fields; see also `getDiscriminants` + # maybe candidate for std/typetraits + var a = T.getTypeImpl + doAssert a.kind == nnkBracketExpr + let sym = a[1] + let t = sym.getTypeImpl + var t2: NimNode + case t.kind + of nnkObjectTy: t2 = t[2] + of nnkRefTy: t2 = t[0].getTypeImpl[2] + else: raiseAssert $t.kind # xxx `nnkPtrTy` could be handled too + doAssert t2.kind == nnkRecList + result = newTree(nnkObjConstr) + result.add sym + for ti in t2: + if ti.kind == nnkRecCase: + let key = ti[0][0] + let typ = ti[0][1] + let key2 = key.strVal + let val = quote do: + `fun`(`key2`, typedesc[`typ`]) + result.add newTree(nnkExprColonExpr, key, val) + +proc raiseJsonException(condStr: string, msg: string) {.noinline.} = + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, condStr & " failed: " & msg) + +template checkJson(cond: untyped, msg = "") = + if not cond: + raiseJsonException(astToStr(cond), msg) + +proc hasField[T](obj: T, field: string): bool = + for k, _ in fieldPairs(obj): + if k == field: + return true + return false + +macro accessField(obj: typed, name: static string): untyped = + newDotExpr(obj, ident(name)) + +template fromJsonFields(newObj, oldObj, json, discKeys, opt) = + type T = typeof(newObj) + # we could customize whether to allow JNull + checkJson json.kind == JObject, $json.kind + var num, numMatched = 0 + for key, val in fieldPairs(newObj): + num.inc + when key notin discKeys: + if json.hasKey key: + numMatched.inc + fromJson(val, json[key], opt) + elif opt.allowMissingKeys: + # if there are no discriminant keys the `oldObj` must always have the + # same keys as the new one. Otherwise we must check, because they could + # be set to different branches. + when typeof(oldObj) isnot typeof(nil): + if discKeys.len == 0 or hasField(oldObj, key): + val = accessField(oldObj, key) + else: + checkJson false, "key '$1' for $2 not in $3" % [key, $T, json.pretty()] + else: + if json.hasKey key: + numMatched.inc + + let ok = + if opt.allowExtraKeys and opt.allowMissingKeys: + true + elif opt.allowExtraKeys: + # This check is redundant because if here missing keys are not allowed, + # and if `num != numMatched` it will fail in the loop above but it is left + # for clarity. + assert num == numMatched + num == numMatched + elif opt.allowMissingKeys: + json.len == numMatched + else: + json.len == num and num == numMatched + + checkJson ok, "There were $1 keys (expecting $2) for $3 with $4" % [$json.len, $num, $T, json.pretty()] + +proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) + +proc discKeyMatch[T](obj: T, json: JsonNode, key: static string): bool = + if not json.hasKey key: + return true + let field = accessField(obj, key) + var jsonVal: typeof(field) + fromJson(jsonVal, json[key]) + if jsonVal != field: + return false + return true + +macro discKeysMatchBodyGen(obj: typed, json: JsonNode, + keys: static seq[string]): untyped = + result = newStmtList() + let r = ident("result") + for key in keys: + let keyLit = newLit key + result.add quote do: + `r` = `r` and discKeyMatch(`obj`, `json`, `keyLit`) + +proc discKeysMatch[T](obj: T, json: JsonNode, keys: static seq[string]): bool = + result = true + discKeysMatchBodyGen(obj, json, keys) + +proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) = + ## inplace version of `jsonTo` + #[ + adding "json path" leading to `b` can be added in future work. + ]# + checkJson b != nil, $($T, b) + when compiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt) + elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + elif T is bool: a = to(b,T) + elif T is enum: + case b.kind + of JInt: a = T(b.getBiggestInt()) + of JString: a = parseEnum[T](b.getStr()) + else: checkJson false, fmt"Expecting int/string for {$T} got {b.pretty()}" + elif T is uint|uint64: a = T(to(b, uint64)) + elif T is Ordinal: a = cast[T](to(b, int)) + elif T is pointer: a = cast[pointer](to(b, int)) + elif T is distinct: a.distinctBase.fromJson(b) + elif T is string|SomeNumber: a = to(b,T) + elif T is cstring: + case b.kind + of JNull: a = nil + of JString: a = b.str + else: checkJson false, fmt"Expecting null/string for {$T} got {b.pretty()}" + elif T is JsonNode: a = b + elif T is ref | ptr: + if b.kind == JNull: a = nil + else: + a = T() + fromJson(a[], b, opt) + elif T is array: + checkJson a.len == b.len, fmt"Json array size doesn't match for {$T}" + var i = 0 + for ai in mitems(a): + fromJson(ai, b[i], opt) + i.inc + elif T is set: + type E = typeof(for ai in a: ai) + for val in b.getElems: + incl a, jsonTo(val, E) + elif T is seq: + a.setLen b.len + for i, val in b.getElems: + fromJson(a[i], val, opt) + elif T is object: + template fun(key, typ): untyped {.used.} = + if b.hasKey key: + jsonTo(b[key], typ) + elif hasField(a, key): + accessField(a, key) + else: + default(typ) + const keys = getDiscriminants(T) + when keys.len == 0: + fromJsonFields(a, nil, b, keys, opt) + else: + if discKeysMatch(a, b, keys): + fromJsonFields(a, nil, b, keys, opt) + else: + var newObj = initCaseObject(T, fun) + fromJsonFields(newObj, a, b, keys, opt) + a = newObj + elif T is tuple: + when isNamedTuple(T): + fromJsonFields(a, nil, b, seq[string].default, opt) + else: + checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + + when compiles(tupleLen(T)): + let tupleSize = tupleLen(T) + else: + # Tuple len isn't in csources_v1 so using tupleLen would fail. + # Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency + var tupleSize = 0 + for val in fields(a): + tupleSize.inc + + checkJson b.len == tupleSize, fmt"Json doesn't match expected length of {tupleSize}, got {b.pretty()}" + var i = 0 + for val in fields(a): + fromJson(val, b[i], opt) + i.inc + else: + # checkJson not appropriate here + static: raiseAssert "not yet implemented: " & $T + +proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = + ## reverse of `toJson` + fromJson(result, b, opt) + +proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = + ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to + ## customize serialization, see strtabs.toJsonHook for an example. + ## + ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can + ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. + ## It is expected that this behavior becomes the new default in upcoming versions. + when compiles(toJsonHook(a, opt)): result = toJsonHook(a, opt) + elif compiles(toJsonHook(a)): result = toJsonHook(a) + elif T is object | tuple: + when T is object or isNamedTuple(T): + result = newJObject() + for k, v in a.fieldPairs: result[k] = toJson(v, opt) + else: + result = newJArray() + for v in a.fields: result.add toJson(v, opt) + elif T is ref | ptr: + template impl = + if system.`==`(a, nil): result = newJNull() + else: result = toJson(a[], opt) + when T is JsonNode: + case opt.jsonNodeMode + of joptJsonNodeAsRef: result = a + of joptJsonNodeAsCopy: result = copy(a) + of joptJsonNodeAsObject: impl() + else: impl() + elif T is array | seq | set: + result = newJArray() + for ai in a: result.add toJson(ai, opt) + elif T is pointer: result = toJson(cast[int](a), opt) + # edge case: `a == nil` could've also led to `newJNull()`, but this results + # in simpler code for `toJson` and `fromJson`. + elif T is distinct: result = toJson(a.distinctBase, opt) + elif T is bool: result = %(a) + elif T is SomeInteger: result = %a + elif T is enum: + case opt.enumMode + of joptEnumOrd: + when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord) + else: toJson($a, opt) + of joptEnumSymbol: + when T is OrdinalEnum: + toJson(symbolName(a), opt) + else: + toJson($a, opt) + of joptEnumString: toJson($a, opt) + elif T is Ordinal: result = %(a.ord) + elif T is cstring: (if a == nil: result = newJNull() else: result = % $a) + else: result = %a + +proc fromJsonHook*[K: string|cstring, V](t: var (Table[K, V] | OrderedTable[K, V]), + jsonNode: JsonNode, opt = Joptions()) = + ## Enables `fromJson` for `Table` and `OrderedTable` types. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook>`_ + runnableExamples: + import std/[tables, json] + var foo: tuple[t: Table[string, int], ot: OrderedTable[string, int]] + fromJson(foo, parseJson(""" + {"t":{"two":2,"one":1},"ot":{"one":1,"three":3}}""")) + assert foo.t == [("one", 1), ("two", 2)].toTable + assert foo.ot == [("one", 1), ("three", 3)].toOrderedTable + + assert jsonNode.kind == JObject, + "The kind of the `jsonNode` must be `JObject`, but its actual " & + "type is `" & $jsonNode.kind & "`." + clear(t) + for k, v in jsonNode: + t[k] = jsonTo(v, V, opt) + +proc toJsonHook*[K: string|cstring, V](t: (Table[K, V] | OrderedTable[K, V]), opt = initToJsonOptions()): JsonNode = + ## Enables `toJson` for `Table` and `OrderedTable` types. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_ + runnableExamples: + import std/[tables, json, sugar] + let foo = ( + t: [("two", 2)].toTable, + ot: [("one", 1), ("three", 3)].toOrderedTable) + assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}""" + # if keys are not string|cstring, you can use this: + let a = {10: "foo", 11: "bar"}.newOrderedTable + let a2 = collect: (for k,v in a: (k,v)) + assert $toJson(a2) == """[[10,"foo"],[11,"bar"]]""" + + result = newJObject() + for k, v in pairs(t): + # not sure if $k has overhead for string + result[(when K is string: k else: $k)] = toJson(v, opt) + +proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode, opt = Joptions()) = + ## Enables `fromJson` for `HashSet` and `OrderedSet` types. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_ + runnableExamples: + import std/[sets, json] + var foo: tuple[hs: HashSet[string], os: OrderedSet[string]] + fromJson(foo, parseJson(""" + {"hs": ["hash", "set"], "os": ["ordered", "set"]}""")) + assert foo.hs == ["hash", "set"].toHashSet + assert foo.os == ["ordered", "set"].toOrderedSet + + assert jsonNode.kind == JArray, + "The kind of the `jsonNode` must be `JArray`, but its actual " & + "type is `" & $jsonNode.kind & "`." + clear(s) + for v in jsonNode: + incl(s, jsonTo(v, A, opt)) + +proc toJsonHook*[A](s: SomeSet[A], opt = initToJsonOptions()): JsonNode = + ## Enables `toJson` for `HashSet` and `OrderedSet` types. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,SomeSet[A],JsonNode>`_ + runnableExamples: + import std/[sets, json] + let foo = (hs: ["hash"].toHashSet, os: ["ordered", "set"].toOrderedSet) + assert $toJson(foo) == """{"hs":["hash"],"os":["ordered","set"]}""" + + result = newJArray() + for k in s: + add(result, toJson(k, opt)) + +proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode, opt = Joptions()) = + ## Enables `fromJson` for `Option` types. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook,Option[T]>`_ + runnableExamples: + import std/[options, json] + var opt: Option[string] + fromJsonHook(opt, parseJson("\"test\"")) + assert get(opt) == "test" + fromJson(opt, parseJson("null")) + assert isNone(opt) + + if jsonNode.kind != JNull: + self = some(jsonTo(jsonNode, T, opt)) + else: + self = none[T]() + +proc toJsonHook*[T](self: Option[T], opt = initToJsonOptions()): JsonNode = + ## Enables `toJson` for `Option` types. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,Option[T],JsonNode>`_ + runnableExamples: + import std/[options, json] + let optSome = some("test") + assert $toJson(optSome) == "\"test\"" + let optNone = none[string]() + assert $toJson(optNone) == "null" + + if isSome(self): + toJson(get(self), opt) + else: + newJNull() + +proc fromJsonHook*(a: var StringTableRef, b: JsonNode) = + ## Enables `fromJson` for `StringTableRef` type. + ## + ## See also: + ## * `toJsonHook proc<#toJsonHook,StringTableRef>`_ + runnableExamples: + import std/[strtabs, json] + var t = newStringTable(modeCaseSensitive) + let jsonStr = """{"mode": 0, "table": {"name": "John", "surname": "Doe"}}""" + fromJsonHook(t, parseJson(jsonStr)) + assert t[] == newStringTable("name", "John", "surname", "Doe", + modeCaseSensitive)[] + + var mode = jsonTo(b["mode"], StringTableMode) + a = newStringTable(mode) + let b2 = b["table"] + for k,v in b2: a[k] = jsonTo(v, string) + +proc toJsonHook*(a: StringTableRef): JsonNode = + ## Enables `toJson` for `StringTableRef` type. + ## + ## See also: + ## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_ + runnableExamples: + import std/[strtabs, json] + let t = newStringTable("name", "John", "surname", "Doe", modeCaseSensitive) + let jsonStr = """{"mode": "modeCaseSensitive", + "table": {"name": "John", "surname": "Doe"}}""" + assert toJson(t) == parseJson(jsonStr) + + result = newJObject() + result["mode"] = toJson($a.mode) + let t = newJObject() + for k,v in a: t[k] = toJson(v) + result["table"] = t diff --git a/lib/std/logic.nim b/lib/std/logic.nim new file mode 100644 index 000000000..84640d380 --- /dev/null +++ b/lib/std/logic.nim @@ -0,0 +1,10 @@ +## This module provides further logic operators like 'forall' and 'exists' +## They are only supported in `.ensures` etc pragmas. + +proc `->`*(a, b: bool): bool {.magic: "Implies".} +proc `<->`*(a, b: bool): bool {.magic: "Iff".} + +proc forall*(args: varargs[untyped]): bool {.magic: "Forall".} +proc exists*(args: varargs[untyped]): bool {.magic: "Exists".} + +proc old*[T](x: T): T {.magic: "Old".} diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index af08c576e..bf6dc776b 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -8,34 +8,35 @@ # ##[ - The ``std/monotimes`` module implements monotonic timestamps. A monotonic - timestamp represents the time that has passed since some system defined - point in time. The monotonic timestamps are guaranteed to always increase, - meaning that that the following is guaranteed to work: - - .. code-block:: nim - let a = getMonoTime() - # ... do some work - let b = getMonoTime() - assert a <= b - - This is not guaranteed for the `times.Time` type! This means that the - `MonoTime` should be used when measuring durations of time with - high precision. - - However, since `MonoTime` represents the time that has passed since some - unknown time origin, it cannot be converted to a human readable timestamp. - If this is required, the `times.Time` type should be used instead. - - The `MonoTime` type stores the timestamp in nanosecond resolution, but note - that the actual supported time resolution differs for different systems. - - See also - ======== - * `times module <times.html>`_ +The `std/monotimes` module implements monotonic timestamps. A monotonic +timestamp represents the time that has passed since some system defined +point in time. The monotonic timestamps are guaranteed not to decrease, +meaning that that the following is guaranteed to work: ]## -import times +runnableExamples: + let a = getMonoTime() + let b = getMonoTime() + assert a <= b + +##[ +This is not guaranteed for the `times.Time` type! This means that the +`MonoTime` should be used when measuring durations of time with +high precision. + +However, since `MonoTime` represents the time that has passed since some +unknown time origin, it cannot be converted to a human readable timestamp. +If this is required, the `times.Time` type should be used instead. + +The `MonoTime` type stores the timestamp in nanosecond resolution, but note +that the actual supported time resolution differs for different systems. + +See also +======== +* `times module <times.html>`_ +]## + +import std/times type MonoTime* = object ## Represents a monotonic timestamp. @@ -51,25 +52,18 @@ when defined(macosx): proc mach_timebase_info(info: var MachTimebaseInfoData) {.importc, header: "<mach/mach_time.h>".} - let machAbsoluteTimeFreq = block: - var freq: MachTimebaseInfoData - mach_timebase_info(freq) - freq - when defined(js): proc getJsTicks: float = - ## Returns ticks in the unit seconds - {.emit: """ - var isNode = typeof module !== 'undefined' && module.exports - - if (isNode) { - var process = require('process'); - var time = process.hrtime() - return time[0] + time[1] / 1000000000; - } else { - return window.performance.now() / 1000; - } - """.} + ## Returns ticks in the unit seconds. + when defined(nodejs): + {.emit: """ + let process = require('process'); + let time = process.hrtime(); + `result` = time[0] + time[1] / 1000000000; + """.} + else: + proc jsNow(): float {.importjs: "window.performance.now()".} + result = jsNow() / 1000 # Workaround for #6752. {.push overflowChecks: off.} @@ -79,8 +73,12 @@ when defined(js): system.`+`(a, b) {.pop.} -elif defined(posix): - import posix +elif defined(posix) and not defined(osx): + import std/posix + +when defined(zephyr): + proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "<kernel.h>".} + proc k_ticks_to_ns_floor64(ticks: int64): int64 {.importc: "k_ticks_to_ns_floor64", header: "<kernel.h>".} elif defined(windows): proc QueryPerformanceCounter(res: var uint64) {. @@ -88,25 +86,25 @@ elif defined(windows): proc QueryPerformanceFrequency(res: var uint64) {. importc: "QueryPerformanceFrequency", stdcall, dynlib: "kernel32".} - let queryPerformanceCounterFreq = block: - var freq: uint64 - QueryPerformanceFrequency(freq) - 1_000_000_000'u64 div freq - proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = - ## Get the current `MonoTime` timestamp. + ## Returns the current `MonoTime` timestamp. ## ## When compiled with the JS backend and executed in a browser, - ## this proc calls `window.performance.now()`, which is not supported by - ## older browsers. See [MDN](https://developer.mozilla.org/en-US/docs/Web/API/Performance/now) + ## this proc calls `window.performance.now()`. + ## See [MDN](https://developer.mozilla.org/en-US/docs/Web/API/Performance/now) ## for more information. when defined(js): let ticks = getJsTicks() result = MonoTime(ticks: (ticks * 1_000_000_000).int64) elif defined(macosx): let ticks = mach_absolute_time() + var machAbsoluteTimeFreq: MachTimebaseInfoData + mach_timebase_info(machAbsoluteTimeFreq) result = MonoTime(ticks: ticks * machAbsoluteTimeFreq.numer div machAbsoluteTimeFreq.denom) + elif defined(zephyr): + let ticks = k_ticks_to_ns_floor64(k_uptime_ticks()) + result = MonoTime(ticks: ticks) elif defined(posix): var ts: Timespec discard clock_gettime(CLOCK_MONOTONIC, ts) @@ -115,6 +113,10 @@ proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = elif defined(windows): var ticks: uint64 QueryPerformanceCounter(ticks) + + var freq: uint64 + QueryPerformanceFrequency(freq) + let queryPerformanceCounterFreq = 1_000_000_000'u64 div freq result = MonoTime(ticks: (ticks * queryPerformanceCounterFreq).int64) proc ticks*(t: MonoTime): int64 = @@ -156,19 +158,3 @@ proc high*(typ: typedesc[MonoTime]): MonoTime = proc low*(typ: typedesc[MonoTime]): MonoTime = ## Returns the lowest representable `MonoTime`. MonoTime(ticks: low(int64)) - -when isMainModule: - let d = initDuration(nanoseconds = 10) - let t1 = getMonoTime() - let t2 = t1 + d - - doAssert t2 - t1 == d - doAssert t1 == t1 - doAssert t1 != t2 - doAssert t2 - d == t1 - doAssert t1 < t2 - doAssert t1 <= t2 - doAssert t1 <= t1 - doAssert not(t2 < t1) - doAssert t1 < high(MonoTime) - doAssert low(MonoTime) < t1 diff --git a/lib/std/objectdollar.nim b/lib/std/objectdollar.nim new file mode 100644 index 000000000..86ce9afc8 --- /dev/null +++ b/lib/std/objectdollar.nim @@ -0,0 +1,13 @@ +## This module implements a generic `$` operator to convert objects to strings. + +import std/private/miscdollars + +proc `$`*[T: object](x: T): string = + ## Generic `$` operator for objects with similar output to + ## `$` for named tuples. + runnableExamples: + type Foo = object + a, b: int + let x = Foo(a: 23, b: 45) + assert $x == "(a: 23, b: 45)" + tupleObjectDollar(result, x) diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim new file mode 100644 index 000000000..7b11c5e8e --- /dev/null +++ b/lib/std/oserrors.nim @@ -0,0 +1,117 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/oserrors` module implements OS error reporting. + +type + OSErrorCode* = distinct int32 ## Specifies an OS Error Code. + +when not defined(nimscript): + when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + else: + var errno {.importc, header: "<errno.h>".}: cint + + proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "<string.h>".} + +proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} +proc `$`*(err: OSErrorCode): string {.borrow.} + +proc osErrorMsg*(errorCode: OSErrorCode): string = + ## Converts an OS error code into a human readable string. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If conversion fails, or `errorCode` is `0` then `""` will be + ## returned. + ## + ## See also: + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ + runnableExamples: + when defined(linux): + assert osErrorMsg(OSErrorCode(0)) == "" + assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" + assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" + + result = "" + when defined(nimscript): + discard + elif defined(windows): + if errorCode != OSErrorCode(0'i32): + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: + result = $msgbuf + if msgbuf != nil: localFree(cast[pointer](msgbuf)) + else: + if errorCode != OSErrorCode(0'i32): + result = $c_strerror(errorCode.int32) + +proc newOSError*( + errorCode: OSErrorCode, additionalInfo = "" +): owned(ref OSError) {.noinline.} = + ## Creates a new `OSError exception <system.html#OSError>`_. + ## + ## The `errorCode` will determine the + ## message, `osErrorMsg proc`_ will be used + ## to get this message. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If the error code is `0` or an error message could not be retrieved, + ## the message `unknown OS error` will be used. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) + if additionalInfo.len > 0: + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. + if result.msg == "": + result.msg = "unknown OS error" + +proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = + ## Raises an `OSError exception <system.html#OSError>`_. + ## + ## Read the description of the `newOSError proc`_ to learn + ## how the exception object is created. + raise newOSError(errorCode, additionalInfo) + +{.push stackTrace:off.} +proc osLastError*(): OSErrorCode {.sideEffect.} = + ## Retrieves the last operating system error code. + ## + ## This procedure is useful in the event when an OS call fails. In that case + ## this procedure will return the error code describing the reason why the + ## OS call failed. The `OSErrorMsg` procedure can then be used to convert + ## this code into a string. + ## + ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. + ## On Windows some OS calls can reset the error code to `0` causing this + ## procedure to return `0`. It is therefore advised to call this procedure + ## immediately after an OS call fails. On POSIX systems this is not a problem. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ + when defined(nimscript): + discard + elif defined(windows): + result = cast[OSErrorCode](getLastError()) + else: + result = OSErrorCode(errno) +{.pop.} diff --git a/lib/std/outparams.nim b/lib/std/outparams.nim new file mode 100644 index 000000000..a471fbaa7 --- /dev/null +++ b/lib/std/outparams.nim @@ -0,0 +1,38 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## `outParamsAt` macro for easy writing code that works with both 2.0 and 1.x. + +import std/macros + +macro outParamsAt*(positions: static openArray[int]; n: untyped): untyped = + ## Use this macro to annotate `out` parameters in a portable way. + runnableExamples: + proc p(x: var int) {.outParamsAt: [1].} = + discard "x is really an 'out int' if the Nim compiler supports 'out' parameters" + + result = n + when defined(nimHasOutParams): + var p = n.params + for po in positions: + p[po][^2].expectKind nnkVarTy + p[po][^2] = newTree(nnkOutTy, p[po][^2][0]) + +when isMainModule: + {.experimental: "strictDefs".} + + proc main(x: var int) {.outParamsAt: [1].} = + x = 3 + + proc us = + var x: int + main x + echo x + + us() diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim new file mode 100644 index 000000000..3320558f2 --- /dev/null +++ b/lib/std/packedsets.nim @@ -0,0 +1,601 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## The `packedsets` module implements an efficient `Ordinal` set implemented as a +## `sparse bit set`:idx:. +## +## Supports any Ordinal type. +## +## See also +## ======== +## * `sets module <sets.html>`_ for more general hash sets + +import std/private/since +import std/hashes + +when defined(nimPreviewSlimSystem): + import std/assertions + +type + BitScalar = uint + +const + InitIntSetSize = 8 # must be a power of two! + TrunkShift = 9 + BitsPerTrunk = 1 shl TrunkShift # needs to be a power of 2 and + # divisible by 64 + TrunkMask = BitsPerTrunk - 1 + IntsPerTrunk = BitsPerTrunk div (sizeof(BitScalar) * 8) + IntShift = 5 + ord(sizeof(BitScalar) == 8) # 5 or 6, depending on int width + IntMask = 1 shl IntShift - 1 + +type + Trunk {.acyclic.} = ref object + next: Trunk # all nodes are connected with this pointer + key: int # start address at bit 0 + bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector + + TrunkSeq = seq[Trunk] + + PackedSet*[A: Ordinal] = object + ## An efficient set of `Ordinal` types implemented as a sparse bit set. + elems: int # only valid for small numbers + counter, max: int + head: Trunk + data: TrunkSeq + a: array[0..33, int] # profiling shows that 34 elements are enough + +proc mustRehash[T](t: T): bool {.inline.} = + let length = t.max + 1 + assert length > t.counter + result = (length * 2 < t.counter * 3) or (length - t.counter < 4) + +proc nextTry(h, maxHash: Hash, perturb: var Hash): Hash {.inline.} = + const PERTURB_SHIFT = 5 + var perturb2 = cast[uint](perturb) shr PERTURB_SHIFT + perturb = cast[Hash](perturb2) + result = ((5 * h) + 1 + perturb) and maxHash + +proc packedSetGet[A](t: PackedSet[A], key: int): Trunk = + var h = key and t.max + var perturb = key + while t.data[h] != nil: + if t.data[h].key == key: + return t.data[h] + h = nextTry(h, t.max, perturb) + result = nil + +proc intSetRawInsert[A](t: PackedSet[A], data: var TrunkSeq, desc: Trunk) = + var h = desc.key and t.max + var perturb = desc.key + while data[h] != nil: + assert data[h] != desc + h = nextTry(h, t.max, perturb) + assert data[h] == nil + data[h] = desc + +proc intSetEnlarge[A](t: var PackedSet[A]) = + var n: TrunkSeq + var oldMax = t.max + t.max = ((t.max + 1) * 2) - 1 + newSeq(n, t.max + 1) + for i in countup(0, oldMax): + if t.data[i] != nil: intSetRawInsert(t, n, t.data[i]) + swap(t.data, n) + +proc intSetPut[A](t: var PackedSet[A], key: int): Trunk = + var h = key and t.max + var perturb = key + while t.data[h] != nil: + if t.data[h].key == key: + return t.data[h] + h = nextTry(h, t.max, perturb) + if mustRehash(t): intSetEnlarge(t) + inc(t.counter) + h = key and t.max + perturb = key + while t.data[h] != nil: h = nextTry(h, t.max, perturb) + assert t.data[h] == nil + new(result) + result.next = t.head + result.key = key + t.head = result + t.data[h] = result + +proc bitincl[A](s: var PackedSet[A], key: int) {.inline.} = + var t = intSetPut(s, key shr TrunkShift) + var u = key and TrunkMask + t.bits[u shr IntShift] = t.bits[u shr IntShift] or + (BitScalar(1) shl (u and IntMask)) + +proc exclImpl[A](s: var PackedSet[A], key: int) = + if s.elems <= s.a.len: + for i in 0..<s.elems: + if s.a[i] == key: + s.a[i] = s.a[s.elems - 1] + dec(s.elems) + return + else: + var t = packedSetGet(s, key shr TrunkShift) + if t != nil: + var u = key and TrunkMask + t.bits[u shr IntShift] = t.bits[u shr IntShift] and + not(BitScalar(1) shl (u and IntMask)) + +template dollarImpl(): untyped = + result = "{" + for key in items(s): + if result.len > 1: result.add(", ") + result.add $key + result.add("}") + +iterator items*[A](s: PackedSet[A]): A {.inline.} = + ## Iterates over any included element of `s`. + if s.elems <= s.a.len: + for i in 0..<s.elems: + yield A(s.a[i]) + else: + var r = s.head + while r != nil: + var i = 0 + while i <= high(r.bits): + var w: uint = r.bits[i] + # taking a copy of r.bits[i] here is correct, because + # modifying operations are not allowed during traversation + var j = 0 + while w != 0: # test all remaining bits for zero + if (w and 1) != 0: # the bit is set! + yield A((r.key shl TrunkShift) or (i shl IntShift +% j)) + inc(j) + w = w shr 1 + inc(i) + r = r.next + +proc initPackedSet*[A]: PackedSet[A] = + ## Returns an empty `PackedSet[A]`. + ## `A` must be `Ordinal`. + ## + ## **See also:** + ## * `toPackedSet proc <#toPackedSet,openArray[A]>`_ + runnableExamples: + let a = initPackedSet[int]() + assert len(a) == 0 + + type Id = distinct int + var ids = initPackedSet[Id]() + ids.incl(3.Id) + + result = PackedSet[A]( + elems: 0, + counter: 0, + max: 0, + head: nil, + data: @[]) + # a: array[0..33, int] # profiling shows that 34 elements are enough + +proc contains*[A](s: PackedSet[A], key: A): bool = + ## Returns true if `key` is in `s`. + ## + ## This allows the usage of the `in` operator. + runnableExamples: + type ABCD = enum A, B, C, D + + let a = [1, 3, 5].toPackedSet + assert a.contains(3) + assert 3 in a + assert not a.contains(8) + assert 8 notin a + + let letters = [A, C].toPackedSet + assert A in letters + assert C in letters + assert B notin letters + + if s.elems <= s.a.len: + result = false + for i in 0..<s.elems: + if s.a[i] == ord(key): return true + else: + var t = packedSetGet(s, ord(key) shr TrunkShift) + if t != nil: + var u = ord(key) and TrunkMask + result = (t.bits[u shr IntShift] and + (BitScalar(1) shl (u and IntMask))) != 0 + else: + result = false + +proc incl*[A](s: var PackedSet[A], key: A) = + ## Includes an element `key` in `s`. + ## + ## This doesn't do anything if `key` is already in `s`. + ## + ## **See also:** + ## * `excl proc <#excl,PackedSet[A],A>`_ for excluding an element + ## * `incl proc <#incl,PackedSet[A],PackedSet[A]>`_ for including a set + ## * `containsOrIncl proc <#containsOrIncl,PackedSet[A],A>`_ + runnableExamples: + var a = initPackedSet[int]() + a.incl(3) + a.incl(3) + assert len(a) == 1 + + if s.elems <= s.a.len: + for i in 0..<s.elems: + if s.a[i] == ord(key): return + if s.elems < s.a.len: + s.a[s.elems] = ord(key) + inc(s.elems) + return + newSeq(s.data, InitIntSetSize) + s.max = InitIntSetSize - 1 + for i in 0..<s.elems: + bitincl(s, s.a[i]) + s.elems = s.a.len + 1 + # fall through: + bitincl(s, ord(key)) + +proc incl*[A](s: var PackedSet[A], other: PackedSet[A]) = + ## Includes all elements from `other` into `s`. + ## + ## This is the in-place version of `s + other <#+,PackedSet[A],PackedSet[A]>`_. + ## + ## **See also:** + ## * `excl proc <#excl,PackedSet[A],PackedSet[A]>`_ for excluding a set + ## * `incl proc <#incl,PackedSet[A],A>`_ for including an element + ## * `containsOrIncl proc <#containsOrIncl,PackedSet[A],A>`_ + runnableExamples: + var a = [1].toPackedSet + a.incl([5].toPackedSet) + assert len(a) == 2 + assert 5 in a + + for item in other.items: incl(s, item) + +proc toPackedSet*[A](x: openArray[A]): PackedSet[A] {.since: (1, 3).} = + ## Creates a new `PackedSet[A]` that contains the elements of `x`. + ## + ## Duplicates are removed. + ## + ## **See also:** + ## * `initPackedSet proc <#initPackedSet>`_ + runnableExamples: + let a = [5, 6, 7, 8, 8].toPackedSet + assert len(a) == 4 + assert $a == "{5, 6, 7, 8}" + + result = initPackedSet[A]() + for item in x: + result.incl(item) + +proc containsOrIncl*[A](s: var PackedSet[A], key: A): bool = + ## Includes `key` in the set `s` and tells if `key` was already in `s`. + ## + ## The difference with regards to the `incl proc <#incl,PackedSet[A],A>`_ is + ## that this proc returns true if `s` already contained `key`. The + ## proc will return false if `key` was added as a new value to `s` during + ## this call. + ## + ## **See also:** + ## * `incl proc <#incl,PackedSet[A],A>`_ for including an element + ## * `missingOrExcl proc <#missingOrExcl,PackedSet[A],A>`_ + runnableExamples: + var a = initPackedSet[int]() + assert a.containsOrIncl(3) == false + assert a.containsOrIncl(3) == true + assert a.containsOrIncl(4) == false + + if s.elems <= s.a.len: + for i in 0..<s.elems: + if s.a[i] == ord(key): + return true + incl(s, key) + result = false + else: + var t = packedSetGet(s, ord(key) shr TrunkShift) + if t != nil: + var u = ord(key) and TrunkMask + result = (t.bits[u shr IntShift] and BitScalar(1) shl (u and IntMask)) != 0 + if not result: + t.bits[u shr IntShift] = t.bits[u shr IntShift] or + (BitScalar(1) shl (u and IntMask)) + else: + incl(s, key) + result = false + +proc excl*[A](s: var PackedSet[A], key: A) = + ## Excludes `key` from the set `s`. + ## + ## This doesn't do anything if `key` is not found in `s`. + ## + ## **See also:** + ## * `incl proc <#incl,PackedSet[A],A>`_ for including an element + ## * `excl proc <#excl,PackedSet[A],PackedSet[A]>`_ for excluding a set + ## * `missingOrExcl proc <#missingOrExcl,PackedSet[A],A>`_ + runnableExamples: + var a = [3].toPackedSet + a.excl(3) + a.excl(3) + a.excl(99) + assert len(a) == 0 + + exclImpl[A](s, ord(key)) + +proc excl*[A](s: var PackedSet[A], other: PackedSet[A]) = + ## Excludes all elements from `other` from `s`. + ## + ## This is the in-place version of `s - other <#-,PackedSet[A],PackedSet[A]>`_. + ## + ## **See also:** + ## * `incl proc <#incl,PackedSet[A],PackedSet[A]>`_ for including a set + ## * `excl proc <#excl,PackedSet[A],A>`_ for excluding an element + ## * `missingOrExcl proc <#missingOrExcl,PackedSet[A],A>`_ + runnableExamples: + var a = [1, 5].toPackedSet + a.excl([5].toPackedSet) + assert len(a) == 1 + assert 5 notin a + + for item in other.items: + excl(s, item) + +proc len*[A](s: PackedSet[A]): int {.inline.} = + ## Returns the number of elements in `s`. + runnableExamples: + let a = [1, 3, 5].toPackedSet + assert len(a) == 3 + + if s.elems < s.a.len: + result = s.elems + else: + result = 0 + for _ in s.items: + # pending bug #11167; when fixed, check each explicit `items` to see if it can be removed + inc(result) + +proc missingOrExcl*[A](s: var PackedSet[A], key: A): bool = + ## Excludes `key` from the set `s` and tells if `key` was already missing from `s`. + ## + ## The difference with regards to the `excl proc <#excl,PackedSet[A],A>`_ is + ## that this proc returns true if `key` was missing from `s`. + ## The proc will return false if `key` was in `s` and it was removed + ## during this call. + ## + ## **See also:** + ## * `excl proc <#excl,PackedSet[A],A>`_ for excluding an element + ## * `excl proc <#excl,PackedSet[A],PackedSet[A]>`_ for excluding a set + ## * `containsOrIncl proc <#containsOrIncl,PackedSet[A],A>`_ + runnableExamples: + var a = [5].toPackedSet + assert a.missingOrExcl(5) == false + assert a.missingOrExcl(5) == true + + var count = s.len + exclImpl(s, ord(key)) + result = count == s.len + +proc clear*[A](result: var PackedSet[A]) = + ## Clears the `PackedSet[A]` back to an empty state. + runnableExamples: + var a = [5, 7].toPackedSet + clear(a) + assert len(a) == 0 + + # setLen(result.data, InitIntSetSize) + # for i in 0..InitIntSetSize - 1: result.data[i] = nil + # result.max = InitIntSetSize - 1 + result.data = @[] + result.max = 0 + result.counter = 0 + result.head = nil + result.elems = 0 + +proc isNil*[A](x: PackedSet[A]): bool {.inline.} = + ## Returns true if `x` is empty, false otherwise. + runnableExamples: + var a = initPackedSet[int]() + assert a.isNil + a.incl(2) + assert not a.isNil + a.excl(2) + assert a.isNil + + x.head.isNil and x.elems == 0 + +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + if src.elems <= src.a.len: + dest.data = @[] + dest.max = 0 + dest.counter = src.counter + dest.head = nil + dest.elems = src.elems + dest.a = src.a + else: + dest.counter = src.counter + dest.max = src.max + dest.elems = src.elems + newSeq(dest.data, src.data.len) + + var it = src.head + while it != nil: + var h = it.key and dest.max + var perturb = it.key + while dest.data[h] != nil: h = nextTry(h, dest.max, perturb) + assert dest.data[h] == nil + var n: Trunk + new(n) + n.next = dest.head + n.key = it.key + n.bits = it.bits + dest.head = n + dest.data[h] = n + it = it.next + +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + +proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the union of the sets `s1` and `s2`. + ## + ## The same as `s1 + s2 <#+,PackedSet[A],PackedSet[A]>`_. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = union(a, b) + assert c.len == 5 + assert c == [1, 2, 3, 4, 5].toPackedSet + + result.assign(s1) + incl(result, s2) + +proc intersection*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the intersection of the sets `s1` and `s2`. + ## + ## The same as `s1 * s2 <#*,PackedSet[A],PackedSet[A]>`_. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = intersection(a, b) + assert c.len == 1 + assert c == [3].toPackedSet + + result = initPackedSet[A]() + for item in s1.items: + if contains(s2, item): + incl(result, item) + +proc difference*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the difference of the sets `s1` and `s2`. + ## + ## The same as `s1 - s2 <#-,PackedSet[A],PackedSet[A]>`_. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = difference(a, b) + assert c.len == 2 + assert c == [1, 2].toPackedSet + + result = initPackedSet[A]() + for item in s1.items: + if not contains(s2, item): + incl(result, item) + +proc symmetricDifference*[A](s1, s2: PackedSet[A]): PackedSet[A] = + ## Returns the symmetric difference of the sets `s1` and `s2`. + runnableExamples: + let + a = [1, 2, 3].toPackedSet + b = [3, 4, 5].toPackedSet + c = symmetricDifference(a, b) + assert c.len == 4 + assert c == [1, 2, 4, 5].toPackedSet + + result.assign(s1) + for item in s2.items: + if containsOrIncl(result, item): + excl(result, item) + +proc `+`*[A](s1, s2: PackedSet[A]): PackedSet[A] {.inline.} = + ## Alias for `union(s1, s2) <#union,PackedSet[A],PackedSet[A]>`_. + result = union(s1, s2) + +proc `*`*[A](s1, s2: PackedSet[A]): PackedSet[A] {.inline.} = + ## Alias for `intersection(s1, s2) <#intersection,PackedSet[A],PackedSet[A]>`_. + result = intersection(s1, s2) + +proc `-`*[A](s1, s2: PackedSet[A]): PackedSet[A] {.inline.} = + ## Alias for `difference(s1, s2) <#difference,PackedSet[A],PackedSet[A]>`_. + result = difference(s1, s2) + +proc disjoint*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if the sets `s1` and `s2` have no items in common. + runnableExamples: + let + a = [1, 2].toPackedSet + b = [2, 3].toPackedSet + c = [3, 4].toPackedSet + assert disjoint(a, b) == false + assert disjoint(a, c) == true + + for item in s1.items: + if contains(s2, item): + return false + return true + +proc card*[A](s: PackedSet[A]): int {.inline.} = + ## Alias for `len() <#len,PackedSet[A]>`_. + ## + ## Card stands for the [cardinality](http://en.wikipedia.org/wiki/Cardinality) + ## of a set. + result = s.len() + +proc `<=`*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if `s1` is a subset of `s2`. + ## + ## A subset `s1` has all of its elements in `s2`, but `s2` doesn't necessarily + ## have more elements than `s1`. That is, `s1` can be equal to `s2`. + runnableExamples: + let + a = [1].toPackedSet + b = [1, 2].toPackedSet + c = [1, 3].toPackedSet + assert a <= b + assert b <= b + assert not (c <= b) + + for item in s1.items: + if not s2.contains(item): + return false + return true + +proc `<`*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if `s1` is a proper subset of `s2`. + ## + ## A strict or proper subset `s1` has all of its elements in `s2`, but `s2` has + ## more elements than `s1`. + runnableExamples: + let + a = [1].toPackedSet + b = [1, 2].toPackedSet + c = [1, 3].toPackedSet + assert a < b + assert not (b < b) + assert not (c < b) + + return s1 <= s2 and not (s2 <= s1) + +proc `==`*[A](s1, s2: PackedSet[A]): bool = + ## Returns true if both `s1` and `s2` have the same elements and set size. + runnableExamples: + assert [1, 2].toPackedSet == [2, 1].toPackedSet + assert [1, 2].toPackedSet == [2, 1, 2].toPackedSet + + return s1 <= s2 and s2 <= s1 + +proc `$`*[A](s: PackedSet[A]): string = + ## Converts `s` to a string. + runnableExamples: + let a = [1, 2, 3].toPackedSet + assert $a == "{1, 2, 3}" + + dollarImpl() diff --git a/lib/std/paths.nim b/lib/std/paths.nim new file mode 100644 index 000000000..664dedd31 --- /dev/null +++ b/lib/std/paths.nim @@ -0,0 +1,302 @@ +## This module implements path handling. +## +## **See also:** +## * `files module <files.html>`_ for file access + +import std/private/osseps +export osseps + +import std/envvars +import std/private/osappdirs + +import std/[pathnorm, hashes, sugar, strutils] + +from std/private/ospaths2 import joinPath, splitPath, + ReadDirEffect, WriteDirEffect, + isAbsolute, relativePath, + normalizePathEnd, isRelativeTo, parentDir, + tailDir, isRootDir, parentDirs, `/../`, + extractFilename, lastPathPart, + changeFileExt, addFileExt, cmpPaths, splitFile, + unixToNativePath, absolutePath, normalizeExe, + normalizePath +export ReadDirEffect, WriteDirEffect + +type + Path* = distinct string + +func hash*(x: Path): Hash = + let x = x.string.dup(normalizePath) + if FileSystemCaseSensitive: + result = x.hash + else: + result = x.toLowerAscii.hash + +template `$`*(x: Path): string = + string(x) + +func `==`*(x, y: Path): bool {.inline.} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. + result = cmpPaths(x.string, y.string) == 0 + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +func add(x: var string, tail: string) = + var state = 0 + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and x.endsWith({DirSep, AltSep}) + normalizePathEnd(x, trailingSep=false) + addNormalizePath(tail, x, state, DirSep) + normalizePathEnd(x, trailingSep=trailingSep) + +func add*(x: var Path, y: Path) {.borrow.} + +func `/`*(head, tail: Path): Path {.inline.} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + Path(joinPath(head.string, tail.string)) + +func splitPath*(path: Path): tuple[head, tail: Path] {.inline.} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `add proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + let res = splitPath(path.string) + result = (Path(res.head), Path(res.tail)) + +func splitFile*(path: Path): tuple[dir, name: Path, ext: string] {.inline.} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + let res = splitFile(path.string) + result = (Path(res.dir), Path(res.name), res.ext) + +func isAbsolute*(path: Path): bool {.inline, raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + result = isAbsolute(path.string) + +proc relativePath*(path, base: Path, sep = DirSep): Path {.inline.} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + result = Path(relativePath(path.string, base.string, sep)) + +proc isRelativeTo*(path: Path, base: Path): bool {.inline.} = + ## Returns true if `path` is relative to `base`. + result = isRelativeTo(path.string, base.string) + + +func parentDir*(path: Path): Path {.inline.} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + result = Path(parentDir(path.string)) + +func tailDir*(path: Path): Path {.inline.} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + result = Path(tailDir(path.string)) + +func isRootDir*(path: Path): bool {.inline.} = + ## Checks whether a given `path` is a root directory. + result = isRootDir(path.string) + +iterator parentDirs*(path: Path, fromRoot=false, inclusive=true): Path = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + for p in parentDirs(path.string, fromRoot, inclusive): + yield Path(p) + +func `/../`*(head, tail: Path): Path {.inline.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + Path(`/../`(head.string, tail.string)) + +func extractFilename*(path: Path): Path {.inline.} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(extractFilename(path.string)) + +func lastPathPart*(path: Path): Path {.inline.} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + result = Path(lastPathPart(path.string)) + +func changeFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + result = Path(changeFileExt(filename.string, ext)) + +func addFileExt*(filename: Path, ext: string): Path {.inline.} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + result = Path(addFileExt(filename.string, ext)) + +func unixToNativePath*(path: Path, drive=Path("")): Path {.inline.} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + result = Path(unixToNativePath(path.string, drive.string)) + +proc getCurrentDir*(): Path {.inline, tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc <appdirs.html#getHomeDir>`_ + ## * `getConfigDir proc <appdirs.html#getConfigDir>`_ + ## * `getTempDir proc <appdirs.html#getTempDir>`_ + ## * `setCurrentDir proc <dirs.html#setCurrentDir>`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + result = Path(ospaths2.getCurrentDir()) + +proc normalizeExe*(file: var Path) {.borrow.} + +proc normalizePath*(path: var Path) {.borrow.} + +proc normalizePathEnd*(path: var Path, trailingSep = false) {.borrow.} + +proc absolutePath*(path: Path, root = getCurrentDir()): Path = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizePath proc`_ + result = Path(absolutePath(path.string, root.string)) + +proc expandTildeImpl(path: string): string {. + tags: [ReadEnvEffect, ReadIOEffect].} = + if len(path) == 0 or path[0] != '~': + result = path + elif len(path) == 1: + result = getHomeDir() + elif (path[1] in {DirSep, AltSep}): + result = joinPath(getHomeDir(), path.substr(2)) + else: + # TODO: handle `~bob` and `~bob/` which means home of bob + result = path + +proc expandTilde*(path: Path): Path {.inline, + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing + ## ``~`` with `getHomeDir() <appdirs.html#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## + ## Windows: this is still supported despite the Windows platform not having this + ## convention; also, both ``~/`` and ``~\`` are handled. + runnableExamples: + import std/appdirs + assert expandTilde(Path("~") / Path("appname.cfg")) == getHomeDir() / Path("appname.cfg") + assert expandTilde(Path("~/foo/bar")) == getHomeDir() / Path("foo/bar") + assert expandTilde(Path("/foo/bar")) == Path("/foo/bar") + result = Path(expandTildeImpl(path.string)) diff --git a/lib/std/private/asciitables.nim b/lib/std/private/asciitables.nim new file mode 100644 index 000000000..cbc595651 --- /dev/null +++ b/lib/std/private/asciitables.nim @@ -0,0 +1,83 @@ +#[ +move to std/asciitables.nim once stable, or to a fusion package +once compiler can depend on fusion +]# + +type Cell* = object + text*: string + width*, row*, col*, ncols*, nrows*: int + +iterator parseTableCells*(s: string, delim = '\t'): Cell = + ## Iterates over all cells in a `delim`-delimited `s`, after a 1st + ## pass that computes number of rows, columns, and width of each column. + var widths: seq[int] + var cell: Cell + template update() = + if widths.len<=cell.col: + widths.setLen cell.col+1 + widths[cell.col] = cell.width + else: + widths[cell.col] = max(widths[cell.col], cell.width) + cell.width = 0 + + for a in s: + case a + of '\n': + update() + cell.col = 0 + cell.row.inc + elif a == delim: + update() + cell.col.inc + else: + # todo: consider multi-width chars when porting to non-ascii implementation + cell.width.inc + if s.len > 0 and s[^1] != '\n': + update() + + cell.ncols = widths.len + cell.nrows = cell.row + 1 + cell.row = 0 + cell.col = 0 + cell.width = 0 + + template update2() = + cell.width = widths[cell.col] + yield cell + cell.text = "" + cell.width = 0 + cell.col.inc + + template finishRow() = + for col in cell.col..<cell.ncols: + cell.col = col + update2() + cell.col = 0 + + for a in s: + case a + of '\n': + finishRow() + cell.row.inc + elif a == delim: + update2() + else: + cell.width.inc + cell.text.add a + + if s.len > 0 and s[^1] != '\n': + finishRow() + +proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string = + ## Formats a `delim`-delimited `s` representing a table; each cell is aligned + ## to a width that's computed for each column; consecutive columns are + ## delimited by `sep`, and alignment space is filled using `fill`. + ## More customized formatting can be done by calling `parseTableCells` directly. + for cell in parseTableCells(s, delim): + result.add cell.text + for i in cell.text.len..<cell.width: + result.add fill + if cell.col < cell.ncols-1: + result.add sep + if cell.col == cell.ncols-1 and cell.row < cell.nrows - 1: + result.add '\n' diff --git a/lib/std/private/bitops_utils.nim b/lib/std/private/bitops_utils.nim new file mode 100644 index 000000000..0b9484416 --- /dev/null +++ b/lib/std/private/bitops_utils.nim @@ -0,0 +1,22 @@ +template forwardImpl*(impl, arg) {.dirty.} = + when sizeof(x) <= 4: + when x is SomeSignedInt: + impl(cast[uint32](x.int32)) + else: + impl(x.uint32) + else: + when x is SomeSignedInt: + impl(cast[uint64](x.int64)) + else: + impl(x.uint64) + +# this could also be implemented via: +# import std/typetraits +# template castToUnsigned*(x: SomeInteger): auto = cast[toUnsigned(typeof(x))](x) + +template castToUnsigned*(x: int8): uint8 = cast[uint8](x) +template castToUnsigned*(x: int16): uint16 = cast[uint16](x) +template castToUnsigned*(x: int32): uint32 = cast[uint32](x) +template castToUnsigned*(x: int64): uint64 = cast[uint64](x) +template castToUnsigned*(x: int): uint = cast[uint](x) +template castToUnsigned*[T: SomeUnsignedInt](x: T): T = x diff --git a/lib/std/private/decode_helpers.nim b/lib/std/private/decode_helpers.nim new file mode 100644 index 000000000..f11e3060a --- /dev/null +++ b/lib/std/private/decode_helpers.nim @@ -0,0 +1,42 @@ +proc handleHexChar*(c: char, x: var int): bool {.inline.} = + ## Converts `%xx` hexadecimal to the ordinal number and adds the result to `x`. + ## Returns `true` if `c` is hexadecimal. + ## + ## When `c` is hexadecimal, the proc is equal to `x = x shl 4 + hex2Int(c)`. + runnableExamples: + var x = 0 + assert handleHexChar('a', x) + assert x == 10 + + assert handleHexChar('B', x) + assert x == 171 # 10 shl 4 + 11 + + assert not handleHexChar('?', x) + assert x == 171 # unchanged + result = true + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: + result = false + +proc handleHexChar*(c: char): int {.inline.} = + case c + of '0'..'9': result = (ord(c) - ord('0')) + of 'a'..'f': result = (ord(c) - ord('a') + 10) + of 'A'..'F': result = (ord(c) - ord('A') + 10) + else: discard + +proc decodePercent*(s: openArray[char], i: var int): char = + ## Converts `%xx` hexadecimal to the character with ordinal number `xx`. + ## + ## If `xx` is not a valid hexadecimal value, it is left intact: only the + ## leading `%` is returned as-is, and `xx` characters will be processed in the + ## next step (e.g. in `uri.decodeUrl`) as regular characters. + result = '%' + if i+2 < s.len: + var x = 0 + if handleHexChar(s[i+1], x) and handleHexChar(s[i+2], x): + result = chr(x) + inc(i, 2) diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim new file mode 100644 index 000000000..f2d0d25cb --- /dev/null +++ b/lib/std/private/digitsutils.nim @@ -0,0 +1,116 @@ +const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + +# Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c +# Generates: +# ```nim +# var res = "" +# for i in 0 .. 99: +# if i < 10: +# res.add "0" & $i +# else: +# res.add $i +# doAssert res == digits100 +# ``` + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int {.inline.} = + trailingZeros100[digits] + +when defined(js): + proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} + +func addChars[T](result: var string, x: T, start: int, n: int) {.inline.} = + let old = result.len + result.setLen old + n + template impl = + for i in 0..<n: result[old + i] = x[start + i] + when nimvm: impl + else: + when defined(js) or defined(nimscript): impl + else: + {.noSideEffect.}: + copyMem result[old].addr, x[start].unsafeAddr, n + +func addChars[T](result: var string, x: T) {.inline.} = + addChars(result, x, 0, x.len) + +func addIntImpl(result: var string, x: uint64) {.inline.} = + var tmp {.noinit.}: array[24, char] + var num = x + var next = tmp.len - 1 + const nbatch = 100 + + while num >= nbatch: + let originNum = num + num = num div nbatch + let index = int16((originNum - num * nbatch) shl 1) + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec(next, 2) + + # process last 1-2 digits + if num < 10: + tmp[next] = chr(ord('0') + num.uint8) + else: + let index = num * 2 + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec next + addChars(result, tmp, next, tmp.len - next) + +when not defined(nimHasEnforceNoRaises): + {.pragma: enforceNoRaises.} + +func addInt*(result: var string, x: uint64) {.enforceNoRaises.} = + when nimvm: addIntImpl(result, x) + else: + when not defined(js): addIntImpl(result, x) + else: + addChars(result, numToString(x)) + +proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = + ## Converts integer to its string representation and appends it to `result`. + runnableExamples: + var s = "foo" + s.addInt(45) + assert s == "foo45" + template impl = + var num: uint64 + if x < 0: + if x == low(int64): + num = cast[uint64](x) + else: + num = uint64(-x) + result.add '-' + else: + num = uint64(x) + addInt(result, num) + when nimvm: impl() + else: + when defined(js): + addChars(result, numToString(x)) + else: impl() + +proc addInt*(result: var string; x: int) {.inline, enforceNoRaises.} = + addInt(result, int64(x)) diff --git a/lib/std/private/dragonbox.nim b/lib/std/private/dragonbox.nim new file mode 100644 index 000000000..85ffea84a --- /dev/null +++ b/lib/std/private/dragonbox.nim @@ -0,0 +1,1325 @@ +## Copyright 2020 Junekey Jeon +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## char* output_end = Dtoa(buffer, value); +## +## Converts the given double-precision number into decimal form and stores the result in the given +## buffer. +## +## The buffer must be large enough, i.e. >= DtoaMinBufferLength. +## The output format is similar to printf("%g"). +## The output is _not_ null-terminted. +## +## The output is optimal, i.e. the output string +## 1. rounds back to the input number when read in (using round-to-nearest-even) +## 2. is as short as possible, +## 3. is as close to the input number as possible. +## +## Note: +## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + +const + dtoaMinBufferLength*: cint = 64 + +## This file contains an implementation of Junekey Jeon's Dragonbox algorithm. +## +## It is a simplified version of the reference implementation found here: +## https://github.com/jk-jeon/dragonbox +## +## The reference implementation also works with single-precision floating-point numbers and +## has options to configure the rounding mode. + +template dragonbox_Assert*(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType* = float + BitsType* = uint64 + +type + Double* = object + bits*: BitsType + +const ## = p (includes the hidden bit) + significandSize*: int32 = 53 + +const ## static constexpr int32_t MaxExponent = 1024 - 1 - (SignificandSize - 1); + ## static constexpr int32_t MinExponent = std::numeric_limits<value_type>::min_exponent - 1 - (SignificandSize - 1); + exponentBias*: int32 = 1024 - 1 + (significandSize - 1) + +const + maxIeeeExponent*: BitsType = BitsType(2 * 1024 - 1) + +const ## = 2^(p-1) + hiddenBit*: BitsType = BitsType(1) shl (significandSize - 1) + +const ## = 2^(p-1) - 1 + significandMask*: BitsType = hiddenBit - 1 + +const + exponentMask*: BitsType = maxIeeeExponent shl (significandSize - 1) + +const + signMask*: BitsType = not (not BitsType(0) shr 1) + +proc constructDouble*(bits: BitsType): Double = + result.bits = bits + +proc constructDouble*(value: ValueType): Double = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent*(this: Double): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero*(this: Double): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit*(this: Double): int {.noSideEffect.} = + return ord((this.bits and signMask) != 0) + + +# ================================================================================================== +# +# ================================================================================================== +## namespace +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2*(x: int32; n: int32): int32 {.inline.} = + return x shr n + +proc floorLog2Pow10*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1233) + dragonbox_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc floorLog10Pow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611, 22) + +proc floorLog10ThreeQuartersPow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611 - 524031, 22) + +# ================================================================================================== +# +# ================================================================================================== + +type + uint64x2* {.bycopy.} = object + hi*: uint64 + lo*: uint64 + + +proc computePow10*(k: int32): uint64x2 {.inline.} = + const + kMin: int32 = -292 + const + kMax: int32 = 326 + const + pow10: array[kMax - kMin + 1, uint64x2] = [ + uint64x2(hi: 0xFF77B1FCBEBCDC4F'u, lo: 0x25E8E89C13BB0F7B'u), + uint64x2(hi: 0x9FAACF3DF73609B1'u, lo: 0x77B191618C54E9AD'u), + uint64x2(hi: 0xC795830D75038C1D'u, lo: 0xD59DF5B9EF6A2418'u), + uint64x2(hi: 0xF97AE3D0D2446F25'u, lo: 0x4B0573286B44AD1E'u), + uint64x2(hi: 0x9BECCE62836AC577'u, lo: 0x4EE367F9430AEC33'u), + uint64x2(hi: 0xC2E801FB244576D5'u, lo: 0x229C41F793CDA740'u), + uint64x2(hi: 0xF3A20279ED56D48A'u, lo: 0x6B43527578C11110'u), + uint64x2(hi: 0x9845418C345644D6'u, lo: 0x830A13896B78AAAA'u), + uint64x2(hi: 0xBE5691EF416BD60C'u, lo: 0x23CC986BC656D554'u), + uint64x2(hi: 0xEDEC366B11C6CB8F'u, lo: 0x2CBFBE86B7EC8AA9'u), + uint64x2(hi: 0x94B3A202EB1C3F39'u, lo: 0x7BF7D71432F3D6AA'u), + uint64x2(hi: 0xB9E08A83A5E34F07'u, lo: 0xDAF5CCD93FB0CC54'u), + uint64x2(hi: 0xE858AD248F5C22C9'u, lo: 0xD1B3400F8F9CFF69'u), + uint64x2(hi: 0x91376C36D99995BE'u, lo: 0x23100809B9C21FA2'u), + uint64x2(hi: 0xB58547448FFFFB2D'u, lo: 0xABD40A0C2832A78B'u), + uint64x2(hi: 0xE2E69915B3FFF9F9'u, lo: 0x16C90C8F323F516D'u), + uint64x2(hi: 0x8DD01FAD907FFC3B'u, lo: 0xAE3DA7D97F6792E4'u), + uint64x2(hi: 0xB1442798F49FFB4A'u, lo: 0x99CD11CFDF41779D'u), + uint64x2(hi: 0xDD95317F31C7FA1D'u, lo: 0x40405643D711D584'u), + uint64x2(hi: 0x8A7D3EEF7F1CFC52'u, lo: 0x482835EA666B2573'u), + uint64x2(hi: 0xAD1C8EAB5EE43B66'u, lo: 0xDA3243650005EED0'u), + uint64x2(hi: 0xD863B256369D4A40'u, lo: 0x90BED43E40076A83'u), + uint64x2(hi: 0x873E4F75E2224E68'u, lo: 0x5A7744A6E804A292'u), + uint64x2(hi: 0xA90DE3535AAAE202'u, lo: 0x711515D0A205CB37'u), + uint64x2(hi: 0xD3515C2831559A83'u, lo: 0x0D5A5B44CA873E04'u), + uint64x2(hi: 0x8412D9991ED58091'u, lo: 0xE858790AFE9486C3'u), + uint64x2(hi: 0xA5178FFF668AE0B6'u, lo: 0x626E974DBE39A873'u), + uint64x2(hi: 0xCE5D73FF402D98E3'u, lo: 0xFB0A3D212DC81290'u), + uint64x2(hi: 0x80FA687F881C7F8E'u, lo: 0x7CE66634BC9D0B9A'u), + uint64x2(hi: 0xA139029F6A239F72'u, lo: 0x1C1FFFC1EBC44E81'u), + uint64x2(hi: 0xC987434744AC874E'u, lo: 0xA327FFB266B56221'u), + uint64x2(hi: 0xFBE9141915D7A922'u, lo: 0x4BF1FF9F0062BAA9'u), + uint64x2(hi: 0x9D71AC8FADA6C9B5'u, lo: 0x6F773FC3603DB4AA'u), + uint64x2(hi: 0xC4CE17B399107C22'u, lo: 0xCB550FB4384D21D4'u), + uint64x2(hi: 0xF6019DA07F549B2B'u, lo: 0x7E2A53A146606A49'u), + uint64x2(hi: 0x99C102844F94E0FB'u, lo: 0x2EDA7444CBFC426E'u), + uint64x2(hi: 0xC0314325637A1939'u, lo: 0xFA911155FEFB5309'u), + uint64x2(hi: 0xF03D93EEBC589F88'u, lo: 0x793555AB7EBA27CB'u), + uint64x2(hi: 0x96267C7535B763B5'u, lo: 0x4BC1558B2F3458DF'u), + uint64x2(hi: 0xBBB01B9283253CA2'u, lo: 0x9EB1AAEDFB016F17'u), + uint64x2(hi: 0xEA9C227723EE8BCB'u, lo: 0x465E15A979C1CADD'u), + uint64x2(hi: 0x92A1958A7675175F'u, lo: 0x0BFACD89EC191ECA'u), + uint64x2(hi: 0xB749FAED14125D36'u, lo: 0xCEF980EC671F667C'u), + uint64x2(hi: 0xE51C79A85916F484'u, lo: 0x82B7E12780E7401B'u), + uint64x2(hi: 0x8F31CC0937AE58D2'u, lo: 0xD1B2ECB8B0908811'u), + uint64x2(hi: 0xB2FE3F0B8599EF07'u, lo: 0x861FA7E6DCB4AA16'u), + uint64x2(hi: 0xDFBDCECE67006AC9'u, lo: 0x67A791E093E1D49B'u), + uint64x2(hi: 0x8BD6A141006042BD'u, lo: 0xE0C8BB2C5C6D24E1'u), + uint64x2(hi: 0xAECC49914078536D'u, lo: 0x58FAE9F773886E19'u), + uint64x2(hi: 0xDA7F5BF590966848'u, lo: 0xAF39A475506A899F'u), + uint64x2(hi: 0x888F99797A5E012D'u, lo: 0x6D8406C952429604'u), + uint64x2(hi: 0xAAB37FD7D8F58178'u, lo: 0xC8E5087BA6D33B84'u), + uint64x2(hi: 0xD5605FCDCF32E1D6'u, lo: 0xFB1E4A9A90880A65'u), + uint64x2(hi: 0x855C3BE0A17FCD26'u, lo: 0x5CF2EEA09A550680'u), + uint64x2(hi: 0xA6B34AD8C9DFC06F'u, lo: 0xF42FAA48C0EA481F'u), + uint64x2(hi: 0xD0601D8EFC57B08B'u, lo: 0xF13B94DAF124DA27'u), + uint64x2(hi: 0x823C12795DB6CE57'u, lo: 0x76C53D08D6B70859'u), + uint64x2(hi: 0xA2CB1717B52481ED'u, lo: 0x54768C4B0C64CA6F'u), + uint64x2(hi: 0xCB7DDCDDA26DA268'u, lo: 0xA9942F5DCF7DFD0A'u), + uint64x2(hi: 0xFE5D54150B090B02'u, lo: 0xD3F93B35435D7C4D'u), + uint64x2(hi: 0x9EFA548D26E5A6E1'u, lo: 0xC47BC5014A1A6DB0'u), + uint64x2(hi: 0xC6B8E9B0709F109A'u, lo: 0x359AB6419CA1091C'u), + uint64x2(hi: 0xF867241C8CC6D4C0'u, lo: 0xC30163D203C94B63'u), + uint64x2(hi: 0x9B407691D7FC44F8'u, lo: 0x79E0DE63425DCF1E'u), + uint64x2(hi: 0xC21094364DFB5636'u, lo: 0x985915FC12F542E5'u), + uint64x2(hi: 0xF294B943E17A2BC4'u, lo: 0x3E6F5B7B17B2939E'u), + uint64x2(hi: 0x979CF3CA6CEC5B5A'u, lo: 0xA705992CEECF9C43'u), + uint64x2(hi: 0xBD8430BD08277231'u, lo: 0x50C6FF782A838354'u), + uint64x2(hi: 0xECE53CEC4A314EBD'u, lo: 0xA4F8BF5635246429'u), + uint64x2(hi: 0x940F4613AE5ED136'u, lo: 0x871B7795E136BE9A'u), + uint64x2(hi: 0xB913179899F68584'u, lo: 0x28E2557B59846E40'u), + uint64x2(hi: 0xE757DD7EC07426E5'u, lo: 0x331AEADA2FE589D0'u), + uint64x2(hi: 0x9096EA6F3848984F'u, lo: 0x3FF0D2C85DEF7622'u), + uint64x2(hi: 0xB4BCA50B065ABE63'u, lo: 0x0FED077A756B53AA'u), + uint64x2(hi: 0xE1EBCE4DC7F16DFB'u, lo: 0xD3E8495912C62895'u), + uint64x2(hi: 0x8D3360F09CF6E4BD'u, lo: 0x64712DD7ABBBD95D'u), + uint64x2(hi: 0xB080392CC4349DEC'u, lo: 0xBD8D794D96AACFB4'u), + uint64x2(hi: 0xDCA04777F541C567'u, lo: 0xECF0D7A0FC5583A1'u), + uint64x2(hi: 0x89E42CAAF9491B60'u, lo: 0xF41686C49DB57245'u), + uint64x2(hi: 0xAC5D37D5B79B6239'u, lo: 0x311C2875C522CED6'u), + uint64x2(hi: 0xD77485CB25823AC7'u, lo: 0x7D633293366B828C'u), + uint64x2(hi: 0x86A8D39EF77164BC'u, lo: 0xAE5DFF9C02033198'u), + uint64x2(hi: 0xA8530886B54DBDEB'u, lo: 0xD9F57F830283FDFD'u), + uint64x2(hi: 0xD267CAA862A12D66'u, lo: 0xD072DF63C324FD7C'u), + uint64x2(hi: 0x8380DEA93DA4BC60'u, lo: 0x4247CB9E59F71E6E'u), + uint64x2(hi: 0xA46116538D0DEB78'u, lo: 0x52D9BE85F074E609'u), + uint64x2(hi: 0xCD795BE870516656'u, lo: 0x67902E276C921F8C'u), + uint64x2(hi: 0x806BD9714632DFF6'u, lo: 0x00BA1CD8A3DB53B7'u), + uint64x2(hi: 0xA086CFCD97BF97F3'u, lo: 0x80E8A40ECCD228A5'u), + uint64x2(hi: 0xC8A883C0FDAF7DF0'u, lo: 0x6122CD128006B2CE'u), + uint64x2(hi: 0xFAD2A4B13D1B5D6C'u, lo: 0x796B805720085F82'u), + uint64x2(hi: 0x9CC3A6EEC6311A63'u, lo: 0xCBE3303674053BB1'u), + uint64x2(hi: 0xC3F490AA77BD60FC'u, lo: 0xBEDBFC4411068A9D'u), + uint64x2(hi: 0xF4F1B4D515ACB93B'u, lo: 0xEE92FB5515482D45'u), + uint64x2(hi: 0x991711052D8BF3C5'u, lo: 0x751BDD152D4D1C4B'u), + uint64x2(hi: 0xBF5CD54678EEF0B6'u, lo: 0xD262D45A78A0635E'u), + uint64x2(hi: 0xEF340A98172AACE4'u, lo: 0x86FB897116C87C35'u), + uint64x2(hi: 0x9580869F0E7AAC0E'u, lo: 0xD45D35E6AE3D4DA1'u), + uint64x2(hi: 0xBAE0A846D2195712'u, lo: 0x8974836059CCA10A'u), + uint64x2(hi: 0xE998D258869FACD7'u, lo: 0x2BD1A438703FC94C'u), + uint64x2(hi: 0x91FF83775423CC06'u, lo: 0x7B6306A34627DDD0'u), + uint64x2(hi: 0xB67F6455292CBF08'u, lo: 0x1A3BC84C17B1D543'u), + uint64x2(hi: 0xE41F3D6A7377EECA'u, lo: 0x20CABA5F1D9E4A94'u), + uint64x2(hi: 0x8E938662882AF53E'u, lo: 0x547EB47B7282EE9D'u), + uint64x2(hi: 0xB23867FB2A35B28D'u, lo: 0xE99E619A4F23AA44'u), + uint64x2(hi: 0xDEC681F9F4C31F31'u, lo: 0x6405FA00E2EC94D5'u), + uint64x2(hi: 0x8B3C113C38F9F37E'u, lo: 0xDE83BC408DD3DD05'u), + uint64x2(hi: 0xAE0B158B4738705E'u, lo: 0x9624AB50B148D446'u), + uint64x2(hi: 0xD98DDAEE19068C76'u, lo: 0x3BADD624DD9B0958'u), + uint64x2(hi: 0x87F8A8D4CFA417C9'u, lo: 0xE54CA5D70A80E5D7'u), + uint64x2(hi: 0xA9F6D30A038D1DBC'u, lo: 0x5E9FCF4CCD211F4D'u), + uint64x2(hi: 0xD47487CC8470652B'u, lo: 0x7647C32000696720'u), + uint64x2(hi: 0x84C8D4DFD2C63F3B'u, lo: 0x29ECD9F40041E074'u), + uint64x2(hi: 0xA5FB0A17C777CF09'u, lo: 0xF468107100525891'u), + uint64x2(hi: 0xCF79CC9DB955C2CC'u, lo: 0x7182148D4066EEB5'u), + uint64x2(hi: 0x81AC1FE293D599BF'u, lo: 0xC6F14CD848405531'u), + uint64x2(hi: 0xA21727DB38CB002F'u, lo: 0xB8ADA00E5A506A7D'u), + uint64x2(hi: 0xCA9CF1D206FDC03B'u, lo: 0xA6D90811F0E4851D'u), + uint64x2(hi: 0xFD442E4688BD304A'u, lo: 0x908F4A166D1DA664'u), + uint64x2(hi: 0x9E4A9CEC15763E2E'u, lo: 0x9A598E4E043287FF'u), + uint64x2(hi: 0xC5DD44271AD3CDBA'u, lo: 0x40EFF1E1853F29FE'u), + uint64x2(hi: 0xF7549530E188C128'u, lo: 0xD12BEE59E68EF47D'u), + uint64x2(hi: 0x9A94DD3E8CF578B9'u, lo: 0x82BB74F8301958CF'u), + uint64x2(hi: 0xC13A148E3032D6E7'u, lo: 0xE36A52363C1FAF02'u), + uint64x2(hi: 0xF18899B1BC3F8CA1'u, lo: 0xDC44E6C3CB279AC2'u), + uint64x2(hi: 0x96F5600F15A7B7E5'u, lo: 0x29AB103A5EF8C0BA'u), + uint64x2(hi: 0xBCB2B812DB11A5DE'u, lo: 0x7415D448F6B6F0E8'u), + uint64x2(hi: 0xEBDF661791D60F56'u, lo: 0x111B495B3464AD22'u), + uint64x2(hi: 0x936B9FCEBB25C995'u, lo: 0xCAB10DD900BEEC35'u), + uint64x2(hi: 0xB84687C269EF3BFB'u, lo: 0x3D5D514F40EEA743'u), + uint64x2(hi: 0xE65829B3046B0AFA'u, lo: 0x0CB4A5A3112A5113'u), + uint64x2(hi: 0x8FF71A0FE2C2E6DC'u, lo: 0x47F0E785EABA72AC'u), + uint64x2(hi: 0xB3F4E093DB73A093'u, lo: 0x59ED216765690F57'u), + uint64x2(hi: 0xE0F218B8D25088B8'u, lo: 0x306869C13EC3532D'u), + uint64x2(hi: 0x8C974F7383725573'u, lo: 0x1E414218C73A13FC'u), + uint64x2(hi: 0xAFBD2350644EEACF'u, lo: 0xE5D1929EF90898FB'u), + uint64x2(hi: 0xDBAC6C247D62A583'u, lo: 0xDF45F746B74ABF3A'u), + uint64x2(hi: 0x894BC396CE5DA772'u, lo: 0x6B8BBA8C328EB784'u), + uint64x2(hi: 0xAB9EB47C81F5114F'u, lo: 0x066EA92F3F326565'u), + uint64x2(hi: 0xD686619BA27255A2'u, lo: 0xC80A537B0EFEFEBE'u), + uint64x2(hi: 0x8613FD0145877585'u, lo: 0xBD06742CE95F5F37'u), + uint64x2(hi: 0xA798FC4196E952E7'u, lo: 0x2C48113823B73705'u), + uint64x2(hi: 0xD17F3B51FCA3A7A0'u, lo: 0xF75A15862CA504C6'u), + uint64x2(hi: 0x82EF85133DE648C4'u, lo: 0x9A984D73DBE722FC'u), + uint64x2(hi: 0xA3AB66580D5FDAF5'u, lo: 0xC13E60D0D2E0EBBB'u), + uint64x2(hi: 0xCC963FEE10B7D1B3'u, lo: 0x318DF905079926A9'u), + uint64x2(hi: 0xFFBBCFE994E5C61F'u, lo: 0xFDF17746497F7053'u), + uint64x2(hi: 0x9FD561F1FD0F9BD3'u, lo: 0xFEB6EA8BEDEFA634'u), + uint64x2(hi: 0xC7CABA6E7C5382C8'u, lo: 0xFE64A52EE96B8FC1'u), + uint64x2(hi: 0xF9BD690A1B68637B'u, lo: 0x3DFDCE7AA3C673B1'u), + uint64x2(hi: 0x9C1661A651213E2D'u, lo: 0x06BEA10CA65C084F'u), + uint64x2(hi: 0xC31BFA0FE5698DB8'u, lo: 0x486E494FCFF30A63'u), + uint64x2(hi: 0xF3E2F893DEC3F126'u, lo: 0x5A89DBA3C3EFCCFB'u), + uint64x2(hi: 0x986DDB5C6B3A76B7'u, lo: 0xF89629465A75E01D'u), + uint64x2(hi: 0xBE89523386091465'u, lo: 0xF6BBB397F1135824'u), + uint64x2(hi: 0xEE2BA6C0678B597F'u, lo: 0x746AA07DED582E2D'u), + uint64x2(hi: 0x94DB483840B717EF'u, lo: 0xA8C2A44EB4571CDD'u), + uint64x2(hi: 0xBA121A4650E4DDEB'u, lo: 0x92F34D62616CE414'u), + uint64x2(hi: 0xE896A0D7E51E1566'u, lo: 0x77B020BAF9C81D18'u), + uint64x2(hi: 0x915E2486EF32CD60'u, lo: 0x0ACE1474DC1D122F'u), + uint64x2(hi: 0xB5B5ADA8AAFF80B8'u, lo: 0x0D819992132456BB'u), + uint64x2(hi: 0xE3231912D5BF60E6'u, lo: 0x10E1FFF697ED6C6A'u), + uint64x2(hi: 0x8DF5EFABC5979C8F'u, lo: 0xCA8D3FFA1EF463C2'u), + uint64x2(hi: 0xB1736B96B6FD83B3'u, lo: 0xBD308FF8A6B17CB3'u), + uint64x2(hi: 0xDDD0467C64BCE4A0'u, lo: 0xAC7CB3F6D05DDBDF'u), + uint64x2(hi: 0x8AA22C0DBEF60EE4'u, lo: 0x6BCDF07A423AA96C'u), + uint64x2(hi: 0xAD4AB7112EB3929D'u, lo: 0x86C16C98D2C953C7'u), + uint64x2(hi: 0xD89D64D57A607744'u, lo: 0xE871C7BF077BA8B8'u), + uint64x2(hi: 0x87625F056C7C4A8B'u, lo: 0x11471CD764AD4973'u), + uint64x2(hi: 0xA93AF6C6C79B5D2D'u, lo: 0xD598E40D3DD89BD0'u), + uint64x2(hi: 0xD389B47879823479'u, lo: 0x4AFF1D108D4EC2C4'u), + uint64x2(hi: 0x843610CB4BF160CB'u, lo: 0xCEDF722A585139BB'u), + uint64x2(hi: 0xA54394FE1EEDB8FE'u, lo: 0xC2974EB4EE658829'u), + uint64x2(hi: 0xCE947A3DA6A9273E'u, lo: 0x733D226229FEEA33'u), + uint64x2(hi: 0x811CCC668829B887'u, lo: 0x0806357D5A3F5260'u), + uint64x2(hi: 0xA163FF802A3426A8'u, lo: 0xCA07C2DCB0CF26F8'u), + uint64x2(hi: 0xC9BCFF6034C13052'u, lo: 0xFC89B393DD02F0B6'u), + uint64x2(hi: 0xFC2C3F3841F17C67'u, lo: 0xBBAC2078D443ACE3'u), + uint64x2(hi: 0x9D9BA7832936EDC0'u, lo: 0xD54B944B84AA4C0E'u), + uint64x2(hi: 0xC5029163F384A931'u, lo: 0x0A9E795E65D4DF12'u), + uint64x2(hi: 0xF64335BCF065D37D'u, lo: 0x4D4617B5FF4A16D6'u), + uint64x2(hi: 0x99EA0196163FA42E'u, lo: 0x504BCED1BF8E4E46'u), + uint64x2(hi: 0xC06481FB9BCF8D39'u, lo: 0xE45EC2862F71E1D7'u), + uint64x2(hi: 0xF07DA27A82C37088'u, lo: 0x5D767327BB4E5A4D'u), + uint64x2(hi: 0x964E858C91BA2655'u, lo: 0x3A6A07F8D510F870'u), + uint64x2(hi: 0xBBE226EFB628AFEA'u, lo: 0x890489F70A55368C'u), + uint64x2(hi: 0xEADAB0ABA3B2DBE5'u, lo: 0x2B45AC74CCEA842F'u), + uint64x2(hi: 0x92C8AE6B464FC96F'u, lo: 0x3B0B8BC90012929E'u), + uint64x2(hi: 0xB77ADA0617E3BBCB'u, lo: 0x09CE6EBB40173745'u), + uint64x2(hi: 0xE55990879DDCAABD'u, lo: 0xCC420A6A101D0516'u), + uint64x2(hi: 0x8F57FA54C2A9EAB6'u, lo: 0x9FA946824A12232E'u), + uint64x2(hi: 0xB32DF8E9F3546564'u, lo: 0x47939822DC96ABFA'u), + uint64x2(hi: 0xDFF9772470297EBD'u, lo: 0x59787E2B93BC56F8'u), + uint64x2(hi: 0x8BFBEA76C619EF36'u, lo: 0x57EB4EDB3C55B65B'u), + uint64x2(hi: 0xAEFAE51477A06B03'u, lo: 0xEDE622920B6B23F2'u), + uint64x2(hi: 0xDAB99E59958885C4'u, lo: 0xE95FAB368E45ECEE'u), + uint64x2(hi: 0x88B402F7FD75539B'u, lo: 0x11DBCB0218EBB415'u), + uint64x2(hi: 0xAAE103B5FCD2A881'u, lo: 0xD652BDC29F26A11A'u), + uint64x2(hi: 0xD59944A37C0752A2'u, lo: 0x4BE76D3346F04960'u), + uint64x2(hi: 0x857FCAE62D8493A5'u, lo: 0x6F70A4400C562DDC'u), + uint64x2(hi: 0xA6DFBD9FB8E5B88E'u, lo: 0xCB4CCD500F6BB953'u), + uint64x2(hi: 0xD097AD07A71F26B2'u, lo: 0x7E2000A41346A7A8'u), + uint64x2(hi: 0x825ECC24C873782F'u, lo: 0x8ED400668C0C28C9'u), + uint64x2(hi: 0xA2F67F2DFA90563B'u, lo: 0x728900802F0F32FB'u), + uint64x2(hi: 0xCBB41EF979346BCA'u, lo: 0x4F2B40A03AD2FFBA'u), + uint64x2(hi: 0xFEA126B7D78186BC'u, lo: 0xE2F610C84987BFA9'u), + uint64x2(hi: 0x9F24B832E6B0F436'u, lo: 0x0DD9CA7D2DF4D7CA'u), + uint64x2(hi: 0xC6EDE63FA05D3143'u, lo: 0x91503D1C79720DBC'u), + uint64x2(hi: 0xF8A95FCF88747D94'u, lo: 0x75A44C6397CE912B'u), + uint64x2(hi: 0x9B69DBE1B548CE7C'u, lo: 0xC986AFBE3EE11ABB'u), + uint64x2(hi: 0xC24452DA229B021B'u, lo: 0xFBE85BADCE996169'u), + uint64x2(hi: 0xF2D56790AB41C2A2'u, lo: 0xFAE27299423FB9C4'u), + uint64x2(hi: 0x97C560BA6B0919A5'u, lo: 0xDCCD879FC967D41B'u), + uint64x2(hi: 0xBDB6B8E905CB600F'u, lo: 0x5400E987BBC1C921'u), + uint64x2(hi: 0xED246723473E3813'u, lo: 0x290123E9AAB23B69'u), + uint64x2(hi: 0x9436C0760C86E30B'u, lo: 0xF9A0B6720AAF6522'u), + uint64x2(hi: 0xB94470938FA89BCE'u, lo: 0xF808E40E8D5B3E6A'u), + uint64x2(hi: 0xE7958CB87392C2C2'u, lo: 0xB60B1D1230B20E05'u), + uint64x2(hi: 0x90BD77F3483BB9B9'u, lo: 0xB1C6F22B5E6F48C3'u), + uint64x2(hi: 0xB4ECD5F01A4AA828'u, lo: 0x1E38AEB6360B1AF4'u), + uint64x2(hi: 0xE2280B6C20DD5232'u, lo: 0x25C6DA63C38DE1B1'u), + uint64x2(hi: 0x8D590723948A535F'u, lo: 0x579C487E5A38AD0F'u), + uint64x2(hi: 0xB0AF48EC79ACE837'u, lo: 0x2D835A9DF0C6D852'u), + uint64x2(hi: 0xDCDB1B2798182244'u, lo: 0xF8E431456CF88E66'u), + uint64x2(hi: 0x8A08F0F8BF0F156B'u, lo: 0x1B8E9ECB641B5900'u), + uint64x2(hi: 0xAC8B2D36EED2DAC5'u, lo: 0xE272467E3D222F40'u), + uint64x2(hi: 0xD7ADF884AA879177'u, lo: 0x5B0ED81DCC6ABB10'u), + uint64x2(hi: 0x86CCBB52EA94BAEA'u, lo: 0x98E947129FC2B4EA'u), + uint64x2(hi: 0xA87FEA27A539E9A5'u, lo: 0x3F2398D747B36225'u), + uint64x2(hi: 0xD29FE4B18E88640E'u, lo: 0x8EEC7F0D19A03AAE'u), + uint64x2(hi: 0x83A3EEEEF9153E89'u, lo: 0x1953CF68300424AD'u), + uint64x2(hi: 0xA48CEAAAB75A8E2B'u, lo: 0x5FA8C3423C052DD8'u), + uint64x2(hi: 0xCDB02555653131B6'u, lo: 0x3792F412CB06794E'u), + uint64x2(hi: 0x808E17555F3EBF11'u, lo: 0xE2BBD88BBEE40BD1'u), + uint64x2(hi: 0xA0B19D2AB70E6ED6'u, lo: 0x5B6ACEAEAE9D0EC5'u), + uint64x2(hi: 0xC8DE047564D20A8B'u, lo: 0xF245825A5A445276'u), + uint64x2(hi: 0xFB158592BE068D2E'u, lo: 0xEED6E2F0F0D56713'u), + uint64x2(hi: 0x9CED737BB6C4183D'u, lo: 0x55464DD69685606C'u), + uint64x2(hi: 0xC428D05AA4751E4C'u, lo: 0xAA97E14C3C26B887'u), + uint64x2(hi: 0xF53304714D9265DF'u, lo: 0xD53DD99F4B3066A9'u), + uint64x2(hi: 0x993FE2C6D07B7FAB'u, lo: 0xE546A8038EFE402A'u), + uint64x2(hi: 0xBF8FDB78849A5F96'u, lo: 0xDE98520472BDD034'u), + uint64x2(hi: 0xEF73D256A5C0F77C'u, lo: 0x963E66858F6D4441'u), + uint64x2(hi: 0x95A8637627989AAD'u, lo: 0xDDE7001379A44AA9'u), + uint64x2(hi: 0xBB127C53B17EC159'u, lo: 0x5560C018580D5D53'u), + uint64x2(hi: 0xE9D71B689DDE71AF'u, lo: 0xAAB8F01E6E10B4A7'u), + uint64x2(hi: 0x9226712162AB070D'u, lo: 0xCAB3961304CA70E9'u), + uint64x2(hi: 0xB6B00D69BB55C8D1'u, lo: 0x3D607B97C5FD0D23'u), + uint64x2(hi: 0xE45C10C42A2B3B05'u, lo: 0x8CB89A7DB77C506B'u), + uint64x2(hi: 0x8EB98A7A9A5B04E3'u, lo: 0x77F3608E92ADB243'u), + uint64x2(hi: 0xB267ED1940F1C61C'u, lo: 0x55F038B237591ED4'u), + uint64x2(hi: 0xDF01E85F912E37A3'u, lo: 0x6B6C46DEC52F6689'u), + uint64x2(hi: 0x8B61313BBABCE2C6'u, lo: 0x2323AC4B3B3DA016'u), + uint64x2(hi: 0xAE397D8AA96C1B77'u, lo: 0xABEC975E0A0D081B'u), + uint64x2(hi: 0xD9C7DCED53C72255'u, lo: 0x96E7BD358C904A22'u), + uint64x2(hi: 0x881CEA14545C7575'u, lo: 0x7E50D64177DA2E55'u), + uint64x2(hi: 0xAA242499697392D2'u, lo: 0xDDE50BD1D5D0B9EA'u), + uint64x2(hi: 0xD4AD2DBFC3D07787'u, lo: 0x955E4EC64B44E865'u), + uint64x2(hi: 0x84EC3C97DA624AB4'u, lo: 0xBD5AF13BEF0B113F'u), + uint64x2(hi: 0xA6274BBDD0FADD61'u, lo: 0xECB1AD8AEACDD58F'u), + uint64x2(hi: 0xCFB11EAD453994BA'u, lo: 0x67DE18EDA5814AF3'u), + uint64x2(hi: 0x81CEB32C4B43FCF4'u, lo: 0x80EACF948770CED8'u), + uint64x2(hi: 0xA2425FF75E14FC31'u, lo: 0xA1258379A94D028E'u), + uint64x2(hi: 0xCAD2F7F5359A3B3E'u, lo: 0x096EE45813A04331'u), + uint64x2(hi: 0xFD87B5F28300CA0D'u, lo: 0x8BCA9D6E188853FD'u), + uint64x2(hi: 0x9E74D1B791E07E48'u, lo: 0x775EA264CF55347E'u), + uint64x2(hi: 0xC612062576589DDA'u, lo: 0x95364AFE032A819E'u), + uint64x2(hi: 0xF79687AED3EEC551'u, lo: 0x3A83DDBD83F52205'u), + uint64x2(hi: 0x9ABE14CD44753B52'u, lo: 0xC4926A9672793543'u), + uint64x2(hi: 0xC16D9A0095928A27'u, lo: 0x75B7053C0F178294'u), + uint64x2(hi: 0xF1C90080BAF72CB1'u, lo: 0x5324C68B12DD6339'u), + uint64x2(hi: 0x971DA05074DA7BEE'u, lo: 0xD3F6FC16EBCA5E04'u), + uint64x2(hi: 0xBCE5086492111AEA'u, lo: 0x88F4BB1CA6BCF585'u), + uint64x2(hi: 0xEC1E4A7DB69561A5'u, lo: 0x2B31E9E3D06C32E6'u), + uint64x2(hi: 0x9392EE8E921D5D07'u, lo: 0x3AFF322E62439FD0'u), + uint64x2(hi: 0xB877AA3236A4B449'u, lo: 0x09BEFEB9FAD487C3'u), + uint64x2(hi: 0xE69594BEC44DE15B'u, lo: 0x4C2EBE687989A9B4'u), + uint64x2(hi: 0x901D7CF73AB0ACD9'u, lo: 0x0F9D37014BF60A11'u), + uint64x2(hi: 0xB424DC35095CD80F'u, lo: 0x538484C19EF38C95'u), + uint64x2(hi: 0xE12E13424BB40E13'u, lo: 0x2865A5F206B06FBA'u), + uint64x2(hi: 0x8CBCCC096F5088CB'u, lo: 0xF93F87B7442E45D4'u), + uint64x2(hi: 0xAFEBFF0BCB24AAFE'u, lo: 0xF78F69A51539D749'u), + uint64x2(hi: 0xDBE6FECEBDEDD5BE'u, lo: 0xB573440E5A884D1C'u), + uint64x2(hi: 0x89705F4136B4A597'u, lo: 0x31680A88F8953031'u), + uint64x2(hi: 0xABCC77118461CEFC'u, lo: 0xFDC20D2B36BA7C3E'u), + uint64x2(hi: 0xD6BF94D5E57A42BC'u, lo: 0x3D32907604691B4D'u), + uint64x2(hi: 0x8637BD05AF6C69B5'u, lo: 0xA63F9A49C2C1B110'u), + uint64x2(hi: 0xA7C5AC471B478423'u, lo: 0x0FCF80DC33721D54'u), + uint64x2(hi: 0xD1B71758E219652B'u, lo: 0xD3C36113404EA4A9'u), + uint64x2(hi: 0x83126E978D4FDF3B'u, lo: 0x645A1CAC083126EA'u), + uint64x2(hi: 0xA3D70A3D70A3D70A'u, lo: 0x3D70A3D70A3D70A4'u), + uint64x2(hi: 0xCCCCCCCCCCCCCCCC'u, lo: 0xCCCCCCCCCCCCCCCD'u), + uint64x2(hi: 0x8000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC800000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xFA00000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9C40000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC350000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xF424000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9896800000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBEBC200000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xEE6B280000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9502F90000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBA43B74000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE8D4A51000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9184E72A00000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB5E620F480000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE35FA931A0000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8E1BC9BF04000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB1A2BC2EC5000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xDE0B6B3A76400000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8AC7230489E80000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xAD78EBC5AC620000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD8D726B7177A8000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x878678326EAC9000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA968163F0A57B400'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD3C21BCECCEDA100'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x84595161401484A0'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA56FA5B99019A5C8'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xCECB8F27F4200F3A'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x813F3978F8940984'u, lo: 0x4000000000000000'u), + uint64x2(hi: 0xA18F07D736B90BE5'u, lo: 0x5000000000000000'u), + uint64x2(hi: 0xC9F2C9CD04674EDE'u, lo: 0xA400000000000000'u), + uint64x2(hi: 0xFC6F7C4045812296'u, lo: 0x4D00000000000000'u), + uint64x2(hi: 0x9DC5ADA82B70B59D'u, lo: 0xF020000000000000'u), + uint64x2(hi: 0xC5371912364CE305'u, lo: 0x6C28000000000000'u), + uint64x2(hi: 0xF684DF56C3E01BC6'u, lo: 0xC732000000000000'u), + uint64x2(hi: 0x9A130B963A6C115C'u, lo: 0x3C7F400000000000'u), + uint64x2(hi: 0xC097CE7BC90715B3'u, lo: 0x4B9F100000000000'u), + uint64x2(hi: 0xF0BDC21ABB48DB20'u, lo: 0x1E86D40000000000'u), + uint64x2(hi: 0x96769950B50D88F4'u, lo: 0x1314448000000000'u), + uint64x2(hi: 0xBC143FA4E250EB31'u, lo: 0x17D955A000000000'u), + uint64x2(hi: 0xEB194F8E1AE525FD'u, lo: 0x5DCFAB0800000000'u), + uint64x2(hi: 0x92EFD1B8D0CF37BE'u, lo: 0x5AA1CAE500000000'u), + uint64x2(hi: 0xB7ABC627050305AD'u, lo: 0xF14A3D9E40000000'u), + uint64x2(hi: 0xE596B7B0C643C719'u, lo: 0x6D9CCD05D0000000'u), + uint64x2(hi: 0x8F7E32CE7BEA5C6F'u, lo: 0xE4820023A2000000'u), + uint64x2(hi: 0xB35DBF821AE4F38B'u, lo: 0xDDA2802C8A800000'u), + uint64x2(hi: 0xE0352F62A19E306E'u, lo: 0xD50B2037AD200000'u), + uint64x2(hi: 0x8C213D9DA502DE45'u, lo: 0x4526F422CC340000'u), + uint64x2(hi: 0xAF298D050E4395D6'u, lo: 0x9670B12B7F410000'u), + uint64x2(hi: 0xDAF3F04651D47B4C'u, lo: 0x3C0CDD765F114000'u), + uint64x2(hi: 0x88D8762BF324CD0F'u, lo: 0xA5880A69FB6AC800'u), + uint64x2(hi: 0xAB0E93B6EFEE0053'u, lo: 0x8EEA0D047A457A00'u), + uint64x2(hi: 0xD5D238A4ABE98068'u, lo: 0x72A4904598D6D880'u), + uint64x2(hi: 0x85A36366EB71F041'u, lo: 0x47A6DA2B7F864750'u), + uint64x2(hi: 0xA70C3C40A64E6C51'u, lo: 0x999090B65F67D924'u), + uint64x2(hi: 0xD0CF4B50CFE20765'u, lo: 0xFFF4B4E3F741CF6D'u), + uint64x2(hi: 0x82818F1281ED449F'u, lo: 0xBFF8F10E7A8921A4'u), + uint64x2(hi: 0xA321F2D7226895C7'u, lo: 0xAFF72D52192B6A0D'u), + uint64x2(hi: 0xCBEA6F8CEB02BB39'u, lo: 0x9BF4F8A69F764490'u), + uint64x2(hi: 0xFEE50B7025C36A08'u, lo: 0x02F236D04753D5B4'u), + uint64x2(hi: 0x9F4F2726179A2245'u, lo: 0x01D762422C946590'u), + uint64x2(hi: 0xC722F0EF9D80AAD6'u, lo: 0x424D3AD2B7B97EF5'u), + uint64x2(hi: 0xF8EBAD2B84E0D58B'u, lo: 0xD2E0898765A7DEB2'u), + uint64x2(hi: 0x9B934C3B330C8577'u, lo: 0x63CC55F49F88EB2F'u), + uint64x2(hi: 0xC2781F49FFCFA6D5'u, lo: 0x3CBF6B71C76B25FB'u), + uint64x2(hi: 0xF316271C7FC3908A'u, lo: 0x8BEF464E3945EF7A'u), + uint64x2(hi: 0x97EDD871CFDA3A56'u, lo: 0x97758BF0E3CBB5AC'u), + uint64x2(hi: 0xBDE94E8E43D0C8EC'u, lo: 0x3D52EEED1CBEA317'u), + uint64x2(hi: 0xED63A231D4C4FB27'u, lo: 0x4CA7AAA863EE4BDD'u), + uint64x2(hi: 0x945E455F24FB1CF8'u, lo: 0x8FE8CAA93E74EF6A'u), + uint64x2(hi: 0xB975D6B6EE39E436'u, lo: 0xB3E2FD538E122B44'u), + uint64x2(hi: 0xE7D34C64A9C85D44'u, lo: 0x60DBBCA87196B616'u), + uint64x2(hi: 0x90E40FBEEA1D3A4A'u, lo: 0xBC8955E946FE31CD'u), + uint64x2(hi: 0xB51D13AEA4A488DD'u, lo: 0x6BABAB6398BDBE41'u), + uint64x2(hi: 0xE264589A4DCDAB14'u, lo: 0xC696963C7EED2DD1'u), + uint64x2(hi: 0x8D7EB76070A08AEC'u, lo: 0xFC1E1DE5CF543CA2'u), + uint64x2(hi: 0xB0DE65388CC8ADA8'u, lo: 0x3B25A55F43294BCB'u), + uint64x2(hi: 0xDD15FE86AFFAD912'u, lo: 0x49EF0EB713F39EBE'u), + uint64x2(hi: 0x8A2DBF142DFCC7AB'u, lo: 0x6E3569326C784337'u), + uint64x2(hi: 0xACB92ED9397BF996'u, lo: 0x49C2C37F07965404'u), + uint64x2(hi: 0xD7E77A8F87DAF7FB'u, lo: 0xDC33745EC97BE906'u), + uint64x2(hi: 0x86F0AC99B4E8DAFD'u, lo: 0x69A028BB3DED71A3'u), + uint64x2(hi: 0xA8ACD7C0222311BC'u, lo: 0xC40832EA0D68CE0C'u), + uint64x2(hi: 0xD2D80DB02AABD62B'u, lo: 0xF50A3FA490C30190'u), + uint64x2(hi: 0x83C7088E1AAB65DB'u, lo: 0x792667C6DA79E0FA'u), + uint64x2(hi: 0xA4B8CAB1A1563F52'u, lo: 0x577001B891185938'u), + uint64x2(hi: 0xCDE6FD5E09ABCF26'u, lo: 0xED4C0226B55E6F86'u), + uint64x2(hi: 0x80B05E5AC60B6178'u, lo: 0x544F8158315B05B4'u), + uint64x2(hi: 0xA0DC75F1778E39D6'u, lo: 0x696361AE3DB1C721'u), + uint64x2(hi: 0xC913936DD571C84C'u, lo: 0x03BC3A19CD1E38E9'u), + uint64x2(hi: 0xFB5878494ACE3A5F'u, lo: 0x04AB48A04065C723'u), + uint64x2(hi: 0x9D174B2DCEC0E47B'u, lo: 0x62EB0D64283F9C76'u), + uint64x2(hi: 0xC45D1DF942711D9A'u, lo: 0x3BA5D0BD324F8394'u), + uint64x2(hi: 0xF5746577930D6500'u, lo: 0xCA8F44EC7EE36479'u), + uint64x2(hi: 0x9968BF6ABBE85F20'u, lo: 0x7E998B13CF4E1ECB'u), + uint64x2(hi: 0xBFC2EF456AE276E8'u, lo: 0x9E3FEDD8C321A67E'u), + uint64x2(hi: 0xEFB3AB16C59B14A2'u, lo: 0xC5CFE94EF3EA101E'u), + uint64x2(hi: 0x95D04AEE3B80ECE5'u, lo: 0xBBA1F1D158724A12'u), + uint64x2(hi: 0xBB445DA9CA61281F'u, lo: 0x2A8A6E45AE8EDC97'u), + uint64x2(hi: 0xEA1575143CF97226'u, lo: 0xF52D09D71A3293BD'u), + uint64x2(hi: 0x924D692CA61BE758'u, lo: 0x593C2626705F9C56'u), + uint64x2(hi: 0xB6E0C377CFA2E12E'u, lo: 0x6F8B2FB00C77836C'u), + uint64x2(hi: 0xE498F455C38B997A'u, lo: 0x0B6DFB9C0F956447'u), + uint64x2(hi: 0x8EDF98B59A373FEC'u, lo: 0x4724BD4189BD5EAC'u), + uint64x2(hi: 0xB2977EE300C50FE7'u, lo: 0x58EDEC91EC2CB657'u), + uint64x2(hi: 0xDF3D5E9BC0F653E1'u, lo: 0x2F2967B66737E3ED'u), + uint64x2(hi: 0x8B865B215899F46C'u, lo: 0xBD79E0D20082EE74'u), + uint64x2(hi: 0xAE67F1E9AEC07187'u, lo: 0xECD8590680A3AA11'u), + uint64x2(hi: 0xDA01EE641A708DE9'u, lo: 0xE80E6F4820CC9495'u), + uint64x2(hi: 0x884134FE908658B2'u, lo: 0x3109058D147FDCDD'u), + uint64x2(hi: 0xAA51823E34A7EEDE'u, lo: 0xBD4B46F0599FD415'u), + uint64x2(hi: 0xD4E5E2CDC1D1EA96'u, lo: 0x6C9E18AC7007C91A'u), + uint64x2(hi: 0x850FADC09923329E'u, lo: 0x03E2CF6BC604DDB0'u), + uint64x2(hi: 0xA6539930BF6BFF45'u, lo: 0x84DB8346B786151C'u), + uint64x2(hi: 0xCFE87F7CEF46FF16'u, lo: 0xE612641865679A63'u), + uint64x2(hi: 0x81F14FAE158C5F6E'u, lo: 0x4FCB7E8F3F60C07E'u), + uint64x2(hi: 0xA26DA3999AEF7749'u, lo: 0xE3BE5E330F38F09D'u), + uint64x2(hi: 0xCB090C8001AB551C'u, lo: 0x5CADF5BFD3072CC5'u), + uint64x2(hi: 0xFDCB4FA002162A63'u, lo: 0x73D9732FC7C8F7F6'u), + uint64x2(hi: 0x9E9F11C4014DDA7E'u, lo: 0x2867E7FDDCDD9AFA'u), + uint64x2(hi: 0xC646D63501A1511D'u, lo: 0xB281E1FD541501B8'u), + uint64x2(hi: 0xF7D88BC24209A565'u, lo: 0x1F225A7CA91A4226'u), + uint64x2(hi: 0x9AE757596946075F'u, lo: 0x3375788DE9B06958'u), + uint64x2(hi: 0xC1A12D2FC3978937'u, lo: 0x0052D6B1641C83AE'u), + uint64x2(hi: 0xF209787BB47D6B84'u, lo: 0xC0678C5DBD23A49A'u), + uint64x2(hi: 0x9745EB4D50CE6332'u, lo: 0xF840B7BA963646E0'u), + uint64x2(hi: 0xBD176620A501FBFF'u, lo: 0xB650E5A93BC3D898'u), + uint64x2(hi: 0xEC5D3FA8CE427AFF'u, lo: 0xA3E51F138AB4CEBE'u), + uint64x2(hi: 0x93BA47C980E98CDF'u, lo: 0xC66F336C36B10137'u), + uint64x2(hi: 0xB8A8D9BBE123F017'u, lo: 0xB80B0047445D4184'u), + uint64x2(hi: 0xE6D3102AD96CEC1D'u, lo: 0xA60DC059157491E5'u), + uint64x2(hi: 0x9043EA1AC7E41392'u, lo: 0x87C89837AD68DB2F'u), + uint64x2(hi: 0xB454E4A179DD1877'u, lo: 0x29BABE4598C311FB'u), + uint64x2(hi: 0xE16A1DC9D8545E94'u, lo: 0xF4296DD6FEF3D67A'u), + uint64x2(hi: 0x8CE2529E2734BB1D'u, lo: 0x1899E4A65F58660C'u), + uint64x2(hi: 0xB01AE745B101E9E4'u, lo: 0x5EC05DCFF72E7F8F'u), + uint64x2(hi: 0xDC21A1171D42645D'u, lo: 0x76707543F4FA1F73'u), + uint64x2(hi: 0x899504AE72497EBA'u, lo: 0x6A06494A791C53A8'u), + uint64x2(hi: 0xABFA45DA0EDBDE69'u, lo: 0x0487DB9D17636892'u), + uint64x2(hi: 0xD6F8D7509292D603'u, lo: 0x45A9D2845D3C42B6'u), + uint64x2(hi: 0x865B86925B9BC5C2'u, lo: 0x0B8A2392BA45A9B2'u), + uint64x2(hi: 0xA7F26836F282B732'u, lo: 0x8E6CAC7768D7141E'u), + uint64x2(hi: 0xD1EF0244AF2364FF'u, lo: 0x3207D795430CD926'u), + uint64x2(hi: 0x8335616AED761F1F'u, lo: 0x7F44E6BD49E807B8'u), + uint64x2(hi: 0xA402B9C5A8D3A6E7'u, lo: 0x5F16206C9C6209A6'u), + uint64x2(hi: 0xCD036837130890A1'u, lo: 0x36DBA887C37A8C0F'u), + uint64x2(hi: 0x802221226BE55A64'u, lo: 0xC2494954DA2C9789'u), + uint64x2(hi: 0xA02AA96B06DEB0FD'u, lo: 0xF2DB9BAA10B7BD6C'u), + uint64x2(hi: 0xC83553C5C8965D3D'u, lo: 0x6F92829494E5ACC7'u), + uint64x2(hi: 0xFA42A8B73ABBF48C'u, lo: 0xCB772339BA1F17F9'u), + uint64x2(hi: 0x9C69A97284B578D7'u, lo: 0xFF2A760414536EFB'u), + uint64x2(hi: 0xC38413CF25E2D70D'u, lo: 0xFEF5138519684ABA'u), + uint64x2(hi: 0xF46518C2EF5B8CD1'u, lo: 0x7EB258665FC25D69'u), + uint64x2(hi: 0x98BF2F79D5993802'u, lo: 0xEF2F773FFBD97A61'u), + uint64x2(hi: 0xBEEEFB584AFF8603'u, lo: 0xAAFB550FFACFD8FA'u), + uint64x2(hi: 0xEEAABA2E5DBF6784'u, lo: 0x95BA2A53F983CF38'u), + uint64x2(hi: 0x952AB45CFA97A0B2'u, lo: 0xDD945A747BF26183'u), + uint64x2(hi: 0xBA756174393D88DF'u, lo: 0x94F971119AEEF9E4'u), + uint64x2(hi: 0xE912B9D1478CEB17'u, lo: 0x7A37CD5601AAB85D'u), + uint64x2(hi: 0x91ABB422CCB812EE'u, lo: 0xAC62E055C10AB33A'u), + uint64x2(hi: 0xB616A12B7FE617AA'u, lo: 0x577B986B314D6009'u), + uint64x2(hi: 0xE39C49765FDF9D94'u, lo: 0xED5A7E85FDA0B80B'u), + uint64x2(hi: 0x8E41ADE9FBEBC27D'u, lo: 0x14588F13BE847307'u), + uint64x2(hi: 0xB1D219647AE6B31C'u, lo: 0x596EB2D8AE258FC8'u), + uint64x2(hi: 0xDE469FBD99A05FE3'u, lo: 0x6FCA5F8ED9AEF3BB'u), + uint64x2(hi: 0x8AEC23D680043BEE'u, lo: 0x25DE7BB9480D5854'u), + uint64x2(hi: 0xADA72CCC20054AE9'u, lo: 0xAF561AA79A10AE6A'u), + uint64x2(hi: 0xD910F7FF28069DA4'u, lo: 0x1B2BA1518094DA04'u), + uint64x2(hi: 0x87AA9AFF79042286'u, lo: 0x90FB44D2F05D0842'u), + uint64x2(hi: 0xA99541BF57452B28'u, lo: 0x353A1607AC744A53'u), + uint64x2(hi: 0xD3FA922F2D1675F2'u, lo: 0x42889B8997915CE8'u), + uint64x2(hi: 0x847C9B5D7C2E09B7'u, lo: 0x69956135FEBADA11'u), + uint64x2(hi: 0xA59BC234DB398C25'u, lo: 0x43FAB9837E699095'u), + uint64x2(hi: 0xCF02B2C21207EF2E'u, lo: 0x94F967E45E03F4BB'u), + uint64x2(hi: 0x8161AFB94B44F57D'u, lo: 0x1D1BE0EEBAC278F5'u), + uint64x2(hi: 0xA1BA1BA79E1632DC'u, lo: 0x6462D92A69731732'u), + uint64x2(hi: 0xCA28A291859BBF93'u, lo: 0x7D7B8F7503CFDCFE'u), + uint64x2(hi: 0xFCB2CB35E702AF78'u, lo: 0x5CDA735244C3D43E'u), + uint64x2(hi: 0x9DEFBF01B061ADAB'u, lo: 0x3A0888136AFA64A7'u), + uint64x2(hi: 0xC56BAEC21C7A1916'u, lo: 0x088AAA1845B8FDD0'u), + uint64x2(hi: 0xF6C69A72A3989F5B'u, lo: 0x8AAD549E57273D45'u), + uint64x2(hi: 0x9A3C2087A63F6399'u, lo: 0x36AC54E2F678864B'u), + uint64x2(hi: 0xC0CB28A98FCF3C7F'u, lo: 0x84576A1BB416A7DD'u), + uint64x2(hi: 0xF0FDF2D3F3C30B9F'u, lo: 0x656D44A2A11C51D5'u), + uint64x2(hi: 0x969EB7C47859E743'u, lo: 0x9F644AE5A4B1B325'u), + uint64x2(hi: 0xBC4665B596706114'u, lo: 0x873D5D9F0DDE1FEE'u), + uint64x2(hi: 0xEB57FF22FC0C7959'u, lo: 0xA90CB506D155A7EA'u), + uint64x2(hi: 0x9316FF75DD87CBD8'u, lo: 0x09A7F12442D588F2'u), + uint64x2(hi: 0xB7DCBF5354E9BECE'u, lo: 0x0C11ED6D538AEB2F'u), + uint64x2(hi: 0xE5D3EF282A242E81'u, lo: 0x8F1668C8A86DA5FA'u), + uint64x2(hi: 0x8FA475791A569D10'u, lo: 0xF96E017D694487BC'u), + uint64x2(hi: 0xB38D92D760EC4455'u, lo: 0x37C981DCC395A9AC'u), + uint64x2(hi: 0xE070F78D3927556A'u, lo: 0x85BBE253F47B1417'u), + uint64x2(hi: 0x8C469AB843B89562'u, lo: 0x93956D7478CCEC8E'u), + uint64x2(hi: 0xAF58416654A6BABB'u, lo: 0x387AC8D1970027B2'u), + uint64x2(hi: 0xDB2E51BFE9D0696A'u, lo: 0x06997B05FCC0319E'u), + uint64x2(hi: 0x88FCF317F22241E2'u, lo: 0x441FECE3BDF81F03'u), + uint64x2(hi: 0xAB3C2FDDEEAAD25A'u, lo: 0xD527E81CAD7626C3'u), + uint64x2(hi: 0xD60B3BD56A5586F1'u, lo: 0x8A71E223D8D3B074'u), + uint64x2(hi: 0x85C7056562757456'u, lo: 0xF6872D5667844E49'u), + uint64x2(hi: 0xA738C6BEBB12D16C'u, lo: 0xB428F8AC016561DB'u), + uint64x2(hi: 0xD106F86E69D785C7'u, lo: 0xE13336D701BEBA52'u), + uint64x2(hi: 0x82A45B450226B39C'u, lo: 0xECC0024661173473'u), + uint64x2(hi: 0xA34D721642B06084'u, lo: 0x27F002D7F95D0190'u), + uint64x2(hi: 0xCC20CE9BD35C78A5'u, lo: 0x31EC038DF7B441F4'u), + uint64x2(hi: 0xFF290242C83396CE'u, lo: 0x7E67047175A15271'u), + uint64x2(hi: 0x9F79A169BD203E41'u, lo: 0x0F0062C6E984D386'u), + uint64x2(hi: 0xC75809C42C684DD1'u, lo: 0x52C07B78A3E60868'u), + uint64x2(hi: 0xF92E0C3537826145'u, lo: 0xA7709A56CCDF8A82'u), + uint64x2(hi: 0x9BBCC7A142B17CCB'u, lo: 0x88A66076400BB691'u), + uint64x2(hi: 0xC2ABF989935DDBFE'u, lo: 0x6ACFF893D00EA435'u), + uint64x2(hi: 0xF356F7EBF83552FE'u, lo: 0x0583F6B8C4124D43'u), + uint64x2(hi: 0x98165AF37B2153DE'u, lo: 0xC3727A337A8B704A'u), + uint64x2(hi: 0xBE1BF1B059E9A8D6'u, lo: 0x744F18C0592E4C5C'u), + uint64x2(hi: 0xEDA2EE1C7064130C'u, lo: 0x1162DEF06F79DF73'u), + uint64x2(hi: 0x9485D4D1C63E8BE7'u, lo: 0x8ADDCB5645AC2BA8'u), + uint64x2(hi: 0xB9A74A0637CE2EE1'u, lo: 0x6D953E2BD7173692'u), + uint64x2(hi: 0xE8111C87C5C1BA99'u, lo: 0xC8FA8DB6CCDD0437'u), + uint64x2(hi: 0x910AB1D4DB9914A0'u, lo: 0x1D9C9892400A22A2'u), + uint64x2(hi: 0xB54D5E4A127F59C8'u, lo: 0x2503BEB6D00CAB4B'u), + uint64x2(hi: 0xE2A0B5DC971F303A'u, lo: 0x2E44AE64840FD61D'u), + uint64x2(hi: 0x8DA471A9DE737E24'u, lo: 0x5CEAECFED289E5D2'u), + uint64x2(hi: 0xB10D8E1456105DAD'u, lo: 0x7425A83E872C5F47'u), + uint64x2(hi: 0xDD50F1996B947518'u, lo: 0xD12F124E28F77719'u), + uint64x2(hi: 0x8A5296FFE33CC92F'u, lo: 0x82BD6B70D99AAA6F'u), + uint64x2(hi: 0xACE73CBFDC0BFB7B'u, lo: 0x636CC64D1001550B'u), + uint64x2(hi: 0xD8210BEFD30EFA5A'u, lo: 0x3C47F7E05401AA4E'u), + uint64x2(hi: 0x8714A775E3E95C78'u, lo: 0x65ACFAEC34810A71'u), + uint64x2(hi: 0xA8D9D1535CE3B396'u, lo: 0x7F1839A741A14D0D'u), + uint64x2(hi: 0xD31045A8341CA07C'u, lo: 0x1EDE48111209A050'u), + uint64x2(hi: 0x83EA2B892091E44D'u, lo: 0x934AED0AAB460432'u), + uint64x2(hi: 0xA4E4B66B68B65D60'u, lo: 0xF81DA84D5617853F'u), + uint64x2(hi: 0xCE1DE40642E3F4B9'u, lo: 0x36251260AB9D668E'u), + uint64x2(hi: 0x80D2AE83E9CE78F3'u, lo: 0xC1D72B7C6B426019'u), + uint64x2(hi: 0xA1075A24E4421730'u, lo: 0xB24CF65B8612F81F'u), + uint64x2(hi: 0xC94930AE1D529CFC'u, lo: 0xDEE033F26797B627'u), + uint64x2(hi: 0xFB9B7CD9A4A7443C'u, lo: 0x169840EF017DA3B1'u), + uint64x2(hi: 0x9D412E0806E88AA5'u, lo: 0x8E1F289560EE864E'u), + uint64x2(hi: 0xC491798A08A2AD4E'u, lo: 0xF1A6F2BAB92A27E2'u), + uint64x2(hi: 0xF5B5D7EC8ACB58A2'u, lo: 0xAE10AF696774B1DB'u), + uint64x2(hi: 0x9991A6F3D6BF1765'u, lo: 0xACCA6DA1E0A8EF29'u), + uint64x2(hi: 0xBFF610B0CC6EDD3F'u, lo: 0x17FD090A58D32AF3'u), + uint64x2(hi: 0xEFF394DCFF8A948E'u, lo: 0xDDFC4B4CEF07F5B0'u), + uint64x2(hi: 0x95F83D0A1FB69CD9'u, lo: 0x4ABDAF101564F98E'u), + uint64x2(hi: 0xBB764C4CA7A4440F'u, lo: 0x9D6D1AD41ABE37F1'u), + uint64x2(hi: 0xEA53DF5FD18D5513'u, lo: 0x84C86189216DC5ED'u), + uint64x2(hi: 0x92746B9BE2F8552C'u, lo: 0x32FD3CF5B4E49BB4'u), + uint64x2(hi: 0xB7118682DBB66A77'u, lo: 0x3FBC8C33221DC2A1'u), + uint64x2(hi: 0xE4D5E82392A40515'u, lo: 0x0FABAF3FEAA5334A'u), + uint64x2(hi: 0x8F05B1163BA6832D'u, lo: 0x29CB4D87F2A7400E'u), + uint64x2(hi: 0xB2C71D5BCA9023F8'u, lo: 0x743E20E9EF511012'u), + uint64x2(hi: 0xDF78E4B2BD342CF6'u, lo: 0x914DA9246B255416'u), + uint64x2(hi: 0x8BAB8EEFB6409C1A'u, lo: 0x1AD089B6C2F7548E'u), + uint64x2(hi: 0xAE9672ABA3D0C320'u, lo: 0xA184AC2473B529B1'u), + uint64x2(hi: 0xDA3C0F568CC4F3E8'u, lo: 0xC9E5D72D90A2741E'u), + uint64x2(hi: 0x8865899617FB1871'u, lo: 0x7E2FA67C7A658892'u), + uint64x2(hi: 0xAA7EEBFB9DF9DE8D'u, lo: 0xDDBB901B98FEEAB7'u), + uint64x2(hi: 0xD51EA6FA85785631'u, lo: 0x552A74227F3EA565'u), + uint64x2(hi: 0x8533285C936B35DE'u, lo: 0xD53A88958F87275F'u), + uint64x2(hi: 0xA67FF273B8460356'u, lo: 0x8A892ABAF368F137'u), + uint64x2(hi: 0xD01FEF10A657842C'u, lo: 0x2D2B7569B0432D85'u), + uint64x2(hi: 0x8213F56A67F6B29B'u, lo: 0x9C3B29620E29FC73'u), + uint64x2(hi: 0xA298F2C501F45F42'u, lo: 0x8349F3BA91B47B8F'u), + uint64x2(hi: 0xCB3F2F7642717713'u, lo: 0x241C70A936219A73'u), + uint64x2(hi: 0xFE0EFB53D30DD4D7'u, lo: 0xED238CD383AA0110'u), + uint64x2(hi: 0x9EC95D1463E8A506'u, lo: 0xF4363804324A40AA'u), + uint64x2(hi: 0xC67BB4597CE2CE48'u, lo: 0xB143C6053EDCD0D5'u), + uint64x2(hi: 0xF81AA16FDC1B81DA'u, lo: 0xDD94B7868E94050A'u), + uint64x2(hi: 0x9B10A4E5E9913128'u, lo: 0xCA7CF2B4191C8326'u), + uint64x2(hi: 0xC1D4CE1F63F57D72'u, lo: 0xFD1C2F611F63A3F0'u), + uint64x2(hi: 0xF24A01A73CF2DCCF'u, lo: 0xBC633B39673C8CEC'u), + uint64x2(hi: 0x976E41088617CA01'u, lo: 0xD5BE0503E085D813'u), + uint64x2(hi: 0xBD49D14AA79DBC82'u, lo: 0x4B2D8644D8A74E18'u), + uint64x2(hi: 0xEC9C459D51852BA2'u, lo: 0xDDF8E7D60ED1219E'u), + uint64x2(hi: 0x93E1AB8252F33B45'u, lo: 0xCABB90E5C942B503'u), + uint64x2(hi: 0xB8DA1662E7B00A17'u, lo: 0x3D6A751F3B936243'u), + uint64x2(hi: 0xE7109BFBA19C0C9D'u, lo: 0x0CC512670A783AD4'u), + uint64x2(hi: 0x906A617D450187E2'u, lo: 0x27FB2B80668B24C5'u), + uint64x2(hi: 0xB484F9DC9641E9DA'u, lo: 0xB1F9F660802DEDF6'u), + uint64x2(hi: 0xE1A63853BBD26451'u, lo: 0x5E7873F8A0396973'u), + uint64x2(hi: 0x8D07E33455637EB2'u, lo: 0xDB0B487B6423E1E8'u), + uint64x2(hi: 0xB049DC016ABC5E5F'u, lo: 0x91CE1A9A3D2CDA62'u), + uint64x2(hi: 0xDC5C5301C56B75F7'u, lo: 0x7641A140CC7810FB'u), + uint64x2(hi: 0x89B9B3E11B6329BA'u, lo: 0xA9E904C87FCB0A9D'u), + uint64x2(hi: 0xAC2820D9623BF429'u, lo: 0x546345FA9FBDCD44'u), + uint64x2(hi: 0xD732290FBACAF133'u, lo: 0xA97C177947AD4095'u), + uint64x2(hi: 0x867F59A9D4BED6C0'u, lo: 0x49ED8EABCCCC485D'u), + uint64x2(hi: 0xA81F301449EE8C70'u, lo: 0x5C68F256BFFF5A74'u), + uint64x2(hi: 0xD226FC195C6A2F8C'u, lo: 0x73832EEC6FFF3111'u), + uint64x2(hi: 0x83585D8FD9C25DB7'u, lo: 0xC831FD53C5FF7EAB'u), + uint64x2(hi: 0xA42E74F3D032F525'u, lo: 0xBA3E7CA8B77F5E55'u), + uint64x2(hi: 0xCD3A1230C43FB26F'u, lo: 0x28CE1BD2E55F35EB'u), + uint64x2(hi: 0x80444B5E7AA7CF85'u, lo: 0x7980D163CF5B81B3'u), + uint64x2(hi: 0xA0555E361951C366'u, lo: 0xD7E105BCC332621F'u), + uint64x2(hi: 0xC86AB5C39FA63440'u, lo: 0x8DD9472BF3FEFAA7'u), + uint64x2(hi: 0xFA856334878FC150'u, lo: 0xB14F98F6F0FEB951'u), + uint64x2(hi: 0x9C935E00D4B9D8D2'u, lo: 0x6ED1BF9A569F33D3'u), + uint64x2(hi: 0xC3B8358109E84F07'u, lo: 0x0A862F80EC4700C8'u), + uint64x2(hi: 0xF4A642E14C6262C8'u, lo: 0xCD27BB612758C0FA'u), + uint64x2(hi: 0x98E7E9CCCFBD7DBD'u, lo: 0x8038D51CB897789C'u), + uint64x2(hi: 0xBF21E44003ACDD2C'u, lo: 0xE0470A63E6BD56C3'u), + uint64x2(hi: 0xEEEA5D5004981478'u, lo: 0x1858CCFCE06CAC74'u), + uint64x2(hi: 0x95527A5202DF0CCB'u, lo: 0x0F37801E0C43EBC8'u), + uint64x2(hi: 0xBAA718E68396CFFD'u, lo: 0xD30560258F54E6BA'u), + uint64x2(hi: 0xE950DF20247C83FD'u, lo: 0x47C6B82EF32A2069'u), + uint64x2(hi: 0x91D28B7416CDD27E'u, lo: 0x4CDC331D57FA5441'u), + uint64x2(hi: 0xB6472E511C81471D'u, lo: 0xE0133FE4ADF8E952'u), + uint64x2(hi: 0xE3D8F9E563A198E5'u, lo: 0x58180FDDD97723A6'u), + uint64x2(hi: 0x8E679C2F5E44FF8F'u, lo: 0x570F09EAA7EA7648'u), + uint64x2(hi: 0xB201833B35D63F73'u, lo: 0x2CD2CC6551E513DA'u), + uint64x2(hi: 0xDE81E40A034BCF4F'u, lo: 0xF8077F7EA65E58D1'u), + uint64x2(hi: 0x8B112E86420F6191'u, lo: 0xFB04AFAF27FAF782'u), + uint64x2(hi: 0xADD57A27D29339F6'u, lo: 0x79C5DB9AF1F9B563'u), + uint64x2(hi: 0xD94AD8B1C7380874'u, lo: 0x18375281AE7822BC'u), + uint64x2(hi: 0x87CEC76F1C830548'u, lo: 0x8F2293910D0B15B5'u), + uint64x2(hi: 0xA9C2794AE3A3C69A'u, lo: 0xB2EB3875504DDB22'u), + uint64x2(hi: 0xD433179D9C8CB841'u, lo: 0x5FA60692A46151EB'u), + uint64x2(hi: 0x849FEEC281D7F328'u, lo: 0xDBC7C41BA6BCD333'u), + uint64x2(hi: 0xA5C7EA73224DEFF3'u, lo: 0x12B9B522906C0800'u), + uint64x2(hi: 0xCF39E50FEAE16BEF'u, lo: 0xD768226B34870A00'u), + uint64x2(hi: 0x81842F29F2CCE375'u, lo: 0xE6A1158300D46640'u), + uint64x2(hi: 0xA1E53AF46F801C53'u, lo: 0x60495AE3C1097FD0'u), + uint64x2(hi: 0xCA5E89B18B602368'u, lo: 0x385BB19CB14BDFC4'u), + uint64x2(hi: 0xFCF62C1DEE382C42'u, lo: 0x46729E03DD9ED7B5'u), + uint64x2(hi: 0x9E19DB92B4E31BA9'u, lo: 0x6C07A2C26A8346D1'u), + uint64x2(hi: 0xC5A05277621BE293'u, lo: 0xC7098B7305241885'u), + uint64x2(hi: 0xF70867153AA2DB38'u, lo: 0xB8CBEE4FC66D1EA7'u)] + dragonbox_Assert(k >= kMin) + dragonbox_Assert(k <= kMax) + return pow10[k - kMin] + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2*(value: uint64; e2: int32): bool {.inline.} = + dragonbox_Assert(e2 >= 0) + return e2 < 64 and (value and ((uint64(1) shl e2) - 1)) == 0 + +## Returns whether value is divisible by 5^e5 + +proc multipleOfPow5*(value: uint64; e5: int32): bool {.inline.} = + type + MulCmp {.bycopy.} = object + mul: uint64 + cmp: uint64 + + const + mod5 = [MulCmp(mul: 0x0000000000000001'u, cmp: 0xFFFFFFFFFFFFFFFF'u), + MulCmp(mul: 0xCCCCCCCCCCCCCCCD'u, cmp: 0x3333333333333333'u), + MulCmp(mul: 0x8F5C28F5C28F5C29'u, cmp: 0x0A3D70A3D70A3D70'u), + MulCmp(mul: 0x1CAC083126E978D5'u, cmp: 0x020C49BA5E353F7C'u), + MulCmp(mul: 0xD288CE703AFB7E91'u, cmp: 0x0068DB8BAC710CB2'u), + MulCmp(mul: 0x5D4E8FB00BCBE61D'u, cmp: 0x0014F8B588E368F0'u), + MulCmp(mul: 0x790FB65668C26139'u, cmp: 0x000431BDE82D7B63'u), + MulCmp(mul: 0xE5032477AE8D46A5'u, cmp: 0x0000D6BF94D5E57A'u), + MulCmp(mul: 0xC767074B22E90E21'u, cmp: 0x00002AF31DC46118'u), + MulCmp(mul: 0x8E47CE423A2E9C6D'u, cmp: 0x0000089705F4136B'u), + MulCmp(mul: 0x4FA7F60D3ED61F49'u, cmp: 0x000001B7CDFD9D7B'u), + MulCmp(mul: 0x0FEE64690C913975'u, cmp: 0x00000057F5FF85E5'u), + MulCmp(mul: 0x3662E0E1CF503EB1'u, cmp: 0x000000119799812D'u), + MulCmp(mul: 0xA47A2CF9F6433FBD'u, cmp: 0x0000000384B84D09'u), + MulCmp(mul: 0x54186F653140A659'u, cmp: 0x00000000B424DC35'u), + MulCmp(mul: 0x7738164770402145'u, cmp: 0x0000000024075F3D'u), + MulCmp(mul: 0xE4A4D1417CD9A041'u, cmp: 0x000000000734ACA5'u), + MulCmp(mul: 0xC75429D9E5C5200D'u, cmp: 0x000000000170EF54'u), + MulCmp(mul: 0xC1773B91FAC10669'u, cmp: 0x000000000049C977'u), + MulCmp(mul: 0x26B172506559CE15'u, cmp: 0x00000000000EC1E4'u), + MulCmp(mul: 0xD489E3A9ADDEC2D1'u, cmp: 0x000000000002F394'u), + MulCmp(mul: 0x90E860BB892C8D5D'u, cmp: 0x000000000000971D'u), + MulCmp(mul: 0x502E79BF1B6F4F79'u, cmp: 0x0000000000001E39'u), + MulCmp(mul: 0xDCD618596BE30FE5'u, cmp: 0x000000000000060B'u), + MulCmp(mul: 0x2C2AD1AB7BFA3661'u, cmp: 0x0000000000000135'u)] + dragonbox_Assert(e5 >= 0) + dragonbox_Assert(e5 <= 24) + let m5: MulCmp = mod5[e5] + return value * m5.mul <= m5.cmp + +type + FloatingDecimal64* {.bycopy.} = object + significand*: uint64 + exponent*: int32 + + +proc toDecimal64AsymmetricInterval*(e2: int32): FloatingDecimal64 {.inline.} = + ## NB: + ## accept_lower_endpoint = true + ## accept_upper_endpoint = true + const + P: int32 = significandSize + ## Compute k and beta + let minusK: int32 = floorLog10ThreeQuartersPow2(e2) + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + ## Compute xi and zi + let pow10: uint64x2 = computePow10(-minusK) + let lowerEndpoint: uint64 = (pow10.hi - (pow10.hi shr (P + 1))) shr + (64 - P - betaMinus1) + let upperEndpoint: uint64 = (pow10.hi + (pow10.hi shr (P + 0))) shr + (64 - P - betaMinus1) + ## If we don't accept the left endpoint (but we do!) or + ## if the left endpoint is not an integer, increase it + let lowerEndpointIsInteger: bool = (2 <= e2 and e2 <= 3) + let xi: uint64 = lowerEndpoint + uint64(not lowerEndpointIsInteger) + let zi: uint64 = upperEndpoint + ## Try bigger divisor + var q: uint64 = zi div 10 + if q * 10 >= xi: + return FloatingDecimal64(significand: q, exponent: minusK + 1) + q = ((pow10.hi shr (64 - (P + 1) - betaMinus1)) + 1) div 2 + ## When tie occurs, choose one of them according to the rule + if e2 == -77: + dec(q, ord(q mod 2 != 0)) + ## Round to even. + else: + inc(q, ord(q < xi)) + return FloatingDecimal64(significand: q, exponent: minusK) + +proc computeDelta*(pow10: uint64x2; betaMinus1: int32): uint32 {.inline.} = + dragonbox_Assert(betaMinus1 >= 0) + dragonbox_Assert(betaMinus1 <= 63) + return cast[uint32](pow10.hi shr (64 - 1 - betaMinus1)) + +when defined(sizeof_Int128): + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + ## 1 mulx + type + uint128T = uint128 + let p: uint128T = uint128T(x) * y + let hi: uint64 = cast[uint64](p shr 64) + let lo: uint64 = cast[uint64](p) + return (hi, lo) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + var hi: uint64 = 0 + var lo: uint64 = umul128(x, y, addr(hi)) + return uint64x2(hi: hi, lo: lo) + +else: + proc lo32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + + proc hi32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + + proc mul128*(a: uint64; b: uint64): uint64x2 {.inline.} = + let b00: uint64 = uint64(lo32(a)) * lo32(b) + let b01: uint64 = uint64(lo32(a)) * hi32(b) + let b10: uint64 = uint64(hi32(a)) * lo32(b) + let b11: uint64 = uint64(hi32(a)) * hi32(b) + let mid1: uint64 = b10 + hi32(b00) + let mid2: uint64 = b01 + lo32(mid1) + let hi: uint64 = b11 + hi32(mid1) + hi32(mid2) + let lo: uint64 = lo32(b00) or uint64(lo32(mid2)) shl 32 + return uint64x2(hi: hi, lo: lo) + +## Returns (x * y) / 2^128 + +proc mulShift*(x: uint64; y: uint64x2): uint64 {.inline.} = + ## 2 mulx + var p1: uint64x2 = mul128(x, y.hi) + var p0: uint64x2 = mul128(x, y.lo) + p1.lo += p0.hi + inc(p1.hi, ord(p1.lo < p0.hi)) + return p1.hi + +proc mulParity*(twoF: uint64; pow10: uint64x2; betaMinus1: int32): bool {.inline.} = + ## 1 mulx, 1 mul + dragonbox_Assert(betaMinus1 >= 1) + dragonbox_Assert(betaMinus1 <= 63) + let p01: uint64 = twoF * pow10.hi + let p10: uint64 = mul128(twoF, pow10.lo).hi + let mid: uint64 = p01 + p10 + return (mid and (uint64(1) shl (64 - betaMinus1))) != 0 + +proc isIntegralEndpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -2: + return false + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc isIntegralMidpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -4: + return multipleOfPow2(twoF, minusK - e2 + 1) + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecimal64 {. + inline.} = + const + kappa: int32 = 2 + const + bigDivisor: uint32 = 1000 + ## 10^(kappa + 1) + const + smallDivisor: uint32 = 100 + ## 10^(kappa) + ## + ## Step 1: + ## integer promotion & Schubfach multiplier calculation. + ## + var m2: uint64 + var e2: int32 + if ieeeExponent != 0: + m2 = hiddenBit or ieeeSignificand + e2 = cast[int32](ieeeExponent) - exponentBias + if 0 <= -e2 and -e2 < significandSize and multipleOfPow2(m2, -e2): + ## Small integer. + return FloatingDecimal64(significand: m2 shr -e2, exponent: 0) + if ieeeSignificand == 0 and ieeeExponent > 1: + ## Shorter interval case; proceed like Schubfach. + return toDecimal64AsymmetricInterval(e2) + else: + ## Subnormal case; interval is always regular. + m2 = ieeeSignificand + e2 = 1 - exponentBias + let isEven: bool = (m2 mod 2 == 0) + let acceptLower: bool = isEven + let acceptUpper: bool = isEven + ## Compute k and beta. + let minusK: int32 = floorLog10Pow2(e2) - kappa + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + dragonbox_Assert(betaMinus1 >= 6) + dragonbox_Assert(betaMinus1 <= 9) + let pow10: uint64x2 = computePow10(-minusK) + ## Compute delta + ## 10^kappa <= delta < 10^(kappa + 1) + ## 100 <= delta < 1000 + let delta: uint32 = computeDelta(pow10, betaMinus1) + dragonbox_Assert(delta >= smallDivisor) + dragonbox_Assert(delta < bigDivisor) + let twoFl: uint64 = 2 * m2 - 1 + let twoFc: uint64 = 2 * m2 + let twoFr: uint64 = 2 * m2 + 1 + ## (54 bits) + ## Compute zi + ## (54 + 9 = 63 bits) + let zi: uint64 = mulShift(twoFr shl betaMinus1, pow10) + ## 2 mulx + ## + ## Step 2: + ## Try larger divisor. + ## + var q: uint64 = zi div bigDivisor + ## uint64_t q = Mul128(zi, 0x83126E978D4FDF3Cu).hi >> 9; // 1 mulx + var r: uint32 = cast[uint32](zi) - bigDivisor * cast[uint32](q) + ## r = zi % BigDivisor + ## 0 <= r < 1000 + if r < delta: ## likely ~50% ?! + ## (r > deltai) + ## Exclude the right endpoint if necessary + if r != 0 or acceptUpper or not isIntegralEndpoint(twoFr, e2, minusK): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + dragonbox_Assert(q != 0) + dec(q) + r = bigDivisor + elif r == delta: ## unlikely + ## Compare fractional parts. + ## Check conditions in the order different from the paper + ## to take advantage of short-circuiting + if (acceptLower and isIntegralEndpoint(twoFl, e2, minusK)) or + mulParity(twoFl, pow10, betaMinus1): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + else: + discard + ## + ## Step 3: + ## Find the significand with the smaller divisor + ## + q = q * 10 + ## 1 hmul + ## 0 <= r <= 1000 + let dist: uint32 = r - (delta div 2) + (smallDivisor div 2) + let distQ: uint32 = dist div 100 + ## 1 mul + ## const uint32_t dist_r = dist % 100; + q += distQ + ## if /*likely*/ (dist_r == 0) + if dist == distQ * 100: + ## const bool approx_y_parity = ((dist ^ (SmallDivisor / 2)) & 1) != 0; + let approxYParity: bool = (dist and 1) != 0 + ## Check z^(f) >= epsilon^(f) + ## We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1, + ## where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f) + ## Since there are only 2 possibilities, we only need to care about the + ## parity. Also, zi and r should have the same parity since the divisor + ## is an even number + if mulParity(twoFc, pow10, betaMinus1) != approxYParity: + dec(q) + elif q mod 2 != 0 and isIntegralMidpoint(twoFc, e2, minusK): + dec(q) + return FloatingDecimal64(significand: q, exponent: minusK + kappa) + +# ================================================================================================== +# ToChars +# ================================================================================================== + +when false: + template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) + + template dec(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) - uint(offset)) + template inc(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) + uint(offset)) + + proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.} + proc memmove(a, b: cstring; L: int) {.importc, nodecl.} + +proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int {.inline.} = + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999'u32) + let q: uint32 = digits div 10000 + let r: uint32 = digits mod 10000 + let qH: uint32 = q div 100 + let qL: uint32 = q mod 100 + utoa2Digits(buf, pos, qH) + utoa2Digits(buf, pos + 2, qL) + if r == 0: + return trailingZeros2Digits(if qL == 0: qH else: qL) + (if qL == 0: 6 else: 4) + else: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos + 4, rH) + utoa2Digits(buf, pos + 6, rL) + return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + +proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int {.inline.} = + var pos = pos + var output64 = output64 + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 17 digits remaining + if output64 >= 100000000'u64: + let q: uint64 = output64 div 100000000'u64 + let r: uint32 = cast[uint32](output64 mod 100000000'u64) + output64 = q + dec(pos, 8) + if r != 0: + tz = utoa8DigitsSkipTrailingZeros(buf, pos, r) + dragonbox_Assert(tz >= 0) + dragonbox_Assert(tz <= 7) + else: + tz = 8 + nd = 8 + dragonbox_Assert(output64 <= high(uint32)) + var output = cast[uint32](output64) + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + if tz == nd: + inc(tz, trailingZeros2Digits(if rL == 0: rH else: rL) + + (if rL == 0: 2 else: 0)) + else: + if tz == nd: + inc(tz, 4) + else: + for i in 0..3: buf[pos+i] = '0' + ## (actually not required...) + inc(nd, 4) + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + dragonbox_Assert(output >= 1) + dragonbox_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + dragonbox_Assert(q >= 1) + dragonbox_Assert(q <= 9) + dec(pos) + buf[pos] = chr(ord('0') + q) + return tz + +proc decimalLength*(v: uint64): int {.inline.} = + dragonbox_Assert(v >= 1) + dragonbox_Assert(v <= 99999999999999999'u64) + if cast[uint32](v shr 32) != 0: + if v >= 10000000000000000'u64: + return 17 + if v >= 1000000000000000'u64: + return 16 + if v >= 100000000000000'u64: + return 15 + if v >= 10000000000000'u64: + return 14 + if v >= 1000000000000'u64: + return 13 + if v >= 100000000000'u64: + return 12 + if v >= 10000000000'u64: + return 11 + return 10 + let v32: uint32 = cast[uint32](v) + if v32 >= 1000000000'u32: + return 10 + if v32 >= 100000000'u32: + return 9 + if v32 >= 10000000'u32: + return 8 + if v32 >= 1000000'u32: + return 7 + if v32 >= 100000'u32: + return 6 + if v32 >= 10000'u32: + return 5 + if v32 >= 1000'u32: + return 4 + if v32 >= 100'u32: + return 3 + if v32 >= 10'u32: + return 2 + return 1 + +proc formatDigits*[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint64; decimalExponent: int; + forceTrailingDotZero = false): int {.inline.} = + const + minFixedDecimalPoint = -6 + const + maxFixedDecimalPoint = 17 + var pos:int = pos.int + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 17, "internal error") + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999999999999'u64) + dragonbox_Assert(decimalExponent >= -999) + dragonbox_Assert(decimalExponent <= 999) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + int(decimalDigitsPosition + numDigits) + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + when true: #defined(vcc) and not defined(clang): + ## VC does not inline the memmove call below. (Even if compiled with /arch:AVX2.) + ## However, memcpy will be inlined. + var tmp: array[16, char] + for i in 0..<16: tmp[i] = buffer[i+pos+decimalPoint] + for i in 0..<16: buffer[i+pos+decimalPoint+1] = tmp[i] + else: + memmove(buffer +! (decimalPoint + 1), buffer +! decimalPoint, 16) + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + ## Copy the first digit one place to the left. + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(ord('0') + k) + inc(pos) + elif k < 100: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + else: + let q: uint32 = k div 100 + let r: uint32 = k mod 100 + buffer[pos] = chr(ord('0') + q) + inc(pos) + utoa2Digits(buffer, pos, r) + inc(pos, 2) + return pos + +proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = false): int {. + inline.} = + var pos = 0 + let significand: uint64 = physicalSignificand(constructDouble(v)) + let exponent: uint64 = physicalExponent(constructDouble(v)) + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec = toDecimal64(significand, exponent) + return formatDigits(buffer, pos, dec.significand, dec.exponent.int, + forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 + +when false: + proc toString*(value: float): string = + var buffer: array[dtoaMinBufferLength, char] + let last = toChars(addr buffer, value) + let L = cast[int](last) - cast[int](addr(buffer)) + result = newString(L) + copyMem(addr result[0], addr buffer[0], L) + diff --git a/lib/std/private/gitutils.nim b/lib/std/private/gitutils.nim new file mode 100644 index 000000000..6dc9c8f3b --- /dev/null +++ b/lib/std/private/gitutils.nim @@ -0,0 +1,74 @@ +##[ +internal API for now, API subject to change +]## + +# xxx move other git utilities here; candidate for stdlib. + +import std/[os, paths, osproc, strutils, tempfiles] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +const commitHead* = "HEAD" + +template retryCall*(maxRetry = 3, backoffDuration = 1.0, call: untyped): bool = + ## Retry `call` up to `maxRetry` times with exponential backoff and initial + ## duraton of `backoffDuration` seconds. + ## This is in particular useful for network commands that can fail. + runnableExamples: + doAssert not retryCall(maxRetry = 2, backoffDuration = 0.1, false) + var i = 0 + doAssert: retryCall(maxRetry = 3, backoffDuration = 0.1, (i.inc; i >= 3)) + doAssert retryCall(call = true) + var result = false + var t = backoffDuration + for i in 0..<maxRetry: + if call: + result = true + break + if i == maxRetry - 1: break + sleep(int(t * 1000)) + t = t * 2 # exponential backoff + result + +proc isGitRepo*(dir: string): bool = + ## Avoid calling git since it depends on /bin/sh existing and fails in Nix. + return fileExists(dir/".git/HEAD") + +proc diffFiles*(path1, path2: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of files `path1`, `path2`, the exact form of + ## which is implementation defined. + # This could be customized, e.g. non-git diff with `diff -uNdr`, or with + # git diff options (e.g. --color-moved, --word-diff). + # in general, `git diff` has more options than `diff`. + var status = 0 + (result.output, status) = execCmdEx("git diff --no-index $1 $2" % [path1.quoteShell, path2.quoteShell]) + doAssert (status == 0) or (status == 1) + result.same = status == 0 + +proc diffStrings*(a, b: string): tuple[output: string, same: bool] = + ## Returns a human readable diff of `a`, `b`, the exact form of which is + ## implementation defined. + ## See also `experimental.diff`. + runnableExamples: + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + let (c, same) = diffStrings(a, b) + doAssert not same + let (c2, same2) = diffStrings(a, a) + doAssert same2 + runnableExamples("-r:off"): + let a = "ok1\nok2\nok3\n" + let b = "ok1\nok2 alt\nok3\nok4\n" + echo diffStrings(a, b).output + + template tmpFileImpl(prefix, str): auto = + let path = genTempPath(prefix, "") + writeFile(path, str) + path + let patha = tmpFileImpl("diffStrings_a_", a) + let pathb = tmpFileImpl("diffStrings_b_", b) + defer: + removeFile(patha) + removeFile(pathb) + result = diffFiles(patha, pathb) diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim new file mode 100644 index 000000000..a6d088558 --- /dev/null +++ b/lib/std/private/globs.nim @@ -0,0 +1,70 @@ +##[ +unstable API, internal use only for now. +this can eventually be moved to std/os and `walkDirRec` can be implemented in terms of this +to avoid duplication +]## + +import std/os +when defined(windows): + from std/strutils import replace + +when defined(nimPreviewSlimSystem): + import std/[assertions, objectdollar] + + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +type + PathEntry* = object + kind*: PathComponent + path*: string + +iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect], effectsOf: follow.} = + ## Improved `os.walkDirRec`. + #[ + note: a yieldFilter isn't needed because caller can filter at call site, without + loss of generality, unlike `follow`. + + Future work: + * need to document + * add a `sort` option, which can be implemented efficiently only here, not at call site. + * provide a way to do error reporting, which is tricky because iteration cannot be resumed + ]# + var stack = @["."] + var checkDir = checkDir + var entry: PathEntry + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): + let rel = d / p + entry.kind = k + if relative: entry.path = rel + else: entry.path = dir / rel + if k in {pcDir, pcLinkToDir}: + if follow == nil or follow(entry): stack.add rel + yield entry + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + +proc nativeToUnixPath*(path: string): string = + # pending https://github.com/nim-lang/Nim/pull/13265 + result = path + when defined(windows): + if path.len >= 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':': + result[0] = '/' + result[1] = path[0] + if path.len > 2 and path[2] != '\\': + raiseAssert "paths like `C:foo` are currently unsupported, path: " & path + when DirSep == '\\': + result = replace(result, '\\', '/') + +when isMainModule: + import std/sugar + for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]): + echo a diff --git a/lib/std/private/jsutils.nim b/lib/std/private/jsutils.nim new file mode 100644 index 000000000..5f79eab27 --- /dev/null +++ b/lib/std/private/jsutils.nim @@ -0,0 +1,96 @@ +when defined(js): + import std/jsbigints + + type + ArrayBuffer* = ref object of JsRoot + Float64Array* = ref object of JsRoot + Uint32Array* = ref object of JsRoot + Uint8Array* = ref object of JsRoot + BigUint64Array* = ref object of JsRoot + + + func newArrayBuffer*(n: int): ArrayBuffer {.importjs: "new ArrayBuffer(#)".} + func newFloat64Array*(buffer: ArrayBuffer): Float64Array {.importjs: "new Float64Array(#)".} + func newUint32Array*(buffer: ArrayBuffer): Uint32Array {.importjs: "new Uint32Array(#)".} + func newBigUint64Array*(buffer: ArrayBuffer): BigUint64Array {.importjs: "new BigUint64Array(#)".} + + func newUint8Array*(n: int): Uint8Array {.importjs: "new Uint8Array(#)".} + + func `[]`*(arr: Uint32Array, i: int): uint32 {.importjs: "#[#]".} + func `[]`*(arr: Uint8Array, i: int): uint8 {.importjs: "#[#]".} + func `[]`*(arr: BigUint64Array, i: int): JsBigInt {.importjs: "#[#]".} + func `[]=`*(arr: Float64Array, i: int, v: float) {.importjs: "#[#] = #".} + + proc jsTypeOf*[T](x: T): cstring {.importjs: "typeof(#)".} = + ## Returns the name of the JsObject's JavaScript type as a cstring. + # xxx replace jsffi.jsTypeOf with this definition and add tests + runnableExamples: + import std/[jsffi, jsbigints] + assert jsTypeOf(1.toJs) == "number" + assert jsTypeOf(false.toJs) == "boolean" + assert [1].toJs.jsTypeOf == "object" # note the difference with `getProtoName` + assert big"1".toJs.jsTypeOf == "bigint" + + proc jsConstructorName*[T](a: T): cstring = + runnableExamples: + import std/jsffi + let a = array[2, float64].default + assert jsConstructorName(a) == "Float64Array" + assert jsConstructorName(a.toJs) == "Float64Array" + {.emit: """`result` = `a`.constructor.name;""".} + + proc hasJsBigInt*(): bool = + {.emit: """`result` = typeof BigInt != 'undefined';""".} + + proc hasBigUint64Array*(): bool = + {.emit: """`result` = typeof BigUint64Array != 'undefined';""".} + + proc getProtoName*[T](a: T): cstring {.importjs: "Object.prototype.toString.call(#)".} = + runnableExamples: + import std/[jsffi, jsbigints] + type A = ref object + assert 1.toJs.getProtoName == "[object Number]" + assert "a".toJs.getProtoName == "[object String]" + assert big"1".toJs.getProtoName == "[object BigInt]" + assert false.toJs.getProtoName == "[object Boolean]" + assert (a: 1).toJs.getProtoName == "[object Object]" + assert A.default.toJs.getProtoName == "[object Null]" + assert [1].toJs.getProtoName == "[object Int32Array]" # implementation defined + assert @[1].toJs.getProtoName == "[object Array]" # ditto + + const maxSafeInteger* = 9007199254740991 + ## The same as `Number.MAX_SAFE_INTEGER` or `2^53 - 1`. + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + runnableExamples: + let a {.importjs: "Number.MAX_SAFE_INTEGER".}: int64 + assert a == maxSafeInteger + + proc isInteger*[T](x: T): bool {.importjs: "Number.isInteger(#)".} = + runnableExamples: + import std/jsffi + assert 1.isInteger + assert not 1.5.isInteger + assert 1.toJs.isInteger + assert not 1.5.toJs.isInteger + + proc isSafeInteger*[T](x: T): bool {.importjs: "Number.isSafeInteger(#)".} = + runnableExamples: + import std/jsffi + assert not "123".toJs.isSafeInteger + assert 123.isSafeInteger + assert 123.toJs.isSafeInteger + when false: + assert 9007199254740991.toJs.isSafeInteger + assert not 9007199254740992.toJs.isSafeInteger + +template whenJsNoBigInt64*(no64, yes64): untyped = + when defined(js): + when compiles(compileOption("jsbigint64")): + when compileOption("jsbigint64"): + yes64 + else: + no64 + else: + no64 + else: + no64 diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim new file mode 100644 index 000000000..06fda6fa1 --- /dev/null +++ b/lib/std/private/miscdollars.nim @@ -0,0 +1,39 @@ +from std/private/digitsutils import addInt + +template toLocation*(result: var string, file: string | cstring, line: int, col: int) = + ## avoids spurious allocations + # Hopefully this can be re-used everywhere so that if a user needs to customize, + # it can be done in a single place. + result.add file + if line > 0: + result.add "(" + addInt(result, line) + if col > 0: + result.add ", " + addInt(result, col) + result.add ")" + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = + result = "(" + const isNamed = T is object or isNamedTuple(typeof(T)) + var count {.used.} = 0 + for name, value in fieldPairs(x): + if count > 0: result.add(", ") + when isNamed: + result.add(name) + result.add(": ") + count.inc + when compiles($value): + when value isnot string and value isnot seq and compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.addQuoted(value) + else: + result.addQuoted(value) + else: + result.add("...") + when not isNamed: + if count == 1: + result.add(",") # $(1,) should print as the semantically legal (1,) + result.add(")") diff --git a/lib/std/private/ntpath.nim b/lib/std/private/ntpath.nim new file mode 100644 index 000000000..7c8661bb7 --- /dev/null +++ b/lib/std/private/ntpath.nim @@ -0,0 +1,61 @@ +# This module is inspired by Python's `ntpath.py` module. + +import std/[ + strutils, +] + +# Adapted `splitdrive` function from the following commits in Python source +# code: +# 5a607a3ee5e81bdcef3f886f9d20c1376a533df4 (2009): Initial UNC handling (by Mark Hammond) +# 2ba0fd5767577954f331ecbd53596cd8035d7186 (2022): Support for "UNC"-device paths (by Barney Gale) +# +# FAQ: Why use `strip` below? `\\?\UNC` is the start of a "UNC symbolic link", +# which is a special UNC form. Running `strip` differentiates `\\?\UNC\` (a UNC +# symbolic link) from e.g. `\\?\UNCD` (UNCD is the server in the UNC path). +func splitDrive*(p: string): tuple[drive, path: string] = + ## Splits a Windows path into a drive and path part. The drive can be e.g. + ## `C:`. It can also be a UNC path (`\\server\drive\path`). + ## + ## The equivalent `splitDrive` for POSIX systems always returns empty drive. + ## Therefore this proc is only necessary on DOS-like file systems (together + ## with Nim's `doslikeFileSystem` conditional variable). + ## + ## This proc's use case is to extract `path` such that it can be manipulated + ## like a POSIX path. + runnableExamples: + doAssert splitDrive("C:") == ("C:", "") + doAssert splitDrive(r"C:\") == (r"C:", r"\") + doAssert splitDrive(r"\\server\drive\foo\bar") == (r"\\server\drive", r"\foo\bar") + doAssert splitDrive(r"\\?\UNC\server\share\dir") == (r"\\?\UNC\server\share", r"\dir") + + result = ("", p) + if p.len < 2: + return + const sep = '\\' + let normp = p.replace('/', sep) + if p.len > 2 and normp[0] == sep and normp[1] == sep and normp[2] != sep: + + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path + # \\machine\mountpoint\directory\etc\... + # directory ^^^^^^^^^^^^^^^ + let start = block: + const unc = "\\\\?\\UNC" # Length is 7 + let idx = min(8, normp.len) + if unc == normp[0..<idx].strip(chars = {sep}, leading = false).toUpperAscii: + 8 + else: + 2 + let index = normp.find(sep, start) + if index == -1: + return + var index2 = normp.find(sep, index + 1) + + # a UNC path can't have two slashes in a row (after the initial two) + if index2 == index + 1: + return + if index2 == -1: + index2 = p.len + return (p[0..<index2], p[index2..^1]) + if p[1] == ':': + return (p[0..1], p[2..^1]) diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim new file mode 100644 index 000000000..07a6809bb --- /dev/null +++ b/lib/std/private/osappdirs.nim @@ -0,0 +1,176 @@ +## .. importdoc:: paths.nim, dirs.nim + +include system/inclrtl +import std/envvars +import std/private/ospaths2 + +proc getHomeDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the home directory of the current user. + ## + ## This proc is wrapped by the `expandTilde proc`_ + ## for the convenience of processing paths coming from user configuration files. + ## + ## See also: + ## * `getDataDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + runnableExamples: + import std/os + assert getHomeDir() == expandTilde("~") + + when defined(windows): return getEnv("USERPROFILE") & "\\" + else: return getEnv("HOME") & "/" + +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + +proc getConfigDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the config directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.config/"). + ## + ## An OS-dependent trailing slash is always present at the end of the + ## returned string: `\\` on Windows and `/` on all other OSs. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + else: + result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config") + result.normalizePathEnd(trailingSep = true) + +proc getCacheDir*(): string = + ## Returns the cache directory of the current user for applications. + ## + ## This makes use of the following environment variables: + ## + ## * On Windows: `getEnv("LOCALAPPDATA")` + ## + ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")` + ## + ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` + ## + ## **See also:** + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ + # follows https://crates.io/crates/platform-dirs + when defined(windows): + result = getEnv("LOCALAPPDATA") + elif defined(osx): + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches") + else: + result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache") + result.normalizePathEnd(false) + +proc getCacheDir*(app: string): string = + ## Returns the cache directory for an application `app`. + ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` + ## * On other platforms, this uses: `getCacheDir() / app` + when defined(windows): + getCacheDir() / app / "cache" + else: + getCacheDir() / app + + +when defined(windows): + type DWORD = uint32 + + when defined(nimPreviewSlimSystem): + import std/widestrs + + proc getTempPath( + nBufferLength: DWORD, lpBuffer: WideCString + ): DWORD {.stdcall, dynlib: "kernel32.dll", importc: "GetTempPathW".} = + ## Retrieves the path of the directory designated for temporary files. + +template getEnvImpl(result: var string, tempDirList: openArray[string]) = + for dir in tempDirList: + if existsEnv(dir): + result = getEnv(dir) + break + +template getTempDirImpl(result: var string) = + when defined(windows): + getEnvImpl(result, ["TMP", "TEMP", "USERPROFILE"]) + else: + getEnvImpl(result, ["TMPDIR", "TEMP", "TMP", "TEMPDIR"]) + +proc getTempDir*(): string {.rtl, extern: "nos$1", + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the temporary directory of the current user for applications to + ## save temporary files in. + ## + ## On Windows, it calls [GetTempPath](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppathw). + ## On Posix based platforms, it will check `TMPDIR`, `TEMP`, `TMP` and `TEMPDIR` environment variables in order. + ## On all platforms, `/tmp` will be returned if the procs fails. + ## + ## You can override this implementation + ## by adding `-d:tempDir=mytempname` to your compiler invocation. + ## + ## **Note:** This proc does not check whether the returned path exists. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + const tempDirDefault = "/tmp" + when defined(tempDir): + const tempDir {.strdefine.}: string = tempDirDefault + result = tempDir + else: + when nimvm: + getTempDirImpl(result) + else: + when defined(windows): + let size = getTempPath(0, nil) + # If the function fails, the return value is zero. + if size > 0: + let buffer = newWideCString(size.int) + if getTempPath(size, buffer) > 0: + result = $buffer + elif defined(android): result = "/data/local/tmp" + else: + getTempDirImpl(result) + if result.len == 0: + result = tempDirDefault + normalizePathEnd(result, trailingSep=true) diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim new file mode 100644 index 000000000..c49d52ef2 --- /dev/null +++ b/lib/std/private/oscommon.nim @@ -0,0 +1,186 @@ +include system/inclrtl + +import std/[oserrors] + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osdirs.nim, os.nim + +const weirdTarget* = defined(nimscript) or defined(js) + + +type + ReadDirEffect* = object of ReadIOEffect ## Effect that denotes a read + ## operation from the directory + ## structure. + WriteDirEffect* = object of WriteIOEffect ## Effect that denotes a write + ## operation to + ## the directory structure. + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix + proc c_rename(oldname, newname: cstring): cint {. + importc: "rename", header: "<stdio.h>".} +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +when defined(windows) and not weirdTarget: + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) + + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) + + proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = + # Note - takes advantage of null delimiter in the cstring + const dot = ord('.') + result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or + f.cFileName[1].int == dot and f.cFileName[2].int == 0) + + +type + PathComponent* = enum ## Enumeration specifying a path component. + ## + ## See also: + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ + pcFile, ## path refers to a file + pcLinkToFile, ## path refers to a symbolic link to a file + pcDir, ## path refers to a directory + pcLinkToDir ## path refers to a symbolic link to a directory + + +when defined(posix) and not weirdTarget: + proc getSymlinkFileKind*(path: string): + tuple[pc: PathComponent, isSpecial: bool] = + # Helper function. + var s: Stat + assert(path != "") + result = (pcLinkToFile, false) + if stat(path, s) == 0'i32: + if S_ISDIR(s.st_mode): + result = (pcLinkToDir, false) + elif not S_ISREG(s.st_mode): + result = (pcLinkToFile, true) + +proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. + ## + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). + ## In case of other errors `OSError` is raised. + ## Returns true in case of success. + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + else: + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError + else: + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) + +when not defined(windows): + const maxSymlinkLen* = 1024 + +proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noNimJs, sideEffect.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + ## + ## See also: + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, filename) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 + else: + var res: Stat + return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) + + +proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], + noNimJs, sideEffect.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + ## + ## See also: + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, dir) + if a != -1'i32: + result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + else: + var res: Stat + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + + +proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], + noWeirdTarget, sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + ## + ## See also: + ## * `fileExists proc`_ + ## * `dirExists proc`_ + when defined(windows): + wrapUnary(a, getFileAttributesW, link) + if a != -1'i32: + # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` + # may also be needed. + result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 + else: + var res: Stat + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + +when defined(windows) and not weirdTarget: + proc openHandle*(path: string, followSymlink=true, writeAccess=false): Handle = + var flags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + if not followSymlink: + flags = flags or FILE_FLAG_OPEN_REPARSE_POINT + let access = if writeAccess: GENERIC_WRITE else: 0'i32 + + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim new file mode 100644 index 000000000..a44cad7d9 --- /dev/null +++ b/lib/std/private/osdirs.nim @@ -0,0 +1,570 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + +include system/inclrtl +import std/oserrors + + +import ospaths2, osfiles +import oscommon +export dirExists, PathComponent + + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/[posix, times] + +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +# Templates for filtering directories and files +when defined(windows) and not weirdTarget: + template isDir(f: WIN32_FIND_DATA): bool = + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 + template isFile(f: WIN32_FIND_DATA): bool = + not isDir(f) +else: + template isDir(f: string): bool {.dirty.} = + dirExists(f) + template isFile(f: string): bool {.dirty.} = + fileExists(f) + +template defaultWalkFilter(item): bool = + ## Walk filter used to return true on both + ## files and directories + true + +template walkCommon(pattern: string, filter) = + ## Common code for getting the files and directories with the + ## specified `pattern` + when defined(windows): + var + f: WIN32_FIND_DATA + res: int + res = findFirstFile(pattern, f) + if res != -1: + defer: findClose(res) + let dotPos = searchExtPos(pattern) + while true: + if not skipFindData(f) and filter(f): + # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check + # that the file extensions have the same length ... + let ff = getFilename(f) + let idx = ff.len - pattern.len + dotPos + if dotPos < 0 or idx >= ff.len or (idx >= 0 and ff[idx] == '.') or + (dotPos >= 0 and dotPos+1 < pattern.len and pattern[dotPos+1] == '*'): + yield splitFile(pattern).dir / extractFilename(ff) + if findNextFile(res, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: # here we use glob + var + f: Glob + res: int + f.gl_offs = 0 + f.gl_pathc = 0 + f.gl_pathv = nil + res = glob(pattern, 0, nil, addr(f)) + defer: globfree(addr(f)) + if res == 0: + for i in 0.. f.gl_pathc - 1: + assert(f.gl_pathv[i] != nil) + let path = $f.gl_pathv[i] + if filter(path): + yield path + +iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files and directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + assert "lib/pure/os.nim".unixToNativePath in paths + walkCommon(pattern, defaultWalkFilter) + +iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the files that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too + walkCommon(pattern, isFile) + +iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = + ## Iterate over all the directories that match the `pattern`. + ## + ## On POSIX this uses the `glob`:idx: call. + ## `pattern` is OS dependent, but at least the `"*.ext"` + ## notation is supported. + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ + runnableExamples: + import std/os + import std/sequtils + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too + assert "lib/pure/concurrency".unixToNativePath in paths + walkCommon(pattern, isDir) + +proc staticWalkDir(dir: string; relative: bool): seq[ + tuple[kind: PathComponent, path: string]] = + discard + +iterator walkDir*(dir: string; relative = false, checkDir = false, + skipSpecial = false): + tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. + ## * If `relative` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``, + ## otherwise the full path is returned. + ## * If `checkDir` is true, `OSError` is raised when `dir` + ## doesn't exist. + ## * If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be yielded on Unix. + ## + ## **Example:** + ## + ## This directory structure: + ## + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + runnableExamples("-r:off"): + import std/[strutils, sugar] + # note: order is not guaranteed + # this also works at compile time + assert collect(for k in walkDir("dirA"): k.path).join(" ") == + "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ + + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + else: + when weirdTarget: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + elif defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h == -1: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: findClose(h) + while true: + var k = pcFile + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: + var d = opendir(dir) + if d == nil: + if checkDir: + raiseOSError(osLastError(), dir) + else: + defer: discard closedir(d) + while true: + var x = readdir(d) + if x == nil: break + var y = $cast[cstring](addr x.d_name) + if y != "." and y != "..": + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + template resolveSymlink() = + var isSpecial: bool + (k, isSpecial) = getSymlinkFileKind(path) + if skipSpecial and isSpecial: continue + + template kSetGeneric() = # pure Posix component `k` resolution + if lstat(path.cstring, s) < 0'i32: continue # don't yield + elif S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + resolveSymlink() + elif skipSpecial and not S_ISREG(s.st_mode): continue + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + case x.d_type + of DT_DIR: k = pcDir + of DT_LNK: + resolveSymlink() + of DT_UNKNOWN: + kSetGeneric() + else: # DT_REG or special "files" like FIFOs + if skipSpecial and x.d_type != DT_REG: continue + else: discard # leave it as pcFile + else: # assuming that field `d_type` is not present + kSetGeneric() + + yield (k, y) + +iterator walkDirRec*(dir: string, + yieldFilter = {pcFile}, followFilter = {pcDir}, + relative = false, checkDir = false, skipSpecial = false): + string {.tags: [ReadDirEffect].} = + ## Recursively walks over the directory `dir` and yields for each file + ## or directory in `dir`. + ## + ## Options `relative`, `checkdir`, `skipSpecial` are explained in + ## [walkDir iterator] description. + ## + ## .. warning:: Modifying the directory structure while the iterator + ## is traversing may result in undefined behavior! + ## + ## Walking is recursive. `followFilter` controls the behaviour of the iterator: + ## + ## ===================== ============================================= + ## yieldFilter meaning + ## ===================== ============================================= + ## ``pcFile`` yield real files (default) + ## ``pcLinkToFile`` yield symbolic links to files + ## ``pcDir`` yield real directories + ## ``pcLinkToDir`` yield symbolic links to directories + ## ===================== ============================================= + ## + ## ===================== ============================================= + ## followFilter meaning + ## ===================== ============================================= + ## ``pcDir`` follow real directories (default) + ## ``pcLinkToDir`` follow symbolic links to directories + ## ===================== ============================================= + ## + ## + ## See also: + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + + var stack = @[""] + var checkDir = checkDir + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir, + skipSpecial = skipSpecial): + let rel = d / p + if k in {pcDir, pcLinkToDir} and k in followFilter: + stack.add rel + if k in yieldFilter: + yield if relative: rel else: dir / rel + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. + # Future work can provide a way to customize this and do error reporting. + + +proc rawRemoveDir(dir: string) {.noWeirdTarget.} = + when defined(windows): + wrapUnary(res, removeDirectoryW, dir) + let lastError = osLastError() + if res == 0'i32 and lastError.int32 != 3'i32 and + lastError.int32 != 18'i32 and lastError.int32 != 2'i32: + raiseOSError(lastError, dir) + else: + if rmdir(dir) != 0'i32 and errno != ENOENT: raiseOSError(osLastError(), dir) + +proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ + WriteDirEffect, ReadDirEffect], benign, noWeirdTarget.} = + ## Removes the directory `dir` including all subdirectories and files + ## in `dir` (recursively). + ## + ## If this fails, `OSError` is raised. This does not fail if the directory never + ## existed in the first place, unless `checkDir` = true. + ## + ## See also: + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + for kind, path in walkDir(dir, checkDir = checkDir): + case kind + of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) + of pcDir: removeDir(path, true) + # for subdirectories there is no benefit in `checkDir = false` + # (unless perhaps for edge case of concurrent processes also deleting + # the same files) + rawRemoveDir(dir) + +proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = + # Try to create one directory (not the whole path). + # returns `true` for success, `false` if the path has previously existed + # + # This is a thin wrapper over mkDir (or alternatives on other systems), + # so in case of a pre-existing path we don't check that it is a directory. + when defined(solaris): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno in {EEXIST, ENOSYS}: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(haiku): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST or errno == EROFS: + result = false + else: + raiseOSError(osLastError(), dir) + elif defined(posix): + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST: + result = false + else: + #echo res + raiseOSError(osLastError(), dir) + else: + wrapUnary(res, createDirectoryW, dir) + + if res != 0'i32: + result = true + elif getLastError() == 183'i32: + result = false + else: + raiseOSError(osLastError(), dir) + +proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Checks if a `directory`:idx: `dir` exists, and creates it otherwise. + ## + ## Does not create parent directories (raises `OSError` if parent directories do not exist). + ## Returns `true` if the directory already exists, and `false` otherwise. + ## + ## See also: + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + result = not rawCreateDir(dir) + if result: + # path already exists - need to check that it is indeed a directory + if not dirExists(dir): + raise newException(IOError, "Failed to create '" & dir & "'") + +proc createDir*(dir: string) {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect], noWeirdTarget.} = + ## Creates the `directory`:idx: `dir`. + ## + ## The directory may contain several subdirectories that do not exist yet. + ## The full path is created. If this fails, `OSError` is raised. + ## + ## It does **not** fail if the directory already exists because for + ## most usages this does not indicate an error. + ## + ## See also: + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ + if dir == "": + return + var omitNext = isAbsolute(dir) + for p in parentDirs(dir, fromRoot=true): + if omitNext: + omitNext = false + else: + discard existsOrCreateDir(p) + +proc copyDir*(source, dest: string, skipSpecial = false) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest`. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will copy the attributes from + ## `source` into `dest`. + ## + ## On other platforms created files and directories will inherit the + ## default permissions of a newly created file/directory for the user. + ## Use `copyDirWithPermissions proc`_ + ## to preserve attributes recursively on these platforms. + ## + ## See also: + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ + createDir(dest) + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDir(path, dest / noSource, skipSpecial = skipSpecial) + else: + copyFile(path, dest / noSource, {cfSymlinkAsIs}) + + +proc copyDirWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + skipSpecial = false) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteIOEffect, ReadIOEffect], + benign, noWeirdTarget.} = + ## Copies a directory from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks + ## are skipped. + ## + ## If `skipSpecial` is true, then (besides all directories) only *regular* + ## files (**without** special "file" objects like FIFOs, device files, + ## etc) will be copied on Unix. + ## + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs + ## on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file or directory + ## itself has been copied, which won't happen atomically and could lead to a + ## race condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + createDir(dest) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + false) + except: + if not ignorePermissionErrors: + raise + for kind, path in walkDir(source, skipSpecial = skipSpecial): + var noSource = splitPath(path).tail + if kind == pcDir: + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors, skipSpecial = skipSpecial) + else: + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, {cfSymlinkAsIs}) + +proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a directory from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` contains symlinks, they themself are + ## moved, not their target. + ## + ## If this fails, `OSError` is raised. + ## + ## See also: + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) + +proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = + ## Sets the `current working directory`:idx:; `OSError` + ## is raised if `newDir` cannot been set. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + when defined(windows): + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) + else: + if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim new file mode 100644 index 000000000..37d8eabca --- /dev/null +++ b/lib/std/private/osfiles.nim @@ -0,0 +1,416 @@ +include system/inclrtl +import std/private/since +import std/oserrors + +import oscommon +export fileExists + +import ospaths2, ossymlinks + +## .. importdoc:: osdirs.nim, os.nim + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/[posix, times] + + proc toTime(ts: Timespec): times.Time {.inline.} = + result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +type + FilePermission* = enum ## File access permission, modelled after UNIX. + ## + ## See also: + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ + fpUserExec, ## execute access for the file owner + fpUserWrite, ## write access for the file owner + fpUserRead, ## read access for the file owner + fpGroupExec, ## execute access for the group + fpGroupWrite, ## write access for the group + fpGroupRead, ## read access for the group + fpOthersExec, ## execute access for others + fpOthersWrite, ## write access for others + fpOthersRead ## read access for others + +proc getFilePermissions*(filename: string): set[FilePermission] {. + rtl, extern: "nos$1", tags: [ReadDirEffect], noWeirdTarget.} = + ## Retrieves file permissions for `filename`. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is checked, every other + ## permission is available in any case. + ## + ## See also: + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var a: Stat + if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) + result = {} + if (a.st_mode and S_IRUSR.Mode) != 0.Mode: result.incl(fpUserRead) + if (a.st_mode and S_IWUSR.Mode) != 0.Mode: result.incl(fpUserWrite) + if (a.st_mode and S_IXUSR.Mode) != 0.Mode: result.incl(fpUserExec) + + if (a.st_mode and S_IRGRP.Mode) != 0.Mode: result.incl(fpGroupRead) + if (a.st_mode and S_IWGRP.Mode) != 0.Mode: result.incl(fpGroupWrite) + if (a.st_mode and S_IXGRP.Mode) != 0.Mode: result.incl(fpGroupExec) + + if (a.st_mode and S_IROTH.Mode) != 0.Mode: result.incl(fpOthersRead) + if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) + if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: + result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, + fpOthersExec, fpOthersRead} + else: + result = {fpUserExec..fpOthersRead} + +proc setFilePermissions*(filename: string, permissions: set[FilePermission], + followSymlinks = true) + {.rtl, extern: "nos$1", tags: [ReadDirEffect, WriteDirEffect], + noWeirdTarget.} = + ## Sets the file permissions for `filename`. + ## + ## If `followSymlinks` set to true (default) and ``filename`` points to a + ## symlink, permissions are set to the file symlink points to. + ## `followSymlinks` set to false is a noop on Windows and some POSIX + ## systems (including Linux) on which `lchmod` is either unavailable or always + ## fails, given that symlinks permissions there are not observed. + ## + ## `OSError` is raised in case of an error. + ## On Windows, only the ``readonly`` flag is changed, depending on + ## ``fpUserWrite`` permission. + ## + ## See also: + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ + when defined(posix): + var p = 0.Mode + if fpUserRead in permissions: p = p or S_IRUSR.Mode + if fpUserWrite in permissions: p = p or S_IWUSR.Mode + if fpUserExec in permissions: p = p or S_IXUSR.Mode + + if fpGroupRead in permissions: p = p or S_IRGRP.Mode + if fpGroupWrite in permissions: p = p or S_IWGRP.Mode + if fpGroupExec in permissions: p = p or S_IXGRP.Mode + + if fpOthersRead in permissions: p = p or S_IROTH.Mode + if fpOthersWrite in permissions: p = p or S_IWOTH.Mode + if fpOthersExec in permissions: p = p or S_IXOTH.Mode + + if not followSymlinks and filename.symlinkExists: + when declared(lchmod): + if lchmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + if chmod(filename, cast[Mode](p)) != 0: + raiseOSError(osLastError(), $(filename, permissions)) + else: + wrapUnary(res, getFileAttributesW, filename) + if res == -1'i32: raiseOSError(osLastError(), filename) + if fpUserWrite in permissions: + res = res and not FILE_ATTRIBUTE_READONLY + else: + res = res or FILE_ATTRIBUTE_READONLY + wrapBinary(res2, setFileAttributesW, filename, res) + if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) + + +const hasCCopyfile = defined(osx) and not defined(nimLegacyCopyFile) + # xxx instead of `nimLegacyCopyFile`, support something like: `when osxVersion >= (10, 5)` + +when hasCCopyfile: + # `copyfile` API available since osx 10.5. + {.push nodecl, header: "<copyfile.h>".} + type + copyfile_state_t {.nodecl.} = pointer + copyfile_flags_t = cint + proc copyfile_state_alloc(): copyfile_state_t + proc copyfile_state_free(state: copyfile_state_t): cint + proc c_copyfile(src, dst: cstring, state: copyfile_state_t, flags: copyfile_flags_t): cint {.importc: "copyfile".} + when (NimMajor, NimMinor) >= (1, 4): + let + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + else: + var + COPYFILE_DATA {.nodecl.}: copyfile_flags_t + COPYFILE_XATTR {.nodecl.}: copyfile_flags_t + {.pop.} + +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks + +const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} + +proc copyFile*(source, dest: string, options = {cfSymlinkFollow}; bufferSize = 16_384) {.rtl, + extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], + noWeirdTarget.} = + ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## If this fails, `OSError` is raised. + ## + ## On the Windows platform this proc will + ## copy the source file's attributes into dest. + ## + ## On other platforms you need + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ + ## procs + ## to copy them by hand (or use the convenience `copyFileWithPermissions + ## proc`_), + ## otherwise `dest` will inherit the default permissions of a newly + ## created file for the user. + ## + ## If `dest` already exists, the file attributes + ## will be preserved and the content overwritten. + ## + ## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless + ## `-d:nimLegacyCopyFile` is used. + ## + ## `copyFile` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + + doAssert card(copyFlagSymlink * options) == 1, "There should be exactly one cfSymlink* in options" + let isSymlink = source.symlinkExists + if isSymlink and (cfSymlinkIgnore in options or defined(windows)): + return + when defined(windows): + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) + else: + if isSymlink and cfSymlinkAsIs in options: + createSymlink(expandSymlink(source), dest) + else: + when hasCCopyfile: + let state = copyfile_state_alloc() + # xxx `COPYFILE_STAT` could be used for one-shot + # `copyFileWithPermissions`. + let status = c_copyfile(source.cstring, dest.cstring, state, + COPYFILE_DATA) + if status != 0: + let err = osLastError() + discard copyfile_state_free(state) + raiseOSError(err, $(source, dest)) + let status2 = copyfile_state_free(state) + if status2 != 0: raiseOSError(osLastError(), $(source, dest)) + else: + # generic version of copyFile which works for any platform: + var d, s: File + if not open(s, source): raiseOSError(osLastError(), source) + if not open(d, dest, fmWrite): + close(s) + raiseOSError(osLastError(), dest) + + # Hints for kernel-level aggressive sequential low-fragmentation read-aheads: + # https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_fadvise.html + when defined(linux) or defined(osx): + discard posix_fadvise(getFileHandle(d), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + discard posix_fadvise(getFileHandle(s), 0.cint, 0.cint, POSIX_FADV_SEQUENTIAL) + + var buf = alloc(bufferSize) + while true: + var bytesread = readBuffer(s, buf, bufferSize) + if bytesread > 0: + var byteswritten = writeBuffer(d, buf, bytesread) + if bytesread != byteswritten: + dealloc(buf) + close(s) + close(d) + raiseOSError(osLastError(), dest) + if bytesread != bufferSize: break + dealloc(buf) + close(s) + flushFile(d) + close(d) + +proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}; bufferSize = 16_384) + {.noWeirdTarget, since: (1,3,7).} = + ## Copies a file `source` into directory `dir`, which must exist. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + if dir.len == 0: # treating "" as "." is error prone + raise newException(ValueError, "dest is empty") + copyFile(source, dir / source.lastPathPart, options, bufferSize) + + +proc copyFileWithPermissions*(source, dest: string, + ignorePermissionErrors = true, + options = {cfSymlinkFollow}) {.noWeirdTarget.} = + ## Copies a file from `source` to `dest` preserving file permissions. + ## + ## On non-Windows OSes, `options` specify the way file is copied; by default, + ## if `source` is a symlink, copies the file symlink points to. `options` is + ## ignored on Windows: symlinks are skipped. + ## + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ + ## procs on non-Windows platforms. + ## + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. + ## + ## On non-Windows systems permissions are copied after the file itself has + ## been copied, which won't happen atomically and could lead to a race + ## condition. If `ignorePermissionErrors` is true (default), errors while + ## reading/setting file attributes will be ignored, otherwise will raise + ## `OSError`. + ## + ## See also: + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ + copyFile(source, dest, options) + when not defined(windows): + try: + setFilePermissions(dest, getFilePermissions(source), followSymlinks = + (cfSymlinkFollow in options)) + except: + if not ignorePermissionErrors: + raise + +when not declared(ENOENT) and not defined(windows): + when defined(nimscript): + when not defined(haiku): + const ENOENT = cint(2) # 2 on most systems including Solaris + else: + const ENOENT = cint(-2147459069) + else: + var ENOENT {.importc, header: "<errno.h>".}: cint + +when defined(windows) and not weirdTarget: + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) + +proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, returns `false`. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + result = true + when defined(windows): + let f = newWideCString(file) + if deleteFile(f) == 0: + result = false + let err = getLastError() + if err == ERROR_FILE_NOT_FOUND or err == ERROR_PATH_NOT_FOUND: + result = true + elif err == ERROR_ACCESS_DENIED and + setFileAttributes(f, FILE_ATTRIBUTE_NORMAL) != 0 and + deleteFile(f) != 0: + result = true + else: + if unlink(file) != 0'i32 and errno != ENOENT: + result = false + +proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = + ## Removes the `file`. + ## + ## If this fails, `OSError` is raised. This does not fail + ## if the file never existed in the first place. + ## + ## On Windows, ignores the read-only attribute. + ## + ## See also: + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ + if not tryRemoveFile(file): + raiseOSError(osLastError(), file) + +proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = + ## Moves a file from `source` to `dest`. + ## + ## Symlinks are not followed: if `source` is a symlink, it is itself moved, + ## not its target. + ## + ## If this fails, `OSError` is raised. + ## If `dest` already exists, it will be overwritten. + ## + ## Can be used to `rename files`:idx:. + ## + ## See also: + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ + + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + raiseAssert "unreachable" + else: + # Fallback to copy & del + copyFileWithPermissions(source, dest, options={cfSymlinkAsIs}) + try: + removeFile(source) + except: + discard tryRemoveFile(dest) + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim new file mode 100644 index 000000000..bc69ff725 --- /dev/null +++ b/lib/std/private/ospaths2.nim @@ -0,0 +1,1030 @@ +include system/inclrtl +import std/private/since + +import std/[strutils, pathnorm] +import std/oserrors + +import oscommon +export ReadDirEffect, WriteDirEffect + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + +const weirdTarget = defined(nimscript) or defined(js) + +when weirdTarget: + discard +elif defined(windows): + import std/winlean +elif defined(posix): + import std/posix, system/ansi_c +else: + {.error: "OS module not ported to your operating system!".} + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + + +proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} + + +import std/private/osseps +export osseps + +proc absolutePathInternal(path: string): string {.gcsafe.} + +proc normalizePathEnd*(path: var string, trailingSep = false) = + ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on + ## ``trailingSep``, and taking care of edge cases: it preservers whether + ## a path is absolute or relative, and makes sure trailing sep is `DirSep`, + ## not `AltSep`. Trailing `/.` are compressed, see examples. + if path.len == 0: return + var i = path.len + while i >= 1: + if path[i-1] in {DirSep, AltSep}: dec(i) + elif path[i-1] == '.' and i >= 2 and path[i-2] in {DirSep, AltSep}: dec(i) + else: break + if trailingSep: + # foo// => foo + path.setLen(i) + # foo => foo/ + path.add DirSep + elif i > 0: + # foo// => foo + path.setLen(i) + else: + # // => / (empty case was already taken care of) + path = $DirSep + +proc normalizePathEnd*(path: string, trailingSep = false): string = + ## outplace overload + runnableExamples: + when defined(posix): + assert normalizePathEnd("/lib//.//", trailingSep = true) == "/lib/" + assert normalizePathEnd("lib/./.", trailingSep = false) == "lib" + assert normalizePathEnd(".//./.", trailingSep = false) == "." + assert normalizePathEnd("", trailingSep = true) == "" # not / ! + assert normalizePathEnd("/", trailingSep = false) == "/" # not "" ! + result = path + result.normalizePathEnd(trailingSep) + +template endsWith(a: string, b: set[char]): bool = + a.len > 0 and a[^1] in b + +proc joinPathImpl(result: var string, state: var int, tail: string) = + let trailingSep = tail.endsWith({DirSep, AltSep}) or tail.len == 0 and result.endsWith({DirSep, AltSep}) + normalizePathEnd(result, trailingSep=false) + addNormalizePath(tail, result, state, DirSep) + normalizePathEnd(result, trailingSep=trailingSep) + +proc joinPath*(head, tail: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Joins two directory names to one. + ## + ## returns normalized path concatenation of `head` and `tail`, preserving + ## whether or not `tail` has a trailing slash (or, if tail if empty, whether + ## head has one). + ## + ## See also: + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert joinPath("usr", "lib") == "usr/lib" + assert joinPath("usr", "lib/") == "usr/lib/" + assert joinPath("usr", "") == "usr" + assert joinPath("usr/", "") == "usr/" + assert joinPath("", "") == "" + assert joinPath("", "lib") == "lib" + assert joinPath("", "/lib") == "/lib" + assert joinPath("usr/", "/lib") == "usr/lib" + assert joinPath("usr/lib", "../bin") == "usr/bin" + + result = newStringOfCap(head.len + tail.len) + var state = 0 + joinPathImpl(result, state, head) + joinPathImpl(result, state, tail) + when false: + if len(head) == 0: + result = tail + elif head[len(head)-1] in {DirSep, AltSep}: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & substr(tail, 1) + else: + result = head & tail + else: + if tail.len > 0 and tail[0] in {DirSep, AltSep}: + result = head & tail + else: + result = head & DirSep & tail + +proc joinPath*(parts: varargs[string]): string {.noSideEffect, + rtl, extern: "nos$1OpenArray".} = + ## The same as `joinPath(head, tail) proc`_, + ## but works with any number of directory parts. + ## + ## You need to pass at least one element or the proc + ## will assert in debug builds and crash on release builds. + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ + runnableExamples: + when defined(posix): + assert joinPath("a") == "a" + assert joinPath("a", "b", "c") == "a/b/c" + assert joinPath("usr/lib", "../../var", "log") == "var/log" + + var estimatedLen = 0 + for p in parts: estimatedLen += p.len + result = newStringOfCap(estimatedLen) + var state = 0 + for i in 0..high(parts): + joinPathImpl(result, state, parts[i]) + +proc `/`*(head, tail: string): string {.noSideEffect, inline.} = + ## The same as `joinPath(head, tail) proc`_. + ## + ## See also: + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ + ## * `uri.combine proc <uri.html#combine,Uri,Uri>`_ + ## * `uri./ proc <uri.html#/,Uri,string>`_ + runnableExamples: + when defined(posix): + assert "usr" / "" == "usr" + assert "" / "lib" == "lib" + assert "" / "/lib" == "/lib" + assert "usr/" / "/lib/" == "usr/lib/" + assert "usr" / "lib" / "../bin" == "usr/bin" + + result = joinPath(head, tail) + +when doslikeFileSystem: + import std/private/ntpath + +proc splitPath*(path: string): tuple[head, tail: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a directory into `(head, tail)` tuple, so that + ## ``head / tail == path`` (except for edge cases like "/usr"). + ## + ## See also: + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ + runnableExamples: + assert splitPath("usr/local/bin") == ("usr/local", "bin") + assert splitPath("usr/local/bin/") == ("usr/local/bin", "") + assert splitPath("/bin/") == ("/bin", "") + when (NimMajor, NimMinor) <= (1, 0): + assert splitPath("/bin") == ("", "bin") + else: + assert splitPath("/bin") == ("/", "bin") + assert splitPath("bin") == ("", "bin") + assert splitPath("") == ("", "") + + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(path) + let stop = drive.len + else: + const stop = 0 + + var sepPos = -1 + for i in countdown(len(path)-1, stop): + if path[i] in {DirSep, AltSep}: + sepPos = i + break + if sepPos >= 0: + result.head = substr(path, 0, + when (NimMajor, NimMinor) <= (1, 0): + sepPos-1 + else: + if likely(sepPos >= 1): sepPos-1 else: 0 + ) + result.tail = substr(path, sepPos+1) + else: + when doslikeFileSystem: + result.head = drive + result.tail = splitpath + else: + result.head = "" + result.tail = path + +proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raises: [].} = + ## Checks whether a given `path` is absolute. + ## + ## On Windows, network paths are considered absolute too. + runnableExamples: + assert not "".isAbsolute + assert not ".".isAbsolute + when defined(posix): + assert "/".isAbsolute + assert not "a/".isAbsolute + assert "/a/".isAbsolute + + if len(path) == 0: return false + + when doslikeFileSystem: + var len = len(path) + result = (path[0] in {'/', '\\'}) or + (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') + elif defined(macos): + # according to https://perldoc.perl.org/File/Spec/Mac.html `:a` is a relative path + result = path[0] != ':' + elif defined(RISCOS): + result = path[0] == '$' + elif defined(posix): + result = path[0] == '/' + elif defined(nodejs): + {.emit: [result," = require(\"path\").isAbsolute(",path.cstring,");"].} + else: + raiseAssert "unreachable" # if ever hits here, adapt as needed + +when FileSystemCaseSensitive: + template `!=?`(a, b: char): bool = a != b +else: + template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) + +when doslikeFileSystem: + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = + ## An absolute path from the root of the current drive (e.g. "\foo") + path.len > 0 and + (path[0] == AltSep or + (path[0] == DirSep and + (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) + + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = + ## Return true if path1 and path2 have a same root. + ## + ## Detail of Windows path formats: + ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats + + assert(isAbsolute(path1)) + assert(isAbsolute(path2)) + + if isAbsFromCurrentDrive(path1) and isAbsFromCurrentDrive(path2): + result = true + elif cmpIgnoreCase(splitDrive(path1).drive, splitDrive(path2).drive) == 0: + result = true + else: + result = false + +proc relativePath*(path, base: string, sep = DirSep): string {. + rtl, extern: "nos$1".} = + ## Converts `path` to a path relative to `base`. + ## + ## The `sep` (default: DirSep_) is used for the path normalizations, + ## this can be useful to ensure the relative path only contains `'/'` + ## so that it can be used for URL constructions. + ## + ## On Windows, if a root of `path` and a root of `base` are different, + ## returns `path` as is because it is impossible to make a relative path. + ## That means an absolute path can be returned. + ## + ## See also: + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ + runnableExamples: + assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" + when not doslikeFileSystem: # On Windows, UNC-paths start with `//` + assert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim" + assert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim" + assert relativePath("", "/users/moo", '/') == "" + assert relativePath("foo", ".", '/') == "foo" + assert relativePath("foo", "foo", '/') == "." + + if path.len == 0: return "" + var base = if base == ".": "" else: base + var path = path + path.normalizePathAux + base.normalizePathAux + let a1 = isAbsolute(path) + let a2 = isAbsolute(base) + if a1 and not a2: + base = absolutePathInternal(base) + elif a2 and not a1: + path = absolutePathInternal(path) + + when doslikeFileSystem: + if isAbsolute(path) and isAbsolute(base): + if not sameRoot(path, base): + return path + + var f = default PathIter + var b = default PathIter + var ff = (0, -1) + var bb = (0, -1) # (int, int) + result = newStringOfCap(path.len) + # skip the common prefix: + while f.hasNext(path) and b.hasNext(base): + ff = next(f, path) + bb = next(b, base) + let diff = ff[1] - ff[0] + if diff != bb[1] - bb[0]: break + var same = true + for i in 0..diff: + if path[i + ff[0]] !=? base[i + bb[0]]: + same = false + break + if not same: break + ff = (0, -1) + bb = (0, -1) + # for i in 0..diff: + # result.add base[i + bb[0]] + + # /foo/bar/xxx/ -- base + # /foo/bar/baz -- path path + # ../baz + # every directory that is in 'base', needs to add '..' + while true: + if bb[1] >= bb[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + result.add ".." + if not b.hasNext(base): break + bb = b.next(base) + + # add the rest of 'path': + while true: + if ff[1] >= ff[0]: + if result.len > 0 and result[^1] != sep: + result.add sep + for i in 0..ff[1] - ff[0]: + result.add path[i + ff[0]] + if not f.hasNext(path): break + ff = f.next(path) + + when not defined(nimOldRelativePathBehavior): + if result.len == 0: result.add "." + +proc isRelativeTo*(path: string, base: string): bool {.since: (1, 1).} = + ## Returns true if `path` is relative to `base`. + runnableExamples: + doAssert isRelativeTo("./foo//bar", "foo") + doAssert isRelativeTo("foo/bar", ".") + doAssert isRelativeTo("/foo/bar.nim", "/foo/bar.nim") + doAssert not isRelativeTo("foo/bar.nims", "foo/bar.nim") + let path = path.normalizePath + let base = base.normalizePath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc parentDirPos(path: string): int = + var q = 1 + if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2 + for i in countdown(len(path)-q, 0): + if path[i] in {DirSep, AltSep}: return i + result = -1 + +proc parentDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the parent directory of `path`. + ## + ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end + ## in a dir separator, but also takes care of path normalizations. + ## The remainder can be obtained with `lastPathPart(path) proc`_. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ + runnableExamples: + assert parentDir("") == "" + when defined(posix): + assert parentDir("/usr/local/bin") == "/usr/local" + assert parentDir("foo/bar//") == "foo" + assert parentDir("//foo//bar//.") == "/foo" + assert parentDir("./foo") == "." + assert parentDir("/./foo//./") == "/" + assert parentDir("a//./") == "." + assert parentDir("a/b/c/..") == "a" + result = pathnorm.normalizePath(path) + when doslikeFileSystem: + let (drive, splitpath) = splitDrive(result) + result = splitpath + var sepPos = parentDirPos(result) + if sepPos >= 0: + result = substr(result, 0, sepPos) + normalizePathEnd(result) + elif result == ".." or result == "." or result.len == 0 or result[^1] in {DirSep, AltSep}: + # `.` => `..` and .. => `../..`(etc) would be a sensible alternative + # `/` => `/` (as done with splitFile) would be a sensible alternative + result = "" + else: + result = "." + when doslikeFileSystem: + if result.len == 0: + discard + elif drive.len > 0 and result.len == 1 and result[0] in {DirSep, AltSep}: + result = drive + else: + result = drive & result + +proc tailDir*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Returns the tail part of `path`. + ## + ## See also: + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + runnableExamples: + assert tailDir("/bin") == "bin" + assert tailDir("bin") == "" + assert tailDir("bin/") == "" + assert tailDir("/usr/local/bin") == "usr/local/bin" + assert tailDir("//usr//local//bin//") == "usr//local//bin//" + assert tailDir("./usr/local/bin") == "usr/local/bin" + assert tailDir("usr/local/bin") == "local/bin" + + var i = 0 + when doslikeFileSystem: + let (drive, splitpath) = path.splitDrive + if drive != "": + return splitpath.strip(chars = {DirSep, AltSep}, trailing = false) + while i < len(path): + if path[i] in {DirSep, AltSep}: + while i < len(path) and path[i] in {DirSep, AltSep}: inc i + return substr(path, i) + inc i + result = "" + +proc isRootDir*(path: string): bool {. + noSideEffect, rtl, extern: "nos$1".} = + ## Checks whether a given `path` is a root directory. + runnableExamples: + assert isRootDir("") + assert isRootDir(".") + assert isRootDir("/") + assert isRootDir("a") + assert not isRootDir("/a") + assert not isRootDir("a/b/c") + + when doslikeFileSystem: + if splitDrive(path).path == "": + return true + result = parentDirPos(path) < 0 + +iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = + ## Walks over all parent directories of a given `path`. + ## + ## If `fromRoot` is true (default: false), the traversal will start from + ## the file system root directory. + ## If `inclusive` is true (default), the original argument will be included + ## in the traversal. + ## + ## Relative paths won't be expanded by this iterator. Instead, it will traverse + ## only the directories appearing in the relative path. + ## + ## See also: + ## * `parentDir proc`_ + ## + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a + + if not fromRoot: + var current = path + if inclusive: yield path + while true: + if current.isRootDir: break + current = current.parentDir + yield current + else: + when doslikeFileSystem: + let start = path.splitDrive.drive.len + else: + const start = 0 + for i in countup(start, path.len - 2): # ignore the last / + # deal with non-normalized paths such as /foo//bar//baz + if path[i] in {DirSep, AltSep} and + (i == 0 or path[i-1] notin {DirSep, AltSep}): + yield path.substr(0, i) + + if inclusive: yield path + +proc `/../`*(head, tail: string): string {.noSideEffect.} = + ## The same as ``parentDir(head) / tail``, unless there is no parent + ## directory. Then ``head / tail`` is performed instead. + ## + ## See also: + ## * `/ proc`_ + ## * `parentDir proc`_ + runnableExamples: + when defined(posix): + assert "a/b/c" /../ "d/e" == "a/b/d/e" + assert "a" /../ "d/e" == "a/d/e" + + when doslikeFileSystem: + let (drive, head) = splitDrive(head) + let sepPos = parentDirPos(head) + if sepPos >= 0: + result = substr(head, 0, sepPos-1) / tail + else: + result = head / tail + when doslikeFileSystem: + result = drive / result + +proc normExt(ext: string): string = + if ext == "" or ext[0] == ExtSep: result = ext # no copy needed here + else: result = ExtSep & ext + +proc searchExtPos*(path: string): int = + ## Returns index of the `'.'` char in `path` if it signifies the beginning + ## of the file extension. Returns -1 otherwise. + ## + ## See also: + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert searchExtPos("a/b/c") == -1 + assert searchExtPos("c.nim") == 1 + assert searchExtPos("a/b/c.nim") == 5 + assert searchExtPos("a.b.c.nim") == 5 + assert searchExtPos(".nim") == -1 + assert searchExtPos("..nim") == -1 + assert searchExtPos("a..nim") == 2 + + # Unless there is any char that is not `ExtSep` before last `ExtSep` in the file name, + # it is not a file extension. + const DirSeps = when doslikeFileSystem: {DirSep, AltSep, ':'} else: {DirSep, AltSep} + result = -1 + var i = path.high + while i >= 1: + if path[i] == ExtSep: + break + elif path[i] in DirSeps: + return -1 # do not skip over path + dec i + + for j in countdown(i - 1, 0): + if path[j] in DirSeps: + return -1 + elif path[j] != ExtSep: + result = i + break + +proc splitFile*(path: string): tuple[dir, name, ext: string] {. + noSideEffect, rtl, extern: "nos$1".} = + ## Splits a filename into `(dir, name, extension)` tuple. + ## + ## `dir` does not end in DirSep_ unless it's `/`. + ## `extension` includes the leading dot. + ## + ## If `path` has no extension, `ext` is the empty string. + ## If `path` has no directory component, `dir` is the empty string. + ## If `path` has no filename component, `name` and `ext` are empty strings. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + var (dir, name, ext) = splitFile("usr/local/nimc.html") + assert dir == "usr/local" + assert name == "nimc" + assert ext == ".html" + (dir, name, ext) = splitFile("/usr/local/os") + assert dir == "/usr/local" + assert name == "os" + assert ext == "" + (dir, name, ext) = splitFile("/usr/local/") + assert dir == "/usr/local" + assert name == "" + assert ext == "" + (dir, name, ext) = splitFile("/tmp.txt") + assert dir == "/" + assert name == "tmp" + assert ext == ".txt" + + var namePos = 0 + var dotPos = 0 + when doslikeFileSystem: + let (drive, _) = splitDrive(path) + let stop = len(drive) + result.dir = drive + else: + const stop = 0 + for i in countdown(len(path) - 1, stop): + if path[i] in {DirSep, AltSep} or i == 0: + if path[i] in {DirSep, AltSep}: + result.dir = substr(path, 0, if likely(i >= 1): i - 1 else: 0) + namePos = i + 1 + if dotPos > i: + result.name = substr(path, namePos, dotPos - 1) + result.ext = substr(path, dotPos) + else: + result.name = substr(path, namePos) + break + elif path[i] == ExtSep and i > 0 and i < len(path) - 1 and + path[i - 1] notin {DirSep, AltSep} and + path[i + 1] != ExtSep and dotPos == 0: + dotPos = i + +proc extractFilename*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Extracts the filename of a given `path`. + ## + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert extractFilename("foo/bar/") == "" + assert extractFilename("foo/bar") == "bar" + assert extractFilename("foo/bar.baz") == "bar.baz" + + if path.len == 0 or path[path.len-1] in {DirSep, AltSep}: + result = "" + else: + result = splitPath(path).tail + +proc lastPathPart*(path: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Like `extractFilename proc`_, but ignores + ## trailing dir separator; aka: `baseName`:idx: in some other languages. + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert lastPathPart("foo/bar/") == "bar" + assert lastPathPart("foo/bar") == "bar" + + let path = path.normalizePathEnd(trailingSep = false) + result = extractFilename(path) + +proc changeFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Changes the file extension to `ext`. + ## + ## If the `filename` has no extension, `ext` will be added. + ## If `ext` == "" then any extension is removed. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. (Although I know + ## of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ + runnableExamples: + assert changeFileExt("foo.bar", "baz") == "foo.baz" + assert changeFileExt("foo.bar", "") == "foo" + assert changeFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = substr(filename, 0, extPos-1) & normExt(ext) + +proc addFileExt*(filename, ext: string): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Adds the file extension `ext` to `filename`, unless + ## `filename` already has an extension. + ## + ## `Ext` should be given without the leading `'.'`, because some + ## filesystems may use a different character. + ## (Although I know of none such beast.) + ## + ## See also: + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + runnableExamples: + assert addFileExt("foo.bar", "baz") == "foo.bar" + assert addFileExt("foo.bar", "") == "foo.bar" + assert addFileExt("foo", "baz") == "foo.baz" + + var extPos = searchExtPos(filename) + if extPos < 0: result = filename & normExt(ext) + else: result = filename + +proc cmpPaths*(pathA, pathB: string): int {. + noSideEffect, rtl, extern: "nos$1".} = + ## Compares two paths. + ## + ## On a case-sensitive filesystem this is done + ## case-sensitively otherwise case-insensitively. Returns: + ## + ## | `0` if pathA == pathB + ## | `< 0` if pathA < pathB + ## | `> 0` if pathA > pathB + runnableExamples: + when defined(macosx): + assert cmpPaths("foo", "Foo") == 0 + elif defined(posix): + assert cmpPaths("foo", "Foo") > 0 + + let a = normalizePath(pathA) + let b = normalizePath(pathB) + if FileSystemCaseSensitive: + result = cmp(a, b) + else: + when defined(nimscript): + result = cmpic(a, b) + elif defined(nimdoc): discard + else: + result = cmpIgnoreCase(a, b) + +proc unixToNativePath*(path: string, drive=""): string {. + noSideEffect, rtl, extern: "nos$1".} = + ## Converts an UNIX-like path to a native one. + ## + ## On an UNIX system this does nothing. Else it converts + ## `'/'`, `'.'`, `'..'` to the appropriate things. + ## + ## On systems with a concept of "drives", `drive` is used to determine + ## which drive label to use during absolute path conversion. + ## `drive` defaults to the drive of the current working directory, and is + ## ignored on systems that do not have a concept of "drives". + when defined(unix): + result = path + else: + if path.len == 0: return "" + + var start: int + if path[0] == '/': + # an absolute path + when doslikeFileSystem: + if drive != "": + result = drive & ":" & DirSep + else: + result = $DirSep + elif defined(macos): + result = "" # must not start with ':' + else: + result = $DirSep + start = 1 + elif path[0] == '.' and (path.len == 1 or path[1] == '/'): + # current directory + result = $CurDir + start = when doslikeFileSystem: 1 else: 2 + else: + result = "" + start = 0 + + var i = start + while i < len(path): # ../../../ --> :::: + if i+2 < path.len and path[i] == '.' and path[i+1] == '.' and path[i+2] == '/': + # parent directory + when defined(macos): + if result[high(result)] == ':': + add result, ':' + else: + add result, ParDir + else: + add result, ParDir & DirSep + inc(i, 3) + elif path[i] == '/': + add result, DirSep + inc(i) + else: + add result, path[i] + inc(i) + + +when not defined(nimscript): + proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns the `current working directory`:idx: i.e. where the built + ## binary is run. + ## + ## So the path returned by this proc is determined at run time. + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ + ## * `currentSourcePath template <system.html#currentSourcePath.t>`_ + ## * `getProjectPath proc <macros.html#getProjectPath>`_ + when defined(nodejs): + var ret: cstring + {.emit: "`ret` = process.cwd();".} + return $ret + elif defined(js): + raiseAssert "use -d:nodejs to have `getCurrentDir` defined" + elif defined(windows): + var bufsize = MAX_PATH.int32 + var res = newWideCString(bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString(L) + bufsize = L + else: + result = res$L + break + else: + var bufsize = 1024 # should be enough + result = newString(bufsize) + while true: + if getcwd(result.cstring, bufsize) != nil: + setLen(result, c_strlen(result.cstring)) + break + else: + let err = osLastError() + if err.int32 == ERANGE: + bufsize = bufsize shl 1 + doAssert(bufsize >= 0) + result = newString(bufsize) + else: + raiseOSError(osLastError()) + +proc absolutePath*(path: string, root = getCurrentDir()): string = + ## Returns the absolute path of `path`, rooted at `root` (which must be absolute; + ## default: current directory). + ## If `path` is absolute, return it, ignoring `root`. + ## + ## See also: + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ + runnableExamples: + assert absolutePath("a") == getCurrentDir() / "a" + + if isAbsolute(path): path + else: + if not root.isAbsolute: + raise newException(ValueError, "The specified root is not absolute: " & root) + joinPath(root, path) + +proc absolutePathInternal(path: string): string = + absolutePath(path, getCurrentDir()) + + +proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = + ## Normalize a path. + ## + ## Consecutive directory separators are collapsed, including an initial double slash. + ## + ## On relative paths, double dot (`..`) sequences are collapsed if possible. + ## On absolute paths they are always collapsed. + ## + ## .. warning:: URL-encoded and Unicode attempts at directory traversal are not detected. + ## Triple dot is not handled. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ + runnableExamples: + when defined(posix): + var a = "a///b//..//c///d" + a.normalizePath() + assert a == "a/c/d" + + path = pathnorm.normalizePath(path) + when false: + let isAbs = isAbsolute(path) + var stack: seq[string] = @[] + for p in split(path, {DirSep}): + case p + of "", ".": + continue + of "..": + if stack.len == 0: + if isAbs: + discard # collapse all double dots on absoluta paths + else: + stack.add(p) + elif stack[^1] == "..": + stack.add(p) + else: + discard stack.pop() + else: + stack.add(p) + + if isAbs: + path = DirSep & join(stack, $DirSep) + elif stack.len > 0: + path = join(stack, $DirSep) + else: + path = "." + +proc normalizePathAux(path: var string) = normalizePath(path) + +proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = + ## Returns a normalized path for the current OS. + ## + ## See also: + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version + runnableExamples: + when defined(posix): + assert normalizedPath("a///b//..//c///d") == "a/c/d" + result = pathnorm.normalizePath(path) + +proc normalizeExe*(file: var string) {.since: (1, 3, 5).} = + ## on posix, prepends `./` if `file` doesn't contain `/` and is not `"", ".", ".."`. + runnableExamples: + import std/sugar + when defined(posix): + doAssert "foo".dup(normalizeExe) == "./foo" + doAssert "foo/../bar".dup(normalizeExe) == "foo/../bar" + doAssert "".dup(normalizeExe) == "" + when defined(posix): + if file.len > 0 and DirSep notin file and file != "." and file != "..": + file = "./" & file + +proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", + tags: [ReadDirEffect], noWeirdTarget.} = + ## Returns true if both pathname arguments refer to the same physical + ## file or directory. + ## + ## Raises `OSError` if any of the files does not + ## exist or information about it can not be obtained. + ## + ## This proc will return true if given two alternative hard-linked or + ## sym-linked paths to the same file or directory. + ## + ## See also: + ## * `sameFileContent proc`_ + when defined(windows): + var success = true + var f1 = openHandle(path1) + var f2 = openHandle(path2) + + var lastErr: OSErrorCode + if f1 != INVALID_HANDLE_VALUE and f2 != INVALID_HANDLE_VALUE: + var fi1, fi2: BY_HANDLE_FILE_INFORMATION + + if getFileInformationByHandle(f1, addr(fi1)) != 0 and + getFileInformationByHandle(f2, addr(fi2)) != 0: + result = fi1.dwVolumeSerialNumber == fi2.dwVolumeSerialNumber and + fi1.nFileIndexHigh == fi2.nFileIndexHigh and + fi1.nFileIndexLow == fi2.nFileIndexLow + else: + lastErr = osLastError() + success = false + else: + lastErr = osLastError() + success = false + + discard closeHandle(f1) + discard closeHandle(f2) + + if not success: raiseOSError(lastErr, $(path1, path2)) + else: + var a, b: Stat + if stat(path1, a) < 0'i32 or stat(path2, b) < 0'i32: + raiseOSError(osLastError(), $(path1, path2)) + else: + result = a.st_dev == b.st_dev and a.st_ino == b.st_ino diff --git a/lib/pure/includes/osseps.nim b/lib/std/private/osseps.nim index bafef686d..f2d49d886 100644 --- a/lib/pure/includes/osseps.nim +++ b/lib/std/private/osseps.nim @@ -1,8 +1,10 @@ # Include file that implements 'DirSep' and friends. Do not import this when -# you also import ``os.nim``! +# you also import `os.nim`! # Improved based on info in 'compiler/platform.nim' +## .. importdoc:: ospaths2.nim + const doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) @@ -37,9 +39,9 @@ const when doslikeFileSystem: '/' else: DirSep ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep <#DirSep>`_ if only one separator + ## pathname components, or the same as DirSep_ if only one separator ## character exists. This is set to `'/'` on Windows systems - ## where `DirSep <#DirSep>`_ is a backslash (`'\\'`). + ## where DirSep_ is a backslash (`'\\'`). PathSep* = when defined(macos) or defined(RISCOS): ',' @@ -47,7 +49,7 @@ const elif defined(PalmOS) or defined(MorphOS): ':' # platform has ':' but osseps has ';' else: ':' ## The character conventionally used by the operating system to separate - ## search patch components (as in PATH), such as `':'` for POSIX + ## search path components (as in PATH), such as `':'` for POSIX ## or `';'` for Windows. FileSystemCaseSensitive* = @@ -55,7 +57,7 @@ const defined(PalmOS) or defined(MorphOS): false else: true ## True if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths proc <#cmpPaths,string,string>`_ to compare filenames properly. + ## `cmpPaths proc`_ to compare filenames properly. ExeExt* = when doslikeFileSystem: "exe" @@ -88,7 +90,7 @@ const ExtSep* = '.' ## The character which separates the base filename from the extension; - ## for example, the `'.'` in ``os.nim``. + ## for example, the `'.'` in `os.nim`. # MacOS paths # =========== diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim new file mode 100644 index 000000000..c1760c42e --- /dev/null +++ b/lib/std/private/ossymlinks.nim @@ -0,0 +1,78 @@ +include system/inclrtl +import std/oserrors + +import oscommon +export symlinkExists + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions, widestrs] + +when weirdTarget: + discard +elif defined(windows): + import std/[winlean, times] +elif defined(posix): + import std/posix +else: + {.error: "OS module not ported to your operating system!".} + + +when weirdTarget: + {.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".} +else: + {.pragma: noWeirdTarget.} + + +when defined(nimscript): + # for procs already defined in scriptconfig.nim + template noNimJs(body): untyped = discard +elif defined(js): + {.pragma: noNimJs, error: "this proc is not available on the js target".} +else: + {.pragma: noNimJs.} + +## .. importdoc:: os.nim + +proc createSymlink*(src, dest: string) {.noWeirdTarget.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + + when defined(windows): + const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 + # allows anyone with developer mode on to create a link + let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) + else: + if symlink(src, dest) != 0: + raiseOSError(osLastError(), $(src, dest)) + +proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + when defined(windows) or defined(nintendoswitch): + result = symlinkPath + else: + var bufLen = 1024 + while true: + result = newString(bufLen) + let len = readlink(symlinkPath.cstring, result.cstring, bufLen) + if len < 0: + raiseOSError(osLastError(), symlinkPath) + if len < bufLen: + result.setLen(len) + break + bufLen = bufLen shl 1 diff --git a/lib/std/private/schubfach.nim b/lib/std/private/schubfach.nim new file mode 100644 index 000000000..b8c85d2bc --- /dev/null +++ b/lib/std/private/schubfach.nim @@ -0,0 +1,436 @@ +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +# -------------------------------------------------------------------------------------------------- +## This file contains an implementation of the Schubfach algorithm as described in +## +## \[1] Raffaello Giulietti, "The Schubfach way to render doubles", +## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN +# -------------------------------------------------------------------------------------------------- + +import std/private/digitsutils + +when defined(nimPreviewSlimSystem): + import std/assertions + + +template sf_Assert(x: untyped): untyped = + assert(x) + +# ================================================================================================== +# +# ================================================================================================== + +type + ValueType = float32 + BitsType = uint32 + Single {.bycopy.} = object + bits: BitsType + +const + significandSize: int32 = 24 + MaxExponent = 128 + exponentBias: int32 = MaxExponent - 1 + (significandSize - 1) + maxIeeeExponent: BitsType = BitsType(2 * MaxExponent - 1) + hiddenBit: BitsType = BitsType(1) shl (significandSize - 1) + significandMask: BitsType = hiddenBit - 1 + exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) + signMask: BitsType = not (not BitsType(0) shr 1) + +proc constructSingle(bits: BitsType): Single = + result.bits = bits + +proc constructSingle(value: ValueType): Single = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent(this: Single): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero(this: Single): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit(this: Single): int {.noSideEffect.} = + return int((this.bits and signMask) != 0) + +# ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2(x: int32; n: int32): int32 {.inline.} = + return x shr n + +## Returns floor(log_10(2^e)) +## ```c +## static inline int32_t FloorLog10Pow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611, 22); +## } +## ``` +## Returns floor(log_10(3/4 2^e)) +## ```c +## static inline int32_t FloorLog10ThreeQuartersPow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611 - 524031, 22); +## } +## ``` +## Returns floor(log_2(10^e)) + +proc floorLog2Pow10(e: int32): int32 {.inline.} = + sf_Assert(e >= -1233) + sf_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + +proc computePow10Single(k: int32): uint64 {.inline.} = + ## There are unique beta and r such that 10^k = beta 2^r and + ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and + ## beta = 2^-r 10^k. + ## Let g = ceil(beta), so (g-1) 2^r < 10^k <= g 2^r, with the latter + ## value being a pretty good overestimate for 10^k. + ## NB: Since for all the required exponents k, we have g < 2^64, + ## all constants can be stored in 128-bit integers. + sf_Assert(k >= kMin) + sf_Assert(k <= kMax) + return g[k - kMin] + +proc lo32(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + +proc hi32(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + +when defined(sizeof_Int128): + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let p: uint128 = uint128(g) * cp + let y1: uint32 = lo32(cast[uint64](p shr 64)) + let y0: uint32 = hi32(cast[uint64](p)) + return y1 or uint32(y0 > 1) + +elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "<intrin.h>".} + proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = + var p1: uint64 = 0 + var p0: uint64 = umul128(g, cpHi, addr(p1)) + let y1: uint32 = lo32(p1) + let y0: uint32 = hi32(p0) + return y1 or uint32(y0 > 1) + +else: + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let b01: uint64 = uint64(lo32(g)) * cp + let b11: uint64 = uint64(hi32(g)) * cp + let hi: uint64 = b11 + hi32(b01) + let y1: uint32 = hi32(hi) + let y0: uint32 = lo32(hi) + return y1 or uint32(y0 > 1) + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2(value: uint32; e2: int32): bool {.inline.} = + sf_Assert(e2 >= 0) + sf_Assert(e2 <= 31) + return (value and ((uint32(1) shl e2) - 1)) == 0 + +type + FloatingDecimal32 {.bycopy.} = object + digits: uint32 ## num_digits <= 9 + exponent: int32 + +proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal32 {. + inline.} = + var c: uint32 + var q: int32 + if ieeeExponent != 0: + c = hiddenBit or ieeeSignificand + q = cast[int32](ieeeExponent) - exponentBias + if 0 <= -q and -q < significandSize and multipleOfPow2(c, -q): + return FloatingDecimal32(digits: c shr -q, exponent: 0'i32) + else: + c = ieeeSignificand + q = 1 - exponentBias + let isEven: bool = (c mod 2 == 0) + let lowerBoundaryIsCloser: bool = (ieeeSignificand == 0 and ieeeExponent > 1) + ## const int32_t qb = q - 2; + let cbl: uint32 = 4 * c - 2 + uint32(lowerBoundaryIsCloser) + let cb: uint32 = 4 * c + let cbr: uint32 = 4 * c + 2 + ## (q * 1262611 ) >> 22 == floor(log_10( 2^q)) + ## (q * 1262611 - 524031) >> 22 == floor(log_10(3/4 2^q)) + sf_Assert(q >= -1500) + sf_Assert(q <= 1500) + let k: int32 = floorDivPow2(q * 1262611 - (if lowerBoundaryIsCloser: 524031 else: 0), 22) + let h: int32 = q + floorLog2Pow10(-k) + 1 + sf_Assert(h >= 1) + sf_Assert(h <= 4) + let pow10: uint64 = computePow10Single(-k) + let vbl: uint32 = roundToOdd(pow10, cbl shl h) + let vb: uint32 = roundToOdd(pow10, cb shl h) + let vbr: uint32 = roundToOdd(pow10, cbr shl h) + let lower: uint32 = vbl + uint32(not isEven) + let upper: uint32 = vbr - uint32(not isEven) + ## See Figure 4 in [1]. + ## And the modifications in Figure 6. + let s: uint32 = vb div 4 + ## NB: 4 * s == vb & ~3 == vb & -4 + if s >= 10: + let sp: uint32 = s div 10 + ## = vb / 40 + let upInside: bool = lower <= 40 * sp + let wpInside: bool = 40 * sp + 40 <= upper + ## if (up_inside || wp_inside) // NB: At most one of u' and w' is in R_v. + if upInside != wpInside: + return FloatingDecimal32(digits: sp + uint32(wpInside), exponent: k + 1) + let uInside: bool = lower <= 4 * s + let wInside: bool = 4 * s + 4 <= upper + if uInside != wInside: + return FloatingDecimal32(digits: s + uint32(wInside), exponent: k) + let mid: uint32 = 4 * s + 2 + ## = 2(s + t) + let roundUp: bool = vb > mid or (vb == mid and (s and 1) != 0) + return FloatingDecimal32(digits: s + uint32(roundUp), exponent: k) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc printDecimalDigitsBackwards[T: Ordinal](buf: var openArray[char]; pos: T; output: uint32): int {.inline.} = + var output = output + var pos = pos + var tz = 0 + ## number of trailing zeros removed. + var nd = 0 + ## number of decimal digits processed. + ## At most 9 digits remaining + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + tz = trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + else: + tz = 4 + nd = 4 + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + sf_Assert(output >= 1) + sf_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + sf_Assert(q >= 1) + sf_Assert(q <= 9) + dec(pos) + buf[pos] = chr(uint32('0') + q) + return tz + +proc decimalLength(v: uint32): int {.inline.} = + sf_Assert(v >= 1) + sf_Assert(v <= 999999999'u) + if v >= 100000000'u: + return 9 + if v >= 10000000'u: + return 8 + if v >= 1000000'u: + return 7 + if v >= 100000'u: + return 6 + if v >= 10000'u: + return 5 + if v >= 1000'u: + return 4 + if v >= 100'u: + return 3 + if v >= 10'u: + return 2 + return 1 + +proc formatDigits[T: Ordinal](buffer: var openArray[char]; pos: T; digits: uint32; decimalExponent: int; + forceTrailingDotZero: bool = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -4 + maxFixedDecimalPoint: int32 = 9 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 1, "internal error") + sf_Assert(digits >= 1) + sf_Assert(digits <= 999999999'u) + sf_Assert(decimalExponent >= -99) + sf_Assert(decimalExponent <= 99) + var numDigits = decimalLength(digits) + let decimalPoint = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + ## Avoid calling memset/memcpy with variable arguments below... + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + decimalDigitsPosition + numDigits + let tz = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + for i in countdown(7, 0): + buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(uint32('0') + k) + inc pos + else: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + return pos + +proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZero = false): int {. + inline.} = + let significand: uint32 = physicalSignificand(constructSingle(v)) + let exponent: uint32 = physicalExponent(constructSingle(v)) + var pos = 0 + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec: auto = toDecimal32(significand, exponent) + return formatDigits(buffer, pos, dec.digits, dec.exponent.int, forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 diff --git a/lib/std/private/since.nim b/lib/std/private/since.nim new file mode 100644 index 000000000..720120f11 --- /dev/null +++ b/lib/std/private/since.nim @@ -0,0 +1,33 @@ +##[ +`since` is used to emulate older versions of nim stdlib, +see `tuse_version.nim`. + +If a symbol `foo` is added in version `(1,3,5)`, use `{.since: (1.3.5).}`, not +`{.since: (1.4).}`, so that it works in devel in between releases. + +The emulation cannot be 100% faithful and to avoid adding too much complexity, +`since` is not needed in those cases: +* if a new module is added +* if an overload is added +* if an extra parameter to an existing routine is added +]## + +template since*(version: (int, int), body: untyped) {.dirty.} = + ## Evaluates `body` if the ``(NimMajor, NimMinor)`` is greater than + ## or equal to `version`. Usage: + ## ```Nim + ## proc fun*() {.since: (1, 3).} + ## since (1, 3): fun() + ## ``` + when (NimMajor, NimMinor) >= version: + body + +template since*(version: (int, int, int), body: untyped) {.dirty.} = + ## Evaluates `body` if ``(NimMajor, NimMinor, NimPatch)`` is greater than + ## or equal to `version`. Usage: + ## ```Nim + ## proc fun*() {.since: (1, 3, 1).} + ## since (1, 3, 1): fun() + ## ``` + when (NimMajor, NimMinor, NimPatch) >= version: + body diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim new file mode 100644 index 000000000..f8c9236a5 --- /dev/null +++ b/lib/std/private/strimpl.nim @@ -0,0 +1,113 @@ +func toLowerAscii*(c: char): char {.inline.} = + if c in {'A'..'Z'}: + result = chr(ord(c) + (ord('a') - ord('A'))) + else: + result = c + +template firstCharCaseSensitiveImpl[T: string | cstring](a, b: T, aLen, bLen: int) = + if aLen == 0 or bLen == 0: + return aLen - bLen + if a[0] != b[0]: return ord(a[0]) - ord(b[0]) + +template cmpIgnoreStyleImpl*[T: string | cstring](a, b: T, + firstCharCaseSensitive: static bool = false) = + let aLen = a.len + let bLen = b.len + var i = 0 + var j = 0 + when firstCharCaseSensitive: + firstCharCaseSensitiveImpl(a, b, aLen, bLen) + inc i + inc j + while true: + while i < aLen and a[i] == '_': inc i + while j < bLen and b[j] == '_': inc j + let aa = if i < aLen: toLowerAscii(a[i]) else: '\0' + let bb = if j < bLen: toLowerAscii(b[j]) else: '\0' + result = ord(aa) - ord(bb) + if result != 0: return result + # the characters are identical: + if i >= aLen: + # both cursors at the end: + if j >= bLen: return 0 + # not yet at the end of 'b': + return -1 + elif j >= bLen: + return 1 + inc i + inc j + +template cmpIgnoreCaseImpl*[T: string | cstring](a, b: T, + firstCharCaseSensitive: static bool = false) = + let aLen = a.len + let bLen = b.len + var i = 0 + when firstCharCaseSensitive: + firstCharCaseSensitiveImpl(a, b, aLen, bLen) + inc i + var m = min(aLen, bLen) + while i < m: + result = ord(toLowerAscii(a[i])) - ord(toLowerAscii(b[i])) + if result != 0: return + inc i + result = aLen - bLen + +template startsWithImpl*[T: string | cstring](s, prefix: T) = + let prefixLen = prefix.len + let sLen = s.len + var i = 0 + while true: + if i >= prefixLen: return true + if i >= sLen or s[i] != prefix[i]: return false + inc(i) + +template endsWithImpl*[T: string | cstring](s, suffix: T) = + let suffixLen = suffix.len + let sLen = s.len + var i = 0 + var j = sLen - suffixLen + while i+j >= 0 and i+j < sLen: + if s[i+j] != suffix[i]: return false + inc(i) + if i >= suffixLen: return true + + +func cmpNimIdentifier*[T: string | cstring](a, b: T): int = + cmpIgnoreStyleImpl(a, b, true) + +func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. + importc: "memchr", header: "<string.h>".} +func c_strstr(haystack, needle: cstring): cstring {. + importc: "strstr", header: "<string.h>".} + + +func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].rfind` for a `start`-origin index. + let last = if last == 0: s.high else: last + let L = last-start+1 + if L > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + if not found.isNil: + return cast[int](found) -% cast[int](s) + return -1 + +func find*(s, sub: cstring, start: Natural = 0, last = 0): int = + ## Searches for `sub` in `s` inside the range `start..last` (both ends included). + ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## + ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## Otherwise the index returned is relative to `s[0]`, not `start`. + ## Use `s[start..last].find` for a `start`-origin index. + if sub.len > s.len - start: return -1 + if sub.len == 1: return find(s, sub[0], start, last) + if last == 0 and s.len > start: + let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) + if not found.isNil: + result = cast[int](found) -% cast[int](s) + else: + result = -1 diff --git a/lib/std/private/syslocks.nim b/lib/std/private/syslocks.nim new file mode 100644 index 000000000..e19ec2c04 --- /dev/null +++ b/lib/std/private/syslocks.nim @@ -0,0 +1,234 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level system locks and condition vars. + +{.push stackTrace: off.} + +when defined(windows): + type + Handle = int + + SysLock* {.importc: "CRITICAL_SECTION", + header: "<windows.h>", final, pure, byref.} = object # CRITICAL_SECTION in WinApi + DebugInfo: pointer + LockCount: int32 + RecursionCount: int32 + OwningThread: int + LockSemaphore: int + SpinCount: int + + SysCond* {.importc: "RTL_CONDITION_VARIABLE", header: "<windows.h>", byref.} = object + thePtr {.importc: "Ptr".} : Handle + + proc initSysLock*(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "<windows.h>".} + ## Initializes the lock `L`. + + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "<windows.h>".} + ## Tries to acquire the lock `L`. + + proc tryAcquireSys*(L: var SysLock): bool {.inline.} = + result = tryAcquireSysAux(L) != 0'i32 + + proc acquireSys*(L: var SysLock) {.importc: "EnterCriticalSection", + header: "<windows.h>".} + ## Acquires the lock `L`. + + proc releaseSys*(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "<windows.h>".} + ## Releases the lock `L`. + + proc deinitSys*(L: SysLock) {.importc: "DeleteCriticalSection", + header: "<windows.h>".} + + proc initializeConditionVariable( + conditionVariable: var SysCond + ) {.stdcall, noSideEffect, dynlib: "kernel32", importc: "InitializeConditionVariable".} + + proc sleepConditionVariableCS( + conditionVariable: var SysCond, + PCRITICAL_SECTION: var SysLock, + dwMilliseconds: int + ): int32 {.stdcall, noSideEffect, dynlib: "kernel32", importc: "SleepConditionVariableCS".} + + + proc signalSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeConditionVariable".} + + proc broadcastSysCond*(hEvent: var SysCond) {.stdcall, noSideEffect, + dynlib: "kernel32", importc: "WakeAllConditionVariable".} + + proc initSysCond*(cond: var SysCond) {.inline.} = + initializeConditionVariable(cond) + proc deinitSysCond*(cond: SysCond) {.inline.} = + discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard sleepConditionVariableCS(cond, lock, -1'i32) + +elif defined(genode): + const + Header = "genode_cpp/syslocks.h" + type + SysLock* {.importcpp: "Nim::SysLock", pure, final, + header: Header.} = object + SysCond* {.importcpp: "Nim::SysCond", pure, final, + header: Header.} = object + + proc initSysLock*(L: var SysLock) = discard + proc deinitSys*(L: SysLock) = discard + proc acquireSys*(L: var SysLock) {.noSideEffect, importcpp.} + proc tryAcquireSys*(L: var SysLock): bool {.noSideEffect, importcpp.} + proc releaseSys*(L: var SysLock) {.noSideEffect, importcpp.} + + proc initSysCond*(L: var SysCond) = discard + proc deinitSysCond*(L: SysCond) = discard + proc waitSysCond*(cond: var SysCond, lock: var SysLock) {. + noSideEffect, importcpp.} + proc signalSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + proc broadcastSysCond*(cond: var SysCond) {. + noSideEffect, importcpp.} + +else: + type + SysLockObj {.importc: "pthread_mutex_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[40 div sizeof(clong), clong] + + SysLockAttr* {.importc: "pthread_mutexattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysCondObj {.importc: "pthread_cond_t", pure, final, + header: """#include <sys/types.h> + #include <pthread.h>""", byref.} = object + when defined(linux) and defined(amd64): + abi: array[48 div sizeof(clonglong), clonglong] + + SysCondAttr {.importc: "pthread_condattr_t", pure, final + header: """#include <sys/types.h> + #include <pthread.h>""".} = object + when defined(linux) and defined(amd64): + abi: array[4 div sizeof(cint), cint] # actually a cint + + SysLockType = distinct cint + + proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. + importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysAux(L: SysLockObj) {.noSideEffect, + importc: "pthread_mutex_destroy", header: "<pthread.h>".} + + proc acquireSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_lock", header: "<pthread.h>".} + proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, + importc: "pthread_mutex_trylock", header: "<pthread.h>".} + + proc releaseSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_unlock", header: "<pthread.h>".} + + when defined(ios): + # iOS will behave badly if sync primitives are moved in memory. In order + # to prevent this once and for all, we're doing an extra malloc when + # initializing the primitive. + type + SysLock* = ptr SysLockObj + SysCond* = ptr SysCondObj + + when not declared(c_malloc): + proc c_malloc(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + + proc initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj)))) + initSysLockAux(L[], attr) + + proc deinitSys*(L: SysLock) = + deinitSysAux(L[]) + c_free(L) + + template acquireSys*(L: var SysLock) = + acquireSysAux(L[]) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L[]) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L[]) + else: + type + SysLock* = SysLockObj + SysCond* = SysCondObj + + template initSysLock*(L: var SysLock, attr: ptr SysLockAttr = nil) = + initSysLockAux(L, attr) + template deinitSys*(L: SysLock) = + deinitSysAux(L) + template acquireSys*(L: var SysLock) = + acquireSysAux(L) + template tryAcquireSys*(L: var SysLock): bool = + tryAcquireSysAux(L) == 0'i32 + template releaseSys*(L: var SysLock) = + releaseSysAux(L) + + # rlocks + var SysLockType_Reentrant* {.importc: "PTHREAD_MUTEX_RECURSIVE", + header: "<pthread.h>".}: SysLockType + proc initSysLockAttr*(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} + proc setSysLockType*(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} + + # locks + proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. + importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} + proc deinitSysCondAux(cond: SysCondObj) {.noSideEffect, + importc: "pthread_cond_destroy", header: "<pthread.h>".} + + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj): cint {. + importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} + proc signalSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} + proc broadcastSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_broadcast", header: "<pthread.h>", noSideEffect.} + + when defined(ios): + proc initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj)))) + initSysCondAux(cond[], cond_attr) + + proc deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond[]) + c_free(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond[], lock[]) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond[]) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond[]) + else: + template initSysCond*(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = + initSysCondAux(cond, cond_attr) + template deinitSysCond*(cond: SysCond) = + deinitSysCondAux(cond) + + template waitSysCond*(cond: var SysCond, lock: var SysLock) = + discard waitSysCondAux(cond, lock) + template signalSysCond*(cond: var SysCond) = + signalSysCondAux(cond) + template broadcastSysCond*(cond: var SysCond) = + broadcastSysCondAux(cond) + +{.pop.} diff --git a/lib/std/private/threadtypes.nim b/lib/std/private/threadtypes.nim new file mode 100644 index 000000000..a1cdf21dc --- /dev/null +++ b/lib/std/private/threadtypes.nim @@ -0,0 +1,176 @@ +include system/inclrtl + +const hasSharedHeap* = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own + +when defined(windows): + type + Handle* = int + SysThread* = Handle + WinThreadProc* = proc (x: pointer): int32 {.stdcall.} + + proc createThread*(lpThreadAttributes: pointer, dwStackSize: int32, + lpStartAddress: WinThreadProc, + lpParameter: pointer, + dwCreationFlags: int32, + lpThreadId: var int32): SysThread {. + stdcall, dynlib: "kernel32", importc: "CreateThread".} + + proc winSuspendThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "SuspendThread".} + + proc winResumeThread*(hThread: SysThread): int32 {. + stdcall, dynlib: "kernel32", importc: "ResumeThread".} + + proc waitForSingleObject*(hHandle: SysThread, dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + + proc waitForMultipleObjects*(nCount: int32, + lpHandles: ptr SysThread, + bWaitAll: int32, + dwMilliseconds: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} + + proc terminateThread*(hThread: SysThread, dwExitCode: int32): int32 {. + stdcall, dynlib: "kernel32", importc: "TerminateThread".} + + proc setThreadAffinityMask*(hThread: SysThread, dwThreadAffinityMask: uint) {. + importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + +elif defined(genode): + const + GenodeHeader* = "genode_cpp/threads.h" + type + SysThread* {.importcpp: "Nim::SysThread", + header: GenodeHeader, final, pure.} = object + GenodeThreadProc* = proc (x: pointer) {.noconv.} + + proc initThread*(s: var SysThread, + env: GenodeEnv, + stackSize: culonglong, + entry: GenodeThreadProc, + arg: pointer, + affinity: cuint) {. + importcpp: "#.initThread(@)".} + + +else: + when not (defined(macosx) or defined(haiku)): + {.passl: "-pthread".} + + when not defined(haiku): + {.passc: "-pthread".} + + const + schedh = "#define _GNU_SOURCE\n#include <sched.h>" + pthreadh* = "#define _GNU_SOURCE\n#include <pthread.h>" + + when not declared(Time): + when defined(linux): + type Time = clong + else: + type Time = int + + when (defined(linux) or defined(nintendoswitch)) and defined(amd64): + type + SysThread* {.importc: "pthread_t", + header: "<sys/types.h>" .} = distinct culong + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + abi: array[56 div sizeof(clong), clong] + elif defined(openbsd) and defined(amd64): + type + SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object + Pthread_attr* {.importc: "pthread_attr_t", + header: "<pthread.h>".} = object + else: + type + SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = int + Pthread_attr* {.importc: "pthread_attr_t", + header: "<sys/types.h>".} = object + type + Timespec* {.importc: "struct timespec", header: "<time.h>".} = object + tv_sec*: Time + tv_nsec*: clong + + proc pthread_attr_init*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_setstacksize*(a1: var Pthread_attr, a2: int): cint {. + importc, header: pthreadh.} + proc pthread_attr_destroy*(a1: var Pthread_attr): cint {. + importc, header: pthreadh.} + + proc pthread_create*(a1: var SysThread, a2: var Pthread_attr, + a3: proc (x: pointer): pointer {.noconv.}, + a4: pointer): cint {.importc: "pthread_create", + header: pthreadh.} + proc pthread_join*(a1: SysThread, a2: ptr pointer): cint {. + importc, header: pthreadh.} + + proc pthread_cancel*(a1: SysThread): cint {. + importc: "pthread_cancel", header: pthreadh.} + + type CpuSet* {.importc: "cpu_set_t", header: schedh.} = object + when defined(linux) and defined(amd64): + abi: array[1024 div (8 * sizeof(culong)), culong] + + proc cpusetZero*(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} + proc cpusetIncl*(cpu: cint; s: var CpuSet) {. + importc: "CPU_SET", header: schedh.} + + when defined(android): + # libc of android doesn't implement pthread_setaffinity_np, + # it exposes pthread_gettid_np though, so we can use that in combination + # with sched_setaffinity to set the thread affinity. + type Pid* {.importc: "pid_t", header: "<sys/types.h>".} = int32 # From posix_other.nim + + proc setAffinityTID*(tid: Pid; setsize: csize_t; s: var CpuSet) {. + importc: "sched_setaffinity", header: schedh.} + + proc pthread_gettid_np*(thread: SysThread): Pid {. + importc: "pthread_gettid_np", header: pthreadh.} + + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) = + setAffinityTID(pthread_gettid_np(thread), setsize, s) + else: + proc setAffinity*(thread: SysThread; setsize: csize_t; s: var CpuSet) {. + importc: "pthread_setaffinity_np", header: pthreadh.} + + +const + emulatedThreadVars* = compileOption("tlsEmulation") +# we preallocate a fixed size for thread local storage, so that no heap +# allocations are needed. Currently less than 16K are used on a 64bit machine. +# We use `float` for proper alignment: +const nimTlsSize {.intdefine.} = 16000 +type + ThreadLocalStorage* = array[0..(nimTlsSize div sizeof(float)), float] + PGcThread* = ptr GcThread + GcThread* {.pure, inheritable.} = object + when emulatedThreadVars: + tls*: ThreadLocalStorage + else: + nil + when hasSharedHeap: + next*, prev*: PGcThread + stackBottom*, stackTop*: pointer + stackSize*: int + else: + nil + +const hasAllocStack* = defined(zephyr) # maybe freertos too? + +type + Thread*[TArg] = object + core*: PGcThread + sys*: SysThread + when TArg is void: + dataFn*: proc () {.nimcall, gcsafe.} + else: + dataFn*: proc (m: TArg) {.nimcall, gcsafe.} + data*: TArg + when hasAllocStack: + rawStack*: pointer + +proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} diff --git a/lib/std/private/underscored_calls.nim b/lib/std/private/underscored_calls.nim new file mode 100644 index 000000000..f853572b5 --- /dev/null +++ b/lib/std/private/underscored_calls.nim @@ -0,0 +1,56 @@ + +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This is an internal helper module. Do not use. + +import std/macros + +proc underscoredCalls*(result, calls, arg0: NimNode) + +proc underscoredCall(n, arg0: NimNode): NimNode = + proc underscorePos(n: NimNode): int = + for i in 1 ..< n.len: + if n[i].eqIdent("_"): return i + return 0 + + if n.kind in nnkCallKinds: + if n[0].kind in {nnkIdent, nnkSym} and n[0].eqIdent("with"): + expectKind n[1], {nnkIdent, nnkSym} + + result = newStmtList() + underscoredCalls(result, n[2 .. ^1].newStmtList, newDotExpr(arg0, n[1])) + else: + result = copyNimNode(n) + result.add n[0] + + let u = underscorePos(n) + for i in 1..u-1: result.add n[i] + result.add arg0 + for i in u+1..n.len-1: result.add n[i] + elif n.kind in {nnkAsgn, nnkExprEqExpr}: + var field = n[0] + if n[0].kind == nnkDotExpr and n[0][0].eqIdent("_"): + # handle _.field = ... + field = n[0][1] + result = newDotExpr(arg0, field).newAssignment n[1] + else: + # handle e.g. 'x.dup(sort)' + result = newNimNode(nnkCall, n) + result.add n + result.add arg0 + +proc underscoredCalls*(result, calls, arg0: NimNode) = + expectKind calls, {nnkArgList, nnkStmtList, nnkStmtListExpr} + + for call in calls: + if call.kind in {nnkStmtList, nnkStmtListExpr}: + underscoredCalls(result, call, arg0) + else: + result.add underscoredCall(call, arg0) diff --git a/lib/std/private/win_getsysteminfo.nim b/lib/std/private/win_getsysteminfo.nim new file mode 100644 index 000000000..b98478231 --- /dev/null +++ b/lib/std/private/win_getsysteminfo.nim @@ -0,0 +1,15 @@ +type + SystemInfo* = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors*: uint32 + dwProcessorType: uint32 + dwAllocationGranularity*: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + +proc getSystemInfo*(lpSystemInfo: ptr SystemInfo) {.stdcall, + dynlib: "kernel32", importc: "GetSystemInfo".} diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim new file mode 100644 index 000000000..66e199dfe --- /dev/null +++ b/lib/std/private/win_setenv.nim @@ -0,0 +1,106 @@ +#[ +Copyright (c) Facebook, Inc. and its affiliates. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Adapted `setenv` from https://github.com/facebook/folly/blob/master/folly/portability/Stdlib.cpp +translated from C to nim. +]# + +#[ +Introduced in https://github.com/facebook/folly/commit/5d8ca09a3f96afefb44e35808f03651a096ab9c7 + +TODO: +check errno_t vs cint +]# + +when not defined(windows): discard +else: + when defined(nimPreviewSlimSystem): + import std/widestrs + + type wchar_t {.importc: "wchar_t".} = int16 + + proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} + # same as winlean.setEnvironmentVariableA + + proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "<stdlib.h>".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "<stdlib.h>".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "<stdlib.h>".} + + var errno {.importc, header: "<errno.h>".}: cint + var genviron {.importc: "_environ".}: ptr ptr char + # xxx `ptr UncheckedArray[WideCString]` did not work + + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "<stdlib.h>".} + # xxx cint vs errno_t? + + proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = + const EINVAL = cint(22) + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: + return 0 + + if value != "": + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) + if e != 0: + errno = EINVAL + return -1 + return 0 + #[ + We are trying to set the value to an empty string, but `_putenv` deletes + entries if the value is an empty string, and just calling + SetEnvironmentVariableA doesn't update `_environ`, + so we have to do these terrible things. + ]# + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: + errno = EINVAL + return -1 + # Here lies the documentation we blatently ignore to make this work. + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[0] = Utf16Char('\0') + #[ + This would result in a double null termination, which normally signifies the + end of the environment variable list, so we stick a completely empty + environment variable into the list instead. + ]# + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) + s[1] = Utf16Char('=') + #[ + If genviron is null, the MBCS environment has not been initialized + yet, and we don't need to try to update it. We have to do this otherwise + we'd be forcing the initialization and maintenance of the MBCS environment + even though it's never actually used in most programs. + ]# + if genviron != nil: + + # wcstombs returns `high(csize_t)` if any characters cannot be represented + # in the current codepage. Skip updating MBCS environment in this case. + # For some reason, second `wcstombs` can find non-convertible characters + # that the first `wcstombs` cannot. + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) + if requiredSizeS != high(csize_t): + let requiredSize = requiredSizeS.int + var buf = newSeq[char](requiredSize + 1) + let buf2 = buf[0].addr + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): + var ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[0] = '\0' + ptrToEnv = c_getenv(cast[cstring](buf2)) + ptrToEnv[1] = '=' + + # And now, we have to update the outer environment to have a proper empty value. + if setEnvironmentVariableW(wideName, value.newWideCString) == 0: + errno = EINVAL + return -1 + return 0 diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim new file mode 100644 index 000000000..8e7bc6a92 --- /dev/null +++ b/lib/std/setutils.nim @@ -0,0 +1,77 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module adds functionality for the built-in `set` type. +## +## See also +## ======== +## * `std/packedsets <packedsets.html>`_ +## * `std/sets <sets.html>`_ + +import std/[typetraits, macros] + +#[ + type SetElement* = char|byte|bool|int16|uint16|enum|uint8|int8 + ## The allowed types of a built-in set. +]# + +template toSet*(iter: untyped): untyped = + ## Returns a built-in set from the elements of the iterable `iter`. + runnableExamples: + assert "helloWorld".toSet == {'W', 'd', 'e', 'h', 'l', 'o', 'r'} + assert toSet([10u16, 20, 30]) == {10u16, 20, 30} + assert [30u8, 100, 10].toSet == {10u8, 30, 100} + assert toSet(@[1321i16, 321, 90]) == {90i16, 321, 1321} + assert toSet([false]) == {false} + assert toSet(0u8..10) == {0u8..10} + + var result: set[elementType(iter)] + for x in iter: + incl(result, x) + result + +macro enumElementsAsSet(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) + +# func fullSet*(T: typedesc): set[T] {.inline.} = # xxx would give: Error: ordinal type expected +func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = + ## Returns a set containing all elements in `U`. + runnableExamples: + assert bool.fullSet == {true, false} + type A = range[1..3] + assert A.fullSet == {1.A, 2, 3} + assert int8.fullSet.len == 256 + when T is Ordinal: + {T.low..T.high} + else: # Hole filled enum + enumElementsAsSet(T) + +func complement*[T](s: set[T]): set[T] {.inline.} = + ## Returns the set complement of `a`. + runnableExamples: + type Colors = enum + red, green = 3, blue + assert complement({red, blue}) == {green} + assert complement({red, green, blue}).card == 0 + assert complement({range[0..10](0), 1, 2, 3}) == {range[0..10](4), 5, 6, 7, 8, 9, 10} + assert complement({'0'..'9'}) == {0.char..255.char} - {'0'..'9'} + fullSet(T) - s + +func `[]=`*[T](t: var set[T], key: T, val: bool) {.inline.} = + ## Syntax sugar for `if val: t.incl key else: t.excl key` + runnableExamples: + type A = enum + a0, a1, a2, a3 + var s = {a0, a3} + s[a0] = false + s[a1] = false + assert s == {a3} + s[a2] = true + s[a3] = true + assert s == {a2, a3} + if val: t.incl key else: t.excl key diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 8f35a44ff..213af4229 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -1,46 +1,44 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # - -## **Note:** Import ``std/sha1`` to use this module -## -## SHA-1 (Secure Hash Algorithm 1) is a cryptographic hash function which -## takes an input and produces a 160-bit (20-byte) hash value known as a -## message digest. -## -## .. code-block:: -## import std/sha1 -## -## let accessName = secureHash("John Doe") -## assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" -## -## .. code-block:: -## import std/sha1 -## -## let -## a = secureHashFile("myFile.nim") -## b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") -## -## if a == b: -## echo "Files match" +## [SHA-1 (Secure Hash Algorithm 1)](https://en.wikipedia.org/wiki/SHA-1) +## is a cryptographic hash function which takes an input and produces +## a 160-bit (20-byte) hash value known as a message digest. ## -## **See also:** -## * `base64 module<base64.html>`_ implements a base64 encoder and decoder +## See also +## ======== +## * `base64 module<base64.html>`_ for a Base64 encoder and decoder ## * `hashes module<hashes.html>`_ for efficient computations of hash values for diverse Nim types -## * `md5 module<md5.html>`_ implements the MD5 checksum algorithm +## * `md5 module<md5.html>`_ for the MD5 checksum algorithm -import strutils -from endians import bigEndian32, bigEndian64 +runnableExamples: + let accessName = secureHash("John Doe") + assert $accessName == "AE6E4D1209F17B460503904FAD297B31E9CF6362" + +runnableExamples("-r:off"): + let + a = secureHashFile("myFile.nim") + b = parseSecureHash("10DFAEBF6BFDBC7939957068E2EFACEC4972933C") + assert a == b, "files don't match" + + +{.deprecated: "use command `nimble install checksums` and import `checksums/sha1` instead".} + +import std/strutils +from std/endians import bigEndian32, bigEndian64 + +when defined(nimPreviewSlimSystem): + import std/syncio const Sha1DigestSize = 20 type - Sha1Digest* = array[0 .. Sha1DigestSize-1, uint8] + Sha1Digest* = array[0 .. Sha1DigestSize - 1, uint8] SecureHash* = distinct Sha1Digest type @@ -49,10 +47,14 @@ type state: array[5, uint32] buf: array[64, byte] -# This implementation of the SHA1 algorithm was ported from the Chromium OS one +# This implementation of the SHA-1 algorithm was ported from the Chromium OS one # with minor modifications that should not affect its functionality. proc newSha1State*(): Sha1State = + ## Creates a `Sha1State`. + ## + ## If you use the `secureHash proc <#secureHash,openArray[char]>`_, + ## there's no need to call this function explicitly. result.count = 0 result.state[0] = 0x67452301'u32 result.state[1] = 0xEFCDAB89'u32 @@ -146,6 +148,10 @@ proc transform(ctx: var Sha1State) = ctx.state[4] += e proc update*(ctx: var Sha1State, data: openArray[char]) = + ## Updates the `Sha1State` with `data`. + ## + ## If you use the `secureHash proc <#secureHash,openArray[char]>`_, + ## there's no need to call this function explicitly. var i = ctx.count mod 64 var j = 0 var len = data.len @@ -177,6 +183,10 @@ proc update*(ctx: var Sha1State, data: openArray[char]) = ctx.count += data.len proc finalize*(ctx: var Sha1State): Sha1Digest = + ## Finalizes the `Sha1State` and returns a `Sha1Digest`. + ## + ## If you use the `secureHash proc <#secureHash,openArray[char]>`_, + ## there's no need to call this function explicitly. var cnt = uint64(ctx.count * 8) # a 1 bit update(ctx, "\x80") @@ -195,54 +205,72 @@ proc finalize*(ctx: var Sha1State): Sha1Digest = # Public API proc secureHash*(str: openArray[char]): SecureHash = - ## Generates a ``SecureHash`` from a ``str``. + ## Generates a `SecureHash` from `str`. ## ## **See also:** - ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a ``SecureHash`` from a file - ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string ``hash`` to ``SecureHash`` + ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a `SecureHash` from a file + ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string `hash` to `SecureHash` runnableExamples: let hash = secureHash("Hello World") assert hash == parseSecureHash("0A4D55A8D778E5022FAB701977C5D840BBC486D0") + var state = newSha1State() state.update(str) SecureHash(state.finalize()) proc secureHashFile*(filename: string): SecureHash = - ## Generates a ``SecureHash`` from a file. + ## Generates a `SecureHash` from a file. ## ## **See also:** - ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a ``SecureHash`` from a string - ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string ``hash`` to ``SecureHash`` - secureHash(readFile(filename)) + ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a `SecureHash` from a string + ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string `hash` to `SecureHash` + const BufferLength = 8192 + + let f = open(filename) + var state = newSha1State() + var buffer = newString(BufferLength) + while true: + let length = readChars(f, buffer) + if length == 0: + break + buffer.setLen(length) + state.update(buffer) + if length != BufferLength: + break + close(f) + + SecureHash(state.finalize()) proc `$`*(self: SecureHash): string = - ## Returns the string representation of a ``SecureHash``. + ## Returns the string representation of a `SecureHash`. ## ## **See also:** - ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a ``SecureHash`` from a string + ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a `SecureHash` from a string runnableExamples: let hash = secureHash("Hello World") assert $hash == "0A4D55A8D778E5022FAB701977C5D840BBC486D0" + result = "" for v in Sha1Digest(self): result.add(toHex(int(v), 2)) proc parseSecureHash*(hash: string): SecureHash = - ## Converts a string ``hash`` to ``SecureHash``. + ## Converts a string `hash` to a `SecureHash`. ## ## **See also:** - ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a ``SecureHash`` from a string - ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a ``SecureHash`` from a file + ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a `SecureHash` from a string + ## * `secureHashFile proc <#secureHashFile,string>`_ for generating a `SecureHash` from a file runnableExamples: let hashStr = "0A4D55A8D778E5022FAB701977C5D840BBC486D0" secureHash = secureHash("Hello World") assert secureHash == parseSecureHash(hashStr) + for i in 0 ..< Sha1DigestSize: Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) proc `==`*(a, b: SecureHash): bool = - ## Checks if two ``SecureHash`` values are identical. + ## Checks if two `SecureHash` values are identical. runnableExamples: let a = secureHash("Hello World") @@ -250,18 +278,10 @@ proc `==`*(a, b: SecureHash): bool = c = parseSecureHash("0A4D55A8D778E5022FAB701977C5D840BBC486D0") assert a != b assert a == c + # Not a constant-time comparison, but that's acceptable in this context Sha1Digest(a) == Sha1Digest(b) -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 - - template checkVector(s, exp: string) = - doAssert secureHash(s) == parseSecureHash(exp) - - checkVector("", "da39a3ee5e6b4b0d3255bfef95601890afd80709") - checkVector("abc", "a9993e364706816aba3e25717850c26c9cd0d89d") - checkVector("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", - "84983e441c3bd26ebaae4aa1f95129e5e54670f1") +proc isValidSha1Hash*(s: string): bool = + ## Checks if a string is a valid sha1 hash sum. + s.len == 40 and allCharsInSet(s, HexDigits) \ No newline at end of file diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim new file mode 100644 index 000000000..45e906795 --- /dev/null +++ b/lib/std/socketstreams.nim @@ -0,0 +1,182 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides an implementation of the streams interface for sockets. +## It contains two separate implementations, a +## `ReadSocketStream <#ReadSocketStream>`_ and a +## `WriteSocketStream <#WriteSocketStream>`_. +## +## The `ReadSocketStream` only supports reading, peeking, and seeking. +## It reads into a buffer, so even by +## seeking backwards it will only read the same position a single time from the +## underlying socket. To clear the buffer and free the data read into it you +## can call `resetStream`, this will also reset the position back to 0 but +## won't do anything to the underlying socket. +## +## The `WriteSocketStream` allows both reading and writing, but it performs the +## reads on the internal buffer. So by writing to the buffer you can then read +## back what was written but without receiving anything from the socket. You +## can also set the position and overwrite parts of the buffer, and to send +## anything over the socket you need to call `flush` at which point you can't +## write anything to the buffer before the point of the flush (but it can still +## be read). Again to empty the underlying buffer you need to call +## `resetStream`. +## +## Examples +## ======== +## +## ```Nim +## import std/socketstreams +## +## var +## socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## stream = newReadSocketStream(socket) +## socket.sendTo("127.0.0.1", Port(12345), "SOME REQUEST") +## echo stream.readLine() # Will call `recv` +## stream.setPosition(0) +## echo stream.readLine() # Will return the read line from the buffer +## stream.resetStream() # Buffer is now empty, position is 0 +## echo stream.readLine() # Will call `recv` again +## stream.close() # Closes the socket +## ``` +## +## ```Nim +## import std/socketstreams +## +## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) +## socket.connect("127.0.0.1", Port(12345)) +## var sendStream = newWriteSocketStream(socket) +## sendStream.write "NOM" +## sendStream.setPosition(1) +## echo sendStream.peekStr(2) # OM +## sendStream.write "I" +## sendStream.setPosition(0) +## echo sendStream.readStr(3) # NIM +## echo sendStream.getPosition() # 3 +## sendStream.flush() # This actually performs the writing to the socket +## sendStream.setPosition(1) +## sendStream.write "I" # Throws an error as we can't write into an already sent buffer +## ``` + +import std/[net, streams] + +type + ReadSocketStream* = ref ReadSocketStreamObj + ReadSocketStreamObj* = object of StreamObj + data: Socket + pos: int + buf: seq[byte] + WriteSocketStream* = ref WriteSocketStreamObj + WriteSocketStreamObj* = object of ReadSocketStreamObj + lastFlush: int + +proc rsAtEnd(s: Stream): bool = + return false + +proc rsSetPosition(s: Stream, pos: int) = + var s = ReadSocketStream(s) + s.pos = pos + +proc rsGetPosition(s: Stream): int = + var s = ReadSocketStream(s) + return s.pos + +proc rsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + let s = ReadSocketStream(s) + if bufLen > 0: + let oldLen = s.buf.len + s.buf.setLen(max(s.pos + bufLen, s.buf.len)) + if s.pos + bufLen > oldLen: + result = s.data.recv(s.buf[oldLen].addr, s.buf.len - oldLen) + if result > 0: + result += oldLen - s.pos + else: + result = bufLen + copyMem(buffer, s.buf[s.pos].addr, result) + +proc rsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = s.rsPeekData(buffer, bufLen) + var s = ReadSocketStream(s) + s.pos += bufLen + +proc rsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = + var s = ReadSocketStream(s) + result = slice.b + 1 - slice.a + if result > 0: + result = s.rsReadData(buffer[slice.a].addr, result) + inc(s.pos, result) + else: + result = 0 + +proc wsWriteData(s: Stream, buffer: pointer, bufLen: int) = + var s = WriteSocketStream(s) + if s.pos < s.lastFlush: + raise newException(IOError, "Unable to write into buffer that has already been sent") + if s.buf.len < s.pos + bufLen: + s.buf.setLen(s.pos + bufLen) + copyMem(s.buf[s.pos].addr, buffer, bufLen) + s.pos += bufLen + +proc wsPeekData(s: Stream, buffer: pointer, bufLen: int): int = + var s = WriteSocketStream(s) + result = bufLen + if result > 0: + if s.pos > s.buf.len or s.pos == s.buf.len or s.pos + bufLen > s.buf.len: + raise newException(IOError, "Unable to read past end of write buffer") + else: + copyMem(buffer, s.buf[s.pos].addr, bufLen) + +proc wsReadData(s: Stream, buffer: pointer, bufLen: int): int = + result = s.wsPeekData(buffer, bufLen) + var s = ReadSocketStream(s) + s.pos += bufLen + +proc wsAtEnd(s: Stream): bool = + var s = WriteSocketStream(s) + return s.pos == s.buf.len + +proc wsFlush(s: Stream) = + var s = WriteSocketStream(s) + discard s.data.send(s.buf[s.lastFlush].addr, s.buf.len - s.lastFlush) + s.lastFlush = s.buf.len + +proc rsClose(s: Stream) = + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? + var s = ReadSocketStream(s) + s.data.close() + +proc newReadSocketStream*(s: Socket): owned ReadSocketStream = + result = ReadSocketStream(data: s, pos: 0, + closeImpl: rsClose, + atEndImpl: rsAtEnd, + setPositionImpl: rsSetPosition, + getPositionImpl: rsGetPosition, + readDataImpl: rsReadData, + peekDataImpl: rsPeekData, + readDataStrImpl: rsReadDataStr) + +proc resetStream*(s: ReadSocketStream) = + s.buf = @[] + s.pos = 0 + +proc newWriteSocketStream*(s: Socket): owned WriteSocketStream = + result = WriteSocketStream(data: s, pos: 0, + closeImpl: rsClose, + atEndImpl: wsAtEnd, + setPositionImpl: rsSetPosition, + getPositionImpl: rsGetPosition, + writeDataImpl: wsWriteData, + readDataImpl: wsReadData, + peekDataImpl: wsPeekData, + flushImpl: wsFlush) + +proc resetStream*(s: WriteSocketStream) = + s.buf = @[] + s.pos = 0 + s.lastFlush = 0 diff --git a/lib/std/stackframes.nim b/lib/std/stackframes.nim new file mode 100644 index 000000000..28be7ce11 --- /dev/null +++ b/lib/std/stackframes.nim @@ -0,0 +1,30 @@ +const NimStackTrace = compileOption("stacktrace") +const NimStackTraceMsgs = compileOption("stacktraceMsgs") + +template procName*(): string = + ## returns current C/C++ function name + when defined(c) or defined(cpp): + var name {.inject, noinit.}: cstring + {.emit: "`name` = __func__;".} + $name + +template getPFrame*(): PFrame = + ## avoids a function call (unlike `getFrame()`) + block: + when NimStackTrace: + var framePtr {.inject, noinit.}: PFrame + {.emit: "`framePtr` = &FR_;".} + framePtr + +template setFrameMsg*(msg: string, prefix = " ") = + ## attach a msg to current `PFrame`. This can be called multiple times + ## in a given PFrame. Noop unless passing --stacktraceMsgs and --stacktrace + when NimStackTrace and NimStackTraceMsgs: + block: + var fr {.inject, noinit.}: PFrame + {.emit: "`fr` = &FR_;".} + # consider setting a custom upper limit on size (analog to stack overflow) + frameMsgBuf.setLen fr.frameMsgLen + frameMsgBuf.add prefix + frameMsgBuf.add msg + fr.frameMsgLen += prefix.len + msg.len diff --git a/lib/std/staticos.nim b/lib/std/staticos.nim new file mode 100644 index 000000000..2617c6913 --- /dev/null +++ b/lib/std/staticos.nim @@ -0,0 +1,13 @@ +## This module implements path handling like os module but works at only compile-time. +## This module works even when cross compiling to OS that is not supported by os module. + +proc staticFileExists*(filename: string): bool {.compileTime.} = + ## Returns true if `filename` exists and is a regular file or symlink. + ## + ## Directories, device files, named pipes and sockets return false. + discard + +proc staticDirExists*(dir: string): bool {.compileTime.} = + ## Returns true if the directory `dir` exists. If `dir` is a file, false + ## is returned. Follows symlinks. + discard diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim new file mode 100644 index 000000000..b2c36a4be --- /dev/null +++ b/lib/std/strbasics.nim @@ -0,0 +1,119 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides some high performance string operations. +## +## Experimental API, subject to change. + +when defined(nimPreviewSlimSystem): + import std/assertions + + +const whitespaces = {' ', '\t', '\v', '\r', '\l', '\f'} + +proc add*(x: var string, y: openArray[char]) = + ## Concatenates `x` and `y` in place. `y` must not overlap with `x` to + ## allow future `memcpy` optimizations. + # Use `{.noalias.}` ? + let n = x.len + x.setLen n + y.len + # pending #19727 + # setLen unnecessarily zeros memory + var i = 0 + while i < y.len: + x[n + i] = y[i] + i.inc + # xxx use `nimCopyMem(x[n].addr, y[0].addr, y.len)` after some refactoring + +func stripSlice(s: openArray[char], leading = true, trailing = true, chars: set[char] = whitespaces): Slice[int] = + ## Returns the slice range of `s` which is stripped `chars`. + runnableExamples: + assert stripSlice(" abc ") == 1 .. 3 + var + first = 0 + last = high(s) + if leading: + while first <= last and s[first] in chars: inc(first) + if trailing: + while last >= first and s[last] in chars: dec(last) + result = first .. last + +func setSlice*(s: var string, slice: Slice[int]) = + ## Inplace version of `substr`. + runnableExamples: + import std/sugar + + var a = "Hello, Nim!" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(0 .. 0)) == "H" + doAssert a.dup(setSlice(0 .. 1)) == "He" + doAssert a.dup(setSlice(0 .. 10)) == a + doAssert a.dup(setSlice(1 .. 0)).len == 0 + doAssert a.dup(setSlice(20 .. -1)).len == 0 + + + doAssertRaises(AssertionDefect): + discard a.dup(setSlice(-1 .. 1)) + + doAssertRaises(AssertionDefect): + discard a.dup(setSlice(1 .. 11)) + + + let first = slice.a + let last = slice.b + + assert first >= 0 + assert last <= s.high + + if first > last: + s.setLen(0) + return + template impl = + for index in first .. last: + s[index - first] = s[index] + if first > 0: + when nimvm: impl() + else: + # not JS and not Nimscript + when not declared(moveMem): + impl() + else: + when defined(nimSeqsV2): + prepareMutation(s) + moveMem(addr s[0], addr s[first], last - first + 1) + s.setLen(last - first + 1) + +func strip*(a: var string, leading = true, trailing = true, chars: set[char] = whitespaces) {.inline.} = + ## Inplace version of `strip`. Strips leading or + ## trailing `chars` (default: whitespace characters). + ## + ## If `leading` is true (default), leading `chars` are stripped. + ## If `trailing` is true (default), trailing `chars` are stripped. + ## If both are false, the string is unchanged. + runnableExamples: + var a = " vhellov " + strip(a) + assert a == "vhellov" + + a = " vhellov " + a.strip(leading = false) + assert a == " vhellov" + + a = " vhellov " + a.strip(trailing = false) + assert a == "vhellov " + + var c = "blaXbla" + c.strip(chars = {'b', 'a'}) + assert c == "laXbl" + c = "blaXbla" + c.strip(chars = {'b', 'a', 'l'}) + assert c == "X" + + setSlice(a, stripSlice(a, leading, trailing, chars)) diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim new file mode 100644 index 000000000..dbe908612 --- /dev/null +++ b/lib/std/symlinks.nim @@ -0,0 +1,33 @@ +## This module implements symlink (symbolic link) handling. + +## .. importdoc:: os.nim + +from std/paths import Path, ReadDirEffect + +from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink + +proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect], sideEffect.} = + ## Returns true if the symlink `link` exists. Will return true + ## regardless of whether the link points to a directory or file. + result = symlinkExists(link.string) + +proc createSymlink*(src, dest: Path) {.inline.} = + ## Create a symbolic link at `dest` which points to the item specified + ## by `src`. On most operating systems, will fail if a link already exists. + ## + ## .. warning:: Some OS's (such as Microsoft Windows) restrict the creation + ## of symlinks to root users (administrators) or users with developer mode enabled. + ## + ## See also: + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ + createSymlink(src.string, dest.string) + +proc expandSymlink*(symlinkPath: Path): Path {.inline.} = + ## Returns a string representing the path to which the symbolic link points. + ## + ## On Windows this is a noop, `symlinkPath` is simply returned. + ## + ## See also: + ## * `createSymlink proc`_ + result = Path(expandSymlink(symlinkPath.string)) diff --git a/lib/system/io.nim b/lib/std/syncio.nim index 8e3acd0b9..c34a025af 100644 --- a/lib/system/io.nim +++ b/lib/std/syncio.nim @@ -1,14 +1,19 @@ # # # Nim's Runtime Library -# (c) Copyright 2019 Nim contributors +# (c) Copyright 2022 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -include inclrtl -import formatfloat +## This module implements various synchronized I/O operations. + +include system/inclrtl +import std/private/since +import std/formatfloat +when defined(windows): + import std/widestrs # ----------------- IO Part ------------------------------------------------ type @@ -16,27 +21,38 @@ type incompleteStruct.} = object File* = ptr CFile ## The type representing a file handle. - FileMode* = enum ## The file mode when opening a file. - fmRead, ## Open the file for read access only. - fmWrite, ## Open the file for write access only. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWrite, ## Open the file for read and write access. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWriteExisting, ## Open the file for read and write access. - ## If the file does not exist, it will not be - ## created. The existing file will not be cleared. - fmAppend ## Open the file for writing only; append data - ## at the end. - - FileHandle* = cint ## type that represents an OS file handle; this is - ## useful for low-level file access + FileMode* = enum ## The file mode when opening a file. + fmRead, ## Open the file for read access only. + ## If the file does not exist, it will not + ## be created. + fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWrite, ## Open the file for read and write access. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWriteExisting, ## Open the file for read and write access. + ## If the file does not exist, it will not be + ## created. The existing file will not be cleared. + fmAppend ## Open the file for writing only; append data + ## at the end. If the file does not exist, it + ## will be created. + + FileHandle* = cint ## The type that represents an OS file handle; this is + ## useful for low-level file access. + + FileSeekPos* = enum ## Position relative to which seek should happen. + # The values are ordered so that they match with stdio + # SEEK_SET, SEEK_CUR and SEEK_END respectively. + fspSet ## Seek to absolute value + fspCur ## Seek relative to current position + fspEnd ## Seek relative to end # text file handling: when not defined(nimscript) and not defined(js): # duplicated between io and ansi_c - const stdioUsesMacros = defined(osx) and not defined(emscripten) + const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined( + dragonfly)) and not defined(emscripten) const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" @@ -74,7 +90,7 @@ proc c_fputs(c: cstring, f: File): cint {. proc c_fgets(c: cstring, n: cint, f: File): cstring {. importc: "fgets", header: "<stdio.h>", tags: [ReadIOEffect].} proc c_fgetc(stream: File): cint {. - importc: "fgetc", header: "<stdio.h>", tags: [ReadIOEffect].} + importc: "fgetc", header: "<stdio.h>", tags: [].} proc c_ungetc(c: cint, f: File): cint {. importc: "ungetc", header: "<stdio.h>", tags: [].} proc c_putc(c: cint, stream: File): cint {. @@ -89,7 +105,7 @@ proc c_feof(f: File): cint {. importc: "feof", header: "<stdio.h>".} when not declared(c_fwrite): - proc c_fwrite(buf: pointer, size, n: csize_t, f: File): cint {. + proc c_fwrite(buf: pointer, size, n: csize_t, f: File): csize_t {. importc: "fwrite", header: "<stdio.h>".} # C routine that is used here: @@ -104,8 +120,22 @@ when defined(windows): else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "_fseeki64", header: "<stdio.h>", tags: [].} - proc c_ftell(f: File): int64 {. - importc: "_ftelli64", header: "<stdio.h>", tags: [].} + when defined(tcc): + proc c_fsetpos(f: File, pos: var int64): int32 {. + importc: "fsetpos", header: "<stdio.h>", tags: [].} + proc c_fgetpos(f: File, pos: var int64): int32 {. + importc: "fgetpos", header: "<stdio.h>", tags: [].} + proc c_telli64(f: cint): int64 {. + importc: "_telli64", header: "<io.h>", tags: [].} + proc c_ftell(f: File): int64 = + # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c + result = -1'i64 + var pos: int64 + if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0: + result = c_telli64(c_fileno(f)) + else: + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "<stdio.h>", tags: [].} else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "fseeko", header: "<stdio.h>", tags: [].} @@ -121,30 +151,21 @@ proc c_fprintf(f: File, frmt: cstring): cint {. proc c_fputc(c: char, f: File): cint {. importc: "fputc", header: "<stdio.h>".} -# When running nim in android app, stdout goes nowhere, so echo gets ignored -# To redirect echo to the android logcat, use -d:androidNDK -when defined(androidNDK): - const ANDROID_LOG_VERBOSE = 2.cint - proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint - {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.} - -template sysFatal(exc, msg) = - raise newException(exc, msg) - proc raiseEIO(msg: string) {.noinline, noreturn.} = - sysFatal(IOError, msg) + raise newException(IOError, msg) proc raiseEOF() {.noinline, noreturn.} = - sysFatal(EOFError, "EOF reached") + raise newException(EOFError, "EOF reached") proc strerror(errnum: cint): cstring {.importc, header: "<string.h>".} -when not defined(NimScript): +when not defined(nimscript): var errno {.importc, header: "<errno.h>".}: cint ## error variable + EINTR {.importc: "EINTR", header: "<errno.h>".}: cint proc checkErr(f: File) = - when not defined(NimScript): + when not defined(nimscript): if c_ferror(f) != 0: let msg = "errno: " & $errno & " `" & $strerror(errno) & "`" c_clearerr(f) @@ -153,30 +174,35 @@ proc checkErr(f: File) = # shouldn't happen quit(1) -{.push stackTrace:off, profiler:off.} +{.push stackTrace: off, profiler: off.} proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer pointed to by `buffer`. Returns + ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) if result != len: checkErr(f) -proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. +proc readBytes*(f: File, a: var openArray[int8|uint8], start, + len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[start]), len) +proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} = + ## Reads up to `a.len` bytes into the buffer `a`. Returns + ## the actual number of bytes that have been read which may be less than + ## `a.len` (if not as many bytes are remaining), but not greater. + result = readBuffer(f, addr(a[0]), a.len) + proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. - tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns + tags: [ReadIOEffect], benign, deprecated: + "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} = + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. - ## - ## **Warning:** The buffer `a` must be pre-allocated. This can be done - ## using, for example, ``newString``. if (start + len) > len(a): raiseEIO("buffer overflow: (start+len) > length of openarray buffer") result = readBuffer(f, addr(a[start]), len) @@ -188,7 +214,7 @@ proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} = proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of buffer pointed to by the parameter `buffer` to the + ## Writes the bytes of buffer pointed to by the parameter `buffer` to the ## file `f`. Returns the number of actual written bytes, which may be less ## than `len` in case of an error. result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f)) @@ -196,7 +222,7 @@ proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of ``a[start..start+len-1]`` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -204,7 +230,7 @@ proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of ``a[start..start+len-1]`` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -217,7 +243,7 @@ when defined(windows): # machine. We also enable `setConsoleOutputCP(65001)` now by default. # But we cannot call printf directly as the string might contain \0. # So we have to loop over all the sections separated by potential \0s. - var i = c_fprintf(f, "%s", s) + var i = int c_fprintf(f, "%s", s) while i < s.len: if s[i] == '\0': let w = c_fputc('\0', f) @@ -240,7 +266,7 @@ proc write*(f: File, s: string) {.tags: [WriteIOEffect], benign.} = raiseEIO("cannot write string to file") {.pop.} -when NoFakeVars: +when defined(nimscript): when defined(windows): const IOFBF = cint(0) @@ -255,10 +281,46 @@ else: IOFBF {.importc: "_IOFBF", nodecl.}: cint IONBF {.importc: "_IONBF", nodecl.}: cint +const SupportIoctlInheritCtl = (defined(linux) or defined(bsd)) and + not defined(nimscript) +when SupportIoctlInheritCtl: + var + FIOCLEX {.importc, header: "<sys/ioctl.h>".}: cint + FIONCLEX {.importc, header: "<sys/ioctl.h>".}: cint + + proc c_ioctl(fd: cint, request: cint): cint {. + importc: "ioctl", header: "<sys/ioctl.h>", varargs.} +elif defined(posix) and not defined(lwip) and not defined(nimscript): + var + F_GETFD {.importc, header: "<fcntl.h>".}: cint + F_SETFD {.importc, header: "<fcntl.h>".}: cint + FD_CLOEXEC {.importc, header: "<fcntl.h>".}: cint + + proc c_fcntl(fd: cint, cmd: cint): cint {. + importc: "fcntl", header: "<fcntl.h>", varargs.} +elif defined(windows): + type + WinDWORD = culong + WinBOOL = cint + + const HANDLE_FLAG_INHERIT = 1.WinDWORD + + proc getOsfhandle(fd: cint): int {. + importc: "_get_osfhandle", header: "<io.h>".} + + type + IoHandle = distinct pointer + ## Windows' HANDLE type. Defined as an untyped pointer but is **not** + ## one. Named like this to avoid collision with other `system` modules. + + proc setHandleInformation(hObject: IoHandle, dwMask, dwFlags: WinDWORD): + WinBOOL {.stdcall, dynlib: "kernel32", + importc: "SetHandleInformation".} + const BufSize = 4000 -proc close*(f: File) {.tags: [], gcsafe.} = +proc close*(f: File) {.tags: [], gcsafe, sideEffect.} = ## Closes the file. if not f.isNil: discard c_fclose(f) @@ -277,7 +339,7 @@ proc flushFile*(f: File) {.tags: [WriteIOEffect].} = discard c_fflush(f) proc getFileHandle*(f: File): FileHandle = - ## returns the file handle of the file ``f``. This is only useful for + ## Returns the file handle of the file `f`. This is only useful for ## platform specific programming. ## Note that on Windows this doesn't return the Windows-specific handle, ## but the C library's notion of a handle, whatever that means. @@ -285,53 +347,148 @@ proc getFileHandle*(f: File): FileHandle = c_fileno(f) proc getOsFileHandle*(f: File): FileHandle = - ## returns the OS file handle of the file ``f``. This is only useful for + ## Returns the OS file handle of the file `f`. This is only useful for ## platform specific programming. when defined(windows): - proc getOsfhandle(fd: cint): FileHandle {. - importc: "_get_osfhandle", header: "<io.h>".} - result = getOsfhandle(getFileHandle(f)) + result = FileHandle getOsfhandle(cint getFileHandle(f)) else: result = c_fileno(f) -proc readLine*(f: File, line: var TaintedString): bool {.tags: [ReadIOEffect], +when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(windows): + proc setInheritable*(f: FileHandle, inheritable: bool): bool = + ## Controls whether a file handle can be inherited by child processes. Returns + ## `true` on success. This requires the OS file handle, which can be + ## retrieved via `getOsFileHandle <#getOsFileHandle,File>`_. + ## + ## This procedure is not guaranteed to be available for all platforms. Test for + ## availability with `declared() <system.html#declared,untyped>`_. + when SupportIoctlInheritCtl: + result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 + elif defined(freertos) or defined(zephyr): + result = true + elif defined(posix): + var flags = c_fcntl(f, F_GETFD) + if flags == -1: + return false + flags = if inheritable: flags and not FD_CLOEXEC else: flags or FD_CLOEXEC + result = c_fcntl(f, F_SETFD, flags) != -1 + else: + result = setHandleInformation(cast[IoHandle](f), HANDLE_FLAG_INHERIT, + inheritable.WinDWORD) != 0 + +proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f` into `line`. May throw an IO + ## Reads a line of text from the file `f` into `line`. May throw an IO ## exception. - ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline - ## character(s) are not part of the returned string. Returns ``false`` - ## if the end of the file has been reached, ``true`` otherwise. If - ## ``false`` is returned `line` contains no new data. + ## A line of text may be delimited by `LF` or `CRLF`. The newline + ## character(s) are not part of the returned string. Returns `false` + ## if the end of the file has been reached, `true` otherwise. If + ## `false` is returned `line` contains no new data. proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} + when defined(windows): + proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, + nNumberOfCharsToRead: int32, + lpNumberOfCharsRead: ptr int32, + pInputControl: pointer): int32 {. + importc: "ReadConsoleW", stdcall, dynlib: "kernel32".} + + proc getLastError(): int32 {. + importc: "GetLastError", stdcall, dynlib: "kernel32", sideEffect.} + + proc formatMessageW(dwFlags: int32, lpSource: pointer, + dwMessageId, dwLanguageId: int32, + lpBuffer: pointer, nSize: int32, + arguments: pointer): int32 {. + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} + + proc localFree(p: pointer) {. + importc: "LocalFree", stdcall, dynlib: "kernel32".} + + proc isatty(f: File): bool = + when defined(posix): + proc isatty(fildes: FileHandle): cint {. + importc: "isatty", header: "<unistd.h>".} + else: + proc isatty(fildes: FileHandle): cint {. + importc: "_isatty", header: "<io.h>".} + result = isatty(getFileHandle(f)) != 0'i32 + + # this implies the file is open + if f.isatty: + const numberOfCharsToRead = 2048 + var numberOfCharsRead = 0'i32 + var buffer = newWideCString(numberOfCharsToRead) + if readConsole(getOsFileHandle(f), addr(buffer[0]), + numberOfCharsToRead, addr(numberOfCharsRead), nil) == 0: + var error = getLastError() + var errorMsg: string + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, error, 0, addr(msgbuf), 0, nil) != 0'i32: + errorMsg = $msgbuf + if msgbuf != nil: + localFree(cast[pointer](msgbuf)) + raiseEIO("error: " & $error & " `" & errorMsg & "`") + # input always ends with "\r\n" + numberOfCharsRead -= 2 + # handle Ctrl+Z as EOF + for i in 0..<numberOfCharsRead: + if buffer[i].uint16 == 26: #Ctrl+Z + close(f) #has the same effect as setting EOF + if i == 0: + line = "" + return false + numberOfCharsRead = i + break + buffer[numberOfCharsRead] = 0.Utf16Char + when defined(nimv2): + line = $toWideCString(buffer) + else: + line = $buffer + return(true) + var pos = 0 # Use the currently reserved space for a first try - var sp = max(line.string.len, 80) - line.string.setLen(sp) + var sp = max(line.len, 80) + line.setLen(sp) while true: # memset to \L so that we can tell how far fgets wrote, even on EOF, where # fgets doesn't append an \L - for i in 0..<sp: line.string[pos+i] = '\L' + for i in 0..<sp: line[pos+i] = '\L' + + var fgetsSuccess: bool + while true: + # fixes #9634; this pattern may need to be abstracted as a template if reused; + # likely other io procs need this for correctness. + fgetsSuccess = c_fgets(cast[cstring](addr line[pos]), sp.cint, f) != nil + if fgetsSuccess: break + when not defined(nimscript): + if errno == EINTR: + errno = 0 + c_clearerr(f) + continue + checkErr(f) + break - var fgetsSuccess = c_fgets(addr line.string[pos], sp.cint, f) != nil - if not fgetsSuccess: checkErr(f) - let m = c_memchr(addr line.string[pos], '\L'.ord, cast[csize_t](sp)) + let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done - var last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0]) - if last > 0 and line.string[last-1] == '\c': - line.string.setLen(last-1) + var last = cast[int](m) - cast[int](addr line[0]) + if last > 0 and line[last-1] == '\c': + line.setLen(last-1) return last > 1 or fgetsSuccess - # We have to distinguish between two possible cases: + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: # \0\l\0 => line ending in a null character. # \0\l\l => last line without newline, null was put there by fgets. - elif last > 0 and line.string[last-1] == '\0': - if last < pos + sp - 1 and line.string[last+1] != '\0': + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 dec last - line.string.setLen(last) + line.setLen(last) return last > 0 or fgetsSuccess else: # fgets will have inserted a null byte at the end of the string. @@ -339,13 +496,13 @@ proc readLine*(f: File, line: var TaintedString): bool {.tags: [ReadIOEffect], # No \l found: Increase buffer and read more inc pos, sp sp = 128 # read in 128 bytes at a time - line.string.setLen(pos+sp) + line.setLen(pos+sp) -proc readLine*(f: File): TaintedString {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f`. May throw an IO exception. - ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline +proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = + ## Reads a line of text from the file `f`. May throw an IO exception. + ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. - result = TaintedString(newStringOfCap(80)) + result = newStringOfCap(80) if not readLine(f, result): raiseEOF() proc write*(f: File, i: int) {.tags: [WriteIOEffect], benign.} = @@ -365,12 +522,12 @@ proc write*(f: File, b: bool) {.tags: [WriteIOEffect], benign.} = else: write(f, "false") proc write*(f: File, r: float32) {.tags: [WriteIOEffect], benign.} = - var buffer: array[65, char] + var buffer {.noinit.}: array[65, char] discard writeFloatToBuffer(buffer, r) if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f) proc write*(f: File, r: BiggestFloat) {.tags: [WriteIOEffect], benign.} = - var buffer: array[65, char] + var buffer {.noinit.}: array[65, char] discard writeFloatToBuffer(buffer, r) if c_fprintf(f, "%s", buffer[0].addr) < 0: checkErr(f) @@ -402,7 +559,7 @@ proc rawFileSize(file: File): int64 = discard c_fseek(file, oldPos, 0) proc endOfFile*(f: File): bool {.tags: [], benign.} = - ## Returns true iff `f` is at the end. + ## Returns true if `f` is at the end. var c = c_fgetc(f) discard c_ungetc(c, f) return c < 0'i32 @@ -414,7 +571,7 @@ proc readAllFile(file: File, len: int64): string = result = newString(len) let bytes = readBuffer(file, addr(result[0]), len) if endOfFile(file): - if bytes < len: + if bytes.int64 < len: result.setLen(bytes) else: # We read all the bytes but did not reach the EOF @@ -425,7 +582,7 @@ proc readAllFile(file: File): string = var len = rawFileSize(file) result = readAllFile(file, len) -proc readAll*(file: File): TaintedString {.tags: [ReadIOEffect], benign.} = +proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} = ## Reads all data from the stream `file`. ## ## Raises an IO exception in case of an error. It is an error if the @@ -438,18 +595,13 @@ proc readAll*(file: File): TaintedString {.tags: [ReadIOEffect], benign.} = else: let len = rawFileSize(file) if len > 0: - result = readAllFile(file, len).TaintedString + result = readAllFile(file, len) else: - result = readAllBuffer(file).TaintedString - -proc writeLn[Ty](f: File, x: varargs[Ty, `$`]) = - for i in items(x): - write(f, i) - write(f, "\n") + result = readAllBuffer(file) proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, tags: [WriteIOEffect], benign.} = - ## writes the values `x` to `f` and then writes "\\n". + ## Writes the values `x` to `f` and then writes "\\n". ## May throw an IO exception. for i in items(x): write(f, i) @@ -457,7 +609,7 @@ proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, # interface to the C procs: -when defined(windows) and not defined(useWinAnsi): +when defined(windows): when defined(cpp): proc wfopen(filename, mode: WideCString): pointer {. importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} @@ -485,7 +637,24 @@ else: importc: "freopen", nodecl.} const - FormatOpen: array[FileMode, string] = ["rb", "wb", "w+b", "r+b", "ab"] + NoInheritFlag = + # Platform specific flag for creating a File without inheritance. + when not defined(nimInheritHandles): + when defined(windows): + "N" + elif defined(linux) or defined(bsd): + "e" + else: + "" + else: + "" + RawFormatOpen: array[FileMode, cstring] = [ + # used for open by FileHandle, which calls `fdopen` + cstring("rb"), "wb", "w+b", "r+b", "ab"] + FormatOpen: array[FileMode, cstring] = [ + cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, + "r+b" & NoInheritFlag, "ab" & NoInheritFlag + ] #"rt", "wt", "w+t", "r+t", "at" # we always use binary here as for Nim the OS line ending # should not be translated. @@ -499,7 +668,7 @@ when defined(posix) and not defined(nimscript): Stat {.importc: "struct stat", header: "<sys/stat.h>", final, pure.} = object ## struct stat filler_1: array[24, char] - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file filler_2: array[144 - 24 - 4, char] proc modeIsDir(m: Mode): bool = @@ -512,7 +681,7 @@ when defined(posix) and not defined(nimscript): Stat {.importc: "struct stat", header: "<sys/stat.h>", final, pure.} = object ## struct stat - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "<sys/stat.h>".} ## Test for a directory. @@ -523,69 +692,94 @@ when defined(posix) and not defined(nimscript): proc open*(f: var File, filename: string, mode: FileMode = fmRead, - bufSize: int = -1): bool {.tags: [], raises: [], benign.} = + bufSize: int = -1): bool {.tags: [], raises: [], benign.} = ## Opens a file named `filename` with given `mode`. ## - ## Default mode is readonly. Returns true iff the file could be opened. + ## Default mode is readonly. Returns true if the file could be opened. ## This throws no exception if the file could not be opened. - var p = fopen(filename, FormatOpen[mode]) + ## + ## The file handle associated with the resulting `File` is not inheritable. + var p = fopen(filename.cstring, FormatOpen[mode]) if p != nil: + var f2 = cast[File](p) when defined(posix) and not defined(nimscript): # How `fopen` handles opening a directory is not specified in ISO C and # POSIX. We do not want to handle directories as regular files that can # be opened. - var f2 = cast[File](p) - var res: Stat + var res {.noinit.}: Stat if c_fstat(getFileHandle(f2), res) >= 0'i32 and modeIsDir(res.st_mode): close(f2) return false + when not defined(nimInheritHandles) and declared(setInheritable) and + NoInheritFlag.len == 0: + if not setInheritable(getOsFileHandle(f2), false): + close(f2) + return false + result = true f = cast[File](p) - if bufSize > 0 and bufSize <= high(cint).int: + if bufSize > 0 and bufSize.uint <= high(uint): discard c_setvbuf(f, nil, IOFBF, cast[csize_t](bufSize)) elif bufSize == 0: discard c_setvbuf(f, nil, IONBF, 0) proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. tags: [], benign.} = - ## reopens the file `f` with given `filename` and `mode`. This + ## Reopens the file `f` with given `filename` and `mode`. This ## is often used to redirect the `stdin`, `stdout` or `stderr` ## file variables. ## - ## Default mode is readonly. Returns true iff the file could be reopened. - result = freopen(filename, FormatOpen[mode], f) != nil + ## Default mode is readonly. Returns true if the file could be reopened. + ## + ## The file handle associated with `f` won't be inheritable. + if freopen(filename.cstring, FormatOpen[mode], f) != nil: + when not defined(nimInheritHandles) and declared(setInheritable) and + NoInheritFlag.len == 0: + if not setInheritable(getOsFileHandle(f), false): + close(f) + return false + result = true proc open*(f: var File, filehandle: FileHandle, mode: FileMode = fmRead): bool {.tags: [], raises: [], benign.} = - ## Creates a ``File`` from a `filehandle` with given `mode`. + ## Creates a `File` from a `filehandle` with given `mode`. + ## + ## Default mode is readonly. Returns true if the file could be opened. ## - ## Default mode is readonly. Returns true iff the file could be opened. - f = c_fdopen(filehandle, FormatOpen[mode]) + ## The passed file handle will no longer be inheritable. + when not defined(nimInheritHandles) and declared(setInheritable): + let oshandle = when defined(windows): FileHandle getOsfhandle( + filehandle) else: filehandle + if not setInheritable(oshandle, false): + return false + f = c_fdopen(filehandle, RawFormatOpen[mode]) result = f != nil proc open*(filename: string, mode: FileMode = fmRead, bufSize: int = -1): File = ## Opens a file named `filename` with given `mode`. ## - ## Default mode is readonly. Raises an ``IOError`` if the file + ## Default mode is readonly. Raises an `IOError` if the file ## could not be opened. + ## + ## The file handle associated with the resulting `File` is not inheritable. if not open(result, filename, mode, bufSize): - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) -proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} = - ## sets the position of the file pointer that is used for read/write +proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign, sideEffect.} = + ## Sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos*(f: File): int64 {.benign.} = - ## retrieves the current position of the file pointer that is used to + ## Retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. result = c_ftell(f) if result < 0: raiseEIO("cannot retrieve file position") proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} = - ## retrieves the file size (in bytes) of `f`. + ## Retrieves the file size (in bytes) of `f`. let oldPos = getFilePos(f) discard c_fseek(f, 0, 2) # seek the end of the file result = getFilePos(f) @@ -600,43 +794,8 @@ proc setStdIoUnbuffered*() {.tags: [], benign.} = when declared(stdin): discard c_setvbuf(stdin, nil, IONBF, 0) -when declared(stdout): - when defined(windows) and compileOption("threads"): - const insideRLocksModule = false - include "system/syslocks" - var echoLock: SysLock - initSysLock echoLock - - proc echoBinSafe(args: openArray[string]) {.compilerproc.} = - when defined(androidNDK): - var s = "" - for arg in args: - s.add arg - android_log_print(ANDROID_LOG_VERBOSE, "nim", s) - else: - # flockfile deadlocks some versions of Android 5.x.x - when not defined(windows) and not defined(android) and not defined(nintendoswitch) and hostOS != "any": - proc flockfile(f: File) {.importc, nodecl.} - proc funlockfile(f: File) {.importc, nodecl.} - flockfile(stdout) - when defined(windows) and compileOption("threads"): - acquireSys echoLock - for s in args: - when defined(windows): - writeWindows(stdout, s) - else: - discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) - const linefeed = "\n" - discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) - discard c_fflush(stdout) - when not defined(windows) and not defined(android) and not defined(nintendoswitch) and hostOS != "any": - funlockfile(stdout) - when defined(windows) and compileOption("threads"): - releaseSys echoLock - - -when defined(windows) and not defined(nimscript): +when defined(windows) and not defined(nimscript) and not defined(js): # work-around C's sucking abstraction: # BUGFIX: stdin and stdout should be binary files! proc c_setmode(handle, mode: cint) {. @@ -652,109 +811,132 @@ when defined(windows) and not defined(nimscript): when defined(windows) and appType == "console" and not defined(nimDontSetUtf8CodePage) and not defined(nimscript): + import std/exitprocs + proc setConsoleOutputCP(codepage: cuint): int32 {.stdcall, dynlib: "kernel32", importc: "SetConsoleOutputCP".} proc setConsoleCP(wCodePageID: cuint): int32 {.stdcall, dynlib: "kernel32", importc: "SetConsoleCP".} + proc getConsoleOutputCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleOutputCP".} + proc getConsoleCP(): cuint {.stdcall, dynlib: "kernel32", + importc: "GetConsoleCP".} + + const Utf8codepage = 65001'u32 + + let + consoleOutputCP = getConsoleOutputCP() + consoleCP = getConsoleCP() - const Utf8codepage = 65001 - discard setConsoleOutputCP(Utf8codepage) - discard setConsoleCP(Utf8codepage) + proc restoreConsoleOutputCP() = discard setConsoleOutputCP(consoleOutputCP) + proc restoreConsoleCP() = discard setConsoleCP(consoleCP) -proc readFile*(filename: string): TaintedString {.tags: [ReadIOEffect], benign.} = + if consoleOutputCP != Utf8codepage: + discard setConsoleOutputCP(Utf8codepage) + addExitProc(restoreConsoleOutputCP) + + if consoleCP != Utf8codepage: + discard setConsoleCP(Utf8codepage) + addExitProc(restoreConsoleCP) + +proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} = ## Opens a file named `filename` for reading, calls `readAll ## <#readAll,File>`_ and closes the file afterwards. Returns the string. ## Raises an IO exception in case of an error. If you need to call ## this inside a compile time macro you can use `staticRead ## <system.html#staticRead,string>`_. - var f: File + var f: File = nil if open(f, filename): try: result = readAll(f) finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} = ## Opens a file named `filename` for writing. Then writes the ## `content` completely to the file and closes the file afterwards. ## Raises an IO exception in case of an error. - var f: File + var f: File = nil if open(f, filename, fmWrite): try: f.write(content) finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = ## Opens a file named `filename` for writing. Then writes the ## `content` completely to the file and closes the file afterwards. ## Raises an IO exception in case of an error. - var f: File + var f: File = nil if open(f, filename, fmWrite): try: - f.writeBuffer(unsafeAddr content[0], content.len) + discard f.writeBuffer(unsafeAddr content[0], content.len) finally: close(f) else: raise newException(IOError, "cannot open: " & filename) -proc readLines*(filename: string, n: Natural): seq[TaintedString] = - ## read `n` lines from the file named `filename`. Raises an IO exception +proc readLines*(filename: string, n: Natural): seq[string] = + ## Reads `n` lines from the file named `filename`. Raises an IO exception ## in case of an error. Raises EOF if file does not contain at least `n` lines. - ## Available at compile time. A line of text may be delimited by ``LF`` or ``CRLF``. + ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`. ## The newline character(s) are not part of the returned strings. - var f: File + var f: File = nil if open(f, filename): try: - result = newSeq[TaintedString](n) + result = newSeq[string](n) for i in 0 .. n - 1: if not readLine(f, result[i]): raiseEOF() finally: close(f) else: - sysFatal(IOError, "cannot open: " & filename) + raise newException(IOError, "cannot open: " & filename) -proc readLines*(filename: string): seq[TaintedString] {.deprecated: "use readLines with two arguments".} = +template readLines*(filename: string): seq[ + string] {.deprecated: "use readLines with two arguments".} = readLines(filename, 1) -iterator lines*(filename: string): TaintedString {.tags: [ReadIOEffect].} = +iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = ## Iterates over any line in the file named `filename`. ## ## If the file does not exist `IOError` is raised. The trailing newline ## character(s) are removed from the iterated lines. Example: ## - ## .. code-block:: nim - ## import strutils - ## - ## proc transformLetters(filename: string) = - ## var buffer = "" - ## for line in filename.lines: - ## buffer.add(line.replace("a", "0") & '\x0A') - ## writeFile(filename, buffer) - var f = open(filename, bufSize=8000) + runnableExamples: + import std/strutils + + proc transformLetters(filename: string) = + var buffer = "" + for line in filename.lines: + buffer.add(line.replace("a", "0") & '\n') + writeFile(filename, buffer) + var f = open(filename, bufSize = 8000) try: - var res = TaintedString(newStringOfCap(80)) + var res = newStringOfCap(80) while f.readLine(res): yield res finally: close(f) -iterator lines*(f: File): TaintedString {.tags: [ReadIOEffect].} = - ## Iterate over any line in the file `f`. +iterator lines*(f: File): string {.tags: [ReadIOEffect].} = + ## Iterates over any line in the file `f`. ## ## The trailing newline character(s) are removed from the iterated lines. - ## Example: ## - ## .. code-block:: nim - ## proc countZeros(filename: File): tuple[lines, zeros: int] = - ## for line in filename.lines: - ## for letter in line: - ## if letter == '0': - ## result.zeros += 1 - ## result.lines += 1 - var res = TaintedString(newStringOfCap(80)) + runnableExamples: + proc countZeros(filename: File): tuple[lines, zeros: int] = + for line in filename.lines: + for letter in line: + if letter == '0': + result.zeros += 1 + result.lines += 1 + var res = newStringOfCap(80) while f.readLine(res): yield res + +template `&=`*(f: File, x: typed) = + ## An alias for `write`. + write(f, x) diff --git a/lib/system/atomics.nim b/lib/std/sysatomics.nim index 146bb859f..2f203b3eb 100644 --- a/lib/system/atomics.nim +++ b/lib/std/sysatomics.nim @@ -7,17 +7,22 @@ # distribution, for details about the copyright. # +when defined(nimPreviewSlimSystem): + {.deprecated: "use `std/atomics` instead".} + # Atomic operations for Nim. {.push stackTrace:off, profiler:off.} -const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) +const + hasThreadSupport = compileOption("threads") and not defined(nimscript) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) const someVcc = defined(vcc) or defined(clang_cl) type AtomType* = SomeNumber|pointer|ptr|char|bool ## Type Class representing valid types for use with atomic procs -when someGcc and hasThreadSupport: +when someGcc: type AtomMemModel* = distinct cint var ATOMIC_RELAXED* {.importc: "__ATOMIC_RELAXED", nodecl.}: AtomMemModel @@ -73,7 +78,7 @@ when someGcc and hasThreadSupport: proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. - importc: "__atomic_compare_exchange_n ", nodecl.} + importc: "__atomic_compare_exchange_n", nodecl.} ## This proc implements an atomic compare and exchange operation. This compares the ## contents at p with the contents at expected and if equal, writes desired at p. ## If they are not equal, the current contents at p is written into expected. @@ -100,13 +105,13 @@ when someGcc and hasThreadSupport: proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_sub_fetch", nodecl.} proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_or_fetch ", nodecl.} + importc: "__atomic_or_fetch", nodecl.} proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_and_fetch", nodecl.} proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_xor_fetch", nodecl.} proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_nand_fetch ", nodecl.} + importc: "__atomic_nand_fetch", nodecl.} ## Perform the operation return the old value, all memory models are valid proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. @@ -125,8 +130,8 @@ when someGcc and hasThreadSupport: proc atomicTestAndSet*(p: pointer, mem: AtomMemModel): bool {. importc: "__atomic_test_and_set", nodecl.} ## This built-in function performs an atomic test-and-set operation on the byte at p. - ## The byte is set to some implementation defined nonzero “set” value and the return - ## value is true if and only if the previous contents were “set”. + ## The byte is set to some implementation defined nonzero "set" value and the return + ## value is true if and only if the previous contents were "set". ## All memory models are valid. proc atomicClear*(p: pointer, mem: AtomMemModel) {. @@ -164,23 +169,48 @@ when someGcc and hasThreadSupport: ## ignore this parameter. template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) -elif someVcc and hasThreadSupport: +elif someVcc: type AtomMemModel* = distinct cint const - ATOMIC_RELAXED = 0.AtomMemModel - ATOMIC_CONSUME = 1.AtomMemModel - ATOMIC_ACQUIRE = 2.AtomMemModel - ATOMIC_RELEASE = 3.AtomMemModel - ATOMIC_ACQ_REL = 4.AtomMemModel - ATOMIC_SEQ_CST = 5.AtomMemModel + ATOMIC_RELAXED* = 0.AtomMemModel + ATOMIC_CONSUME* = 1.AtomMemModel + ATOMIC_ACQUIRE* = 2.AtomMemModel + ATOMIC_RELEASE* = 3.AtomMemModel + ATOMIC_ACQ_REL* = 4.AtomMemModel + ATOMIC_SEQ_CST* = 5.AtomMemModel proc `==`(x1, x2: AtomMemModel): bool {.borrow.} - proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} - proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} + proc readBarrier() {.importc: "_ReadBarrier", header: "<intrin.h>".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "<intrin.h>".} proc fence*() {.importc: "_ReadWriteBarrier", header: "<intrin.h>".} + when defined(cpp): + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importcpp: "_InterlockedCompareExchange(static_cast<long volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importcpp: "_InterlockedExchange8(static_cast<NI8 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importcpp: "_InterlockedExchange16(static_cast<NI16 volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importcpp: "_InterlockedExchange(static_cast<long volatile *>(#), #)", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importcpp: "_InterlockedExchange64(static_cast<NI64 volatile *>(#), #)", header: "<intrin.h>".} + else: + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} + + proc interlockedExchange8(location: pointer; desired: int8): int8 {.importc: "_InterlockedExchange8", header: "<intrin.h>".} + proc interlockedExchange16(location: pointer; desired: int16): int16 {.importc: "_InterlockedExchange16", header: "<intrin.h>".} + proc interlockedExchange32(location: pointer; desired: int32): int32 {.importc: "_InterlockedExchange", header: "<intrin.h>".} + proc interlockedExchange64(location: pointer; desired: int64): int64 {.importc: "_InterlockedExchange64", header: "<intrin.h>".} + + template barrier(mem: AtomMemModel) = when mem == ATOMIC_RELAXED: discard elif mem == ATOMIC_CONSUME: readBarrier() @@ -189,10 +219,28 @@ elif someVcc and hasThreadSupport: elif mem == ATOMIC_ACQ_REL: fence() elif mem == ATOMIC_SEQ_CST: fence() + proc atomicStoreN*[T: AtomType](p: ptr T, val: T, mem: static[AtomMemModel]) = + barrier(mem) + p[] = val + proc atomicLoadN*[T: AtomType](p: ptr T, mem: static[AtomMemModel]): T = result = p[] barrier(mem) + proc atomicCompareExchangeN*[T: ptr](p, expected: ptr T, desired: T, + weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool = + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](desired), cast[int64](expected)) == + cast[int64](expected) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](desired), cast[int32](expected)) == + cast[int32](expected) + + proc atomicExchangeN*[T: ptr](p: ptr T, val: T, mem: AtomMemModel): T = + when sizeof(T) == 8: + cast[T](interlockedExchange64(p, cast[int64](val))) + elif sizeof(T) == 4: + cast[T](interlockedExchange32(p, cast[int32](val))) when defined(cpp): when sizeof(int) == 8: proc addAndFetch*(p: ptr int, val: int): int {. @@ -215,9 +263,11 @@ else: inc(p[], val) result = p[] -proc atomicInc*(memLoc: var int, x: int = 1): int = + +proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically increments the integer by some `x`. It returns the new value. when someGcc and hasThreadSupport: - result = atomicAddFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, x) inc(result, x) @@ -225,12 +275,13 @@ proc atomicInc*(memLoc: var int, x: int = 1): int = inc(memLoc, x) result = memLoc -proc atomicDec*(memLoc: var int, x: int = 1): int = +proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable, raises: [], tags: [].} = + ## Atomically decrements the integer by some `x`. It returns the new value. when someGcc and hasThreadSupport: when declared(atomicSubFetch): - result = atomicSubFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) else: - result = atomicAddFetch(memLoc.addr, -x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, -x) dec(result, x) @@ -239,21 +290,6 @@ proc atomicDec*(memLoc: var int, x: int = 1): int = result = memLoc when someVcc: - when defined(cpp): - proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 - {.importcpp: "_InterlockedCompareExchange64(static_cast<NI64 volatile *>(#), #, #)", header: "<intrin.h>".} - proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 - {.importcpp: "_InterlockedCompareExchange(static_cast<NI volatile *>(#), #, #)", header: "<intrin.h>".} - proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte - {.importcpp: "_InterlockedCompareExchange8(static_cast<char volatile *>(#), #, #)", header: "<intrin.h>".} - else: - proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 - {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} - proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 - {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} - proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte - {.importc: "_InterlockedCompareExchange8", header: "<intrin.h>".} - proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = when sizeof(T) == 8: interlockedCompareExchange64(p, cast[int64](newValue), cast[int64](oldValue)) == @@ -281,10 +317,7 @@ static int __tcc_cas(int *ptr, int oldVal, int newVal) : "r" (newVal), "m" (*ptr), "a" (oldVal) : "memory"); - if (ret) - return 0; - else - return 1; + return ret; } """.} else: @@ -301,10 +334,7 @@ static int __tcc_cas(int *ptr, int oldVal, int newVal) : "r" (newVal), "m" (*ptr), "a" (oldVal) : "memory"); - if (ret) - return 0; - else - return 1; + return ret; } """.} @@ -333,7 +363,7 @@ elif someGcc or defined(tcc): elif defined(icl): proc cpuRelax* {.importc: "_mm_pause", header: "xmmintrin.h".} elif false: - from os import sleep + from std/os import sleep proc cpuRelax* {.inline.} = os.sleep(1) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim new file mode 100644 index 000000000..6f2c6b0c1 --- /dev/null +++ b/lib/std/sysrand.nim @@ -0,0 +1,326 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## .. warning:: This module was added in Nim 1.6. If you are using it for cryptographic purposes, +## keep in mind that so far this has not been audited by any security professionals, +## therefore may not be secure. +## +## `std/sysrand` generates random numbers from a secure source provided by the operating system. +## It is a cryptographically secure pseudorandom number generator +## and should be unpredictable enough for cryptographic applications, +## though its exact quality depends on the OS implementation. +## +## | Targets | Implementation | +## | :--- | ----: | +## | Windows | `BCryptGenRandom`_ | +## | Linux | `getrandom`_ | +## | MacOSX | `SecRandomCopyBytes`_ | +## | iOS | `SecRandomCopyBytes`_ | +## | OpenBSD | `getentropy openbsd`_ | +## | FreeBSD | `getrandom freebsd`_ | +## | JS (Web Browser) | `getRandomValues`_ | +## | Node.js | `randomFillSync`_ | +## | Other Unix platforms | `/dev/urandom`_ | +## +## .. _BCryptGenRandom: https://docs.microsoft.com/en-us/windows/win32/api/bcrypt/nf-bcrypt-bcryptgenrandom +## .. _getrandom: https://man7.org/linux/man-pages/man2/getrandom.2.html +## .. _getentropy: https://www.unix.com/man-page/mojave/2/getentropy +## .. _SecRandomCopyBytes: https://developer.apple.com/documentation/security/1399291-secrandomcopybytes?language=objc +## .. _getentropy openbsd: https://man.openbsd.org/getentropy.2 +## .. _getrandom freebsd: https://www.freebsd.org/cgi/man.cgi?query=getrandom&manpath=FreeBSD+12.0-stable +## .. _getRandomValues: https://www.w3.org/TR/WebCryptoAPI/#Crypto-method-getRandomValues +## .. _randomFillSync: https://nodejs.org/api/crypto.html#crypto_crypto_randomfillsync_buffer_offset_size +## .. _/dev/urandom: https://en.wikipedia.org/wiki//dev/random +## +## On a Linux target, a call to the `getrandom` syscall can be avoided (e.g. +## for targets running kernel version < 3.17) by passing a compile flag of +## `-d:nimNoGetRandom`. If this flag is passed, sysrand will use `/dev/urandom` +## as with any other POSIX compliant OS. +## + +runnableExamples: + doAssert urandom(0).len == 0 + doAssert urandom(113).len == 113 + doAssert urandom(1234) != urandom(1234) # unlikely to fail in practice + +## +## See also +## ======== +## * `random module <random.html>`_ +## + + +when not defined(js): + import std/oserrors + +when defined(posix): + import std/posix + +when defined(nimPreviewSlimSystem): + import std/assertions + +const + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) + batchSize {.used.} = 256 + +when batchImplOS: + template batchImpl(result: var int, dest: var openArray[byte], getRandomImpl) = + let size = dest.len + if size == 0: + return + + let + chunks = (size - 1) div batchSize + left = size - chunks * batchSize + + for i in 0 ..< chunks: + let readBytes = getRandomImpl(addr dest[result], batchSize) + if readBytes < 0: + return readBytes + inc(result, batchSize) + + result = getRandomImpl(addr dest[result], left) + +when defined(js): + import std/private/jsutils + + when defined(nodejs): + {.emit: "const _nim_nodejs_crypto = require('crypto');".} + + proc randomFillSync(p: Uint8Array) {.importjs: "_nim_nodejs_crypto.randomFillSync(#)".} + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + var src = newUint8Array(size) + randomFillSync(src) + for i in 0 ..< size: + dest[i] = src[i] + + else: + proc getRandomValues(p: Uint8Array) {.importjs: "window.crypto.getRandomValues(#)".} + # The requested length of `p` must not be more than 65536. + + proc assign(dest: var openArray[byte], src: Uint8Array, base: int, size: int) = + getRandomValues(src) + for j in 0 ..< size: + dest[base + j] = src[j] + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + if size <= batchSize: + var src = newUint8Array(size) + assign(dest, src, 0, size) + return + + let + chunks = (size - 1) div batchSize + left = size - chunks * batchSize + + var srcArray = newUint8Array(batchSize) + for i in 0 ..< chunks: + assign(dest, srcArray, result, batchSize) + inc(result, batchSize) + + var leftArray = newUint8Array(left) + assign(dest, leftArray, result, left) + +elif defined(windows): + type + PVOID = pointer + BCRYPT_ALG_HANDLE = PVOID + PUCHAR = ptr uint8 + NTSTATUS = clong + ULONG = culong + + const + STATUS_SUCCESS = 0x00000000 + BCRYPT_USE_SYSTEM_PREFERRED_RNG = 0x00000002 + + proc bCryptGenRandom( + hAlgorithm: BCRYPT_ALG_HANDLE, + pbBuffer: PUCHAR, + cbBuffer: ULONG, + dwFlags: ULONG + ): NTSTATUS {.stdcall, importc: "BCryptGenRandom", dynlib: "Bcrypt.dll".} + + + proc randomBytes(pbBuffer: pointer, cbBuffer: Natural): int {.inline.} = + bCryptGenRandom(nil, cast[PUCHAR](pbBuffer), ULONG(cbBuffer), + BCRYPT_USE_SYSTEM_PREFERRED_RNG) + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + result = randomBytes(addr dest[0], size) + +elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): + when (NimMajor, NimMinor) >= (1, 4): + let SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + else: + var SYS_getrandom {.importc: "SYS_getrandom", header: "<sys/syscall.h>".}: clong + const syscallHeader = """#include <unistd.h> +#include <sys/syscall.h>""" + + proc syscall(n: clong): clong {. + importc: "syscall", varargs, header: syscallHeader.} + # When reading from the urandom source (GRND_RANDOM is not set), + # getrandom() will block until the entropy pool has been + # initialized (unless the GRND_NONBLOCK flag was specified). If a + # request is made to read a large number of bytes (more than 256), + # getrandom() will block until those bytes have been generated and + # transferred from kernel memory to buf. + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + while result < size: + let readBytes = syscall(SYS_getrandom, addr dest[result], cint(size - result), 0).int + if readBytes == 0: + raiseAssert "unreachable" + elif readBytes > 0: + inc(result, readBytes) + else: + if osLastError().cint in [EINTR, EAGAIN]: discard + else: + result = -1 + break + +elif defined(openbsd): + proc getentropy(p: pointer, size: cint): cint {.importc: "getentropy", header: "<unistd.h>".} + # Fills a buffer with high-quality entropy, + # which can be used as input for process-context pseudorandom generators like `arc4random`. + # The maximum buffer size permitted is 256 bytes. + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + result = getentropy(p, cint(size)).int + +elif defined(zephyr): + proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "<random/rand32.h>".} + # Fill the destination buffer with cryptographically secure + # random data values + # + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + # 0 if success, -EIO if entropy reseed error + result = sys_csrand_get(p, csize_t(size)).int + +elif defined(freebsd): + type cssize_t {.importc: "ssize_t", header: "<sys/types.h>".} = int + + proc getrandom(p: pointer, size: csize_t, flags: cuint): cssize_t {.importc: "getrandom", header: "<sys/random.h>".} + # Upon successful completion, the number of bytes which were actually read + # is returned. For requests larger than 256 bytes, this can be fewer bytes + # than were requested. Otherwise, -1 is returned and the global variable + # errno is set to indicate the error. + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + result = getrandom(p, csize_t(size), 0) + +elif defined(ios) or defined(macosx): + {.passl: "-framework Security".} + + const errSecSuccess = 0 ## No error. + + type + SecRandom {.importc: "struct __SecRandom".} = object + + SecRandomRef = ptr SecRandom + ## An abstract Core Foundation-type object containing information about a random number generator. + + proc secRandomCopyBytes( + rnd: SecRandomRef, count: csize_t, bytes: pointer + ): cint {.importc: "SecRandomCopyBytes", header: "<Security/SecRandom.h>".} + ## https://developer.apple.com/documentation/security/1399291-secrandomcopybytes + + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + result = secRandomCopyBytes(nil, csize_t(size), addr dest[0]) + +else: + template urandomImpl(result: var int, dest: var openArray[byte]) = + let size = dest.len + if size == 0: + return + + # see: https://www.2uo.de/myths-about-urandom/ which justifies using urandom instead of random + let fd = posix.open("/dev/urandom", O_RDONLY) + + if fd < 0: + result = -1 + else: + try: + var stat: Stat + if fstat(fd, stat) != -1 and S_ISCHR(stat.st_mode): + let + chunks = (size - 1) div batchSize + left = size - chunks * batchSize + + for i in 0 ..< chunks: + let readBytes = posix.read(fd, addr dest[result], batchSize) + if readBytes < 0: + return readBytes + inc(result, batchSize) + + result = posix.read(fd, addr dest[result], left) + else: + result = -1 + finally: + discard posix.close(fd) + +proc urandomInternalImpl(dest: var openArray[byte]): int {.inline.} = + when batchImplOS: + batchImpl(result, dest, getRandomImpl) + else: + urandomImpl(result, dest) + +proc urandom*(dest: var openArray[byte]): bool = + ## Fills `dest` with random bytes suitable for cryptographic use. + ## If the call succeeds, returns `true`. + ## + ## If `dest` is empty, `urandom` immediately returns success, + ## without calling the underlying operating system API. + ## + ## .. warning:: The code hasn't been audited by cryptography experts and + ## is provided as-is without guarantees. Use at your own risks. For production + ## systems we advise you to request an external audit. + result = true + when defined(js): discard urandomInternalImpl(dest) + else: + let ret = urandomInternalImpl(dest) + when defined(windows): + if ret != STATUS_SUCCESS: + result = false + else: + if ret < 0: + result = false + +proc urandom*(size: Natural): seq[byte] {.inline.} = + ## Returns random bytes suitable for cryptographic use. + ## + ## .. warning:: The code hasn't been audited by cryptography experts and + ## is provided as-is without guarantees. Use at your own risks. For production + ## systems we advise you to request an external audit. + result = newSeq[byte](size) + when defined(js): discard urandomInternalImpl(result) + else: + if not urandom(result): + raiseOSError(osLastError()) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim new file mode 100644 index 000000000..7e59747f5 --- /dev/null +++ b/lib/std/tasks.nim @@ -0,0 +1,312 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module provides basic primitives for creating parallel programs. +## A `Task` should be only owned by a single Thread, it cannot be shared by threads. + +import std/[macros, isolation, typetraits] + +when defined(nimPreviewSlimSystem): + import std/assertions + +export isolation + + +when compileOption("threads"): + from std/effecttraits import isGcSafe + + +# +# proc hello(a: int, b: string) = +# echo $a & b +# +# let literal = "Nim" +# let t = toTask(hello(521, literal)) +# +# +# is roughly converted to +# +# type +# ScratchObj_369098780 = object +# a: int +# b: string +# +# let scratch_369098762 = cast[ptr ScratchObj_369098780](c_calloc(csize_t 1, +# csize_t sizeof(ScratchObj_369098780))) +# if scratch_369098762.isNil: +# raise newException(OutOfMemDefect, "Could not allocate memory") +# block: +# var isolate_369098776 = isolate(521) +# scratch_369098762.a = extract(isolate_369098776) +# var isolate_369098778 = isolate(literal) +# scratch_369098762.b = extract(isolate_369098778) +# proc hello_369098781(args`gensym3: pointer) {.nimcall.} = +# let objTemp_369098775 = cast[ptr ScratchObj_369098780](args`gensym3) +# let :tmp_369098777 = objTemp_369098775.a +# let :tmp_369098779 = objTemp_369098775.b +# hello(a = :tmp_369098777, b = :tmp_369098779) +# +# proc destroyScratch_369098782(args`gensym3: pointer) {.nimcall.} = +# let obj_369098783 = cast[ptr ScratchObj_369098780](args`gensym3) +# =destroy(obj_369098783[]) +# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782) +# + + +type + Task* = object ## `Task` contains the callback and its arguments. + callback: proc (args, res: pointer) {.nimcall, gcsafe.} + args: pointer + destroy: proc (args: pointer) {.nimcall, gcsafe.} + + +proc `=copy`*(x: var Task, y: Task) {.error.} + +const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) +when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`*(t: Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) +else: + proc `=destroy`*(t: var Task) {.inline, gcsafe.} = + ## Frees the resources allocated for a `Task`. + if t.args != nil: + if t.destroy != nil: + t.destroy(t.args) + deallocShared(t.args) + +proc invoke*(task: Task; res: pointer = nil) {.inline, gcsafe.} = + ## Invokes the `task`. + assert task.callback != nil + task.callback(task.args, res) + +template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr: NimNode) = + # block: + # var isoTempA = isolate(521) + # scratch.a = extract(isolateA) + # var isoTempB = isolate(literal) + # scratch.b = extract(isolateB) + let isolatedTemp = genSym(nskTemp, "isoTemp") + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam)) + scratchAssignList.add newAssignment(scratchDotExpr, + newCall(newIdentNode("extract"), isolatedTemp)) + +template addAllNode(assignParam: NimNode, procParam: NimNode) = + let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) + + checkIsolate(scratchAssignList, procParam, scratchDotExpr) + + let tempNode = genSym(kind = nskTemp, ident = formalParams[i][0].strVal) + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], tempNode) + tempAssignList.add newLetStmt(tempNode, newDotExpr(objTemp, formalParams[i][0])) + scratchRecList.add newIdentDefs(newIdentNode(formalParams[i][0].strVal), assignParam) + +proc analyseRootSym(s: NimNode): NimNode = + result = s + while true: + case result.kind + of nnkBracketExpr, nnkDerefExpr, nnkHiddenDeref, + nnkAddr, nnkHiddenAddr, + nnkObjDownConv, nnkObjUpConv: + result = result[0] + of nnkDotExpr, nnkCheckedFieldExpr, nnkHiddenStdConv, nnkHiddenSubConv: + result = result[1] + else: + break + +macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkCallStrLit}): Task = + ## Converts the call and its arguments to `Task`. + runnableExamples: + proc hello(a: int) = echo a + + let b = toTask hello(13) + assert b is Task + + let retType = getTypeInst(e) + let returnsVoid = retType.typeKind == ntyVoid + + let rootSym = analyseRootSym(e[0]) + expectKind rootSym, nnkSym + + when compileOption("threads"): + if not isGcSafe(rootSym): + error("'toTask' takes a GC safe call expression", e) + + if hasClosure(rootSym): + error("closure call is not allowed", e) + + if e.len > 1: + let scratchIdent = genSym(kind = nskTemp, ident = "scratch") + let impl = e[0].getTypeInst + + when defined(nimTasksDebug): + echo impl.treeRepr + echo e.treeRepr + let formalParams = impl[0] + + var + scratchRecList = newNimNode(nnkRecList) + scratchAssignList: seq[NimNode] + tempAssignList: seq[NimNode] + callNode: seq[NimNode] + + let + objTemp = genSym(nskTemp, ident = "objTemp") + + for i in 1 ..< formalParams.len: + var param = formalParams[i][1] + + if param.kind == nnkBracketExpr and param[0].eqIdent("sink"): + param = param[0] + + if param.typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + + case param.kind + of nnkVarTy: + error("'toTask'ed function cannot have a 'var' parameter", e) + of nnkBracketExpr: + if param[0].typeKind == ntyTypeDesc: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: + if param[1].typeKind in {ntyExpr, ntyStmt}: + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) + let + seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) + seqCallNode = newCall("@", e[i]) + addAllNode(seqType, seqCallNode) + else: + addAllNode(param, e[i]) + of nnkBracket, nnkObjConstr: + # passing by static parameters + # so we pass them directly instead of passing by scratchObj + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + of nnkSym, nnkPtrTy, nnkProcTy, nnkTupleConstr: + addAllNode(param, e[i]) + of nnkCharLit..nnkNilLit: + callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) + else: + error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e) + + let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") + let scratchObj = nnkTypeSection.newTree( + nnkTypeDef.newTree( + scratchObjType, + newEmptyNode(), + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + scratchRecList + ) + ) + ) + + + let scratchObjPtrType = quote do: + cast[ptr `scratchObjType`](allocShared0(sizeof(`scratchObjType`))) + + let scratchLetSection = newLetStmt(scratchIdent, scratchObjPtrType) + + var stmtList = newStmtList() + stmtList.add(scratchObj) + stmtList.add(scratchLetSection) + stmtList.add(nnkBlockStmt.newTree(newEmptyNode(), newStmtList(scratchAssignList))) + + var functionStmtList = newStmtList() + let funcCall = newCall(e[0], callNode) + functionStmtList.add tempAssignList + + let funcName = genSym(nskProc, rootSym.strVal) + let destroyName = genSym(nskProc, "destroyScratch") + let objTemp2 = genSym(ident = "obj") + let tempNode = quote("@") do: + `=destroy`(@objTemp2[]) + + var funcDecl: NimNode + if returnsVoid: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + `funcCall` + else: + funcDecl = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + let `objTemp` = cast[ptr `scratchObjType`](args) + `functionStmtList` + cast[ptr `retType`](res)[] = `funcCall` + + result = quote do: + `stmtList` + + `funcDecl` + + proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = + let `objTemp2` = cast[ptr `scratchObjType`](args) + `tempNode` + + Task(callback: `funcName`, args: `scratchIdent`, destroy: `destroyName`) + else: + let funcCall = newCall(e[0]) + let funcName = genSym(nskProc, rootSym.strVal) + + if returnsVoid: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + `funcCall` + + Task(callback: `funcName`, args: nil) + else: + result = quote do: + proc `funcName`(args, res: pointer) {.gcsafe, nimcall.} = + cast[ptr `retType`](res)[] = `funcCall` + + Task(callback: `funcName`, args: nil) + + + when defined(nimTasksDebug): + echo result.repr + +runnableExamples: + block: + var num = 0 + proc hello(a: int) = inc num, a + + let b = toTask hello(13) + b.invoke() + assert num == 13 + # A task can be invoked multiple times + b.invoke() + assert num == 26 + + block: + type + Runnable = ref object + data: int + + var data: int + proc hello(a: Runnable) {.nimcall.} = + a.data += 2 + data = a.data + + + when false: + # the parameters of call must be isolated. + let x = Runnable(data: 12) + let b = toTask hello(x) # error ----> expression cannot be isolated: x + b.invoke() + + let b = toTask(hello(Runnable(data: 12))) + b.invoke() + assert data == 14 + b.invoke() + assert data == 16 diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim new file mode 100644 index 000000000..539305bde --- /dev/null +++ b/lib/std/tempfiles.nim @@ -0,0 +1,192 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module creates temporary files and directories. +## +## Experimental API, subject to change. + +#[ +See also: +* `GetTempFileName` (on windows), refs https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettempfilenamea +* `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html +]# + +import std / [os, random] + +when defined(nimPreviewSlimSystem): + import std/syncio + +const + maxRetry = 10000 + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nimTempPathLength {.intdefine.} = 8 + + +when defined(windows): + import std/winlean + when defined(nimPreviewSlimSystem): + import std/widestrs + + var O_RDWR {.importc: "_O_RDWR", header: "<fcntl.h>".}: cint + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "_fdopen",header: "<stdio.h>".} + + proc open_osfhandle(osh: Handle, mode: cint): cint {. + importc: "_open_osfhandle", header: "<io.h>".} + + proc close_osfandle(fd: cint): cint {. + importc: "_close", header: "<io.h>".} +else: + import std/posix + + proc c_fdopen( + filehandle: cint, + mode: cstring + ): File {.importc: "fdopen",header: "<stdio.h>".} + + +proc safeOpen(filename: string): File = + ## Open files exclusively; returns `nil` if the file already exists. + # xxx this should be clarified; it doesn't in particular prevent other processes + # from opening the file, at least currently. + when defined(windows): + let dwShareMode = FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE + let dwCreation = CREATE_NEW + let dwFlags = FILE_FLAG_BACKUP_SEMANTICS or FILE_ATTRIBUTE_NORMAL + let handle = createFileW(newWideCString(filename), GENERIC_READ or GENERIC_WRITE, dwShareMode, + nil, dwCreation, dwFlags, Handle(0)) + + if handle == INVALID_HANDLE_VALUE: + if getLastError() == ERROR_FILE_EXISTS: + return nil + else: + raiseOSError(osLastError(), filename) + + let fileHandle = open_osfhandle(handle, O_RDWR) + if fileHandle == -1: + discard closeHandle(handle) + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard close_osfandle(fileHandle) + raiseOSError(osLastError(), filename) + else: + # xxx we need a `proc toMode(a: FilePermission): Mode`, possibly by + # exposing fusion/filepermissions.fromFilePermissions to stdlib; then we need + # to expose a `perm` param so users can customize this (e.g. the temp file may + # need execute permissions), and figure out how to make the API cross platform. + let mode = Mode(S_IRUSR or S_IWUSR) + let flags = posix.O_RDWR or posix.O_CREAT or posix.O_EXCL + let fileHandle = posix.open(filename, flags, mode) + if fileHandle == -1: + if errno == EEXIST: + # xxx `getLastError()` should be defined on posix too and resolve to `errno`? + return nil + else: + raiseOSError(osLastError(), filename) + + result = c_fdopen(fileHandle, "w+") + if result == nil: + discard posix.close(fileHandle) # TODO handles failure when closing file + raiseOSError(osLastError(), filename) + + +type + NimTempPathState = object + state: Rand + isInit: bool + +var nimTempPathState {.threadvar.}: NimTempPathState + +template randomPathName(length: Natural): string = + var res = newString(length) + if not nimTempPathState.isInit: + nimTempPathState.isInit = true + nimTempPathState.state = initRand() + + for i in 0 ..< length: + res[i] = nimTempPathState.state.sample(letters) + res + +proc getTempDirImpl(dir: string): string {.inline.} = + result = dir + if result.len == 0: + result = getTempDir() + +proc genTempPath*(prefix, suffix: string, dir = ""): string = + ## Generates a path name in `dir`. + ## + ## The path begins with `prefix` and ends with `suffix`. + ## + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + let dir = getTempDirImpl(dir) + result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) + +proc createTempFile*(prefix, suffix: string, dir = ""): tuple[cfile: File, path: string] = + ## Creates a new temporary file in the directory `dir`. + ## + ## This generates a path name using `genTempPath(prefix, suffix, dir)` and + ## returns a file handle to an open file and the path of that file, possibly after + ## retrying to ensure it doesn't already exist. + ## + ## If failing to create a temporary file, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to close `result.cfile` and + ## remove `result.file` when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempFile("", "", "nonexistent") + let (cfile, path) = createTempFile("tmpprefix_", "_end.tmp") + # path looks like: getTempDir() / "tmpprefix_FDCIRZA0_end.tmp" + cfile.write "foo" + cfile.setFilePos 0 + assert readAll(cfile) == "foo" + close cfile + assert readFile(path) == "foo" + removeFile(path) + # xxx why does above work without `cfile.flushFile` ? + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result.path = genTempPath(prefix, suffix, dir) + result.cfile = safeOpen(result.path) + if result.cfile != nil: + return + + raise newException(OSError, "Failed to create a temporary file under directory " & dir) + +proc createTempDir*(prefix, suffix: string, dir = ""): string = + ## Creates a new temporary directory in the directory `dir`. + ## + ## This generates a dir name using `genTempPath(prefix, suffix, dir)`, creates + ## the directory and returns it, possibly after retrying to ensure it doesn't + ## already exist. + ## + ## If failing to create a temporary directory, `OSError` will be raised. + ## + ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir <os.html#getTempDir>`_). + runnableExamples: + import std/os + doAssertRaises(OSError): discard createTempDir("", "", "nonexistent") + let dir = createTempDir("tmpprefix_", "_end") + # dir looks like: getTempDir() / "tmpprefix_YEl9VuVj_end" + assert dirExists(dir) + removeDir(dir) + let dir = getTempDirImpl(dir) + for i in 0 ..< maxRetry: + result = genTempPath(prefix, suffix, dir) + if not existsOrCreateDir(result): + return + + raise newException(OSError, "Failed to create a temporary directory under directory " & dir) diff --git a/lib/std/time_t.nim b/lib/std/time_t.nim index 37918ee6c..de051b135 100644 --- a/lib/std/time_t.nim +++ b/lib/std/time_t.nim @@ -9,15 +9,15 @@ when defined(nimdoc): type - impl = distinct int64 - Time* = impl ## \ - ## Wrapper for ``time_t``. On posix, this is an alias to ``posix.Time``. + Impl = distinct int64 + Time* = Impl ## \ + ## Wrapper for `time_t`. On posix, this is an alias to `posix.Time`. elif defined(windows): when defined(i386) and defined(gcc): - type Time* {.importc: "time_t", header: "<time.h>".} = distinct int32 + type Time* {.importc: "time_t", header: "<time.h>".} = distinct clong else: # newest version of Visual C++ defines time_t to be of 64 bits type Time* {.importc: "time_t", header: "<time.h>".} = distinct int64 elif defined(posix): - import posix + import std/posix export posix.Time \ No newline at end of file diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim new file mode 100644 index 000000000..7b0b81968 --- /dev/null +++ b/lib/std/typedthreads.nim @@ -0,0 +1,305 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +##[ +Thread support for Nim. Threads allow multiple functions to execute concurrently. + +In Nim, threads are a low-level construct and using a library like `malebolgia`, `taskpools` or `weave` is recommended. + +When creating a thread, you can pass arguments to it. As Nim's garbage collector does not use atomic references, sharing +`ref` and other variables managed by the garbage collector between threads is not supported. +Use global variables to do so, or pointers. + +Memory allocated using [`sharedAlloc`](./system.html#allocShared.t%2CNatural) can be used and shared between threads. + +To communicate between threads, consider using [channels](./system.html#Channel) + +Examples +======== + +```Nim +import std/locks + +var + thr: array[0..4, Thread[tuple[a,b: int]]] + L: Lock + +proc threadFunc(interval: tuple[a,b: int]) {.thread.} = + for i in interval.a..interval.b: + acquire(L) # lock stdout + echo i + release(L) + +initLock(L) + +for i in 0..high(thr): + createThread(thr[i], threadFunc, (i*10, i*10+5)) +joinThreads(thr) + +deinitLock(L) +``` + +When using a memory management strategy that supports shared heaps like `arc` or `boehm`, +you can pass pointer to threads and share memory between them, but the memory must outlive the thread. +The default memory management strategy, `orc`, supports this. +The example below is **not valid** for memory management strategies that use local heaps like `refc`! + +```Nim +import locks + +var l: Lock + +proc threadFunc(obj: ptr seq[int]) {.thread.} = + withLock l: + for i in 0..<100: + obj[].add(obj[].len * obj[].len) + +proc threadHandler() = + var thr: array[0..4, Thread[ptr seq[int]]] + var s = newSeq[int]() + + for i in 0..high(thr): + createThread(thr[i], threadFunc, s.addr) + joinThreads(thr) + echo s + +initLock(l) +threadHandler() +deinitLock(l) +``` +]## + + +import std/private/[threadtypes] +export Thread + +import system/ansi_c + +when defined(nimPreviewSlimSystem): + import std/assertions + +when defined(genode): + import genode/env + +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} + +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize + + +when defined(gcDestructors): + proc allocThreadStorage(size: int): pointer = + result = c_malloc(csize_t size) + zeroMem(result, size) +else: + template allocThreadStorage(size: untyped): untyped = allocShared0(size) + +#const globalsSlot = ThreadVarSlot(0) +#sysAssert checkSlot.int == globalsSlot.int + +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include <pthread.h> + """.} + + +# We jump through some hops here to ensure that Nim thread procs can have +# the Nim calling convention. This is needed because thread procs are +# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just +# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. + + + +{.push stack_trace:off.} +when defined(windows): + proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = + nimThreadProcWrapperBody(closure) + # implicitly return 0 +elif defined(genode): + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + nimThreadProcWrapperBody(closure) +else: + proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = + nimThreadProcWrapperBody(closure) +{.pop.} + +proc running*[TArg](t: Thread[TArg]): bool {.inline.} = + ## Returns true if `t` is running. + result = t.dataFn != nil + +proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = + ## Returns the thread handle of `t`. + result = t.sys + +when hostOS == "windows": + const MAXIMUM_WAIT_OBJECTS = 64 + + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard waitForSingleObject(t.sys, -1'i32) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] + var k = 0 + while k < len(t): + var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) + for i in 0..(count - 1): a[i] = t[i + k].sys + discard waitForMultipleObjects(int32(count), + cast[ptr SysThread](addr(a)), 1, -1) + inc(k, MAXIMUM_WAIT_OBJECTS) + +elif defined(genode): + proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} + ## Waits for the thread `t` to finish. + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +else: + proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = + ## Waits for the thread `t` to finish. + discard pthread_join(t.sys, nil) + + proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = + ## Waits for every thread in `t` to finish. + for i in 0..t.high: joinThread(t[i]) + +when false: + # XXX a thread should really release its heap here somehow: + proc destroyThread*[TArg](t: var Thread[TArg]) = + ## Forces the thread `t` to terminate. This is potentially dangerous if + ## you don't have full control over `t` and its acquired resources. + when hostOS == "windows": + discard TerminateThread(t.sys, 1'i32) + else: + discard pthread_cancel(t.sys) + when declared(registerThread): unregisterThread(addr(t)) + t.dataFn = nil + ## if thread `t` already exited, `t.core` will be `null`. + if not isNil(t.core): + deallocThreadStorage(t.core) + t.core = nil + +when hostOS == "windows": + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. + ## `param` is passed to `tp`. `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var dummyThreadId: int32 + t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], + addr(t), 0'i32, dummyThreadId) + if t.sys <= 0: + raise newException(ResourceExhaustedError, "cannot create thread") + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + setThreadAffinityMask(t.sys, uint(1 shl cpu)) + +elif defined(genode): + var affinityOffset: cuint = 1 + ## CPU affinity offset for next thread, safe to roll-over. + + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.stackSize = ThreadStackSize + t.sys.initThread( + runtimeEnv, + ThreadStackSize.culonglong, + threadProcWrapper[TArg], addr(t), affinityOffset) + inc affinityOffset + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + {.hint: "cannot change Genode thread CPU affinity after initialization".} + discard + +else: + proc createThread*[TArg](t: var Thread[TArg], + tp: proc (arg: TArg) {.thread, nimcall.}, + param: TArg) = + ## Creates a new thread `t` and starts its execution. + ## + ## Entry point is the proc `tp`. `param` is passed to `tp`. + ## `TArg` can be `void` if you + ## don't need to pass any data to the thread. + t.core = cast[PGcThread](allocThreadStorage(sizeof(GcThread))) + + when TArg isnot void: t.data = param + t.dataFn = tp + when hasSharedHeap: t.core.stackSize = ThreadStackSize + var a {.noinit.}: Pthread_attr + doAssert pthread_attr_init(a) == 0 + when hasAllocStack: + var + rawstk = allocThreadStorage(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + + when not defined(ios): + # This fails on iOS + doAssert(setstacksizeResult == 0) + if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: + raise newException(ResourceExhaustedError, "cannot create thread") + doAssert pthread_attr_destroy(a) == 0 + + proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = + ## Pins a thread to a `CPU`:idx:. + ## + ## In other words sets a thread's `affinity`:idx:. + ## If you don't know what this means, you shouldn't use this proc. + when not defined(macosx): + var s {.noinit.}: CpuSet + cpusetZero(s) + cpusetIncl(cpu.cint, s) + setAffinity(t.sys, csize_t(sizeof(s)), s) + +proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = + createThread[void](t, tp) + +when not defined(gcOrc): + include system/threadids diff --git a/lib/std/varints.nim b/lib/std/varints.nim index fe06b478a..32fe2fffb 100644 --- a/lib/std/varints.nim +++ b/lib/std/varints.nim @@ -82,29 +82,29 @@ proc writeVu64*(z: var openArray[byte], x: uint64): int = z[3] = cast[uint8](y) return 4 z[0] = 251 - varintWrite32(toOpenArray(z, 1, z.high-1), y) + varintWrite32(toOpenArray(z, 1, 4), y) return 5 if w <= 255: z[0] = 252 z[1] = cast[uint8](w) - varintWrite32(toOpenArray(z, 2, z.high-2), y) + varintWrite32(toOpenArray(z, 2, 5), y) return 6 if w <= 65535: z[0] = 253 z[1] = cast[uint8](w shr 8) z[2] = cast[uint8](w) - varintWrite32(toOpenArray(z, 3, z.high-3), y) + varintWrite32(toOpenArray(z, 3, 6), y) return 7 if w <= 16777215: z[0] = 254 z[1] = cast[uint8](w shr 16) z[2] = cast[uint8](w shr 8) z[3] = cast[uint8](w) - varintWrite32(toOpenArray(z, 4, z.high-4), y) + varintWrite32(toOpenArray(z, 4, 7), y) return 8 z[0] = 255 - varintWrite32(toOpenArray(z, 1, z.high-1), w) - varintWrite32(toOpenArray(z, 5, z.high-5), y) + varintWrite32(toOpenArray(z, 1, 4), w) + varintWrite32(toOpenArray(z, 5, 8), y) return 9 proc sar(a, b: int64): int64 = @@ -119,36 +119,3 @@ proc encodeZigzag*(x: int64): uint64 {.inline.} = proc decodeZigzag*(x: uint64): int64 {.inline.} = let casted = cast[int64](x) result = (`shr`(casted, 1)) xor (-(casted and 1)) - -when isMainModule: - #import random - - var dest: array[50, byte] - var got: uint64 - - for test in [0xFFFF_FFFF_FFFFF_FFFFu64, 77u64, 0u64, 10_000_000u64, uint64(high(int64)), - uint64(high(int32)),uint64(low(int32)),uint64(low(int64))]: - let wrLen = writeVu64(dest, test) - let rdLen = readVu64(dest, got) - assert wrLen == rdLen - echo(if got == test: "YES" else: "NO") - echo "number is ", got - - if encodeZigzag(decodeZigzag(test)) != test: - echo "Failure for ", test, " ", encodeZigzag(decodeZigzag(test)), " ", decodeZigzag(test) - - for test in 0u64..300u64: - let wrLen = writeVu64(dest, test) - let rdLen = readVu64(dest, got) - assert wrLen == rdLen - if got != test: - echo "BUG! expected: ", test, " got: ", got, " z0: ", dest[0] - - # check this also works for floats: - for test in [0.0, 0.1, 2.0, +Inf, Nan, NegInf]: - let t = cast[uint64](test) - let wrLenB = writeVu64(dest, t) - let rdLenB = readVu64(dest, got) - assert wrLenB == rdLenB - echo rdLenB - echo(if cast[float64](got) == test: "YES" else: "NO") diff --git a/lib/std/vmutils.nim b/lib/std/vmutils.nim new file mode 100644 index 000000000..e16912a3c --- /dev/null +++ b/lib/std/vmutils.nim @@ -0,0 +1,11 @@ +##[ +Experimental API, subject to change. +]## + +proc vmTrace*(on: bool) {.compileTime.} = + runnableExamples: + static: vmTrace(true) + proc fn = + var a = 1 + vmTrace(false) + static: fn() diff --git a/lib/std/widestrs.nim b/lib/std/widestrs.nim new file mode 100644 index 000000000..2ddf80d14 --- /dev/null +++ b/lib/std/widestrs.nim @@ -0,0 +1,239 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nim support for C/C++'s `wide strings`:idx:. + +#when not declared(ThisIsSystem): +# {.error: "You must not import this module explicitly".} + +type + Utf16Char* = distinct int16 + +when not (defined(cpu16) or defined(cpu8)): + when defined(nimv2): + + type + WideCString* = ptr UncheckedArray[Utf16Char] + + WideCStringObj* = object + bytes: int + data: WideCString + + const arcLike = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + when defined(nimAllowNonVarDestructor) and arcLike: + proc `=destroy`(a: WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + else: + proc `=destroy`(a: var WideCStringObj) = + if a.data != nil: + when compileOption("threads"): + deallocShared(a.data) + else: + dealloc(a.data) + + proc `=copy`(a: var WideCStringObj; b: WideCStringObj) {.error.} + + proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = + a.bytes = b.bytes + a.data = b.data + + proc createWide(a: var WideCStringObj; bytes: int) = + a.bytes = bytes + when compileOption("threads"): + a.data = cast[typeof(a.data)](allocShared0(bytes)) + else: + a.data = cast[typeof(a.data)](alloc0(bytes)) + + template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] + template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val + + template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) + + converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = + result = x.data + + else: + template nullWide(): untyped = nil + + type + WideCString* = ref UncheckedArray[Utf16Char] + WideCStringObj* = WideCString + + template createWide(a; L) = + unsafeNew(a, L) + + proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) + + proc len*(w: WideCString): int = + ## returns the length of a widestring. This traverses the whole string to + ## find the binary zero end marker! + result = 0 + while int16(w[result]) != 0'i16: inc result + + const + UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) + UNI_MAX_BMP = 0x0000FFFF + UNI_MAX_UTF16 = 0x0010FFFF + # UNI_MAX_UTF32 = 0x7FFFFFFF + # UNI_MAX_LEGAL_UTF32 = 0x0010FFFF + + halfShift = 10 + halfBase = 0x0010000 + halfMask = 0x3FF + + UNI_SUR_HIGH_START = 0xD800 + UNI_SUR_HIGH_END = 0xDBFF + UNI_SUR_LOW_START = 0xDC00 + UNI_SUR_LOW_END = 0xDFFF + UNI_REPL = 0xFFFD + + template ones(n: untyped): untyped = ((1 shl n)-1) + + template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = + ## Returns the unicode character `s[i]` in `result`. If `doInc == true` + ## `i` is incremented by the number of bytes that have been processed. + bind ones + + if ord(s[i]) <= 127: + result = ord(s[i]) + when doInc: inc(i) + elif ord(s[i]) shr 5 == 0b110: + #assert(ord(s[i+1]) shr 6 == 0b10) + if i <= L - 2: + result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) + when doInc: inc(i, 2) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 4 == 0b1110: + if i <= L - 3: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + result = (ord(s[i]) and ones(4)) shl 12 or + (ord(s[i+1]) and ones(6)) shl 6 or + (ord(s[i+2]) and ones(6)) + when doInc: inc(i, 3) + else: + result = UNI_REPL + when doInc: inc(i) + elif ord(s[i]) shr 3 == 0b11110: + if i <= L - 4: + #assert(ord(s[i+1]) shr 6 == 0b10) + #assert(ord(s[i+2]) shr 6 == 0b10) + #assert(ord(s[i+3]) shr 6 == 0b10) + result = (ord(s[i]) and ones(3)) shl 18 or + (ord(s[i+1]) and ones(6)) shl 12 or + (ord(s[i+2]) and ones(6)) shl 6 or + (ord(s[i+3]) and ones(6)) + when doInc: inc(i, 4) + else: + result = UNI_REPL + when doInc: inc(i) + else: + result = 0xFFFD + when doInc: inc(i) + + iterator runes(s: cstring, L: int): int = + var + i = 0 + result: int + while i < L: + fastRuneAt(s, i, L, result, true) + yield result + + proc newWideCString*(size: int): WideCStringObj = + createWide(result, size * 2 + 2) + + proc newWideCString*(source: cstring, L: int): WideCStringObj = + ## Warning:: `source` needs to be preallocated with the length `L` + createWide(result, L * 2 + 2) + var d = 0 + for ch in runes(source, L): + + if ch <= UNI_MAX_BMP: + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: + result[d] = UNI_REPLACEMENT_CHAR + else: + result[d] = cast[Utf16Char](uint16(ch)) + elif ch > UNI_MAX_UTF16: + result[d] = UNI_REPLACEMENT_CHAR + else: + let ch = ch - halfBase + result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) + inc d + result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) + inc d + result[d] = Utf16Char(0) + + proc newWideCString*(s: cstring): WideCStringObj = + if s.isNil: return nullWide + + result = newWideCString(s, s.len) + + proc newWideCString*(s: string): WideCStringObj = + result = newWideCString(cstring s, s.len) + + proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = + result = newStringOfCap(estimate + estimate shr 2) + + var i = 0 + while w[i].int16 != 0'i16: + var ch = ord(w[i]) + inc i + if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: + # If the 16 bits following the high surrogate are in the source buffer... + let ch2 = ord(w[i]) + + # If it's a low surrogate, convert to UTF32: + if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: + ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase + inc i + else: + #invalid UTF-16 + ch = replacement + elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: + #invalid UTF-16 + ch = replacement + + if ch < 0x80: + result.add chr(ch) + elif ch < 0x800: + result.add chr((ch shr 6) or 0xc0) + result.add chr((ch and 0x3f) or 0x80) + elif ch < 0x10000: + result.add chr((ch shr 12) or 0xe0) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + elif ch <= 0x10FFFF: + result.add chr((ch shr 18) or 0xf0) + result.add chr(((ch shr 12) and 0x3f) or 0x80) + result.add chr(((ch shr 6) and 0x3f) or 0x80) + result.add chr((ch and 0x3f) or 0x80) + else: + # replacement char(in case user give very large number): + result.add chr(0xFFFD shr 12 or 0b1110_0000) + result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) + result.add chr(0xFFFD and ones(6) or 0b10_0000_00) + + proc `$`*(s: WideCString): string = + result = s $ 80 + + when defined(nimv2): + proc `$`*(s: WideCStringObj, estimate: int, replacement: int = 0xFFFD): string = + `$`(s.data, estimate, replacement) + + proc `$`*(s: WideCStringObj): string = + $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/lib/std/with.nim b/lib/std/with.nim new file mode 100644 index 000000000..c2eaa4bef --- /dev/null +++ b/lib/std/with.nim @@ -0,0 +1,48 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the `with` macro for easy +## function chaining. See https://github.com/nim-lang/RFCs/issues/193 +## and https://github.com/nim-lang/RFCs/issues/192 for details leading to this +## particular design. +## +## **Since:** version 1.2. + +import std/[macros, private / underscored_calls] + +macro with*(arg: typed; calls: varargs[untyped]): untyped = + ## This macro provides `chaining`:idx: of function calls. + ## It does so by patching every call in `calls` to + ## use `arg` as the first argument. + ## + ## .. caution:: This evaluates `arg` multiple times! + runnableExamples: + var x = "yay" + with x: + add "abc" + add "efg" + doAssert x == "yayabcefg" + + var a = 44 + with a: + += 4 + -= 5 + doAssert a == 43 + + # Nesting works for object types too! + var foo = (bar: 1, qux: (baz: 2)) + with foo: + bar = 2 + with qux: + baz = 3 + doAssert foo.bar == 2 + doAssert foo.qux.baz == 3 + + result = newNimNode(nnkStmtList, arg) + underscoredCalls(result, calls, arg) diff --git a/lib/std/wordwrap.nim b/lib/std/wordwrap.nim index 4b0dc4417..9333f880b 100644 --- a/lib/std/wordwrap.nim +++ b/lib/std/wordwrap.nim @@ -9,12 +9,12 @@ ## This module contains an algorithm to wordwrap a Unicode string. -import strutils, unicode +import std/[strutils, unicode] -proc olen(s: string): int = - var i = 0 +proc olen(s: string; start, lastExclusive: int): int = + var i = start result = 0 - while i < s.len: + while i < lastExclusive: inc result let L = graphemeLen(s, i) inc i, L @@ -32,64 +32,43 @@ proc wrapWords*(s: string, maxLineWidth = 80, result = newStringOfCap(s.len + s.len shr 6) var spaceLeft = maxLineWidth var lastSep = "" - for word, isSep in tokenize(s, seps): - let wlen = olen(word) + + var i = 0 + while true: + var j = i + let isSep = j < s.len and s[j] in seps + while j < s.len and (s[j] in seps) == isSep: inc(j) + if j <= i: break + #yield (substr(s, i, j-1), isSep) if isSep: - lastSep = word - spaceLeft = spaceLeft - wlen - elif wlen > spaceLeft: - if splitLongWords and wlen > maxLineWidth: - var i = 0 - while i < word.len: - if spaceLeft <= 0: - spaceLeft = maxLineWidth - result.add newLine - dec spaceLeft - let L = graphemeLen(word, i) - for j in 0 ..< L: result.add word[i+j] - inc i, L + lastSep.setLen 0 + for k in i..<j: + if s[k] notin {'\L', '\C'}: lastSep.add s[k] + if lastSep.len == 0: + lastSep.add ' ' + dec spaceLeft else: - spaceLeft = maxLineWidth - wlen - result.add(newLine) - result.add(word) + spaceLeft = spaceLeft - olen(lastSep, 0, lastSep.len) else: - spaceLeft = spaceLeft - wlen - result.add(lastSep) - result.add(word) - lastSep.setLen(0) - -when isMainModule: - - when true: - let - inp = """ this is a long text -- muchlongerthan10chars and here - it goes""" - outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes" - doAssert wrapWords(inp, 10, false) == outp - - let - longInp = """ThisIsOneVeryLongStringWhichWeWillSplitIntoEightSeparatePartsNow""" - longOutp = "ThisIsOn\neVeryLon\ngStringW\nhichWeWi\nllSplitI\nntoEight\nSeparate\nPartsNow" - doAssert wrapWords(longInp, 8, true) == longOutp - - # test we don't break Umlauts into invalid bytes: - let fies = "äöüöäöüöäöüöäöüööäöüöäößßßßüöäößßßßßß" - let fiesRes = "ä\nö\nü\nö\nä\nö\nü\nö\nä\nö\nü\nö\nä\nö\nü\nö\nö\nä\nö\nü\nö\nä\nö\nß\nß\nß\nß\nü\nö\nä\nö\nß\nß\nß\nß\nß\nß" - doAssert wrapWords(fies, 1, true) == fiesRes - - let longlongword = """abc uitdaeröägfßhydüäpydqfü,träpydqgpmüdträpydföägpydörztdüöäfguiaeowäzjdtrüöäp psnrtuiydrözenrüöäpyfdqazpesnrtulocjtüö -äzydgyqgfqfgprtnwjlcydkqgfüöezmäzydydqüüöäpdtrnvwfhgckdumböäpydfgtdgfhtdrntdrntydfogiayqfguiatrnydrntüöärtniaoeydfgaoeiqfglwcßqfgxvlcwgtfhiaoen -rsüöäapmböäptdrniaoydfglckqfhouenrtsüöäptrniaoeyqfgulocfqclgwxßqflgcwßqfxglcwrniatrnmüböäpmöäbpümöäbpüöämpbaoestnriaesnrtdiaesrtdniaesdrtnaetdr -iaoenvlcyfglwckßqfgvwkßqgfvlwkßqfgvlwckßqvlwkgfUIαοιαοιαχολωχσωχνωκψρχκψρτιεαοσηζϵηζιοεννκεωνιαλωσωκνκψρκγτφγτχκγτεκργτιχνκιωχσιλωσλωχξλξλξωχωχ -ξχλωωχαοεοιαεοαεοιαεοαεοιαοεσναοεκνρκψγκψφϵιηαααοε""" - let longlongwordRes = """ -abc uitdaeröägfßhydüäpydqfü,träpydqgpmüdträpydföägpydörztdüöäfguiaeowäzjdtrüöäp -psnrtuiydrözenrüöäpyfdqazpesnrtulocjtüöäzydgyqgfqfgprtnwjlcydkqgfüöezmäzydydqüü -öäpdtrnvwfhgckdumböäpydfgtdgfhtdrntdrntydfogiayqfguiatrnydrntüöärtniaoeydfgaoeiq -fglwcßqfgxvlcwgtfhiaoenrsüöäapmböäptdrniaoydfglckqfhouenrtsüöäptrniaoeyqfgulocf -qclgwxßqflgcwßqfxglcwrniatrnmüböäpmöäbpümöäbpüöämpbaoestnriaesnrtdiaesrtdniaesdr -tnaetdriaoenvlcyfglwckßqfgvwkßqgfvlwkßqfgvlwckßqvlwkgfUIαοιαοιαχολωχσωχνωκψρχκψ -ρτιεαοσηζϵηζιοεννκεωνιαλωσωκνκψρκγτφγτχκγτεκργτιχνκιωχσιλωσλωχξλξλξωχωχ -ξχλωωχαοεοιαεοαεοιαεοαεοιαοεσναοεκνρκψγκψφϵιηαααοε""" - doAssert wrapWords(longlongword) == longlongwordRes - + let wlen = olen(s, i, j) + if wlen > spaceLeft: + if splitLongWords and wlen > maxLineWidth: + var k = 0 + while k < j - i: + if spaceLeft <= 0: + spaceLeft = maxLineWidth + result.add newLine + dec spaceLeft + let L = graphemeLen(s, k+i) + for m in 0 ..< L: result.add s[i+k+m] + inc k, L + else: + spaceLeft = maxLineWidth - wlen + result.add(newLine) + for k in i..<j: result.add(s[k]) + else: + spaceLeft = spaceLeft - wlen + result.add(lastSep) + for k in i..<j: result.add(s[k]) + #lastSep.setLen(0) + i = j diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index b9eb70790..0b75c270e 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -1,9 +1,19 @@ -## This module allows chains of field-access and indexing where the LHS can be nil. -## This simplifies code by reducing need for if-else branches around intermediate values -## that maybe be nil. +## This module allows evaluating expressions safely against the following conditions: +## * nil dereferences +## * field accesses with incorrect discriminant in case objects ## -## Note: experimental module and relies on {.experimental: "dotOperators".} -## Unstable API. +## `default(T)` is returned in those cases when evaluating an expression of type `T`. +## This simplifies code by reducing need for if-else branches. +## +## Note: experimental module, unstable API. + +#[ +TODO: +consider handling indexing operations, eg: +doAssert ?.default(seq[int])[3] == default(int) +]# + +import std/macros runnableExamples: type Foo = ref object @@ -19,91 +29,165 @@ runnableExamples: assert ?.f2.x1 == "a" # same as f2.x1 (no nil LHS in this chain) assert ?.Foo(x1: "a").x1 == "a" # can use constructor inside - # when you know a sub-expression is not nil, you can scope it as follows: - assert ?.(f2.x2.x2).x3[] == 0 # because `f` is nil + # when you know a sub-expression doesn't involve a `nil` (e.g. `f2.x2.x2`), + # you can scope it as follows: + assert ?.(f2.x2.x2).x3[] == 0 -type Wrapnil[T] = object - valueImpl: T - validImpl: bool - -proc wrapnil[T](a: T): Wrapnil[T] = - ## See top-level example. - Wrapnil[T](valueImpl: a, validImpl: true) - -template unwrap(a: Wrapnil): untyped = - ## See top-level example. - a.valueImpl + assert (?.f2.x2.x2).x3 == nil # this terminates ?. early -{.push experimental: "dotOperators".} +runnableExamples: + # ?. also allows case object + type B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + var b = B(cond: false, b0: 3) + doAssertRaises(FieldDefect): discard b.b1 # wrong discriminant + doAssert ?.b.b1 == 0.0 # safe + b = B(cond: true, b1: 4.5) + doAssert ?.b.b1 == 4.5 + + # lvalue semantics are preserved: + if (let p = ?.b.b1.addr; p != nil): p[] = 4.7 + doAssert b.b1 == 4.7 + +proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = + if level == 0: + result = quote: `lhs` = `n` + else: + result = quote: (let `lhs` = `n`) + +proc process(n: NimNode, lhs: NimNode, label: NimNode, level: int): NimNode = + var n = n.copyNimTree + var it = n + let addr2 = bindSym"addr" + var old: tuple[n: NimNode, index: int] + while true: + if it.len == 0: + result = finalize(n, lhs, level) + break + elif it.kind == nnkCheckedFieldExpr: + let dot = it[0] + let obj = dot[0] + let objRef = quote do: `addr2`(`obj`) + # avoids a copy and preserves lvalue semantics, see tests + let check = it[1] + let okSet = check[1] + let kind1 = check[2] + let tmp = genSym(nskLet, "tmpCase") + let body = process(objRef, tmp, label, level + 1) + let tmp3 = nnkDerefExpr.newTree(tmp) + it[0][0] = tmp3 + let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) + if old.n != nil: old.n[old.index] = dot2 + else: n = dot2 + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp3`.`kind1` notin `okSet`: break `label` + `assgn` + break + elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: + let tmp = genSym(nskLet, "tmp") + let body = process(it[0], tmp, label, level + 1) + it[0] = tmp + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp` == nil: break `label` + `assgn` + break + elif it.kind == nnkCall: # consider extending to `nnkCallKinds` + # `copyNimTree` needed to avoid `typ = nil` issues + old = (it, 1) + it = it[1].copyNimTree + else: + old = (it, 0) + it = it[0] -template `.`*(a: Wrapnil, b): untyped = +macro `?.`*(a: typed): auto = + ## Transforms `a` into an expression that can be safely evaluated even in + ## presence of intermediate nil pointers/references, in which case a default + ## value is produced. + let lhs = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs, label, 0) + result = quote do: + var `lhs`: type(`a`) + block `label`: + `body` + `lhs` + +# the code below is not needed for `?.` +from std/options import Option, isSome, get, option, unsafeGet, UnpackDefect + +macro `??.`*(a: typed): Option = + ## Same as `?.` but returns an `Option`. + runnableExamples: + import std/options + type Foo = ref object + x1: ref int + x2: int + # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: + var f1 = Foo(x1: int.new, x2: 2) + doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. + doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain + doAssert (??.f1.x2).get == 2 + + var f2: Foo + doAssert not (??.f2.x1[]).isSome # f2 was nil + + doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get + doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) + + let lhs = genSym(nskVar, "lhs") + let lhs2 = genSym(nskVar, "lhs") + let label = genSym(nskLabel, "label") + let body = process(a, lhs2, label, 0) + result = quote do: + var `lhs`: Option[type(`a`)] + block `label`: + var `lhs2`: type(`a`) + `body` + `lhs` = option(`lhs2`) + `lhs` + +template fakeDot*(a: Option, b): untyped = ## See top-level example. let a1 = a # to avoid double evaluations - let a2 = a1.valueImpl - type T = Wrapnil[type(a2.b)] - if a1.validImpl: - when type(a2) is ref|ptr: + type T = Option[typeof(unsafeGet(a1).b)] + if isSome(a1): + let a2 = unsafeGet(a1) + when typeof(a2) is ref|ptr: if a2 == nil: default(T) else: - wrapnil(a2.b) + option(a2.b) else: - wrapnil(a2.b) + option(a2.b) else: # nil is "sticky"; this is needed, see tests default(T) -{.pop.} +# xxx this should but doesn't work: func `[]`*[T, I](a: Option[T], i: I): Option {.inline.} = -proc isValid(a: Wrapnil): bool = - ## Returns true if `a` didn't contain intermediate `nil` values (note that - ## `a.valueImpl` itself can be nil even in that case) - a.validImpl - -template `[]`*[I](a: Wrapnil, i: I): untyped = +func `[]`*[T, I](a: Option[T], i: I): auto {.inline.} = ## See top-level example. - let a1 = a # to avoid double evaluations - if a1.validImpl: - # correctly will raise IndexError if a is valid but wraps an empty container - wrapnil(a1.valueImpl[i]) - else: - default(Wrapnil[type(a1.valueImpl[i])]) + if isSome(a): + # correctly will raise IndexDefect if a is valid but wraps an empty container + result = option(a.unsafeGet[i]) -template `[]`*(a: Wrapnil): untyped = +func `[]`*[U](a: Option[U]): auto {.inline.} = ## See top-level example. - let a1 = a # to avoid double evaluations - let a2 = a1.valueImpl - type T = Wrapnil[type(a2[])] - if a1.validImpl: - if a2 == nil: - default(T) - else: - wrapnil(a2[]) - else: - default(T) - -import std/macros - -proc replace(n: NimNode): NimNode = - if n.kind == nnkPar: - doAssert n.len == 1 - newCall(bindSym"wrapnil", n[0]) - elif n.kind in {nnkCall, nnkObjConstr}: - newCall(bindSym"wrapnil", n) - elif n.len == 0: - newCall(bindSym"wrapnil", n) - else: - n[0] = replace(n[0]) - n - -macro `?.`*(a: untyped): untyped = - ## Transforms `a` into an expression that can be safely evaluated even in - ## presence of intermediate nil pointers/references, in which case a default - ## value is produced. - #[ - Using a template like this wouldn't work: - template `?.`*(a: untyped): untyped = wrapnil(a)[] - ]# - result = replace(a) - result = quote do: - `result`.valueImpl + if isSome(a): + let a2 = a.unsafeGet + if a2 != nil: + result = option(a2[]) + +when false: + # xxx: expose a way to do this directly in std/options, e.g.: `getAsIs` + proc safeGet[T](a: Option[T]): T {.inline.} = + get(a, default(T)) diff --git a/lib/system.nim b/lib/system.nim index 92470fbce..2f9cdc5f9 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -14,7 +14,7 @@ ## ## Each module implicitly imports the System module; it must not be listed ## explicitly. Because of this there cannot be a user-defined module named -## ``system``. +## `system`. ## ## System module ## ============= @@ -22,40 +22,59 @@ ## .. include:: ./system_overview.rst -type - float* {.magic: Float.} ## Default floating point type. - float32* {.magic: Float32.} ## 32 bit floating point type. - float64* {.magic: Float.} ## 64 bit floating point type. +include "system/basic_types" -# 'float64' is now an alias to 'float'; this solves many problems +func zeroDefault*[T](_: typedesc[T]): T {.magic: "ZeroDefault".} = + ## Returns the binary zeros representation of the type `T`. It ignores + ## default fields of an object. + ## + ## See also: + ## * `default <#default,typedesc[T]>`_ -type - char* {.magic: Char.} ## Built-in 8 bit character type (unsigned). - string* {.magic: String.} ## Built-in string type. - cstring* {.magic: Cstring.} ## Built-in cstring (*compatible string*) type. - pointer* {.magic: Pointer.} ## Built-in pointer type, use the ``addr`` - ## operator to get a pointer to a variable. +include "system/compilation" - typedesc* {.magic: TypeDesc.} ## Meta type to denote a type description. +{.push warning[GcMem]: off, warning[Uninit]: off.} +# {.push hints: off.} type - `ptr`*[T] {.magic: Pointer.} ## Built-in generic untraced pointer type. - `ref`*[T] {.magic: Pointer.} ## Built-in generic traced pointer type. - - `nil` {.magic: "Nil".} - - void* {.magic: "VoidType".} ## Meta type to denote the absence of any type. - auto* {.magic: Expr.} ## Meta type for automatic type determination. - any* = distinct auto ## Meta type for any supported type. - untyped* {.magic: Expr.} ## Meta type to denote an expression that - ## is not resolved (for templates). - typed* {.magic: Stmt.} ## Meta type to denote an expression that - ## is resolved (for templates). + `static`*[T] {.magic: "Static".} + ## Meta type representing all values that can be evaluated at compile-time. + ## + ## The type coercion `static(x)` can be used to force the compile-time + ## evaluation of the given expression `x`. -include "system/basic_types" + `type`*[T] {.magic: "Type".} + ## Meta type representing the type of all type values. + ## + ## The coercion `type(x)` can be used to obtain the type of the given + ## expression `x`. -{.push warning[GcMem]: off, warning[Uninit]: off.} -{.push hints: off.} +type + TypeOfMode* = enum ## Possible modes of `typeof`. + typeOfProc, ## Prefer the interpretation that means `x` is a proc call. + typeOfIter ## Prefer the interpretation that means `x` is an iterator call. + +proc typeof*(x: untyped; mode = typeOfIter): typedesc {. + magic: "TypeOf", noSideEffect, compileTime.} = + ## Builtin `typeof` operation for accessing the type of an expression. + ## Since version 0.20.0. + runnableExamples: + proc myFoo(): float = 0.0 + iterator myFoo(): string = yield "abc" + iterator myFoo2(): string = yield "abc" + iterator myFoo3(): string {.closure.} = yield "abc" + doAssert type(myFoo()) is string + doAssert typeof(myFoo()) is string + doAssert typeof(myFoo(), typeOfIter) is string + doAssert typeof(myFoo3) is iterator + + doAssert typeof(myFoo(), typeOfProc) is float + doAssert typeof(0.0, typeOfProc) is float + doAssert typeof(myFoo3, typeOfProc) is iterator + doAssert not compiles(typeof(myFoo2(), typeOfProc)) + # this would give: Error: attempting to call routine: 'myFoo2' + # since `typeOfProc` expects a typed expression and `myFoo2()` can + # only be used in a `for` context. proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} ## Constructs an `or` meta class. @@ -66,164 +85,85 @@ proc `and`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} proc `not`*(a: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} ## Constructs an `not` meta class. +when defined(nimHasIterable): + type + iterable*[T] {.magic: IterableType.} ## Represents an expression that yields `T` type - SomeFloat* = float|float32|float64 - ## Type class matching all floating point number types. - - SomeNumber* = SomeInteger|SomeFloat - ## Type class matching all number types. + Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, + ## bool, character, and enumeration types + ## as well as their subtypes. See also + ## `SomeOrdinal`. -proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## defined. - ## - ## `x` is an external symbol introduced through the compiler's - ## `-d:x switch <nimc.html#compiler-usage-compile-time-symbols>`_ to enable - ## build time conditionals: - ## - ## .. code-block:: Nim - ## when not defined(release): - ## # Do here programmer friendly expensive sanity checks. - ## # Put here the normal code - -when defined(nimHasRunnableExamples): - proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".} - ## A section you should use to mark `runnable example`:idx: code with. - ## - ## - In normal debug and release builds code within - ## a ``runnableExamples`` section is ignored. - ## - The documentation generator is aware of these examples and considers them - ## part of the ``##`` doc comment. As the last step of documentation - ## generation each runnableExample is put in its own file ``$file_examples$i.nim``, - ## compiled and tested. The collected examples are - ## put into their own module to ensure the examples do not refer to - ## non-exported symbols. - ## - ## Usage: - ## - ## .. code-block:: Nim - ## proc double*(x: int): int = - ## ## This proc doubles a number. - ## runnableExamples: - ## ## at module scope - ## assert double(5) == 10 - ## block: ## at block scope - ## defer: echo "done" - ## - ## result = 2 * x -else: - template runnableExamples*(body: untyped) = - discard -proc declared*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared. `x` has to be an identifier or a qualified identifier. - ## - ## See also: - ## * `declaredInScope <#declaredInScope,untyped>`_ +proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = + ## Builtin `addr` operator for taking the address of a memory location. ## - ## This can be used to check whether a library provides a certain - ## feature or not: + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. ## - ## .. code-block:: Nim - ## when not declared(strutils.toUpper): - ## # provide our own toUpper proc here, because strutils is - ## # missing it. - -proc declaredInScope*(x: untyped): bool {. - magic: "DefinedInScope", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared in the current scope. `x` has to be an identifier. - -proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = - ## Builtin `addr` operator for taking the address of a memory location. ## Cannot be overloaded. ## - ## See also: - ## * `unsafeAddr <#unsafeAddr,T>`_ - ## - ## .. code-block:: Nim - ## var - ## buf: seq[char] = @['a','b','c'] - ## p = buf[1].addr - ## echo p.repr # ref 0x7faa35c40059 --> 'b' - ## echo p[] # b + ## ```nim + ## var + ## buf: seq[char] = @['a','b','c'] + ## p = buf[1].addr + ## echo p.repr # ref 0x7faa35c40059 --> 'b' + ## echo p[] # b + ## ``` discard proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = - ## Builtin `addr` operator for taking the address of a memory - ## location. This works even for ``let`` variables or parameters - ## for better interop with C and so it is considered even more - ## unsafe than the ordinary `addr <#addr,T>`_. - ## - ## **Note**: When you use it to write a wrapper for a C library, you should - ## always check that the original library does never write to data behind the - ## pointer that is returned from this procedure. - ## - ## Cannot be overloaded. + ## .. warning:: `unsafeAddr` is a deprecated alias for `addr`, + ## use `addr` instead. discard -when defined(nimNewTypedesc): - type - `static`*[T] {.magic: "Static".} - ## Meta type representing all values that can be evaluated at compile-time. - ## - ## The type coercion ``static(x)`` can be used to force the compile-time - ## evaluation of the given expression ``x``. - - `type`*[T] {.magic: "Type".} - ## Meta type representing the type of all type values. - ## - ## The coercion ``type(x)`` can be used to obtain the type of the given - ## expression ``x``. -else: - proc `type`*(x: untyped): typedesc {.magic: "TypeOf", noSideEffect, compileTime.} = - ## Builtin `type` operator for accessing the type of an expression. - ## Cannot be overloaded. - discard - -when defined(nimHasTypeof): - type - TypeOfMode* = enum ## Possible modes of `typeof`. - typeOfProc, ## Prefer the interpretation that means `x` is a proc call. - typeOfIter ## Prefer the interpretation that means `x` is an iterator call. - - proc typeof*(x: untyped; mode = typeOfIter): typedesc {. - magic: "TypeOf", noSideEffect, compileTime.} = - ## Builtin `typeof` operation for accessing the type of an expression. - ## Since version 0.20.0. - discard - const ThisIsSystem = true -proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.} - ## Leaked implementation detail. Do not use. +proc new*[T](a: var ref T, finalizer: proc (x: ref T) {.nimcall.}) {. + magic: "NewFinalize", noSideEffect.} + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. + ## + ## When the garbage collector frees the object, `finalizer` is called. + ## The `finalizer` may not keep a reference to the + ## object pointed to by `x`. The `finalizer` cannot prevent the GC from + ## freeing the object. + ## + ## **Note**: The `finalizer` refers to the type `T`, not to the object! + ## This means that for each object of type `T` the finalizer will be called! -when true: - proc new*[T](a: var ref T, finalizer: proc (x: ref T) {.nimcall.}) {. - magic: "NewFinalize", noSideEffect.} - ## Creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. - ## - ## When the garbage collector frees the object, `finalizer` is called. - ## The `finalizer` may not keep a reference to the - ## object pointed to by `x`. The `finalizer` cannot prevent the GC from - ## freeing the object. - ## - ## **Note**: The `finalizer` refers to the type `T`, not to the object! - ## This means that for each object of type `T` the finalizer will be called! +proc `=wasMoved`*[T](obj: var T) {.magic: "WasMoved", noSideEffect.} = + ## Generic `wasMoved`:idx: implementation that can be overridden. -proc wasMoved*[T](obj: var T) {.magic: "WasMoved", noSideEffect.} = +proc wasMoved*[T](obj: var T) {.inline, noSideEffect.} = ## Resets an object `obj` to its initial (binary zero) value to signify ## it was "moved" and to signify its destructor should do nothing and ## ideally be optimized away. - discard + {.cast(raises: []), cast(tags: []).}: + `=wasMoved`(obj) proc move*[T](x: var T): T {.magic: "Move", noSideEffect.} = result = x - wasMoved(x) + {.cast(raises: []), cast(tags: []).}: + `=wasMoved`(x) + +when defined(nimHasEnsureMove): + proc ensureMove*[T](x: T): T {.magic: "EnsureMove", noSideEffect.} = + ## Ensures that `x` is moved to the new location, otherwise it gives + ## an error at the compile time. + runnableExamples: + proc foo = + var x = "Hello" + let y = ensureMove(x) + doAssert y == "Hello" + foo() + discard "implemented in injectdestructors" type range*[T]{.magic: "Range".} ## Generic type to construct range types. @@ -237,72 +177,73 @@ type seq*[T]{.magic: "Seq".} ## Generic type to construct sequences. set*[T]{.magic: "Set".} ## Generic type to construct bit sets. -when defined(nimUncheckedArrayTyp): - type - UncheckedArray*[T]{.magic: "UncheckedArray".} - ## Array with no bounds checking. -else: - type - UncheckedArray*[T]{.unchecked.} = array[0,T] - ## Array with no bounds checking. +type + UncheckedArray*[T]{.magic: "UncheckedArray".} + ## Array with no bounds checking. type sink*[T]{.magic: "BuiltinType".} type lent*[T]{.magic: "BuiltinType".} -proc high*[T: Ordinal|enum|range](x: T): T {.magic: "High", noSideEffect.} +proc high*[T: Ordinal|enum|range](x: T): T {.magic: "High", noSideEffect, + deprecated: "Deprecated since v1.4; there should not be `high(value)`. Use `high(type)`.".} ## Returns the highest possible value of an ordinal value `x`. ## ## As a special semantic rule, `x` may also be a type identifier. ## - ## See also: - ## * `low(T) <#low,T>`_ + ## **This proc is deprecated**, use this one instead: + ## * `high(typedesc) <#high,typedesc[T]>`_ ## - ## .. code-block:: Nim - ## high(2) # => 9223372036854775807 + ## ```nim + ## high(2) # => 9223372036854775807 + ## ``` proc high*[T: Ordinal|enum|range](x: typedesc[T]): T {.magic: "High", noSideEffect.} ## Returns the highest possible value of an ordinal or enum type. ## - ## ``high(int)`` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:. + ## `high(int)` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:. + ## ```nim + ## high(int) # => 9223372036854775807 + ## ``` ## ## See also: ## * `low(typedesc) <#low,typedesc[T]>`_ - ## - ## .. code-block:: Nim - ## high(int) # => 9223372036854775807 proc high*[T](x: openArray[T]): int {.magic: "High", noSideEffect.} ## Returns the highest possible index of a sequence `x`. + ## ```nim + ## var s = @[1, 2, 3, 4, 5, 6, 7] + ## high(s) # => 6 + ## for i in low(s)..high(s): + ## echo s[i] + ## ``` ## ## See also: ## * `low(openArray) <#low,openArray[T]>`_ - ## - ## .. code-block:: Nim - ## var s = @[1, 2, 3, 4, 5, 6, 7] - ## high(s) # => 6 - ## for i in low(s)..high(s): - ## echo s[i] proc high*[I, T](x: array[I, T]): I {.magic: "High", noSideEffect.} ## Returns the highest possible index of an array `x`. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## var arr = [1, 2, 3, 4, 5, 6, 7] + ## high(arr) # => 6 + ## for i in low(arr)..high(arr): + ## echo arr[i] + ## ``` + ## ## See also: ## * `low(array) <#low,array[I,T]>`_ - ## - ## .. code-block:: Nim - ## var arr = [1, 2, 3, 4, 5, 6, 7] - ## high(arr) # => 6 - ## for i in low(arr)..high(arr): - ## echo arr[i] proc high*[I, T](x: typedesc[array[I, T]]): I {.magic: "High", noSideEffect.} ## Returns the highest possible index of an array type. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## high(array[7, int]) # => 6 + ## ``` + ## ## See also: ## * `low(typedesc[array]) <#low,typedesc[array[I,T]]>`_ - ## - ## .. code-block:: Nim - ## high(array[7, int]) # => 6 proc high*(x: cstring): int {.magic: "High", noSideEffect.} ## Returns the highest possible index of a compatible string `x`. @@ -313,67 +254,73 @@ proc high*(x: cstring): int {.magic: "High", noSideEffect.} proc high*(x: string): int {.magic: "High", noSideEffect.} ## Returns the highest possible index of a string `x`. + ## ```nim + ## var str = "Hello world!" + ## high(str) # => 11 + ## ``` ## ## See also: ## * `low(string) <#low,string>`_ - ## - ## .. code-block:: Nim - ## var str = "Hello world!" - ## high(str) # => 11 -proc low*[T: Ordinal|enum|range](x: T): T {.magic: "Low", noSideEffect.} +proc low*[T: Ordinal|enum|range](x: T): T {.magic: "Low", noSideEffect, + deprecated: "Deprecated since v1.4; there should not be `low(value)`. Use `low(type)`.".} ## Returns the lowest possible value of an ordinal value `x`. As a special ## semantic rule, `x` may also be a type identifier. ## - ## See also: - ## * `high(T) <#high,T>`_ + ## **This proc is deprecated**, use this one instead: + ## * `low(typedesc) <#low,typedesc[T]>`_ ## - ## .. code-block:: Nim - ## low(2) # => -9223372036854775808 + ## ```nim + ## low(2) # => -9223372036854775808 + ## ``` proc low*[T: Ordinal|enum|range](x: typedesc[T]): T {.magic: "Low", noSideEffect.} ## Returns the lowest possible value of an ordinal or enum type. ## - ## ``low(int)`` is Nim's way of writing `INT_MIN`:idx: or `MIN_INT`:idx:. + ## `low(int)` is Nim's way of writing `INT_MIN`:idx: or `MIN_INT`:idx:. + ## ```nim + ## low(int) # => -9223372036854775808 + ## ``` ## ## See also: ## * `high(typedesc) <#high,typedesc[T]>`_ - ## - ## .. code-block:: Nim - ## low(int) # => -9223372036854775808 proc low*[T](x: openArray[T]): int {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of a sequence `x`. + ## ```nim + ## var s = @[1, 2, 3, 4, 5, 6, 7] + ## low(s) # => 0 + ## for i in low(s)..high(s): + ## echo s[i] + ## ``` ## ## See also: ## * `high(openArray) <#high,openArray[T]>`_ - ## - ## .. code-block:: Nim - ## var s = @[1, 2, 3, 4, 5, 6, 7] - ## low(s) # => 0 - ## for i in low(s)..high(s): - ## echo s[i] proc low*[I, T](x: array[I, T]): I {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of an array `x`. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## var arr = [1, 2, 3, 4, 5, 6, 7] + ## low(arr) # => 0 + ## for i in low(arr)..high(arr): + ## echo arr[i] + ## ``` + ## ## See also: ## * `high(array) <#high,array[I,T]>`_ - ## - ## .. code-block:: Nim - ## var arr = [1, 2, 3, 4, 5, 6, 7] - ## low(arr) # => 0 - ## for i in low(arr)..high(arr): - ## echo arr[i] proc low*[I, T](x: typedesc[array[I, T]]): I {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of an array type. ## + ## For empty arrays, the return type is `int`. + ## ```nim + ## low(array[7, int]) # => 0 + ## ``` + ## ## See also: ## * `high(typedesc[array]) <#high,typedesc[array[I,T]]>`_ - ## - ## .. code-block:: Nim - ## low(array[7, int]) # => 0 proc low*(x: cstring): int {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of a compatible string `x`. @@ -383,98 +330,120 @@ proc low*(x: cstring): int {.magic: "Low", noSideEffect.} proc low*(x: string): int {.magic: "Low", noSideEffect.} ## Returns the lowest possible index of a string `x`. + ## ```nim + ## var str = "Hello world!" + ## low(str) # => 0 + ## ``` ## ## See also: ## * `high(string) <#high,string>`_ - ## - ## .. code-block:: Nim - ## var str = "Hello world!" - ## low(str) # => 0 -proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} - ## Use this instead of `=` for a `shallow copy`:idx:. - ## - ## The shallow copy only changes the semantics for sequences and strings - ## (and types which contain those). - ## - ## Be careful with the changed semantics though! - ## There is a reason why the default assignment does a deep copy of sequences - ## and strings. +when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} + ## Use this instead of `=` for a `shallow copy`:idx:. + ## + ## The shallow copy only changes the semantics for sequences and strings + ## (and types which contain those). + ## + ## Be careful with the changed semantics though! + ## There is a reason why the default assignment does a deep copy of sequences + ## and strings. + +# :array|openArray|string|seq|cstring|tuple +proc `[]`*[I: Ordinal;T](a: T; i: I): T {. + noSideEffect, magic: "ArrGet".} +proc `[]=`*[I: Ordinal;T,S](a: T; i: I; + x: sink S) {.noSideEffect, magic: "ArrPut".} +proc `=`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} +proc `=copy`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} -when defined(nimArrIdx): - # :array|openArray|string|seq|cstring|tuple - proc `[]`*[I: Ordinal;T](a: T; i: I): T {. - noSideEffect, magic: "ArrGet".} - proc `[]=`*[I: Ordinal;T,S](a: T; i: I; - x: S) {.noSideEffect, magic: "ArrPut".} - proc `=`*[T](dest: var T; src: T) {.noSideEffect, magic: "Asgn".} +proc arrGet[I: Ordinal;T](a: T; i: I): T {. + noSideEffect, magic: "ArrGet".} +proc arrPut[I: Ordinal;T,S](a: T; i: I; + x: S) {.noSideEffect, magic: "ArrPut".} - proc arrGet[I: Ordinal;T](a: T; i: I): T {. - noSideEffect, magic: "ArrGet".} - proc arrPut[I: Ordinal;T,S](a: T; i: I; - x: S) {.noSideEffect, magic: "ArrPut".} +const arcLikeMem = defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) + +when defined(nimAllowNonVarDestructor) and arcLikeMem and defined(nimPreviewNonVarDestructor): + proc `=destroy`*[T](x: T) {.inline, magic: "Destroy".} = + ## Generic `destructor`:idx: implementation that can be overridden. + discard +else: proc `=destroy`*[T](x: var T) {.inline, magic: "Destroy".} = ## Generic `destructor`:idx: implementation that can be overridden. discard - proc `=sink`*[T](x: var T; y: T) {.inline, magic: "Asgn".} = - ## Generic `sink`:idx: implementation that can be overridden. + + when defined(nimAllowNonVarDestructor) and arcLikeMem: + proc `=destroy`*(x: string) {.inline, magic: "Destroy", enforceNoRaises.} = + discard + + proc `=destroy`*[T](x: seq[T]) {.inline, magic: "Destroy".} = + discard + + proc `=destroy`*[T](x: ref T) {.inline, magic: "Destroy".} = + discard + +when defined(nimHasDup): + proc `=dup`*[T](x: T): T {.inline, magic: "Dup".} = + ## Generic `dup`:idx: implementation that can be overridden. + discard + +proc `=sink`*[T](x: var T; y: T) {.inline, nodestroy, magic: "Asgn".} = + ## Generic `sink`:idx: implementation that can be overridden. + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + x = y + else: shallowCopy(x, y) +when defined(nimHasTrace): + proc `=trace`*[T](x: var T; env: pointer) {.inline, magic: "Trace".} = + ## Generic `trace`:idx: implementation that can be overridden. + discard + type HSlice*[T, U] = object ## "Heterogeneous" slice type. a*: T ## The lower bound (inclusive). b*: U ## The upper bound (inclusive). - Slice*[T] = HSlice[T, T] ## An alias for ``HSlice[T, T]``. + Slice*[T] = HSlice[T, T] ## An alias for `HSlice[T, T]`. -proc `..`*[T, U](a: T, b: U): HSlice[T, U] {.noSideEffect, inline, magic: "DotDot".} = - ## Binary `slice`:idx: operator that constructs an interval ``[a, b]``, both `a` +proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, magic: "DotDot".} = + ## Binary `slice`:idx: operator that constructs an interval `[a, b]`, both `a` ## and `b` are inclusive. ## ## Slices can also be used in the set constructor and in ordinal case ## statements, but then they are special-cased by the compiler. - ## - ## .. code-block:: Nim + ## ```nim ## let a = [10, 20, 30, 40, 50] ## echo a[2 .. 3] # @[30, 40] + ## ``` result = HSlice[T, U](a: a, b: b) -proc `..`*[T](b: T): HSlice[int, T] {.noSideEffect, inline, magic: "DotDot".} = - ## Unary `slice`:idx: operator that constructs an interval ``[default(int), b]``. - ## - ## .. code-block:: Nim +proc `..`*[T](b: sink T): HSlice[int, T] + {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} = + ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`. + ## ```nim ## let a = [10, 20, 30, 40, 50] ## echo a[.. 2] # @[10, 20, 30] + ## ``` result = HSlice[int, T](a: 0, b: b) -when not defined(niminheritable): - {.pragma: inheritable.} -when not defined(nimunion): - {.pragma: unchecked.} -when not defined(nimHasHotCodeReloading): - {.pragma: nonReloadable.} when defined(hotCodeReloading): {.pragma: hcrInline, inline.} else: {.pragma: hcrInline.} -{.push profiler: off.} -let nimvm* {.magic: "Nimvm", compileTime.}: bool = false - ## May be used only in `when` expression. - ## It is true in Nim VM context and false otherwise. -{.pop.} - include "system/arithmetics" include "system/comparisons" const - appType* {.magic: "AppType"}: string = "" + appType* {.magic: "AppType".}: string = "" ## A string that describes the application type. Possible values: ## `"console"`, `"gui"`, `"lib"`. include "system/inclrtl" -const NoFakeVars* = defined(nimscript) ## `true` if the backend doesn't support \ +const NoFakeVars = defined(nimscript) ## `true` if the backend doesn't support \ ## "fake variables" like `var EBADF {.importc.}: cint`. const notJSnotNims = not defined(js) and not defined(nimscript) @@ -485,21 +454,18 @@ when not defined(js) and not defined(nimSeqsV2): len, reserved: int when defined(gogc): elemSize: int + elemAlign: int PGenericSeq {.exportc.} = ptr TGenericSeq # len and space without counting the terminating zero: NimStringDesc {.compilerproc, final.} = object of TGenericSeq data: UncheckedArray[char] NimString = ptr NimStringDesc -when notJSnotNims and not defined(nimSeqsV2): - template space(s: PGenericSeq): int {.dirty.} = - s.reserved and not (seqShallowFlag or strlitFlag) - -when notJSnotNims and not defined(nimV2): +when notJSnotNims: include "system/hti" type - byte* = uint8 ## This is an alias for ``uint8``, that is an unsigned + byte* = uint8 ## This is an alias for `uint8`, that is an unsigned ## integer, 8 bits wide. Natural* = range[0..high(int)] @@ -510,6 +476,7 @@ type ## is an `int` type ranging from one to the maximum value ## of an `int`. This type is often useful for documentation and debugging. +type RootObj* {.compilerproc, inheritable.} = object ## The root of Nim's object hierarchy. ## @@ -517,8 +484,64 @@ type ## However, objects that have no ancestor are also allowed. RootRef* = ref RootObj ## Reference to `RootObj`. +const NimStackTraceMsgs = compileOption("stacktraceMsgs") + +type + RootEffect* {.compilerproc.} = object of RootObj ## \ + ## Base effect class. + ## + ## Each effect should inherit from `RootEffect` unless you know what + ## you're doing. + +type + StackTraceEntry* = object ## In debug mode exceptions store the stack trace that led + ## to them. A `StackTraceEntry` is a single entry of the + ## stack trace. + procname*: cstring ## Name of the proc that is currently executing. + line*: int ## Line number of the proc that is currently executing. + filename*: cstring ## Filename of the proc that is currently executing. + when NimStackTraceMsgs: + frameMsg*: string ## When a stacktrace is generated in a given frame and + ## rendered at a later time, we should ensure the stacktrace + ## data isn't invalidated; any pointer into PFrame is + ## subject to being invalidated so shouldn't be stored. + when defined(nimStackTraceOverride): + programCounter*: uint ## Program counter - will be used to get the rest of the info, + ## when `$` is called on this type. We can't use + ## "cuintptr_t" in here. + procnameStr*, filenameStr*: string ## GC-ed alternatives to "procname" and "filename" + + Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ + ## Base exception class. + ## + ## Each exception has to inherit from `Exception`. See the full `exception + ## hierarchy <manual.html#exception-handling-exception-hierarchy>`_. + parent*: ref Exception ## Parent exception (can be used as a stack). + name*: cstring ## The exception's name is its Nim identifier. + ## This field is filled automatically in the + ## `raise` statement. + msg* {.exportc: "message".}: string ## The exception's message. Not + ## providing an exception message + ## is bad style. + when defined(js): + trace*: string + else: + trace*: seq[StackTraceEntry] + up: ref Exception # used for stacking exceptions. Not exported! + + Defect* = object of Exception ## \ + ## Abstract base class for all exceptions that Nim's runtime raises + ## but that are strictly uncatchable as they can also be mapped to + ## a `quit` / `trap` / `exit` operation. + + CatchableError* = object of Exception ## \ + ## Abstract class for all exceptions that are catchable. -include "system/exceptions" +when defined(nimIcIntegrityChecks): + include "system/exceptions" +else: + import system/exceptions + export exceptions when defined(js) or defined(nimdoc): type @@ -526,10 +549,10 @@ when defined(js) or defined(nimdoc): ## Root type of the JavaScript object hierarchy proc unsafeNew*[T](a: var ref T, size: Natural) {.magic: "New", noSideEffect.} - ## Creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. ## - ## This is **unsafe** as it allocates an object of the passed ``size``. + ## This is **unsafe** as it allocates an object of the passed `size`. ## This should only be used for optimization purposes when you know ## what you're doing! ## @@ -537,190 +560,166 @@ proc unsafeNew*[T](a: var ref T, size: Natural) {.magic: "New", noSideEffect.} ## * `new <#new,ref.T,proc(ref.T)>`_ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.} - ## Returns the size of ``x`` in bytes. + ## Returns the size of `x` in bytes. ## ## Since this is a low-level proc, ## its usage is discouraged - using `new <#new,ref.T,proc(ref.T)>`_ for - ## the most cases suffices that one never needs to know ``x``'s size. + ## the most cases suffices that one never needs to know `x`'s size. ## - ## As a special semantic rule, ``x`` may also be a type identifier - ## (``sizeof(int)`` is valid). + ## As a special semantic rule, `x` may also be a type identifier + ## (`sizeof(int)` is valid). ## ## Limitations: If used for types that are imported from C or C++, - ## sizeof should fallback to the ``sizeof`` in the C compiler. The + ## sizeof should fallback to the `sizeof` in the C compiler. The ## result isn't available for the Nim compiler and therefore can't ## be used inside of macros. - ## - ## .. code-block:: Nim - ## sizeof('A') # => 1 - ## sizeof(2) # => 8 + ## ```nim + ## sizeof('A') # => 1 + ## sizeof(2) # => 8 + ## ``` -when defined(nimHasalignOf): - proc alignof*[T](x: T): int {.magic: "AlignOf", noSideEffect.} - proc alignof*(x: typedesc): int {.magic: "AlignOf", noSideEffect.} +proc alignof*[T](x: T): int {.magic: "AlignOf", noSideEffect.} +proc alignof*(x: typedesc): int {.magic: "AlignOf", noSideEffect.} - proc offsetOfDotExpr(typeAccess: typed): int {.magic: "OffsetOf", noSideEffect, compileTime.} +proc offsetOfDotExpr(typeAccess: typed): int {.magic: "OffsetOf", noSideEffect, compileTime.} - template offsetOf*[T](t: typedesc[T]; member: untyped): int = - var tmp {.noinit.}: ptr T - offsetOfDotExpr(tmp[].member) +template offsetOf*[T](t: typedesc[T]; member: untyped): int = + var tmp {.noinit.}: ptr T + offsetOfDotExpr(tmp[].member) - template offsetOf*[T](value: T; member: untyped): int = - offsetOfDotExpr(value.member) +template offsetOf*[T](value: T; member: untyped): int = + offsetOfDotExpr(value.member) - #proc offsetOf*(memberaccess: typed): int {.magic: "OffsetOf", noSideEffect.} +#proc offsetOf*(memberaccess: typed): int {.magic: "OffsetOf", noSideEffect.} -when defined(nimtypedescfixed): - proc sizeof*(x: typedesc): int {.magic: "SizeOf", noSideEffect.} +proc sizeof*(x: typedesc): int {.magic: "SizeOf", noSideEffect.} proc newSeq*[T](s: var seq[T], len: Natural) {.magic: "NewSeq", noSideEffect.} - ## Creates a new sequence of type ``seq[T]`` with length ``len``. + ## Creates a new sequence of type `seq[T]` with length `len`. ## - ## This is equivalent to ``s = @[]; setlen(s, len)``, but more + ## This is equivalent to `s = @[]; setlen(s, len)`, but more ## efficient since no reallocation is needed. ## ## Note that the sequence will be filled with zeroed entries. ## After the creation of the sequence you should assign entries to ## the sequence instead of adding them. Example: - ## - ## .. code-block:: Nim - ## var inputStrings : seq[string] + ## ```nim + ## var inputStrings: seq[string] ## newSeq(inputStrings, 3) ## assert len(inputStrings) == 3 ## inputStrings[0] = "The fourth" ## inputStrings[1] = "assignment" ## inputStrings[2] = "would crash" ## #inputStrings[3] = "out of bounds" + ## ``` proc newSeq*[T](len = 0.Natural): seq[T] = - ## Creates a new sequence of type ``seq[T]`` with length ``len``. + ## Creates a new sequence of type `seq[T]` with length `len`. ## ## Note that the sequence will be filled with zeroed entries. ## After the creation of the sequence you should assign entries to ## the sequence instead of adding them. - ## - ## See also: - ## * `newSeqOfCap <#newSeqOfCap,Natural>`_ - ## * `newSeqUninitialized <#newSeqUninitialized,Natural>`_ - ## - ## .. code-block:: Nim + ## ```nim ## var inputStrings = newSeq[string](3) ## assert len(inputStrings) == 3 ## inputStrings[0] = "The fourth" ## inputStrings[1] = "assignment" ## inputStrings[2] = "would crash" ## #inputStrings[3] = "out of bounds" + ## ``` + ## + ## See also: + ## * `newSeqOfCap <#newSeqOfCap,Natural>`_ + ## * `newSeqUninit <#newSeqUninit,Natural>`_ newSeq(result, len) proc newSeqOfCap*[T](cap: Natural): seq[T] {. magic: "NewSeqOfCap", noSideEffect.} = - ## Creates a new sequence of type ``seq[T]`` with length zero and capacity - ## ``cap``. - ## - ## .. code-block:: Nim + ## Creates a new sequence of type `seq[T]` with length zero and capacity + ## `cap`. Example: + ## ```nim ## var x = newSeqOfCap[int](5) ## assert len(x) == 0 ## x.add(10) ## assert len(x) == 1 + ## ``` discard -when not defined(js): - proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] = - ## Creates a new sequence of type ``seq[T]`` with length ``len``. - ## - ## Only available for numbers types. Note that the sequence will be - ## uninitialized. After the creation of the sequence you should assign - ## entries to the sequence instead of adding them. - ## - ## .. code-block:: Nim - ## var x = newSeqUninitialized[int](3) - ## assert len(x) == 3 - ## x[0] = 10 - result = newSeqOfCap[T](len) - when defined(nimSeqsV2): - cast[ptr int](addr result)[] = len - else: - var s = cast[PGenericSeq](result) - s.len = len - -proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {. - magic: "LengthOpenArray", noSideEffect.} +func len*[TOpenArray: openArray|varargs](x: TOpenArray): int {.magic: "LengthOpenArray".} = ## Returns the length of an openArray. - ## - ## .. code-block:: Nim - ## var s = [1, 1, 1, 1, 1] - ## echo len(s) # => 5 + runnableExamples: + proc bar[T](a: openArray[T]): int = len(a) + assert bar([1,2]) == 2 + assert [1,2].len == 2 -proc len*(x: string): int {.magic: "LengthStr", noSideEffect.} +func len*(x: string): int {.magic: "LengthStr".} = ## Returns the length of a string. - ## - ## .. code-block:: Nim - ## var str = "Hello world!" - ## echo len(str) # => 12 - -proc len*(x: cstring): int {.magic: "LengthStr", noSideEffect.} - ## Returns the length of a compatible string. This is sometimes - ## an O(n) operation. - ## - ## .. code-block:: Nim - ## var str: cstring = "Hello world!" - ## len(str) # => 12 - -proc len*(x: (type array)|array): int {.magic: "LengthArray", noSideEffect.} + runnableExamples: + assert "abc".len == 3 + assert "".len == 0 + assert string.default.len == 0 + +proc len*(x: cstring): int {.magic: "LengthStr", noSideEffect.} = + ## Returns the length of a compatible string. This is an O(n) operation except + ## in js at runtime. + ## + ## **Note:** On the JS backend this currently counts UTF-16 code points + ## instead of bytes at runtime (not at compile time). For now, if you + ## need the byte length of the UTF-8 encoding, convert to string with + ## `$` first then call `len`. + runnableExamples: + doAssert len(cstring"abc") == 3 + doAssert len(cstring r"ab\0c") == 5 # \0 is escaped + doAssert len(cstring"ab\0c") == 5 # ditto + var a: cstring = "ab\0c" + when defined(js): doAssert a.len == 4 # len ignores \0 for js + else: doAssert a.len == 2 # \0 is a null terminator + static: + var a2: cstring = "ab\0c" + doAssert a2.len == 2 # \0 is a null terminator, even in js vm + +func len*(x: (type array)|array): int {.magic: "LengthArray".} = ## Returns the length of an array or an array type. - ## This is roughly the same as ``high(T)-low(T)+1``. - ## - ## .. code-block:: Nim - ## var arr = [1, 1, 1, 1, 1] - ## echo len(arr) # => 5 - ## echo len(array[3..8, int]) # => 6 - -proc len*[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.} - ## Returns the length of a sequence. - ## - ## .. code-block:: Nim - ## var s = @[1, 1, 1, 1, 1] - ## echo len(s) # => 5 - - -proc ord*[T: Ordinal|enum](x: T): int {.magic: "Ord", noSideEffect.} - ## Returns the internal `int` value of an ordinal value ``x``. - ## - ## .. code-block:: Nim - ## echo ord('A') # => 65 - ## echo ord('a') # => 97 - -proc chr*(u: range[0..255]): char {.magic: "Chr", noSideEffect.} - ## Converts an `int` in the range `0..255` to a character. - ## - ## .. code-block:: Nim - ## echo chr(65) # => A - ## echo chr(97) # => a - - -# floating point operations: -proc `+`*(x: float32): float32 {.magic: "UnaryPlusF64", noSideEffect.} -proc `-`*(x: float32): float32 {.magic: "UnaryMinusF64", noSideEffect.} -proc `+`*(x, y: float32): float32 {.magic: "AddF64", noSideEffect.} -proc `-`*(x, y: float32): float32 {.magic: "SubF64", noSideEffect.} -proc `*`*(x, y: float32): float32 {.magic: "MulF64", noSideEffect.} -proc `/`*(x, y: float32): float32 {.magic: "DivF64", noSideEffect.} - -proc `+`*(x: float): float {.magic: "UnaryPlusF64", noSideEffect.} -proc `-`*(x: float): float {.magic: "UnaryMinusF64", noSideEffect.} -proc `+`*(x, y: float): float {.magic: "AddF64", noSideEffect.} -proc `-`*(x, y: float): float {.magic: "SubF64", noSideEffect.} -proc `*`*(x, y: float): float {.magic: "MulF64", noSideEffect.} -proc `/`*(x, y: float): float {.magic: "DivF64", noSideEffect.} - -proc `==`*(x, y: float32): bool {.magic: "EqF64", noSideEffect.} -proc `<=`*(x, y: float32): bool {.magic: "LeF64", noSideEffect.} -proc `<` *(x, y: float32): bool {.magic: "LtF64", noSideEffect.} + ## This is roughly the same as `high(T)-low(T)+1`. + runnableExamples: + var a = [1, 1, 1] + assert a.len == 3 + assert array[0, float].len == 0 + static: assert array[-2..2, float].len == 5 -proc `==`*(x, y: float): bool {.magic: "EqF64", noSideEffect.} -proc `<=`*(x, y: float): bool {.magic: "LeF64", noSideEffect.} -proc `<`*(x, y: float): bool {.magic: "LtF64", noSideEffect.} +func len*[T](x: seq[T]): int {.magic: "LengthSeq".} = + ## Returns the length of `x`. + runnableExamples: + assert @[0, 1].len == 2 + assert seq[int].default.len == 0 + assert newSeq[int](3).len == 3 + let s = newSeqOfCap[int](3) + assert s.len == 0 + # xxx this gives cgen error: assert newSeqOfCap[int](3).len == 0 + +func ord*[T: Ordinal|enum](x: T): int {.magic: "Ord".} = + ## Returns the internal `int` value of `x`, including for enum with holes + ## and distinct ordinal types. + runnableExamples: + assert ord('A') == 65 + type Foo = enum + f0 = 0, f1 = 3 + assert f1.ord == 3 + type Bar = distinct int + assert 3.Bar.ord == 3 + +func chr*(u: range[0..255]): char {.magic: "Chr".} = + ## Converts `u` to a `char`, same as `char(u)`. + runnableExamples: + doAssert chr(65) == 'A' + doAssert chr(255) == '\255' + doAssert chr(255) == char(255) + doAssert not compiles chr(256) + doAssert not compiles char(256) + var x = 256 + doAssertRaises(RangeDefect): discard chr(x) + doAssertRaises(RangeDefect): discard char(x) include "system/setops" @@ -728,33 +727,36 @@ include "system/setops" proc contains*[U, V, W](s: HSlice[U, V], value: W): bool {.noSideEffect, inline.} = ## Checks if `value` is within the range of `s`; returns true if - ## `value >= s.a and value <= s.b` - ## - ## .. code-block:: Nim + ## `value >= s.a and value <= s.b`. + ## ```nim ## assert((1..3).contains(1) == true) ## assert((1..3).contains(2) == true) ## assert((1..3).contains(4) == false) + ## ``` result = s.a <= value and value <= s.b -template `in`*(x, y: untyped): untyped {.dirty.} = contains(y, x) +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template `in`*(x, y: untyped): untyped {.dirty, callsite.} = contains(y, x) ## Sugar for `contains`. - ## - ## .. code-block:: Nim + ## ```nim ## assert(1 in (1..3) == true) ## assert(5 in (1..3) == false) -template `notin`*(x, y: untyped): untyped {.dirty.} = not contains(y, x) + ## ``` +template `notin`*(x, y: untyped): untyped {.dirty, callsite.} = not contains(y, x) ## Sugar for `not contains`. - ## - ## .. code-block:: Nim + ## ```nim ## assert(1 notin (1..3) == false) ## assert(5 notin (1..3) == true) + ## ``` proc `is`*[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## Checks if `T` is of the same type as `S`. ## ## For a negated version, use `isnot <#isnot.t,untyped,untyped>`_. ## - ## .. code-block:: Nim + ## ```nim ## assert 42 is int ## assert @[1, 2] is seq ## @@ -766,12 +768,13 @@ proc `is`*[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.} ## ## assert(test[int](3) == 3) ## assert(test[string]("xyz") == 0) -template `isnot`*(x, y: untyped): untyped = not (x is y) - ## Negated version of `is <#is,T,S>`_. Equivalent to ``not(x is y)``. - ## - ## .. code-block:: Nim + ## ``` +template `isnot`*(x, y: untyped): untyped {.callsite.} = not (x is y) + ## Negated version of `is <#is,T,S>`_. Equivalent to `not(x is y)`. + ## ```nim ## assert 42 isnot float ## assert @[1, 2] isnot enum + ## ``` when (defined(nimOwnedEnabled) and not defined(nimscript)) or defined(nimFixedOwned): type owned*[T]{.magic: "BuiltinType".} ## type constructor to mark a ref/ptr or a closure as `owned`. @@ -780,15 +783,15 @@ else: when defined(nimOwnedEnabled) and not defined(nimscript): proc new*[T](a: var owned(ref T)) {.magic: "New", noSideEffect.} - ## Creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. proc new*(t: typedesc): auto = - ## Creates a new object of type ``T`` and returns a safe (traced) + ## Creates a new object of type `T` and returns a safe (traced) ## reference to it as result value. ## - ## When ``T`` is a ref type then the resulting type will be ``T``, - ## otherwise it will be ``ref T``. + ## When `T` is a ref type then the resulting type will be `T`, + ## otherwise it will be `ref T`. when (t is ref): var r: owned t else: @@ -797,24 +800,22 @@ when defined(nimOwnedEnabled) and not defined(nimscript): return r proc unown*[T](x: T): T {.magic: "Unown", noSideEffect.} - ## Use the expression ``x`` ignoring its ownership attribute. + ## Use the expression `x` ignoring its ownership attribute. - # This is only required to make 0.20 compile with the 0.19 line. - template `<//>`*(t: untyped): untyped = owned(t) else: template unown*(x: typed): untyped = x proc new*[T](a: var ref T) {.magic: "New", noSideEffect.} - ## Creates a new object of type ``T`` and returns a safe (traced) - ## reference to it in ``a``. + ## Creates a new object of type `T` and returns a safe (traced) + ## reference to it in `a`. proc new*(t: typedesc): auto = - ## Creates a new object of type ``T`` and returns a safe (traced) + ## Creates a new object of type `T` and returns a safe (traced) ## reference to it as result value. ## - ## When ``T`` is a ref type then the resulting type will be ``T``, - ## otherwise it will be ``ref T``. + ## When `T` is a ref type then the resulting type will be `T`, + ## otherwise it will be `ref T`. when (t is ref): var r: t else: @@ -822,26 +823,41 @@ else: new(r) return r - # This is only required to make 0.20 compile with the 0.19 line. - template `<//>`*(t: untyped): untyped = t template disarm*(x: typed) = - ## Useful for ``disarming`` dangling pointers explicitly for the - ## --newruntime. Regardless of whether --newruntime is used or not - ## this sets the pointer or callback ``x`` to ``nil``. This is an + ## Useful for `disarming` dangling pointers explicitly for `--newruntime`. + ## Regardless of whether `--newruntime` is used or not + ## this sets the pointer or callback `x` to `nil`. This is an ## experimental API! x = nil -proc `of`*[T, S](x: typedesc[T], y: typedesc[S]): bool {.magic: "Of", noSideEffect.} -proc `of`*[T, S](x: T, y: typedesc[S]): bool {.magic: "Of", noSideEffect.} -proc `of`*[T, S](x: T, y: S): bool {.magic: "Of", noSideEffect.} - ## Checks if `x` has a type of `y`. - ## - ## .. code-block:: Nim - ## assert(FloatingPointError of Exception) - ## assert(DivByZeroError of Exception) - -proc cmp*[T](x, y: T): int {.procvar.} = +proc `of`*[T, S](x: T, y: typedesc[S]): bool {.magic: "Of", noSideEffect.} = + ## Checks if `x` is an instance of `y`. + runnableExamples: + type + Base = ref object of RootObj + Sub1 = ref object of Base + Sub2 = ref object of Base + Unrelated = ref object + + var base: Base = Sub1() # downcast + doAssert base of Base # generates `CondTrue` (statically true) + doAssert base of Sub1 + doAssert base isnot Sub1 + doAssert not (base of Sub2) + + base = Sub2() # re-assign + doAssert base of Sub2 + doAssert Sub2(base) != nil # upcast + doAssertRaises(ObjectConversionDefect): discard Sub1(base) + + var sub1 = Sub1() + doAssert sub1 of Base + doAssert sub1.Base of Sub1 + + doAssert not compiles(base of Unrelated) + +proc cmp*[T](x, y: T): int = ## Generic compare proc. ## ## Returns: @@ -851,159 +867,159 @@ proc cmp*[T](x, y: T): int {.procvar.} = ## ## This is useful for writing generic algorithms without performance loss. ## This generic implementation uses the `==` and `<` operators. - ## - ## .. code-block:: Nim - ## import algorithm - ## echo sorted(@[4, 2, 6, 5, 8, 7], cmp[int]) + ## ```nim + ## import std/algorithm + ## echo sorted(@[4, 2, 6, 5, 8, 7], cmp[int]) + ## ``` if x == y: return 0 if x < y: return -1 return 1 -proc cmp*(x, y: string): int {.noSideEffect, procvar.} +proc cmp*(x, y: string): int {.noSideEffect.} ## Compare proc for strings. More efficient than the generic version. ## ## **Note**: The precise result values depend on the used C runtime library and ## can differ between operating systems! -when defined(nimHasDefault): - proc `@`* [IDX, T](a: sink array[IDX, T]): seq[T] {. - magic: "ArrToSeq", noSideEffect.} - ## Turns an array into a sequence. - ## - ## This most often useful for constructing - ## sequences with the array constructor: ``@[1, 2, 3]`` has the type - ## ``seq[int]``, while ``[1, 2, 3]`` has the type ``array[0..2, int]``. - ## - ## .. code-block:: Nim - ## let - ## a = [1, 3, 5] - ## b = "foo" - ## - ## echo @a # => @[1, 3, 5] - ## echo @b # => @['f', 'o', 'o'] +proc `@`* [IDX, T](a: sink array[IDX, T]): seq[T] {.magic: "ArrToSeq", noSideEffect.} + ## Turns an array into a sequence. + ## + ## This most often useful for constructing + ## sequences with the array constructor: `@[1, 2, 3]` has the type + ## `seq[int]`, while `[1, 2, 3]` has the type `array[0..2, int]`. + ## + ## ```nim + ## let + ## a = [1, 3, 5] + ## b = "foo" + ## + ## echo @a # => @[1, 3, 5] + ## echo @b # => @['f', 'o', 'o'] + ## ``` - proc default*(T: typedesc): T {.magic: "Default", noSideEffect.} - ## returns the default value of the type ``T``. +proc default*[T](_: typedesc[T]): T {.magic: "Default", noSideEffect.} = + ## Returns the default value of the type `T`. Contrary to `zeroDefault`, it takes default fields + ## of an object into consideration. + ## + ## See also: + ## * `zeroDefault <#zeroDefault,typedesc[T]>`_ + ## + runnableExamples("-d:nimPreviewRangeDefault"): + assert (int, float).default == (0, 0.0) + type Foo = object + a: range[2..6] + var x = Foo.default + assert x.a == 2 - proc reset*[T](obj: var T) {.noSideEffect.} = - ## Resets an object `obj` to its default value. - obj = default(typeof(obj)) -else: - proc `@`* [IDX, T](a: array[IDX, T]): seq[T] {. - magic: "ArrToSeq", noSideEffect.} - when defined(nimV2): - proc reset*[T](obj: var T) {.magic: "Destroy", noSideEffect.} +proc reset*[T](obj: var T) {.noSideEffect.} = + ## Resets an object `obj` to its default value. + when nimvm: + obj = default(typeof(obj)) else: - proc reset*[T](obj: var T) {.magic: "Reset", noSideEffect.} + when defined(gcDestructors): + {.cast(noSideEffect), cast(raises: []), cast(tags: []).}: + `=destroy`(obj) + `=wasMoved`(obj) + else: + obj = default(typeof(obj)) proc setLen*[T](s: var seq[T], newlen: Natural) {. - magic: "SetLengthSeq", noSideEffect.} - ## Sets the length of seq `s` to `newlen`. ``T`` may be any sequence type. + magic: "SetLengthSeq", noSideEffect, nodestroy.} + ## Sets the length of seq `s` to `newlen`. `T` may be any sequence type. ## ## If the current length is greater than the new length, - ## ``s`` will be truncated. - ## - ## .. code-block:: Nim + ## `s` will be truncated. + ## ```nim ## var x = @[10, 20] ## x.setLen(5) ## x[4] = 50 ## assert x == @[10, 20, 0, 0, 50] ## x.setLen(1) ## assert x == @[10] + ## ``` proc setLen*(s: var string, newlen: Natural) {. magic: "SetLengthStr", noSideEffect.} ## Sets the length of string `s` to `newlen`. ## ## If the current length is greater than the new length, - ## ``s`` will be truncated. - ## - ## .. code-block:: Nim - ## var myS = "Nim is great!!" - ## myS.setLen(3) # myS <- "Nim" - ## echo myS, " is fantastic!!" + ## `s` will be truncated. + ## ```nim + ## var myS = "Nim is great!!" + ## myS.setLen(3) # myS <- "Nim" + ## echo myS, " is fantastic!!" + ## ``` proc newString*(len: Natural): string {. magic: "NewString", importc: "mnewString", noSideEffect.} - ## Returns a new string of length ``len`` but with uninitialized - ## content. One needs to fill the string character after character - ## with the index operator ``s[i]``. + ## Returns a new string of length `len`. + ## One needs to fill the string character after character + ## with the index operator `s[i]`. ## ## This procedure exists only for optimization purposes; - ## the same effect can be achieved with the ``&`` operator or with ``add``. + ## the same effect can be achieved with the `&` operator or with `add`. proc newStringOfCap*(cap: Natural): string {. magic: "NewStringOfCap", importc: "rawNewString", noSideEffect.} - ## Returns a new string of length ``0`` but with capacity `cap`. + ## Returns a new string of length `0` but with capacity `cap`. ## ## This procedure exists only for optimization purposes; the same effect can - ## be achieved with the ``&`` operator or with ``add``. + ## be achieved with the `&` operator or with `add`. proc `&`*(x: string, y: char): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates `x` with `y`. - ## - ## .. code-block:: Nim + ## ```nim ## assert("ab" & 'c' == "abc") + ## ``` proc `&`*(x, y: char): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates characters `x` and `y` into a string. - ## - ## .. code-block:: Nim + ## ```nim ## assert('a' & 'b' == "ab") + ## ``` proc `&`*(x, y: string): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates strings `x` and `y`. - ## - ## .. code-block:: Nim + ## ```nim ## assert("ab" & "cd" == "abcd") + ## ``` proc `&`*(x: char, y: string): string {. - magic: "ConStrStr", noSideEffect, merge.} + magic: "ConStrStr", noSideEffect.} ## Concatenates `x` with `y`. - ## - ## .. code-block:: Nim + ## ```nim ## assert('a' & "bc" == "abc") + ## ``` # implementation note: These must all have the same magic value "ConStrStr" so # that the merge optimization works properly. proc add*(x: var string, y: char) {.magic: "AppendStrCh", noSideEffect.} ## Appends `y` to `x` in place. - ## - ## .. code-block:: Nim + ## ```nim ## var tmp = "" ## tmp.add('a') ## tmp.add('b') ## assert(tmp == "ab") -proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} + ## ``` + +proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} = ## Concatenates `x` and `y` in place. ## - ## .. code-block:: Nim - ## var tmp = "" - ## tmp.add("ab") - ## tmp.add("cd") - ## assert(tmp == "abcd") - + ## See also `strbasics.add`. + runnableExamples: + var tmp = "" + tmp.add("ab") + tmp.add("cd") + assert tmp == "abcd" type Endianness* = enum ## Type describing the endianness of a processor. littleEndian, bigEndian const - isMainModule* {.magic: "IsMainModule".}: bool = false - ## True only when accessed in the main module. This works thanks to - ## compiler magic. It is useful to embed testing code in a module. - - CompileDate* {.magic: "CompileDate"}: string = "0000-00-00" - ## The date (in UTC) of compilation as a string of the form - ## ``YYYY-MM-DD``. This works thanks to compiler magic. - - CompileTime* {.magic: "CompileTime"}: string = "00:00:00" - ## The time (in UTC) of compilation as a string of the form - ## ``HH:MM:SS``. This works thanks to compiler magic. - - cpuEndian* {.magic: "CpuEndian"}: Endianness = littleEndian + cpuEndian* {.magic: "CpuEndian".}: Endianness = littleEndian ## The endianness of the target CPU. This is a valuable piece of ## information for low-level code only. This works thanks to compiler ## magic. @@ -1021,38 +1037,24 @@ const ## Possible values: ## `"i386"`, `"alpha"`, `"powerpc"`, `"powerpc64"`, `"powerpc64el"`, ## `"sparc"`, `"amd64"`, `"mips"`, `"mipsel"`, `"arm"`, `"arm64"`, - ## `"mips64"`, `"mips64el"`, `"riscv64"`. + ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`, `"loongarch64"`. seqShallowFlag = low(int) strlitFlag = 1 shl (sizeof(int)*8 - 2) # later versions of the codegen \ # emit this flag # for string literals, it allows for some optimizations. -proc compileOption*(option: string): bool {. - magic: "CompileOption", noSideEffect.} - ## Can be used to determine an `on|off` compile-time option. Example: - ## - ## .. code-block:: Nim - ## when compileOption("floatchecks"): - ## echo "compiled with floating point NaN and Inf checks" - -proc compileOption*(option, arg: string): bool {. - magic: "CompileOptionArg", noSideEffect.} - ## Can be used to determine an enum compile-time option. Example: - ## - ## .. code-block:: Nim - ## when compileOption("opt", "size") and compileOption("gc", "boehm"): - ## echo "compiled with optimization for size and uses Boehm's GC" - const hasThreadSupport = compileOption("threads") and not defined(nimscript) hasSharedHeap = defined(boehmgc) or defined(gogc) # don't share heaps; every thread has its own - taintMode = compileOption("taintmode") - nimEnableCovariance* = defined(nimEnableCovariance) # or true + +when notJSnotNims and not defined(nimSeqsV2): + template space(s: PGenericSeq): int = + s.reserved and not (seqShallowFlag or strlitFlag) when hasThreadSupport and defined(tcc) and not compileOption("tlsEmulation"): # tcc doesn't support TLS - {.error: "``--tlsEmulation:on`` must be used when using threads with tcc backend".} + {.error: "`--tlsEmulation:on` must be used when using threads with tcc backend".} when defined(boehmgc): when defined(windows): @@ -1063,29 +1065,16 @@ when defined(boehmgc): elif defined(macosx): const boehmLib = "libgc.dylib" elif defined(openbsd): - const boehmLib = "libgc.so.4.0" + const boehmLib = "libgc.so.(4|5).0" elif defined(freebsd): const boehmLib = "libgc-threaded.so.1" else: const boehmLib = "libgc.so.1" {.pragma: boehmGC, noconv, dynlib: boehmLib.} -when taintMode: - type TaintedString* = distinct string ## A distinct string type that - ## is `tainted`:idx:, see `taint mode - ## <manual_experimental.html#taint-mode>`_ - ## for details. It is an alias for - ## ``string`` if the taint mode is not - ## turned on. +when not defined(nimPreviewSlimSystem): + type TaintedString* {.deprecated: "Deprecated since 1.5".} = string - proc len*(s: TaintedString): int {.borrow.} -else: - type TaintedString* = string ## A distinct string type that - ## is `tainted`:idx:, see `taint mode - ## <manual_experimental.html#taint-mode>`_ - ## for details. It is an alias for - ## ``string`` if the taint mode is not - ## turned on. when defined(profiler) and not defined(nimscript): proc nimProfile() {.compilerproc, noinline.} @@ -1103,56 +1092,24 @@ const ## is the value that should be passed to `quit <#quit,int>`_ to indicate ## failure. -when defined(js) and defined(nodejs) and not defined(nimscript): - var programResult* {.importc: "process.exitCode".}: int - programResult = 0 -elif hostOS != "standalone": +when not defined(js) and hostOS != "standalone": var programResult* {.compilerproc, exportc: "nim_program_result".}: int - ## deprecated, prefer ``quit`` - -when defined(nimdoc): - proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} - ## Stops the program immediately with an exit code. - ## - ## Before stopping the program the "quit procedures" are called in the - ## opposite order they were added with `addQuitProc <#addQuitProc,proc>`_. - ## ``quit`` never returns and ignores any exception that may have been raised - ## by the quit procedures. It does *not* call the garbage collector to free - ## all the memory, unless a quit procedure calls `GC_fullCollect - ## <#GC_fullCollect>`_. - ## - ## The proc ``quit(QuitSuccess)`` is called implicitly when your nim - ## program finishes without incident for platforms where this is the - ## expected behavior. A raised unhandled exception is - ## equivalent to calling ``quit(QuitFailure)``. - ## - ## Note that this is a *runtime* call and using ``quit`` inside a macro won't - ## have any compile time effect. If you need to stop the compiler inside a - ## macro, use the `error <manual.html#pragmas-error-pragma>`_ or `fatal - ## <manual.html#pragmas-fatal-pragma>`_ pragmas. - -elif defined(genode): - include genode/env - - var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr - - type GenodeEnv* = GenodeEnvPtr - ## Opaque type representing Genode environment. - - proc quit*(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, - importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "<base/sleep.h>".} - - proc quit*(errorcode: int = QuitSuccess) = - systemEnv.quit(errorcode) + ## deprecated, prefer `quit` or `exitprocs.getProgramResult`, `exitprocs.setProgramResult`. -elif defined(js) and defined(nodejs) and not defined(nimscript): - proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", - importc: "process.exit", noreturn.} +import std/private/since +import system/ctypes +export ctypes -else: - proc quit*(errorcode: int = QuitSuccess) {. - magic: "Exit", importc: "exit", header: "<stdlib.h>", noreturn.} +proc align(address, alignment: int): int = + if alignment == 0: # Actually, this is illegal. This branch exists to actively + # hide problems. + result = address + else: + result = (address + (alignment - 1)) and not (alignment - 1) +include system/rawquits +when defined(genode): + export GenodeEnv template sysAssert(cond: bool, msg: string) = when defined(useSysAssert): @@ -1160,261 +1117,269 @@ template sysAssert(cond: bool, msg: string) = cstderr.rawWrite "[SYSASSERT] " cstderr.rawWrite msg cstderr.rawWrite "\n" - quit 1 + rawQuit 1 const hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(nimscript) -when notJSnotNims and hostOS != "standalone" and hostOS != "any": - include "system/cgprocs" when notJSnotNims and hasAlloc and not defined(nimSeqsV2): proc addChar(s: NimString, c: char): NimString {.compilerproc, benign.} when defined(nimscript) or not defined(nimSeqsV2): - proc add*[T](x: var seq[T], y: T) {.magic: "AppendSeqElem", noSideEffect.} + proc add*[T](x: var seq[T], y: sink T) {.magic: "AppendSeqElem", noSideEffect.} ## Generic proc for adding a data item `y` to a container `x`. ## ## For containers that have an order, `add` means *append*. New generic ## containers should also call their adding proc `add` for consistency. ## Generic code becomes much easier to write if the Nim naming scheme is ## respected. + ## ```nim + ## var s: seq[string] = @["test2","test2"] + ## s.add("test") + ## assert s == @["test2", "test2", "test"] + ## ``` + +when false: # defined(gcDestructors): + proc add*[T](x: var seq[T], y: sink openArray[T]) {.noSideEffect.} = + ## Generic proc for adding a container `y` to a container `x`. + ## + ## For containers that have an order, `add` means *append*. New generic + ## containers should also call their adding proc `add` for consistency. + ## Generic code becomes much easier to write if the Nim naming scheme is + ## respected. + ## ```nim + ## var s: seq[string] = @["test2","test2"] + ## s.add("test") # s <- @[test2, test2, test] + ## ``` + ## + ## See also: + ## * `& proc <#&,seq[T],seq[T]>`_ + {.noSideEffect.}: + let xl = x.len + setLen(x, xl + y.len) + for i in 0..high(y): + when nimvm: + # workaround the fact that the VM does not yet + # handle sink parameters properly: + x[xl+i] = y[i] + else: + x[xl+i] = move y[i] +else: + proc add*[T](x: var seq[T], y: openArray[T]) {.noSideEffect.} = + ## Generic proc for adding a container `y` to a container `x`. + ## + ## For containers that have an order, `add` means *append*. New generic + ## containers should also call their adding proc `add` for consistency. + ## Generic code becomes much easier to write if the Nim naming scheme is + ## respected. + ## + ## See also: + ## * `& proc <#&,seq[T],seq[T]>`_ + runnableExamples: + var a = @["a1", "a2"] + a.add(["b1", "b2"]) + assert a == @["a1", "a2", "b1", "b2"] + var c = @["c0", "c1", "c2", "c3"] + a.add(c.toOpenArray(1, 2)) + assert a == @["a1", "a2", "b1", "b2", "c1", "c2"] + + {.noSideEffect.}: + let xl = x.len + setLen(x, xl + y.len) + for i in 0..high(y): x[xl+i] = y[i] -proc add*[T](x: var seq[T], y: openArray[T]) {.noSideEffect.} = - ## Generic proc for adding a container `y` to a container `x`. - ## - ## For containers that have an order, `add` means *append*. New generic - ## containers should also call their adding proc `add` for consistency. - ## Generic code becomes much easier to write if the Nim naming scheme is - ## respected. - ## - ## See also: - ## * `& proc <#&,seq[T][T],seq[T][T]>`_ - ## - ## .. code-block:: Nim - ## var s: seq[string] = @["test2","test2"] - ## s.add("test") # s <- @[test2, test2, test] - let xl = x.len - setLen(x, xl + y.len) - for i in 0..high(y): x[xl+i] = y[i] when defined(nimSeqsV2): - template movingCopy(a, b) = + template movingCopy(a, b: typed) = a = move(b) else: - template movingCopy(a, b) = + template movingCopy(a, b: typed) = shallowCopy(a, b) proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## Deletes the item at index `i` by putting ``x[high(x)]`` into position `i`. + ## Deletes the item at index `i` by putting `x[high(x)]` into position `i`. ## ## This is an `O(1)` operation. ## ## See also: - ## * `delete <#delete,seq[T][T],Natural>`_ for preserving the order - ## - ## .. code-block:: Nim - ## var i = @[1, 2, 3, 4, 5] - ## i.del(2) # => @[1, 2, 5, 4] + ## * `delete <#delete,seq[T],Natural>`_ for preserving the order + runnableExamples: + var a = @[10, 11, 12, 13, 14] + a.del(2) + assert a == @[10, 11, 14, 13] let xl = x.len - 1 movingCopy(x[i], x[xl]) setLen(x, xl) -proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## Deletes the item at index `i` by moving all ``x[i+1..]`` items by one position. - ## - ## This is an `O(n)` operation. - ## - ## See also: - ## * `del <#delete,seq[T][T],Natural>`_ for O(1) operation - ## - ## .. code-block:: Nim - ## var i = @[1, 2, 3, 4, 5] - ## i.delete(2) # => @[1, 2, 4, 5] - template defaultImpl = - let xl = x.len - for j in i.int..xl-2: movingCopy(x[j], x[j+1]) - setLen(x, xl-1) - - when nimvm: - defaultImpl() - else: - when defined(js): - {.emit: "`x`.splice(`i`, 1);".} - else: - defaultImpl() - -proc insert*[T](x: var seq[T], item: T, i = 0.Natural) {.noSideEffect.} = +proc insert*[T](x: var seq[T], item: sink T, i = 0.Natural) {.noSideEffect.} = ## Inserts `item` into `x` at position `i`. - ## - ## .. code-block:: Nim - ## var i = @[1, 3, 5] - ## i.insert(99, 0) # i <- @[99, 1, 3, 5] - template defaultImpl = - let xl = x.len - setLen(x, xl+1) - var j = xl-1 - while j >= i: - movingCopy(x[j+1], x[j]) - dec(j) - when nimvm: - defaultImpl() - else: - when defined(js): - var it : T - {.emit: "`x` = `x` || []; `x`.splice(`i`, 0, `it`);".} - else: + ## ```nim + ## var i = @[1, 3, 5] + ## i.insert(99, 0) # i <- @[99, 1, 3, 5] + ## ``` + {.noSideEffect.}: + template defaultImpl = + let xl = x.len + setLen(x, xl+1) + var j = xl-1 + while j >= i: + movingCopy(x[j+1], x[j]) + dec(j) + when nimvm: defaultImpl() - x[i] = item + else: + when defined(js): + var it : T + {.emit: "`x` = `x` || []; `x`.splice(`i`, 0, `it`);".} + else: + defaultImpl() + x[i] = item when not defined(nimV2): proc repr*[T](x: T): string {.magic: "Repr", noSideEffect.} ## Takes any Nim variable and returns its string representation. + ## No trailing newline is inserted (so `echo` won't add an empty newline). + ## Use `-d:nimLegacyReprWithNewline` to revert to old behavior where newlines + ## were added in some cases. ## ## It works even for complex data graphs with cycles. This is a great ## debugging tool. - ## - ## .. code-block:: Nim - ## var s: seq[string] = @["test2", "test2"] - ## var i = @[1, 2, 3, 4, 5] - ## echo repr(s) # => 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] - ## echo repr(i) # => 0x1055ed050[1, 2, 3, 4, 5] - -type - ByteAddress* = int - ## is the signed integer type that should be used for converting - ## pointers to integer addresses for readability. - - BiggestFloat* = float64 - ## is an alias for the biggest floating point type the Nim - ## compiler supports. Currently this is ``float64``, but it is - ## platform-dependent in general. + ## ```nim + ## var s: seq[string] = @["test2", "test2"] + ## var i = @[1, 2, 3, 4, 5] + ## echo repr(s) # => 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] + ## echo repr(i) # => 0x1055ed050[1, 2, 3, 4, 5] + ## ``` + +when not defined(nimPreviewSlimSystem): + type + csize* {.importc: "size_t", nodecl, deprecated: "use `csize_t` instead".} = int + ## This isn't the same as `size_t` in *C*. Don't use it. -when defined(js): - type BiggestUInt* = uint32 - ## is an alias for the biggest unsigned integer type the Nim compiler - ## supports. Currently this is ``uint32`` for JS and ``uint64`` for other - ## targets. -else: - type BiggestUInt* = uint64 - ## is an alias for the biggest unsigned integer type the Nim compiler - ## supports. Currently this is ``uint32`` for JS and ``uint64`` for other - ## targets. +const + Inf* = 0x7FF0000000000000'f64 + ## Contains the IEEE floating point value of positive infinity. + NegInf* = 0xFFF0000000000000'f64 + ## Contains the IEEE floating point value of negative infinity. + NaN* = 0x7FF7FFFFFFFFFFFF'f64 + ## Contains an IEEE floating point value of *Not A Number*. + ## + ## Note that you cannot compare a floating point value to this value + ## and expect a reasonable result - use the `isNaN` or `classify` procedure + ## in the `math module <math.html>`_ for checking for NaN. -when defined(windows): - type - clong* {.importc: "long", nodecl.} = int32 - ## This is the same as the type ``long`` in *C*. - culong* {.importc: "unsigned long", nodecl.} = uint32 - ## This is the same as the type ``unsigned long`` in *C*. -else: - type - clong* {.importc: "long", nodecl.} = int - ## This is the same as the type ``long`` in *C*. - culong* {.importc: "unsigned long", nodecl.} = uint - ## This is the same as the type ``unsigned long`` in *C*. - -type # these work for most platforms: - cchar* {.importc: "char", nodecl.} = char - ## This is the same as the type ``char`` in *C*. - cschar* {.importc: "signed char", nodecl.} = int8 - ## This is the same as the type ``signed char`` in *C*. - cshort* {.importc: "short", nodecl.} = int16 - ## This is the same as the type ``short`` in *C*. - cint* {.importc: "int", nodecl.} = int32 - ## This is the same as the type ``int`` in *C*. - csize* {.importc: "size_t", nodecl, deprecated: "use `csize_t` instead".} = int - ## This isn't the same as ``size_t`` in *C*. Don't use it. - csize_t* {.importc: "size_t", nodecl.} = uint - ## This is the same as the type ``size_t`` in *C*. - clonglong* {.importc: "long long", nodecl.} = int64 - ## This is the same as the type ``long long`` in *C*. - cfloat* {.importc: "float", nodecl.} = float32 - ## This is the same as the type ``float`` in *C*. - cdouble* {.importc: "double", nodecl.} = float64 - ## This is the same as the type ``double`` in *C*. - clongdouble* {.importc: "long double", nodecl.} = BiggestFloat - ## This is the same as the type ``long double`` in *C*. - ## This C type is not supported by Nim's code generator. - - cuchar* {.importc: "unsigned char", nodecl.} = char - ## This is the same as the type ``unsigned char`` in *C*. - cushort* {.importc: "unsigned short", nodecl.} = uint16 - ## This is the same as the type ``unsigned short`` in *C*. - cuint* {.importc: "unsigned int", nodecl.} = uint32 - ## This is the same as the type ``unsigned int`` in *C*. - culonglong* {.importc: "unsigned long long", nodecl.} = uint64 - ## This is the same as the type ``unsigned long long`` in *C*. - - cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring] - ## This is binary compatible to the type ``char**`` in *C*. The array's - ## high value is large enough to disable bounds checking in practice. - ## Use `cstringArrayToSeq proc <#cstringArrayToSeq,cstringArray,Natural>`_ - ## to convert it into a ``seq[string]``. - - PFloat32* = ptr float32 ## An alias for ``ptr float32``. - PFloat64* = ptr float64 ## An alias for ``ptr float64``. - PInt64* = ptr int64 ## An alias for ``ptr int64``. - PInt32* = ptr int32 ## An alias for ``ptr int32``. +proc high*(T: typedesc[SomeFloat]): T = Inf +proc low*(T: typedesc[SomeFloat]): T = NegInf proc toFloat*(i: int): float {.noSideEffect, inline.} = - ## Converts an integer `i` into a ``float``. + ## Converts an integer `i` into a `float`. Same as `float(i)`. ## ## If the conversion fails, `ValueError` is raised. ## However, on most platforms the conversion cannot fail. ## - ## .. code-block:: Nim + ## ```nim ## let ## a = 2 ## b = 3.7 ## ## echo a.toFloat + b # => 5.7 + ## ``` float(i) proc toBiggestFloat*(i: BiggestInt): BiggestFloat {.noSideEffect, inline.} = - ## Same as `toFloat <#toFloat,int>`_ but for ``BiggestInt`` to ``BiggestFloat``. + ## Same as `toFloat <#toFloat,int>`_ but for `BiggestInt` to `BiggestFloat`. BiggestFloat(i) proc toInt*(f: float): int {.noSideEffect.} = - ## Converts a floating point number `f` into an ``int``. + ## Converts a floating point number `f` into an `int`. ## ## Conversion rounds `f` half away from 0, see ## `Round half away from zero - ## <https://en.wikipedia.org/wiki/Rounding#Round_half_away_from_zero>`_. + ## <https://en.wikipedia.org/wiki/Rounding#Round_half_away_from_zero>`_, + ## as opposed to a type conversion which rounds towards zero. ## ## Note that some floating point numbers (e.g. infinity or even 1e19) ## cannot be accurately converted. - ## - ## .. code-block:: Nim + ## ```nim ## doAssert toInt(0.49) == 0 ## doAssert toInt(0.5) == 1 ## doAssert toInt(-0.5) == -1 # rounding is symmetrical + ## ``` if f >= 0: int(f+0.5) else: int(f-0.5) proc toBiggestInt*(f: BiggestFloat): BiggestInt {.noSideEffect.} = - ## Same as `toInt <#toInt,float>`_ but for ``BiggestFloat`` to ``BiggestInt``. + ## Same as `toInt <#toInt,float>`_ but for `BiggestFloat` to `BiggestInt`. if f >= 0: BiggestInt(f+0.5) else: BiggestInt(f-0.5) -proc addQuitProc*(quitProc: proc() {.noconv.}) {. - importc: "atexit", header: "<stdlib.h>".} - ## Adds/registers a quit procedure. +proc `/`*(x, y: int): float {.inline, noSideEffect.} = + ## Division of integers that results in a float. + ## ```nim + ## echo 7 / 5 # => 1.4 + ## ``` + ## + ## See also: + ## * `div <system.html#div,int,int>`_ + ## * `mod <system.html#mod,int,int>`_ + result = toFloat(x) / toFloat(y) + +{.push stackTrace: off.} + +when defined(js): + proc js_abs[T: SomeNumber](x: T): T {.importc: "Math.abs".} +else: + proc c_fabs(x: cdouble): cdouble {.importc: "fabs", header: "<math.h>".} + proc c_fabsf(x: cfloat): cfloat {.importc: "fabsf", header: "<math.h>".} + +proc abs*[T: float64 | float32](x: T): T {.noSideEffect, inline.} = + when nimvm: + if x < 0.0: result = -x + elif x == 0.0: result = 0.0 # handle 0.0, -0.0 + else: result = x # handle NaN, > 0 + else: + when defined(js): result = js_abs(x) + else: + when T is float64: + result = c_fabs(x) + else: + result = c_fabsf(x) + +func abs*(x: int): int {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int8): int8 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int16): int16 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int32): int32 {.magic: "AbsI", inline.} = + if x < 0: -x else: x +func abs*(x: int64): int64 {.magic: "AbsI", inline.} = + ## Returns the absolute value of `x`. ## - ## Each call to ``addQuitProc`` registers another quit procedure. Up to 30 - ## procedures can be registered. They are executed on a last-in, first-out - ## basis (that is, the last function registered is the first to be executed). - ## ``addQuitProc`` raises an EOutOfIndex exception if ``quitProc`` cannot be - ## registered. + ## If `x` is `low(x)` (that is -MININT for its type), + ## an overflow exception is thrown (if overflow checking is turned on). + result = if x < 0: -x else: x -# Support for addQuitProc() is done by Ansi C's facilities here. -# In case of an unhandled exception the exit handlers should -# not be called explicitly! The user may decide to do this manually though. +{.pop.} # stackTrace: off +when not defined(nimPreviewSlimSystem): + proc addQuitProc*(quitProc: proc() {.noconv.}) {. + importc: "atexit", header: "<stdlib.h>", deprecated: "use exitprocs.addExitProc".} + ## Adds/registers a quit procedure. + ## + ## Each call to `addQuitProc` registers another quit procedure. Up to 30 + ## procedures can be registered. They are executed on a last-in, first-out + ## basis (that is, the last function registered is the first to be executed). + ## `addQuitProc` raises an EOutOfIndex exception if `quitProc` cannot be + ## registered. + # Support for addQuitProc() is done by Ansi C's facilities here. + # In case of an unhandled exception the exit handlers should + # not be called explicitly! The user may decide to do this manually though. proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} ## Swaps the values `a` and `b`. ## - ## This is often more efficient than ``tmp = a; a = b; b = tmp``. + ## This is often more efficient than `tmp = a; a = b; b = tmp`. ## Particularly useful for sorting algorithms. ## - ## .. code-block:: Nim + ## ```nim ## var ## a = 5 ## b = 9 @@ -1423,6 +1388,7 @@ proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} ## ## assert a == 9 ## assert b == 5 + ## ``` when not defined(js) and not defined(booting) and defined(nimTrMacros): template swapRefsInArray*{swap(arr[a], arr[b])}(arr: openArray[ref], a, b: int) = @@ -1431,18 +1397,16 @@ when not defined(js) and not defined(booting) and defined(nimTrMacros): # unnecessary slow down in this case. swap(cast[ptr pointer](addr arr[a])[], cast[ptr pointer](addr arr[b])[]) -const - Inf* = 0x7FF0000000000000'f64 - ## Contains the IEEE floating point value of positive infinity. - NegInf* = 0xFFF0000000000000'f64 - ## Contains the IEEE floating point value of negative infinity. - NaN* = 0x7FF7FFFFFFFFFFFF'f64 - ## Contains an IEEE floating point value of *Not A Number*. - ## - ## Note that you cannot compare a floating point value to this value - ## and expect a reasonable result - use the `classify` procedure - ## in the `math module <math.html>`_ for checking for NaN. +when not defined(nimscript): + {.push stackTrace: off, profiler: off.} + when not defined(nimPreviewSlimSystem): + import std/sysatomics + export sysatomics + else: + import std/sysatomics + + {.pop.} include "system/memalloc" @@ -1452,126 +1416,132 @@ proc `|`*(a, b: typedesc): typedesc = discard include "system/iterators_1" -{.push stackTrace: off.} - -proc abs*(x: float64): float64 {.noSideEffect, inline.} = - if x < 0.0: -x else: x -proc abs*(x: float32): float32 {.noSideEffect, inline.} = - if x < 0.0: -x else: x -proc min*(x, y: float32): float32 {.noSideEffect, inline.} = - if x <= y or y != y: x else: y -proc min*(x, y: float64): float64 {.noSideEffect, inline.} = - if x <= y or y != y: x else: y -proc max*(x, y: float32): float32 {.noSideEffect, inline.} = - if y <= x or y != y: x else: y -proc max*(x, y: float64): float64 {.noSideEffect, inline.} = - if y <= x or y != y: x else: y -proc min*[T: not SomeFloat](x, y: T): T {.inline.} = - if x <= y: x else: y -proc max*[T: not SomeFloat](x, y: T): T {.inline.} = - if y <= x: x else: y - -{.pop.} # stackTrace: off - - -proc high*(T: typedesc[SomeFloat]): T = Inf -proc low*(T: typedesc[SomeFloat]): T = NegInf - proc len*[U: Ordinal; V: Ordinal](x: HSlice[U, V]): int {.noSideEffect, inline.} = ## Length of ordinal slice. When x.b < x.a returns zero length. - ## - ## .. code-block:: Nim + ## ```nim ## assert((0..5).len == 6) ## assert((5..2).len == 0) + ## ``` result = max(0, ord(x.b) - ord(x.a) + 1) -when defined(nimNoNilSeqs2): - when not compileOption("nilseqs"): - {.pragma: nilError, error.} - else: - {.pragma: nilError.} -else: - {.pragma: nilError.} - -proc isNil*[T](x: seq[T]): bool {.noSideEffect, magic: "IsNil", nilError.} - ## Requires `--nilseqs:on` since 0.19. - ## - ## Seqs are no longer nil by default, but set and empty. - ## Check for zero length instead. - ## - ## See also: - ## * `isNil(string) <#isNil,string>`_ - proc isNil*[T](x: ref T): bool {.noSideEffect, magic: "IsNil".} -proc isNil*(x: string): bool {.noSideEffect, magic: "IsNil", nilError.} - ## Requires `--nilseqs:on`. - ## - ## See also: - ## * `isNil(seq[T]) <#isNil,seq[T][T]>`_ proc isNil*[T](x: ptr T): bool {.noSideEffect, magic: "IsNil".} proc isNil*(x: pointer): bool {.noSideEffect, magic: "IsNil".} proc isNil*(x: cstring): bool {.noSideEffect, magic: "IsNil".} -proc isNil*[T: proc](x: T): bool {.noSideEffect, magic: "IsNil".} +proc isNil*[T: proc | iterator {.closure.}](x: T): bool {.noSideEffect, magic: "IsNil".} ## Fast check whether `x` is nil. This is sometimes more efficient than - ## ``== nil``. + ## `== nil`. +when defined(nimHasTopDownInference): + # magic used for seq type inference + proc `@`*[T](a: openArray[T]): seq[T] {.magic: "OpenArrayToSeq".} = + ## Turns an *openArray* into a sequence. + ## + ## This is not as efficient as turning a fixed length array into a sequence + ## as it always copies every element of `a`. + newSeq(result, a.len) + for i in 0..a.len-1: result[i] = a[i] +else: + proc `@`*[T](a: openArray[T]): seq[T] = + ## Turns an *openArray* into a sequence. + ## + ## This is not as efficient as turning a fixed length array into a sequence + ## as it always copies every element of `a`. + newSeq(result, a.len) + for i in 0..a.len-1: result[i] = a[i] -proc `@`*[T](a: openArray[T]): seq[T] = - ## Turns an *openArray* into a sequence. - ## - ## This is not as efficient as turning a fixed length array into a sequence - ## as it always copies every element of `a`. - newSeq(result, a.len) - for i in 0..a.len-1: result[i] = a[i] -proc `&`*[T](x, y: seq[T]): seq[T] {.noSideEffect.} = - ## Concatenates two sequences. - ## - ## Requires copying of the sequences. - ## - ## See also: - ## * `add(var seq[T], openArray[T]) <#add,seq[T][T],openArray[T]>`_ - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) - newSeq(result, x.len + y.len) - for i in 0..x.len-1: - result[i] = x[i] - for i in 0..y.len-1: - result[i+x.len] = y[i] +when defined(nimSeqsV2): -proc `&`*[T](x: seq[T], y: T): seq[T] {.noSideEffect.} = - ## Appends element y to the end of the sequence. - ## - ## Requires copying of the sequence. - ## - ## See also: - ## * `add(var seq[T], T) <#add,seq[T][T],T>`_ - ## - ## .. code-block:: Nim - ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) - newSeq(result, x.len + 1) - for i in 0..x.len-1: - result[i] = x[i] - result[x.len] = y + proc `&`*[T](x, y: sink seq[T]): seq[T] {.noSideEffect.} = + ## Concatenates two sequences. + ## + ## Requires copying of the sequences. + ## ```nim + ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], openArray[T]) <#add,seq[T],openArray[T]>`_ + newSeq(result, x.len + y.len) + for i in 0..x.len-1: + result[i] = move(x[i]) + for i in 0..y.len-1: + result[i+x.len] = move(y[i]) + + proc `&`*[T](x: sink seq[T], y: sink T): seq[T] {.noSideEffect.} = + ## Appends element y to the end of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], T) <#add,seq[T],sinkT>`_ + newSeq(result, x.len + 1) + for i in 0..x.len-1: + result[i] = move(x[i]) + result[x.len] = move(y) + + proc `&`*[T](x: sink T, y: sink seq[T]): seq[T] {.noSideEffect.} = + ## Prepends the element x to the beginning of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) + ## ``` + newSeq(result, y.len + 1) + result[0] = move(x) + for i in 0..y.len-1: + result[i+1] = move(y[i]) -proc `&`*[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = - ## Prepends the element x to the beginning of the sequence. - ## - ## Requires copying of the sequence. - ## - ## .. code-block:: Nim - ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) - newSeq(result, y.len + 1) - result[0] = x - for i in 0..y.len-1: - result[i+1] = y[i] +else: + proc `&`*[T](x, y: seq[T]): seq[T] {.noSideEffect.} = + ## Concatenates two sequences. + ## + ## Requires copying of the sequences. + ## ```nim + ## assert(@[1, 2, 3, 4] & @[5, 6] == @[1, 2, 3, 4, 5, 6]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], openArray[T]) <#add,seq[T],openArray[T]>`_ + newSeq(result, x.len + y.len) + for i in 0..x.len-1: + result[i] = x[i] + for i in 0..y.len-1: + result[i+x.len] = y[i] + + proc `&`*[T](x: seq[T], y: T): seq[T] {.noSideEffect.} = + ## Appends element y to the end of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(@[1, 2, 3] & 4 == @[1, 2, 3, 4]) + ## ``` + ## + ## See also: + ## * `add(var seq[T], T) <#add,seq[T],sinkT>`_ + newSeq(result, x.len + 1) + for i in 0..x.len-1: + result[i] = x[i] + result[x.len] = y + + proc `&`*[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = + ## Prepends the element x to the beginning of the sequence. + ## + ## Requires copying of the sequence. + ## ```nim + ## assert(1 & @[2, 3, 4] == @[1, 2, 3, 4]) + ## ``` + newSeq(result, y.len + 1) + result[0] = x + for i in 0..y.len-1: + result[i+1] = y[i] -proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} - ## Converts the AST of `x` into a string representation. This is very useful - ## for debugging. proc instantiationInfo*(index = -1, fullPaths = false): tuple[ filename: string, line: int, column: int] {.magic: "InstantiationInfo", noSideEffect.} @@ -1581,12 +1551,12 @@ proc instantiationInfo*(index = -1, fullPaths = false): tuple[ ## While similar to the `caller info`:idx: of other languages, it is determined ## at compile time. ## - ## This proc is mostly useful for meta programming (eg. ``assert`` template) + ## This proc is mostly useful for meta programming (eg. `assert` template) ## to retrieve information about the current filename and line number. ## Example: ## - ## .. code-block:: nim - ## import strutils + ## ```nim + ## import std/strutils ## ## template testException(exception, code: untyped): typed = ## try: @@ -1604,81 +1574,163 @@ proc instantiationInfo*(index = -1, fullPaths = false): tuple[ ## result = a[pos] ## ## when isMainModule: - ## testException(IndexError, tester(30)) - ## testException(IndexError, tester(1)) + ## testException(IndexDefect, tester(30)) + ## testException(IndexDefect, tester(1)) ## # --> Test failure at example.nim:20 with 'tester(1)' + ## ``` -proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = - ## Special compile-time procedure that checks whether `x` can be compiled - ## without any semantic error. - ## This can be used to check whether a type supports some operation: - ## - ## .. code-block:: Nim - ## when compiles(3 + 4): - ## echo "'+' for integers is available" - discard when notJSnotNims: - import "system/ansi_c" - import "system/memory" + import system/ansi_c + import system/memory {.push stackTrace: off.} when not defined(js) and hasThreadSupport and hostOS != "standalone": - const insideRLocksModule = false - include "system/syslocks" + import std/private/syslocks include "system/threadlocalstorage" when not defined(js) and defined(nimV2): type - TNimNode {.compilerproc.} = object # to keep the code generator simple DestructorProc = proc (p: pointer) {.nimcall, benign, raises: [].} - TNimType {.compilerproc.} = object + TNimTypeV2 {.compilerproc.} = object destructor: pointer size: int - name: cstring + align: int16 + depth: int16 + display: ptr UncheckedArray[uint32] # classToken + when defined(nimTypeNames) or defined(nimArcIds): + name: cstring traceImpl: pointer - disposeImpl: pointer - PNimType = ptr TNimType + typeInfoV1: pointer # for backwards compat, usually nil + flags: int + when defined(gcDestructors): + when defined(cpp): + vTable: ptr UncheckedArray[pointer] # vtable for types + else: + vTable: UncheckedArray[pointer] # vtable for types + PNimTypeV2 = ptr TNimTypeV2 + +proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} when notJSnotNims and defined(nimSeqsV2): include "system/strs_v2" include "system/seqs_v2" -{.pop.} +when not defined(js): + template newSeqImpl(T, len) = + result = newSeqOfCap[T](len) + {.cast(noSideEffect).}: + when defined(nimSeqsV2): + cast[ptr int](addr result)[] = len + else: + var s = cast[PGenericSeq](result) + s.len = len + proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] {.deprecated: "Use `newSeqUninit` instead".} = + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## Only available for numbers types. Note that the sequence will be + ## uninitialized. After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + ## Example: + ## ```nim + ## var x = newSeqUninitialized[int](3) + ## assert len(x) == 3 + ## x[0] = 10 + ## ``` + result = newSeqOfCap[T](len) + when defined(nimSeqsV2): + cast[ptr int](addr result)[] = len + else: + var s = cast[PGenericSeq](result) + s.len = len -when notJSnotNims: + func newSeqUninit*[T](len: Natural): seq[T] = + ## Creates a new sequence of type `seq[T]` with length `len`. + ## + ## Only available for types, which don't contain + ## managed memory or have destructors. + ## Note that the sequence will be uninitialized. + ## After the creation of the sequence you should assign + ## entries to the sequence instead of adding them. + runnableExamples: + var x = newSeqUninit[int](3) + assert len(x) == 3 + x[0] = 10 + when supportsCopyMem(T): + when nimvm: + result = newSeq[T](len) + else: + newSeqImpl(T, len) + else: + {.error: "The type T cannot contain managed memory or have destructors".} + + proc newStringUninit*(len: Natural): string = + ## Returns a new string of length `len` but with uninitialized + ## content. One needs to fill the string character after character + ## with the index operator `s[i]`. + ## + ## This procedure exists only for optimization purposes; + ## the same effect can be achieved with the `&` operator or with `add`. + when nimvm: + result = newString(len) + else: + result = newStringOfCap(len) + when defined(nimSeqsV2): + let s = cast[ptr NimStringV2](addr result) + if len > 0: + s.len = len + s.p.data[len] = '\0' + else: + let s = cast[NimString](result) + s.len = len + s.data[len] = '\0' +else: + proc newStringUninit*(len: Natural): string {. + magic: "NewString", importc: "mnewString", noSideEffect.} + +{.pop.} + +when not defined(nimscript): proc writeStackTrace*() {.tags: [], gcsafe, raises: [].} - ## Writes the current stack trace to ``stderr``. This is only works + ## Writes the current stack trace to `stderr`. This is only works ## for debug builds. Since it's usually used for debugging, this ## is proclaimed to have no IO effect! when not declared(sysFatal): include "system/fatal" -when notJSnotNims: - {.push stackTrace: off, profiler: off.} - - proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, - discardable, benign.} - ## Atomic increment of `memLoc`. Returns the value after the operation. - - proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, - discardable, benign.} - ## Atomic decrement of `memLoc`. Returns the value after the operation. - - include "system/atomics" +type + PFrame* = ptr TFrame ## Represents a runtime frame of the call stack; + ## part of the debugger API. + # keep in sync with nimbase.h `struct TFrame_` + TFrame* {.importc, nodecl, final.} = object ## The frame itself. + prev*: PFrame ## Previous frame; used for chaining the call stack. + procname*: cstring ## Name of the proc that is currently executing. + line*: int ## Line number of the proc that is currently executing. + filename*: cstring ## Filename of the proc that is currently executing. + len*: int16 ## Length of the inspectable slots. + calldepth*: int16 ## Used for max call depth checking. + when NimStackTraceMsgs: + frameMsgLen*: int ## end position in frameMsgBuf for this frame. - {.pop.} +when defined(nimV2): + var + framePtr {.threadvar.}: PFrame + include system/arc -when defined(nimV2): - include system/refs_v2 +template newException*(exceptn: typedesc, message: string; + parentException: ref Exception = nil): untyped = + ## Creates an exception object of type `exceptn` and sets its `msg` field + ## to `message`. Returns the new exception object. + (ref exceptn)(msg: message, parent: parentException) -import system/assertions -export assertions +when not defined(nimPreviewSlimSystem): + import std/assertions + export assertions import system/iterators export iterators @@ -1687,6 +1739,7 @@ export iterators proc find*[T, S](a: T, item: S): int {.inline.}= ## Returns the first index of `item` in `a` or -1 if not found. This requires ## appropriate `items` and `==` operations to work. + result = 0 for i in items(a): if i == item: return inc(result) @@ -1694,21 +1747,23 @@ proc find*[T, S](a: T, item: S): int {.inline.}= proc contains*[T](a: openArray[T], item: T): bool {.inline.}= ## Returns true if `item` is in `a` or false if not found. This is a shortcut - ## for ``find(a, item) >= 0``. + ## for `find(a, item) >= 0`. ## ## This allows the `in` operator: `a.contains(item)` is the same as ## `item in a`. - ## - ## .. code-block:: Nim + ## ```nim ## var a = @[1, 3, 5] ## assert a.contains(5) ## assert 3 in a ## assert 99 notin a + ## ``` return find(a, item) >= 0 proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = - ## Returns the last item of `s` and decreases ``s.len`` by one. This treats + ## Returns the last item of `s` and decreases `s.len` by one. This treats ## `s` as a stack and implements the common *pop* operation. + ## + ## Raises `IndexDefect` if `s` is empty. runnableExamples: var a = @[1, 3, 5, 7] let b = pop(a) @@ -1724,14 +1779,14 @@ proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = setLen(s, L) proc `==`*[T: tuple|object](x, y: T): bool = - ## Generic ``==`` operator for tuples that is lifted from the components. + ## Generic `==` operator for tuples that is lifted from the components. ## of `x` and `y`. for a, b in fields(x, y): if a != b: return false return true proc `<=`*[T: tuple](x, y: T): bool = - ## Generic lexicographic ``<=`` operator for tuples that is lifted from the + ## Generic lexicographic `<=` operator for tuples that is lifted from the ## components of `x` and `y`. This implementation uses `cmp`. for a, b in fields(x, y): var c = cmp(a, b) @@ -1740,7 +1795,7 @@ proc `<=`*[T: tuple](x, y: T): bool = return true proc `<`*[T: tuple](x, y: T): bool = - ## Generic lexicographic ``<`` operator for tuples that is lifted from the + ## Generic lexicographic `<` operator for tuples that is lifted from the ## components of `x` and `y`. This implementation uses `cmp`. for a, b in fields(x, y): var c = cmp(a, b) @@ -1754,25 +1809,7 @@ include "system/gc_interface" # we have to compute this here before turning it off in except.nim anyway ... const NimStackTrace = compileOption("stacktrace") -template coroutinesSupportedPlatform(): bool = - when defined(sparc) or defined(ELATE) or compileOption("gc", "v2") or - defined(boehmgc) or defined(gogc) or defined(nogc) or defined(gcRegions) or - defined(gcMarkAndSweep): - false - else: - true - -when defined(nimCoroutines): - # Explicit opt-in. - when not coroutinesSupportedPlatform(): - {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} - const nimCoroutines* = true -elif defined(noNimCoroutines): - # Explicit opt-out. - const nimCoroutines* = false -else: - # Autodetect coroutine support. - const nimCoroutines* = false +import system/coro_detection {.push checks: off.} # obviously we cannot generate checking operations here :-) @@ -1780,76 +1817,78 @@ else: # however, stack-traces are available for most parts # of the code -var - globalRaiseHook*: proc (e: ref Exception): bool {.nimcall, benign.} - ## With this hook you can influence exception handling on a global level. - ## If not nil, every 'raise' statement ends up calling this hook. - ## - ## **Warning**: Ordinary application code should never set this hook! - ## You better know what you do when setting this. - ## - ## If ``globalRaiseHook`` returns false, the exception is caught and does - ## not propagate further through the call stack. - - localRaiseHook* {.threadvar.}: proc (e: ref Exception): bool {.nimcall, benign.} - ## With this hook you can influence exception handling on a - ## thread local level. - ## If not nil, every 'raise' statement ends up calling this hook. - ## - ## **Warning**: Ordinary application code should never set this hook! - ## You better know what you do when setting this. - ## - ## If ``localRaiseHook`` returns false, the exception - ## is caught and does not propagate further through the call stack. +when notJSnotNims: + var + globalRaiseHook*: proc (e: ref Exception): bool {.nimcall, benign.} + ## With this hook you can influence exception handling on a global level. + ## If not nil, every 'raise' statement ends up calling this hook. + ## + ## .. warning:: Ordinary application code should never set this hook! You better know what you do when setting this. + ## + ## If `globalRaiseHook` returns false, the exception is caught and does + ## not propagate further through the call stack. - outOfMemHook*: proc () {.nimcall, tags: [], benign, raises: [].} - ## Set this variable to provide a procedure that should be called - ## in case of an `out of memory`:idx: event. The standard handler - ## writes an error message and terminates the program. - ## - ## `outOfMemHook` can be used to raise an exception in case of OOM like so: - ## - ## .. code-block:: Nim - ## - ## var gOutOfMem: ref EOutOfMemory - ## new(gOutOfMem) # need to be allocated *before* OOM really happened! - ## gOutOfMem.msg = "out of memory" - ## - ## proc handleOOM() = - ## raise gOutOfMem - ## - ## system.outOfMemHook = handleOOM - ## - ## If the handler does not raise an exception, ordinary control flow - ## continues and the program is terminated. - unhandledExceptionHook*: proc (e: ref Exception) {.nimcall, tags: [], benign, raises: [].} - ## Set this variable to provide a procedure that should be called - ## in case of an `unhandle exception` event. The standard handler - ## writes an error message and terminates the program, except when - ## using `--os:any` + localRaiseHook* {.threadvar.}: proc (e: ref Exception): bool {.nimcall, benign.} + ## With this hook you can influence exception handling on a + ## thread local level. + ## If not nil, every 'raise' statement ends up calling this hook. + ## + ## .. warning:: Ordinary application code should never set this hook! You better know what you do when setting this. + ## + ## If `localRaiseHook` returns false, the exception + ## is caught and does not propagate further through the call stack. -type - PFrame* = ptr TFrame ## Represents a runtime frame of the call stack; - ## part of the debugger API. - TFrame* {.importc, nodecl, final.} = object ## The frame itself. - prev*: PFrame ## Previous frame; used for chaining the call stack. - procname*: cstring ## Name of the proc that is currently executing. - line*: int ## Line number of the proc that is currently executing. - filename*: cstring ## Filename of the proc that is currently executing. - len*: int16 ## Length of the inspectable slots. - calldepth*: int16 ## Used for max call depth checking. + outOfMemHook*: proc () {.nimcall, tags: [], benign, raises: [].} + ## Set this variable to provide a procedure that should be called + ## in case of an `out of memory`:idx: event. The standard handler + ## writes an error message and terminates the program. + ## + ## `outOfMemHook` can be used to raise an exception in case of OOM like so: + ## + ## ```nim + ## var gOutOfMem: ref EOutOfMemory + ## new(gOutOfMem) # need to be allocated *before* OOM really happened! + ## gOutOfMem.msg = "out of memory" + ## + ## proc handleOOM() = + ## raise gOutOfMem + ## + ## system.outOfMemHook = handleOOM + ## ``` + ## + ## If the handler does not raise an exception, ordinary control flow + ## continues and the program is terminated. + unhandledExceptionHook*: proc (e: ref Exception) {.nimcall, tags: [], benign, raises: [].} + ## Set this variable to provide a procedure that should be called + ## in case of an `unhandle exception` event. The standard handler + ## writes an error message and terminates the program, except when + ## using `--os:any` -when defined(js): +when defined(js) or defined(nimdoc): proc add*(x: var string, y: cstring) {.asmNoStackFrame.} = - asm """ + ## Appends `y` to `x` in place. + runnableExamples: + var tmp = "" + tmp.add(cstring("ab")) + tmp.add(cstring("cd")) + doAssert tmp == "abcd" + {.emit: """ if (`x` === null) { `x` = []; } var off = `x`.length; `x`.length += `y`.length; for (var i = 0; i < `y`.length; ++i) { `x`[off+i] = `y`.charCodeAt(i); } - """ - proc add*(x: var cstring, y: cstring) {.magic: "AppendStrStr".} + """.} + proc add*(x: var cstring, y: cstring) {.magic: "AppendStrStr".} = + ## Appends `y` to `x` in place. + ## Only implemented for JS backend. + runnableExamples: + when defined(js): + var tmp: cstring = "" + tmp.add(cstring("ab")) + tmp.add(cstring("cd")) + doAssert tmp == cstring("abcd") elif hasAlloc: {.push stackTrace: off, profiler: off.} @@ -1861,52 +1900,32 @@ elif hasAlloc: inc(i) {.pop.} -when defined(nimvarargstyped): - proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} - ## Writes and flushes the parameters to the standard output. - ## - ## Special built-in that takes a variable number of arguments. Each argument - ## is converted to a string via ``$``, so it works for user-defined - ## types that have an overloaded ``$`` operator. - ## It is roughly equivalent to ``writeLine(stdout, x); flushFile(stdout)``, but - ## available for the JavaScript target too. - ## - ## Unlike other IO operations this is guaranteed to be thread-safe as - ## ``echo`` is very often used for debugging convenience. If you want to use - ## ``echo`` inside a `proc without side effects - ## <manual.html#pragmas-nosideeffect-pragma>`_ you can use `debugEcho - ## <#debugEcho,varargs[typed,]>`_ instead. - - proc debugEcho*(x: varargs[typed, `$`]) {.magic: "Echo", noSideEffect, - tags: [], raises: [].} - ## Same as `echo <#echo,varargs[typed,]>`_, but as a special semantic rule, - ## ``debugEcho`` pretends to be free of side effects, so that it can be used - ## for debugging routines marked as `noSideEffect - ## <manual.html#pragmas-nosideeffect-pragma>`_. -else: - proc echo*(x: varargs[untyped, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} - proc debugEcho*(x: varargs[untyped, `$`]) {.magic: "Echo", noSideEffect, - tags: [], raises: [].} +proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", benign, sideEffect.} + ## Writes and flushes the parameters to the standard output. + ## + ## Special built-in that takes a variable number of arguments. Each argument + ## is converted to a string via `$`, so it works for user-defined + ## types that have an overloaded `$` operator. + ## It is roughly equivalent to `writeLine(stdout, x); flushFile(stdout)`, but + ## available for the JavaScript target too. + ## + ## Unlike other IO operations this is guaranteed to be thread-safe as + ## `echo` is very often used for debugging convenience. If you want to use + ## `echo` inside a `proc without side effects + ## <manual.html#pragmas-nosideeffect-pragma>`_ you can use `debugEcho + ## <#debugEcho,varargs[typed,]>`_ instead. -template newException*(exceptn: typedesc, message: string; - parentException: ref Exception = nil): untyped = - ## Creates an exception object of type ``exceptn`` and sets its ``msg`` field - ## to `message`. Returns the new exception object. - when declared(owned): - var e: owned(ref exceptn) - else: - var e: ref exceptn - new(e) - e.msg = message - e.parent = parentException - e +proc debugEcho*(x: varargs[typed, `$`]) {.magic: "Echo", noSideEffect, + tags: [], raises: [].} + ## Same as `echo <#echo,varargs[typed,]>`_, but as a special semantic rule, + ## `debugEcho` pretends to be free of side effects, so that it can be used + ## for debugging routines marked as `noSideEffect + ## <manual.html#pragmas-nosideeffect-pragma>`_. when hostOS == "standalone" and defined(nogc): proc nimToCStringConv(s: NimString): cstring {.compilerproc, inline.} = if s == nil or s.len == 0: result = cstring"" - else: result = cstring(addr s.data) + else: result = cast[cstring](addr s.data) proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} ## Get type information for `x`. @@ -1914,25 +1933,9 @@ proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} ## Ordinary code should not use this, but the `typeinfo module ## <typeinfo.html>`_ instead. -{.push stackTrace: off.} -proc abs*(x: int): int {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int8): int8 {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int16): int16 {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int32): int32 {.magic: "AbsI", noSideEffect.} = - if x < 0: -x else: x -proc abs*(x: int64): int64 {.magic: "AbsI", noSideEffect.} = - ## Returns the absolute value of `x`. - ## - ## If `x` is ``low(x)`` (that is -MININT for its type), - ## an overflow exception is thrown (if overflow checking is turned on). - result = if x < 0: -x else: x -{.pop.} - when not defined(js): + proc likelyProc(val: bool): bool {.importc: "NIM_LIKELY", nodecl, noSideEffect.} proc unlikelyProc(val: bool): bool {.importc: "NIM_UNLIKELY", nodecl, noSideEffect.} @@ -1942,13 +1945,13 @@ template likely*(val: bool): bool = ## You can use this template to decorate a branch condition. On certain ## platforms this can help the processor predict better which branch is ## going to be run. Example: - ## - ## .. code-block:: Nim + ## ```nim ## for value in inputValues: ## if likely(value <= 100): ## process(value) ## else: ## echo "Value too big!" + ## ``` ## ## On backends without branch prediction (JS and the nimscript VM), this ## template will not affect code execution. @@ -1966,13 +1969,13 @@ template unlikely*(val: bool): bool = ## You can use this proc to decorate a branch condition. On certain ## platforms this can help the processor predict better which branch is ## going to be run. Example: - ## - ## .. code-block:: Nim + ## ```nim ## for value in inputValues: ## if unlikely(value > 100): ## echo "Value too big!" ## else: ## process(value) + ## ``` ## ## On backends without branch prediction (JS and the nimscript VM), this ## template will not affect code execution. @@ -1984,46 +1987,63 @@ template unlikely*(val: bool): bool = else: unlikelyProc(val) - import system/dollars export dollars -const - NimMajor* {.intdefine.}: int = 1 - ## is the major number of Nim's version. +when defined(nimAuditDelete): + {.pragma: auditDelete, deprecated: "review this call for out of bounds behavior".} +else: + {.pragma: auditDelete.} - NimMinor* {.intdefine.}: int = 1 - ## is the minor number of Nim's version. +proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, systemRaisesDefect, auditDelete.} = + ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position. + ## + ## This is an `O(n)` operation. + ## + ## See also: + ## * `del <#del,seq[T],Natural>`_ for O(1) operation + ## + runnableExamples: + var s = @[1, 2, 3, 4, 5] + s.delete(2) + doAssert s == @[1, 2, 4, 5] - NimPatch* {.intdefine.}: int = 1 - ## is the patch number of Nim's version. + when not defined(nimAuditDelete): + if i > high(x): + # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring + raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed") - NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch - ## is the version of Nim as a string. + template defaultImpl = + let xl = x.len + for j in i.int..xl-2: movingCopy(x[j], x[j+1]) + setLen(x, xl-1) + when nimvm: + defaultImpl() + else: + when defined(js): + {.emit: "`x`.splice(`i`, 1);".} + else: + defaultImpl() -type - FileSeekPos* = enum ## Position relative to which seek should happen. - # The values are ordered so that they match with stdio - # SEEK_SET, SEEK_CUR and SEEK_END respectively. - fspSet ## Seek to absolute value - fspCur ## Seek relative to current position - fspEnd ## Seek relative to end +const + NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch + ## is the version of Nim as a string. when not defined(js): {.push stackTrace: off, profiler: off.} when hasAlloc: when not defined(gcRegions) and not usesDestructors: - proc initGC() {.gcsafe.} + proc initGC() {.gcsafe, raises: [].} proc initStackBottom() {.inline, compilerproc.} = # WARNING: This is very fragile! An array size of 8 does not work on my # Linux 64bit system. -- That's because the stack direction is the other # way around. when declared(nimGC_setStackBottom): - var locals {.volatile.}: pointer + var locals {.volatile, noinit.}: pointer locals = addr(locals) nimGC_setStackBottom(locals) @@ -2062,19 +2082,18 @@ when notJSnotNims: memTrackerOp("moveMem", dest, size) proc equalMem(a, b: pointer, size: Natural): bool = nimCmpMem(a, b, size) == 0 + proc cmpMem(a, b: pointer, size: Natural): int = + nimCmpMem(a, b, size).int -when not defined(js): +when not defined(js) or defined(nimscript): + # nimscript can be defined if config file for js compilation proc cmp(x, y: string): int = - when defined(nimscript): + when nimvm: if x < y: result = -1 elif x > y: result = 1 else: result = 0 else: - when nimvm: - if x < y: result = -1 - elif x > y: result = 1 - else: result = 0 - else: + when not defined(nimscript): # avoid semantic checking let minlen = min(x.len, y.len) result = int(nimCmpMem(x.cstring, y.cstring, cast[csize_t](minlen))) if result == 0: @@ -2082,14 +2101,16 @@ when not defined(js): when declared(newSeq): proc cstringArrayToSeq*(a: cstringArray, len: Natural): seq[string] = - ## Converts a ``cstringArray`` to a ``seq[string]``. `a` is supposed to be - ## of length ``len``. + ## Converts a `cstringArray` to a `seq[string]`. `a` is supposed to be + ## of length `len`. + if a == nil: return @[] newSeq(result, len) for i in 0..len-1: result[i] = $a[i] proc cstringArrayToSeq*(a: cstringArray): seq[string] = - ## Converts a ``cstringArray`` to a ``seq[string]``. `a` is supposed to be - ## terminated by ``nil``. + ## Converts a `cstringArray` to a `seq[string]`. `a` is supposed to be + ## terminated by `nil`. + if a == nil: return @[] var L = 0 while a[L] != nil: inc(L) result = cstringArrayToSeq(a, L) @@ -2114,7 +2135,7 @@ when not defined(js) and declared(alloc0) and declared(dealloc): inc(i) dealloc(a) -when notJSnotNims: +when notJSnotNims and not gotoBasedExceptions: type PSafePoint = ptr TSafePoint TSafePoint {.compilerproc, final.} = object @@ -2127,7 +2148,12 @@ when not defined(js): when declared(initAllocator): initAllocator() when hasThreadSupport: - when hostOS != "standalone": include "system/threads" + when hostOS != "standalone": + include system/threadimpl + when not defined(nimPreviewSlimSystem): + import std/typedthreads + export typedthreads + elif not defined(nogc) and not defined(nimscript): when not defined(useNimRtl) and not defined(createNimRtl): initStackBottom() when declared(initGC): initGC() @@ -2136,6 +2162,16 @@ when notJSnotNims: proc setControlCHook*(hook: proc () {.noconv.}) ## Allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. + ## Example: + ## + ## ```nim + ## proc ctrlc() {.noconv.} = + ## echo "Ctrl+C fired!" + ## # do clean up stuff + ## quit() + ## + ## setControlCHook(ctrlc) + ## ``` when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook*() @@ -2147,7 +2183,7 @@ when notJSnotNims: proc getStackTrace*(e: ref Exception): string {.gcsafe.} ## Gets the stack trace associated with `e`, which is the stack that - ## lead to the ``raise`` statement. This only works for debug builds. + ## lead to the `raise` statement. This only works for debug builds. {.push stackTrace: off, profiler: off.} when defined(memtracker): @@ -2161,7 +2197,7 @@ when notJSnotNims: # we cannot compile this with stack tracing on # as it would recurse endlessly! - include "system/arithm" + include "system/integerops" {.pop.} @@ -2174,8 +2210,19 @@ when not defined(js): when notJSnotNims: when hostOS != "standalone" and hostOS != "any": + type + LibHandle = pointer # private type + ProcAddr = pointer # library loading and loading of procs: + + proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} + proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} + + proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} + include "system/dyncalls" + import system/countbits_impl include "system/sets" when defined(gogc): @@ -2183,27 +2230,28 @@ when notJSnotNims: else: const GenericSeqSize = (2 * sizeof(int)) - when not defined(nimV2): - proc getDiscriminant(aa: pointer, n: ptr TNimNode): uint = - sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase") - var d: uint - var a = cast[uint](aa) - case n.typ.size - of 1: d = uint(cast[ptr uint8](a + uint(n.offset))[]) - of 2: d = uint(cast[ptr uint16](a + uint(n.offset))[]) - of 4: d = uint(cast[ptr uint32](a + uint(n.offset))[]) - of 8: d = uint(cast[ptr uint64](a + uint(n.offset))[]) - else: sysAssert(false, "getDiscriminant: invalid n.typ.size") - return d - - proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = - var discr = getDiscriminant(aa, n) - if discr < cast[uint](n.len): - result = n.sons[discr] - if result == nil: result = n.sons[n.len] - # n.sons[n.len] contains the ``else`` part (but may be nil) - else: - result = n.sons[n.len] + proc getDiscriminant(aa: pointer, n: ptr TNimNode): uint = + sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase") + var d: uint + var a = cast[uint](aa) + case n.typ.size + of 1: d = uint(cast[ptr uint8](a + uint(n.offset))[]) + of 2: d = uint(cast[ptr uint16](a + uint(n.offset))[]) + of 4: d = uint(cast[ptr uint32](a + uint(n.offset))[]) + of 8: d = uint(cast[ptr uint64](a + uint(n.offset))[]) + else: + d = 0'u + sysAssert(false, "getDiscriminant: invalid n.typ.size") + return d + + proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = + var discr = getDiscriminant(aa, n) + if discr < cast[uint](n.len): + result = n.sons[discr] + if result == nil: result = n.sons[n.len] + # n.sons[n.len] contains the `else` part (but may be nil) + else: + result = n.sons[n.len] when notJSnotNims and hasAlloc: {.push profiler: off.} @@ -2215,13 +2263,14 @@ when notJSnotNims and hasAlloc: {.pop.} include "system/strmantle" - when not usesDestructors: - include "system/assign" + include "system/assign" + when not defined(nimV2): include "system/repr" when notJSnotNims and hasThreadSupport and hostOS != "standalone": - include "system/channels" + when not defined(nimPreviewSlimSystem): + include "system/channels_builtin" when notJSnotNims and hostOS != "standalone": @@ -2242,9 +2291,10 @@ when notJSnotNims and hostOS != "standalone": proc setCurrentException*(exc: ref Exception) {.inline, benign.} = ## Sets the current exception. ## - ## **Warning**: Only use this if you know what you are doing. + ## .. warning:: Only use this if you know what you are doing. currException = exc - +elif defined(nimscript): + proc getCurrentException*(): ref Exception {.compilerRtl.} = discard when notJSnotNims: {.push stackTrace: off, profiler: off.} @@ -2252,45 +2302,135 @@ when notJSnotNims: include "system/profiler" {.pop.} - proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = + proc rawProc*[T: proc {.closure.} | iterator {.closure.}](x: T): pointer {.noSideEffect, inline.} = ## Retrieves the raw proc pointer of the closure `x`. This is - ## useful for interfacing closures with C. + ## useful for interfacing closures with C/C++, hash computations, etc. + ## If `rawEnv(x)` returns `nil`, the proc which the result points to + ## takes as many parameters as `x`, but with `{.nimcall.}` as its calling + ## convention instead of `{.closure.}`, otherwise it takes one more parameter + ## which is a `pointer`, and it still has `{.nimcall.}` as its calling convention. + ## To invoke the resulted proc, what this returns has to be casted into a `proc`, + ## not a `ptr proc`, and, in a case where `rawEnv(x)` returns non-`nil`, + ## the last and additional argument has to be the result of `rawEnv(x)`. + ## This is not available for the JS target. + #[ + The conversion from function pointer to `void*` is a tricky topic, but this + should work at least for c++ >= c++11, e.g. for `dlsym` support. + refs: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57869, + https://stackoverflow.com/questions/14125474/casts-between-pointer-to-function-and-pointer-to-object-in-c-and-c + ]# + runnableExamples: + proc makeClosure(x: int): (proc(y: int): int) = + var n = x + result = ( + proc(y: int): int = + n += y + return n + ) + + var + c1 = makeClosure(10) + e = c1.rawEnv() + p = c1.rawProc() + + if e.isNil(): + let c2 = cast[proc(y: int): int {.nimcall.}](p) + echo c2(2) + else: + let c3 = cast[proc(y: int; env: pointer): int {.nimcall.}](p) + echo c3(3, e) + {.emit: """ - `result` = `x`.ClP_0; + `result` = (void*)`x`.ClP_0; """.} - proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## Retrieves the raw environment pointer of the closure `x`. This is - ## useful for interfacing closures with C. + proc rawEnv*[T: proc {.closure.} | iterator {.closure.}](x: T): pointer {.noSideEffect, inline.} = + ## Retrieves the raw environment pointer of the closure `x`. See also `rawProc`. + ## This is not available for the JS target. {.emit: """ `result` = `x`.ClE_0; """.} - proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = - ## can be used to determine if a first class iterator has finished. +proc finished*[T: iterator {.closure.}](x: T): bool {.noSideEffect, inline, magic: "Finished".} = + ## It can be used to determine if a first class iterator has finished. + when defined(js): + # TODO: mangle `:state` + {.emit: """ + `result` = (`x`.ClE_0).HEX3Astate < 0; + """.} + else: {.emit: """ `result` = ((NI*) `x`.ClE_0)[1] < 0; """.} -when defined(js): - when not defined(nimscript): - include "system/jssys" - include "system/reprjs" - else: - proc cmp(x, y: string): int = - if x == y: return 0 - if x < y: return -1 - return 1 +from std/private/digitsutils import addInt +export addInt + +when defined(js) and not defined(nimscript): + # nimscript can be defined if config file for js compilation + include "system/jssys" + include "system/reprjs" + + +when defined(nimNoQuit): + proc quit*(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(nimdoc): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} + ## Stops the program immediately with an exit code. + ## + ## Before stopping the program the "exit procedures" are called in the + ## opposite order they were added with `addExitProc <exitprocs.html#addExitProc,proc)>`_. + ## + ## The proc `quit(QuitSuccess)` is called implicitly when your nim + ## program finishes without incident for platforms where this is the + ## expected behavior. A raised unhandled exception is + ## equivalent to calling `quit(QuitFailure)`. + ## + ## Note that this is a *runtime* call and using `quit` inside a macro won't + ## have any compile time effect. If you need to stop the compiler inside a + ## macro, use the `error <manual.html#pragmas-error-pragma>`_ or `fatal + ## <manual.html#pragmas-fatal-pragma>`_ pragmas. + ## + ## .. warning:: `errorcode` gets saturated when it exceeds the valid range + ## on the specific platform. On Posix, the valid range is `low(int8)..high(int8)`. + ## On Windows, the valid range is `low(int32)..high(int32)`. For instance, + ## `quit(int(0x100000000))` is equal to `quit(127)` on Linux. + ## + ## .. danger:: In almost all cases, in particular in library code, prefer + ## alternatives, e.g. `raiseAssert` or raise a `Defect`. + ## `quit` bypasses regular control flow in particular `defer`, + ## `try`, `catch`, `finally` and `destructors`, and exceptions that may have been + ## raised by an `addExitProc` proc, as well as cleanup code in other threads. + ## It does *not* call the garbage collector to free all the memory, + ## unless an `addExitProc` proc calls `GC_fullCollect <#GC_fullCollect>`_. -when defined(js) or defined(nimscript): - proc addInt*(result: var string; x: int64) = - result.add $x +elif defined(genode): + proc quit*(errorcode: int = QuitSuccess) {.inline, noreturn.} = + rawQuit(errorcode) - proc addFloat*(result: var string; x: float) = - result.add $x +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc quit*(errorcode: int = QuitSuccess) {.inline, noreturn.} = + when defined(posix): # posix uses low 8 bits + type ExitCodeRange = int8 + else: # win32 uses low 32 bits + type ExitCodeRange = cint + when sizeof(errorcode) > sizeof(ExitCodeRange): + if errorcode < low(ExitCodeRange): + rawQuit(low(ExitCodeRange).cint) + elif errorcode > high(ExitCodeRange): + rawQuit(high(ExitCodeRange).cint) + else: + rawQuit(errorcode.cint) + else: + rawQuit(errorcode.cint) proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = - ## A shorthand for ``echo(errormsg); quit(errorcode)``. + ## A shorthand for `echo(errormsg); quit(errorcode)`. when defined(nimscript) or defined(js) or (hostOS == "standalone"): echo errormsg else: @@ -2302,312 +2442,57 @@ proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = quit(errorcode) {.pop.} # checks: off -{.pop.} # hints: off - -proc `/`*(x, y: int): float {.inline, noSideEffect.} = - ## Division of integers that results in a float. - ## - ## See also: - ## * `div <#div,int,int>`_ - ## * `mod <#mod,int,int>`_ - ## - ## .. code-block:: Nim - ## echo 7 / 5 # => 1.4 - result = toFloat(x) / toFloat(y) - -type - BackwardsIndex* = distinct int ## Type that is constructed by ``^`` for - ## reversed array accesses. - ## (See `^ template <#^.t,int>`_) - -template `^`*(x: int): BackwardsIndex = BackwardsIndex(x) - ## Builtin `roof`:idx: operator that can be used for convenient array access. - ## ``a[^x]`` is a shortcut for ``a[a.len-x]``. - ## - ## .. code-block:: Nim - ## let - ## a = [1, 3, 5, 7, 9] - ## b = "abcdefgh" - ## - ## echo a[^1] # => 9 - ## echo b[^2] # => g - -template `..^`*(a, b: untyped): untyped = - ## A shortcut for `.. ^` to avoid the common gotcha that a space between - ## '..' and '^' is required. - a .. ^b - -template `..<`*(a, b: untyped): untyped = - ## A shortcut for `a .. pred(b)`. - ## - ## .. code-block:: Nim - ## for i in 5 ..< 9: - ## echo i # => 5; 6; 7; 8 - a .. (when b is BackwardsIndex: succ(b) else: pred(b)) - -template spliceImpl(s, a, L, b: untyped): untyped = - # make room for additional elements or cut: - var shift = b.len - max(0,L) # ignore negative slice size - var newLen = s.len + shift - if shift > 0: - # enlarge: - setLen(s, newLen) - for i in countdown(newLen-1, a+b.len): movingCopy(s[i], s[i-shift]) - else: - for i in countup(a+b.len, newLen-1): movingCopy(s[i], s[i-shift]) - # cut down: - setLen(s, newLen) - # fill the hole: - for i in 0 ..< b.len: s[a+i] = b[i] - -template `^^`(s, i: untyped): untyped = - (when i is BackwardsIndex: s.len - int(i) else: int(i)) - -template `[]`*(s: string; i: int): char = arrGet(s, i) -template `[]=`*(s: string; i: int; val: char) = arrPut(s, i, val) - -proc `[]`*[T, U](s: string, x: HSlice[T, U]): string {.inline.} = - ## Slice operation for strings. - ## Returns the inclusive range `[s[x.a], s[x.b]]`: - ## - ## .. code-block:: Nim - ## var s = "abcdef" - ## assert s[1..3] == "bcd" - let a = s ^^ x.a - let L = (s ^^ x.b) - a + 1 - result = newString(L) - for i in 0 ..< L: result[i] = s[i + a] - -proc `[]=`*[T, U](s: var string, x: HSlice[T, U], b: string) = - ## Slice assignment for strings. - ## - ## If ``b.len`` is not exactly the number of elements that are referred to - ## by `x`, a `splice`:idx: is performed: - ## - runnableExamples: - var s = "abcdefgh" - s[1 .. ^2] = "xyz" - assert s == "axyzh" - - var a = s ^^ x.a - var L = (s ^^ x.b) - a + 1 - if L == b.len: - for i in 0..<L: s[i+a] = b[i] - else: - spliceImpl(s, a, L, b) - -proc `[]`*[Idx, T, U, V](a: array[Idx, T], x: HSlice[U, V]): seq[T] = - ## Slice operation for arrays. - ## Returns the inclusive range `[a[x.a], a[x.b]]`: - ## - ## .. code-block:: Nim - ## var a = [1, 2, 3, 4] - ## assert a[0..2] == @[1, 2, 3] - let xa = a ^^ x.a - let L = (a ^^ x.b) - xa + 1 - result = newSeq[T](L) - for i in 0..<L: result[i] = a[Idx(i + xa)] - -proc `[]=`*[Idx, T, U, V](a: var array[Idx, T], x: HSlice[U, V], b: openArray[T]) = - ## Slice assignment for arrays. - ## - ## .. code-block:: Nim - ## var a = [10, 20, 30, 40, 50] - ## a[1..2] = @[99, 88] - ## assert a == [10, 99, 88, 40, 50] - let xa = a ^^ x.a - let L = (a ^^ x.b) - xa + 1 - if L == b.len: - for i in 0..<L: a[Idx(i + xa)] = b[i] - else: - sysFatal(RangeError, "different lengths for slice assignment") - -proc `[]`*[T, U, V](s: openArray[T], x: HSlice[U, V]): seq[T] = - ## Slice operation for sequences. - ## Returns the inclusive range `[s[x.a], s[x.b]]`: - ## - ## .. code-block:: Nim - ## var s = @[1, 2, 3, 4] - ## assert s[0..2] == @[1, 2, 3] - let a = s ^^ x.a - let L = (s ^^ x.b) - a + 1 - newSeq(result, L) - for i in 0 ..< L: result[i] = s[i + a] - -proc `[]=`*[T, U, V](s: var seq[T], x: HSlice[U, V], b: openArray[T]) = - ## Slice assignment for sequences. - ## - ## If ``b.len`` is not exactly the number of elements that are referred to - ## by `x`, a `splice`:idx: is performed. - runnableExamples: - var s = @"abcdefgh" - s[1 .. ^2] = @"xyz" - assert s == @"axyzh" - - let a = s ^^ x.a - let L = (s ^^ x.b) - a + 1 - if L == b.len: - for i in 0 ..< L: s[i+a] = b[i] - else: - spliceImpl(s, a, L, b) - -proc `[]`*[T](s: openArray[T]; i: BackwardsIndex): T {.inline.} = - system.`[]`(s, s.len - int(i)) - -proc `[]`*[Idx, T](a: array[Idx, T]; i: BackwardsIndex): T {.inline.} = - a[Idx(a.len - int(i) + int low(a))] -proc `[]`*(s: string; i: BackwardsIndex): char {.inline.} = s[s.len - int(i)] - -proc `[]`*[T](s: var openArray[T]; i: BackwardsIndex): var T {.inline.} = - system.`[]`(s, s.len - int(i)) -proc `[]`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex): var T {.inline.} = - a[Idx(a.len - int(i) + int low(a))] - -proc `[]=`*[T](s: var openArray[T]; i: BackwardsIndex; x: T) {.inline.} = - system.`[]=`(s, s.len - int(i), x) -proc `[]=`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex; x: T) {.inline.} = - a[Idx(a.len - int(i) + int low(a))] = x -proc `[]=`*(s: var string; i: BackwardsIndex; x: char) {.inline.} = - s[s.len - int(i)] = x - -proc slurp*(filename: string): string {.magic: "Slurp".} - ## This is an alias for `staticRead <#staticRead,string>`_. - -proc staticRead*(filename: string): string {.magic: "Slurp".} - ## Compile-time `readFile <io.html#readFile,string>`_ proc for easy - ## `resource`:idx: embedding: - ## - ## .. code-block:: Nim - ## const myResource = staticRead"mydatafile.bin" - ## - ## `slurp <#slurp,string>`_ is an alias for ``staticRead``. - -proc gorge*(command: string, input = "", cache = ""): string {. - magic: "StaticExec".} = discard - ## This is an alias for `staticExec <#staticExec,string,string,string>`_. - -proc staticExec*(command: string, input = "", cache = ""): string {. - magic: "StaticExec".} = discard - ## Executes an external process at compile-time and returns its text output - ## (stdout + stderr). - ## - ## If `input` is not an empty string, it will be passed as a standard input - ## to the executed program. - ## - ## .. code-block:: Nim - ## const buildInfo = "Revision " & staticExec("git rev-parse HEAD") & - ## "\nCompiled on " & staticExec("uname -v") - ## - ## `gorge <#gorge,string,string,string>`_ is an alias for ``staticExec``. - ## - ## Note that you can use this proc inside a pragma like - ## `passc <manual.html#implementation-specific-pragmas-passc-pragma>`_ or - ## `passl <manual.html#implementation-specific-pragmas-passl-pragma>`_. - ## - ## If ``cache`` is not empty, the results of ``staticExec`` are cached within - ## the ``nimcache`` directory. Use ``--forceBuild`` to get rid of this caching - ## behaviour then. ``command & input & cache`` (the concatenated string) is - ## used to determine whether the entry in the cache is still valid. You can - ## use versioning information for ``cache``: - ## - ## .. code-block:: Nim - ## const stateMachine = staticExec("dfaoptimizer", "input", "0.8.0") - -proc gorgeEx*(command: string, input = "", cache = ""): tuple[output: string, - exitCode: int] = - ## Similar to `gorge <#gorge,string,string,string>`_ but also returns the - ## precious exit code. - discard - - -proc `+=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Increments in place a floating point number. - x = x + y +# {.pop.} # hints: off -proc `-=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Decrements in place a floating point number. - x = x - y - -proc `*=`*[T: float|float32|float64] (x: var T, y: T) {. - inline, noSideEffect.} = - ## Multiplies in place a floating point number. - x = x * y - -proc `/=`*(x: var float64, y: float64) {.inline, noSideEffect.} = - ## Divides in place a floating point number. - x = x / y - -proc `/=`*[T: float|float32](x: var T, y: T) {.inline, noSideEffect.} = - ## Divides in place a floating point number. - x = x / y +include "system/indices" proc `&=`*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} ## Appends in place to a string. - ## - ## .. code-block:: Nim + ## ```nim ## var a = "abc" ## a &= "de" # a <- "abcde" + ## ``` template `&=`*(x, y: typed) = ## Generic 'sink' operator for Nim. ## - ## For files an alias for ``write``. - ## If not specialized further, an alias for ``add``. + ## If not specialized further, an alias for `add`. add(x, y) -when declared(File): - template `&=`*(f: File, x: typed) = write(f, x) - -template currentSourcePath*: string = instantiationInfo(-1, true).filename - ## Returns the full file-system path of the current source. - ## - ## To get the directory containing the current source, use it with - ## `os.parentDir() <os.html#parentDir%2Cstring>`_ as ``currentSourcePath.parentDir()``. - ## - ## The path returned by this template is set at compile time. - ## - ## See the docstring of `macros.getProjectPath() <macros.html#getProjectPath>`_ - ## for an example to see the distinction between the ``currentSourcePath`` - ## and ``getProjectPath``. - ## - ## See also: - ## * `getCurrentDir proc <os.html#getCurrentDir>`_ when compileOption("rangechecks"): template rangeCheck*(cond) = ## Helper for performing user-defined range checks. - ## Such checks will be performed only when the ``rangechecks`` + ## Such checks will be performed only when the `rangechecks` ## compile-time option is enabled. - if not cond: sysFatal(RangeError, "range check failed") + if not cond: sysFatal(RangeDefect, "range check failed") else: template rangeCheck*(cond) = discard -when not defined(nimhygiene): - {.pragma: inject.} - -proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = - ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not - ## perform deep copies of `s`. - ## - ## This is only useful for optimization purposes. - if s.len == 0: return - when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): - var s = cast[PGenericSeq](s) - s.reserved = s.reserved or seqShallowFlag - -proc shallow*(s: var string) {.noSideEffect, inline.} = - ## Marks a string `s` as `shallow`:idx:. Subsequent assignments will not - ## perform deep copies of `s`. - ## - ## This is only useful for optimization purposes. - when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): - var s = cast[PGenericSeq](s) - if s == nil: - s = cast[PGenericSeq](newString(0)) - # string literals cannot become 'shallow': - if (s.reserved and strlitFlag) == 0: - s.reserved = s.reserved or seqShallowFlag +when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} = + ## Marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not + ## perform deep copies of `s`. + ## + ## This is only useful for optimization purposes. + if s.len == 0: return + when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): + var s = cast[PGenericSeq](s) + {.noSideEffect.}: + s.reserved = s.reserved or seqShallowFlag + + proc shallow*(s: var string) {.noSideEffect, inline.} = + ## Marks a string `s` as `shallow`:idx:. Subsequent assignments will not + ## perform deep copies of `s`. + ## + ## This is only useful for optimization purposes. + when not defined(js) and not defined(nimscript) and not defined(nimSeqsV2): + var s = cast[PGenericSeq](s) + if s == nil: + s = cast[PGenericSeq](newString(0)) + # string literals cannot become 'shallow': + if (s.reserved and strlitFlag) == 0: + {.noSideEffect.}: + s.reserved = s.reserved or seqShallowFlag type NimNodeObj = object @@ -2615,36 +2500,47 @@ type NimNode* {.magic: "PNimrodNode".} = ref NimNodeObj ## Represents a Nim AST node. Macros operate on this type. +type + ForLoopStmt* {.compilerproc.} = object ## \ + ## A special type that marks a macro as a `for-loop macro`:idx:. + ## See `"For Loop Macro" <manual.html#macros-for-loop-macro>`_. + +macro varargsLen*(x: varargs[untyped]): int {.since: (1, 1).} = + ## returns number of variadic arguments in `x` + proc varargsLenImpl(x: NimNode): NimNode {.magic: "LengthOpenArray", noSideEffect.} + varargsLenImpl(x) + when defined(nimV2): import system/repr_v2 export repr_v2 -macro lenVarargs*(x: varargs[untyped]): int {.since: (1, 1).} = - ## returns number of variadic arguments in `x` - proc lenVarargsImpl(x: NimNode): NimNode {.magic: "LengthOpenArray", noSideEffect.} - lenVarargsImpl(x) - -when false: - template eval*(blk: typed): typed = - ## Executes a block of code at compile time just as if it was a macro. - ## - ## Optionally, the block can return an AST tree that will replace the - ## eval expression. - macro payload: typed {.gensym.} = blk - payload() +proc repr*[T, U](x: HSlice[T, U]): string = + ## Generic `repr` operator for slices that is lifted from the components + ## of `x`. Example: + ## ```Nim + ## $(1 .. 5) == "1 .. 5" + ## ``` + result = repr(x.a) + result.add(" .. ") + result.add(repr(x.b)) when hasAlloc or defined(nimscript): proc insert*(x: var string, item: string, i = 0.Natural) {.noSideEffect.} = ## Inserts `item` into `x` at position `i`. - ## - ## .. code-block:: Nim + ## ```nim ## var a = "abc" ## a.insert("zz", 0) # a <- "zzabc" + ## ``` + if item.len == 0: # prevents self-assignment + return var xl = x.len setLen(x, xl+item.len) var j = xl-1 while j >= i: - shallowCopy(x[j+item.len], x[j]) + when defined(gcArc) or defined(gcOrc) or defined(gcAtomicArc): + x[j+item.len] = move x[j] + else: + shallowCopy(x[j+item.len], x[j]) dec(j) j = 0 while j < item.len: @@ -2657,32 +2553,33 @@ when declared(initDebugger): proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} = ## Adds a char to string `s` and applies the following escaping: ## - ## * replaces any ``\`` by ``\\`` - ## * replaces any ``'`` by ``\'`` - ## * replaces any ``"`` by ``\"`` - ## * replaces any ``\a`` by ``\\a`` - ## * replaces any ``\b`` by ``\\b`` - ## * replaces any ``\t`` by ``\\t`` - ## * replaces any ``\n`` by ``\\n`` - ## * replaces any ``\v`` by ``\\v`` - ## * replaces any ``\f`` by ``\\f`` - ## * replaces any ``\c`` by ``\\c`` - ## * replaces any ``\e`` by ``\\e`` - ## * replaces any other character not in the set ``{'\21..'\126'} - ## by ``\xHH`` where ``HH`` is its hexadecimal value. + ## * replaces any ``\`` by `\\` + ## * replaces any `'` by `\'` + ## * replaces any `"` by `\"` + ## * replaces any `\a` by `\\a` + ## * replaces any `\b` by `\\b` + ## * replaces any `\t` by `\\t` + ## * replaces any `\n` by `\\n` + ## * replaces any `\v` by `\\v` + ## * replaces any `\f` by `\\f` + ## * replaces any `\r` by `\\r` + ## * replaces any `\e` by `\\e` + ## * replaces any other character not in the set `{\21..\126}` + ## by `\xHH` where `HH` is its hexadecimal value ## ## The procedure has been designed so that its output is usable for many ## different common syntaxes. ## - ## **Note**: This is **not correct** for producing Ansi C code! + ## .. warning:: This is **not correct** for producing ANSI C code! + ## case c of '\a': s.add "\\a" # \x07 of '\b': s.add "\\b" # \x08 of '\t': s.add "\\t" # \x09 - of '\L': s.add "\\n" # \x0A + of '\n': s.add "\\n" # \x0A of '\v': s.add "\\v" # \x0B of '\f': s.add "\\f" # \x0C - of '\c': s.add "\\c" # \x0D + of '\r': (when defined(nimLegacyAddEscapedCharx0D): s.add "\\c" else: s.add "\\r") # \x0D of '\e': s.add "\\e" # \x1B of '\\': s.add("\\\\") of '\'': s.add("\\'") @@ -2701,9 +2598,9 @@ proc addQuoted*[T](s: var string, x: T) = ## ## See `addEscapedChar <#addEscapedChar,string,char>`_ ## for the escaping scheme. When `x` is a string, characters in the - ## range ``{\128..\255}`` are never escaped so that multibyte UTF-8 + ## range `{\128..\255}` are never escaped so that multibyte UTF-8 ## characters are untouched (note that this behavior is different from - ## ``addEscapedChar``). + ## `addEscapedChar`). ## ## The Nim standard library uses this function on the elements of ## collections when producing a string representation of a collection. @@ -2711,7 +2608,7 @@ proc addQuoted*[T](s: var string, x: T) = ## Users may overload `addQuoted` for custom (string-like) types if ## they want to implement a customized element representation. ## - ## .. code-block:: Nim + ## ```nim ## var tmp = "" ## tmp.addQuoted(1) ## tmp.add(", ") @@ -2719,6 +2616,7 @@ proc addQuoted*[T](s: var string, x: T) = ## tmp.add(", ") ## tmp.addQuoted('c') ## assert(tmp == """1, "string", 'c'""") + ## ``` when T is string or T is cstring: s.add("\"") for c in x: @@ -2734,7 +2632,7 @@ proc addQuoted*[T](s: var string, x: T) = s.addEscapedChar(x) s.add("'") # prevent temporary string allocation - elif T is SomeSignedInt: + elif T is SomeInteger: s.addInt(x) elif T is SomeFloat: s.addFloat(x) @@ -2749,10 +2647,10 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} = ## ## This is quite fast as it does not rely ## on any debug or runtime information. Note that in contrast to what - ## the official signature says, the return type is *not* ``RootObj`` but a + ## the official signature says, the return type is *not* `RootObj` but a ## tuple of a structure that depends on the current scope. Example: ## - ## .. code-block:: Nim + ## ```nim ## proc testLocals() = ## var ## a = "something" @@ -2767,15 +2665,19 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} = ## # -> name a with value something ## # -> name b with value 4 ## # -> B is 1 + ## ``` discard -when hasAlloc and notJSnotNims and not usesDestructors: +when hasAlloc and notJSnotNims: # XXX how to implement 'deepCopy' is an open problem. proc deepCopy*[T](x: var T, y: T) {.noSideEffect, magic: "DeepCopy".} = ## Performs a deep copy of `y` and copies it into `x`. ## ## This is also used by the code generator - ## for the implementation of ``spawn``. + ## for the implementation of `spawn`. + ## + ## For `--mm:arc` or `--mm:orc` deepcopy support has to be enabled + ## via `--deepcopy:on`. discard proc deepCopy*[T](y: T): T = @@ -2787,10 +2689,10 @@ when hasAlloc and notJSnotNims and not usesDestructors: proc procCall*(x: untyped) {.magic: "ProcCall", compileTime.} = ## Special magic to prohibit dynamic binding for `method`:idx: calls. ## This is similar to `super`:idx: in ordinary OO languages. - ## - ## .. code-block:: Nim + ## ```nim ## # 'someMethod' will be resolved fully statically: ## procCall someMethod(a, b) + ## ``` discard @@ -2800,33 +2702,19 @@ proc `==`*(x, y: cstring): bool {.magic: "EqCString", noSideEffect, proc strcmp(a, b: cstring): cint {.noSideEffect, importc, header: "<string.h>".} if pointer(x) == pointer(y): result = true - elif x.isNil or y.isNil: result = false + elif pointer(x) == nil or pointer(y) == nil: result = false else: result = strcmp(x, y) == 0 -when defined(nimNoNilSeqs2) and not compileOption("nilseqs"): - when defined(nimHasUserErrors): - # bug #9149; ensure that 'type(nil)' does not match *too* well by using 'type(nil) | type(nil)'. - # Eventually (in 0.20?) we will be able to remove this hack completely. - proc `==`*(x: string; y: type(nil) | type(nil)): bool {. - error: "'nil' is now invalid for 'string'; compile with --nilseqs:on for a migration period".} = - discard - proc `==`*(x: type(nil) | type(nil); y: string): bool {. - error: "'nil' is now invalid for 'string'; compile with --nilseqs:on for a migration period".} = - discard - else: - proc `==`*(x: string; y: type(nil) | type(nil)): bool {.error.} = discard - proc `==`*(x: type(nil) | type(nil); y: string): bool {.error.} = discard - template closureScope*(body: untyped): untyped = ## Useful when creating a closure in a loop to capture local loop variables by ## their current iteration values. ## ## Note: This template may not work in some cases, use - ## `capture <sugar.html#capture.m,openArray[typed],untyped>`_ instead. + ## `capture <sugar.html#capture.m,varargs[typed],untyped>`_ instead. ## ## Example: ## - ## .. code-block:: Nim + ## ```nim ## var myClosure : proc() ## # without closureScope: ## for i in 0 .. 5: @@ -2841,20 +2729,19 @@ template closureScope*(body: untyped): untyped = ## if j == 3: ## myClosure = proc() = echo j ## myClosure() # outputs 3 + ## ``` (proc() = body)() template once*(body: untyped): untyped = ## Executes a block of code only once (the first time the block is reached). - ## - ## .. code-block:: Nim - ## - ## proc draw(t: Triangle) = - ## once: - ## graphicsInit() - ## line(t.p1, t.p2) - ## line(t.p2, t.p3) - ## line(t.p3, t.p1) - ## + ## ```nim + ## proc draw(t: Triangle) = + ## once: + ## graphicsInit() + ## line(t.p1, t.p2) + ## line(t.p2, t.p3) + ## line(t.p3, t.p1) + ## ``` var alreadyExecuted {.global.} = false if not alreadyExecuted: alreadyExecuted = true @@ -2862,14 +2749,26 @@ template once*(body: untyped): untyped = {.pop.} # warning[GcMem]: off, warning[Uninit]: off -proc substr*(s: string, first, last: int): string = +proc substr*(s: openArray[char]): string = + ## Copies a slice of `s` into a new string and returns this new + ## string. + runnableExamples: + let a = "abcdefgh" + assert a.substr(2, 5) == "cdef" + assert a.substr(2) == "cdefgh" + assert a.substr(5, 99) == "fgh" + result = newString(s.len) + for i, ch in s: + result[i] = ch + +proc substr*(s: string, first, last: int): string = # A bug with `magic: Slice` requires this to exist this way ## Copies a slice of `s` into a new string and returns this new ## string. ## ## The bounds `first` and `last` denote the indices of - ## the first and last characters that shall be copied. If ``last`` - ## is omitted, it is treated as ``high(s)``. If ``last >= s.len``, ``s.len`` - ## is used instead: This means ``substr`` can also be used to `cut`:idx: + ## the first and last characters that shall be copied. If `last` + ## is omitted, it is treated as `high(s)`. If `last >= s.len`, `s.len` + ## is used instead: This means `substr` can also be used to `cut`:idx: ## or `limit`:idx: a string's length. runnableExamples: let a = "abcdefgh" @@ -2892,14 +2791,25 @@ when defined(nimconfig): when not defined(js): proc toOpenArray*[T](x: ptr UncheckedArray[T]; first, last: int): openArray[T] {. magic: "Slice".} - when defined(nimToOpenArrayCString): - proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. - magic: "Slice".} - proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. - magic: "Slice".} + proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. + magic: "Slice".} + proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. + magic: "Slice".} proc toOpenArray*[T](x: seq[T]; first, last: int): openArray[T] {. magic: "Slice".} + ## Allows passing the slice of `x` from the element at `first` to the element + ## at `last` to `openArray[T]` parameters without copying it. + ## + ## Example: + ## ```nim + ## proc test(x: openArray[int]) = + ## doAssert x == [1, 2, 3] + ## + ## let s = @[0, 1, 2, 3, 4] + ## s.toOpenArray(1, 3).test + ## ``` + proc toOpenArray*[T](x: openArray[T]; first, last: int): openArray[T] {. magic: "Slice".} proc toOpenArray*[I, T](x: array[I, T]; first, last: I): openArray[T] {. @@ -2914,10 +2824,8 @@ proc toOpenArrayByte*(x: openArray[char]; first, last: int): openArray[byte] {. proc toOpenArrayByte*(x: seq[char]; first, last: int): openArray[byte] {. magic: "Slice".} -type - ForLoopStmt* {.compilerproc.} = object ## \ - ## A special type that marks a macro as a `for-loop macro`:idx:. - ## See `"For Loop Macro" <manual.html#macros-for-loop-macro>`_. +proc toOpenArrayChar*(x: openArray[byte]; first, last: int): openArray[char] {. + magic: "Slice".} when defined(genode): var componentConstructHook*: proc (env: GenodeEnv) {.nimcall.} @@ -2925,13 +2833,13 @@ when defined(genode): ## ## This hook is called after all globals are initialized. ## When this hook is set the component will not automatically exit, - ## call ``quit`` explicitly to do so. This is the only available method + ## call `quit` explicitly to do so. This is the only available method ## of accessing the initial Genode environment. proc nim_component_construct(env: GenodeEnv) {.exportc.} = - ## Procedure called during ``Component::construct`` by the loader. + ## Procedure called during `Component::construct` by the loader. if componentConstructHook.isNil: - env.quit(programResult) + env.rawQuit(programResult) # No native Genode application initialization, # exit as would POSIX. else: @@ -2940,11 +2848,108 @@ when defined(genode): # and return to thread entrypoint. -import system/widestrs -export widestrs - -import system/io -export io +when not defined(nimPreviewSlimSystem): + import std/widestrs + export widestrs -when not defined(createNimHcr): +when notJSnotNims: + when defined(windows) and compileOption("threads"): + when not declared(addSysExitProc): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} + var echoLock: SysLock + initSysLock echoLock + addSysExitProc(proc() {.noconv.} = deinitSys(echoLock)) + + const stdOutLock = compileOption("threads") and + not defined(windows) and + not defined(android) and + not defined(nintendoswitch) and + not defined(freertos) and + not defined(zephyr) and + not defined(nuttx) and + hostOS != "any" + + proc raiseEIO(msg: string) {.noinline, noreturn.} = + raise newException(IOError, msg) + + proc echoBinSafe(args: openArray[string]) {.compilerproc.} = + when defined(androidNDK): + # When running nim in android app, stdout goes nowhere, so echo gets ignored + # To redirect echo to the android logcat, use -d:androidNDK + const ANDROID_LOG_VERBOSE = 2.cint + proc android_log_print(prio: cint, tag: cstring, fmt: cstring): cint + {.importc: "__android_log_print", header: "<android/log.h>", varargs, discardable.} + var s = "" + for arg in args: + s.add arg + android_log_print(ANDROID_LOG_VERBOSE, "nim", s) + else: + # flockfile deadlocks some versions of Android 5.x.x + when stdOutLock: + proc flockfile(f: CFilePtr) {.importc, nodecl.} + proc funlockfile(f: CFilePtr) {.importc, nodecl.} + flockfile(cstdout) + when defined(windows) and compileOption("threads"): + acquireSys echoLock + for s in args: + when defined(windows): + # equivalent to syncio.writeWindows + proc writeWindows(f: CFilePtr; s: string; doRaise = false) = + # Don't ask why but the 'printf' family of function is the only thing + # that writes utf-8 strings reliably on Windows. At least on my Win 10 + # machine. We also enable `setConsoleOutputCP(65001)` now by default. + # But we cannot call printf directly as the string might contain \0. + # So we have to loop over all the sections separated by potential \0s. + var i = int c_fprintf(f, "%s", s) + while i < s.len: + if s[i] == '\0': + let w = c_fputc('\0', f) + if w != 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i + else: + let w = c_fprintf(f, "%s", unsafeAddr s[i]) + if w <= 0: + if doRaise: raiseEIO("cannot write string to file") + break + inc i, w + writeWindows(cstdout, s) + else: + discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, cstdout) + const linefeed = "\n" + discard c_fwrite(linefeed.cstring, linefeed.len, 1, cstdout) + discard c_fflush(cstdout) + when stdOutLock: + funlockfile(cstdout) + when defined(windows) and compileOption("threads"): + releaseSys echoLock + +when not defined(nimPreviewSlimSystem): + import std/syncio + export syncio + +when not defined(createNimHcr) and not defined(nimscript): include nimhcr + +when notJSnotNims and not defined(nimSeqsV2): + proc prepareMutation*(s: var string) {.inline.} = + ## String literals (e.g. "abc", etc) in the ARC/ORC mode are "copy on write", + ## therefore you should call `prepareMutation` before modifying the strings + ## via `addr`. + runnableExamples: + var x = "abc" + var y = "defgh" + prepareMutation(y) # without this, you may get a `SIGBUS` or `SIGSEGV` + moveMem(addr y[0], addr x[0], x.len) + assert y == "abcgh" + discard + +proc arrayWith*[T](y: T, size: static int): array[size, T] {.raises: [].} = + ## Creates a new array filled with `y`. + for i in 0..size-1: + when nimvm: + result[i] = y + else: + # TODO: fixme it should be `=dup` + result[i] = y diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 95d5ae7ee..3de6d8713 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -11,6 +11,8 @@ {.push profiler:off.} include osalloc +import std/private/syslocks +import std/sysatomics template track(op, address, size) = when defined(memTracker): @@ -18,11 +20,42 @@ template track(op, address, size) = # We manage *chunks* of memory. Each chunk is a multiple of the page size. # Each chunk starts at an address that is divisible by the page size. +# Small chunks may be divided into smaller cells of reusable pointers to reduce the number of page allocations. + +# An allocation of a small pointer looks approximately like this +#[ + + alloc -> rawAlloc -> No free chunk available > Request a new page from tslf -> result = chunk.data -------------+ + | | + v | + Free chunk available | + | | + v v + Fetch shared cells -> No free cells available -> Advance acc -> result = chunk.data + chunk.acc -------> return + (may not add new cells) ^ + | | + v | + Free cells available -> result = chunk.freeList -> Advance chunk.freeList -----------------------------------+ +]# +# so it is split into 3 paths, where the last path is preferred to prevent unnecessary allocations. +# +# +# A deallocation of a small pointer then looks like this +#[ + dealloc -> rawDealloc -> chunk.owner == addr(a) --------------> This thread owns the chunk ------> The current chunk is active -> Chunk is completely unused -----> Chunk references no foreign cells + | | (Add cell into the current chunk) | Return the current chunk back to tlsf + | | | | + v v v v + A different thread owns this chunk. The current chunk is not active. chunk.free was < size Chunk references foreign cells, noop + Add the cell to a.sharedFreeLists Add the cell into the active chunk Activate the chunk (end) + (end) (end) (end) +]# +# So "true" deallocation is delayed for as long as possible in favor of reusing cells. const nimMinHeapPages {.intdefine.} = 128 # 0.5 MB SmallChunkSize = PageSize - MaxFli = 30 + MaxFli = when sizeof(int) > 2: 30 else: 14 MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32' # everywhere! MaxSli = 1 shl MaxLog2Sli @@ -30,7 +63,7 @@ const RealFli = MaxFli - FliOffset # size of chunks in last matrix bin - MaxBigChunkSize = 1 shl MaxFli - 1 shl (MaxFli-MaxLog2Sli-1) + MaxBigChunkSize = int(1'i32 shl MaxFli - 1'i32 shl (MaxFli-MaxLog2Sli-1)) HugeChunkSize = MaxBigChunkSize + 1 type @@ -44,9 +77,33 @@ type IntSet = object data: TrunkBuckets +# ------------- chunk table --------------------------------------------------- +# We use a PtrSet of chunk starts and a table[Page, chunksize] for chunk +# endings of big chunks. This is needed by the merging operation. The only +# remaining operation is best-fit for big chunks. Since there is a size-limit +# for big chunks (because greater than the limit means they are returned back +# to the OS), a fixed size array can be used. + +type + PLLChunk = ptr LLChunk + LLChunk = object ## *low-level* chunk + size: int # remaining size + acc: int # accumulator + next: PLLChunk # next low-level chunk; only needed for dealloc + + PAvlNode = ptr AvlNode + AvlNode = object + link: array[0..1, PAvlNode] # Left (0) and right (1) links + key, upperBound: int + level: int + +const + RegionHasLock = false # hasThreadSupport and defined(gcDestructors) + type - AlignType = BiggestFloat FreeCell {.final, pure.} = object + # A free cell is a pointer that has been freed, meaning it became available for reuse. + # It may become foreign if it is lent to a chunk that did not create it, doing so reduces the amount of needed pages. next: ptr FreeCell # next free cell in chunk (overlaid with refcount) when not defined(gcDestructors): zeroField: int # 0 means cell is not used (overlaid with typ field) @@ -62,42 +119,27 @@ type prevSize: int # size of previous chunk; for coalescing # 0th bit == 1 if 'used size: int # if < PageSize it is a small chunk + owner: ptr MemRegion SmallChunk = object of BaseChunk next, prev: PSmallChunk # chunks of the same size - freeList: ptr FreeCell - free: int # how many bytes remain - acc: int # accumulator for small object allocation - when defined(cpu32): - align: int - data: AlignType # start of usable memory + freeList: ptr FreeCell # Singly linked list of cells. They may be from foreign chunks or from the current chunk. + # Should be `nil` when the chunk isn't active in `a.freeSmallChunks`. + free: int32 # Bytes this chunk is able to provide using both the accumulator and free cells. + # When a cell is considered foreign, its source chunk's free field is NOT adjusted until it + # reaches dealloc while the source chunk is active. + # Instead, the receiving chunk gains the capacity and thus reserves space in the foreign chunk. + acc: uint32 # Offset from data, used when there are no free cells available but the chunk is considered free. + foreignCells: int # When a free cell is given to a chunk that is not its origin, + # both the cell and the source chunk are considered foreign. + # Receiving a foreign cell can happen both when deallocating from another thread or when + # the active chunk in `a.freeSmallChunks` is not the current chunk. + # Freeing a chunk while `foreignCells > 0` leaks memory as all references to it become lost. + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory BigChunk = object of BaseChunk # not necessarily > PageSize! next, prev: PBigChunk # chunks of the same (or bigger) size - data: AlignType # start of usable memory - -template smallChunkOverhead(): untyped = sizeof(SmallChunk)-sizeof(AlignType) -template bigChunkOverhead(): untyped = sizeof(BigChunk)-sizeof(AlignType) - -# ------------- chunk table --------------------------------------------------- -# We use a PtrSet of chunk starts and a table[Page, chunksize] for chunk -# endings of big chunks. This is needed by the merging operation. The only -# remaining operation is best-fit for big chunks. Since there is a size-limit -# for big chunks (because greater than the limit means they are returned back -# to the OS), a fixed size array can be used. - -type - PLLChunk = ptr LLChunk - LLChunk = object ## *low-level* chunk - size: int # remaining size - acc: int # accumulator - next: PLLChunk # next low-level chunk; only needed for dealloc - - PAvlNode = ptr AvlNode - AvlNode = object - link: array[0..1, PAvlNode] # Left (0) and right (1) links - key, upperBound: int - level: int + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory HeapLinks = object len: int @@ -105,23 +147,54 @@ type next: ptr HeapLinks MemRegion = object - minLargeObj, maxLargeObj: int - freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] + when not defined(gcDestructors): + minLargeObj, maxLargeObj: int + freeSmallChunks: array[0..max(1, SmallChunkSize div MemAlign-1), PSmallChunk] + # List of available chunks per size class. Only one is expected to be active per class. + when defined(gcDestructors): + sharedFreeLists: array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell] + # When a thread frees a pointer it did not create, it must not adjust the counters. + # Instead, the cell is placed here and deferred until the next allocation. flBitmap: uint32 slBitmap: array[RealFli, uint32] matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly + when RegionHasLock: + lock: SysLock + when defined(gcDestructors): + sharedFreeListBigChunks: PBigChunk # make no attempt at avoiding false sharing for now for this object field + chunkStarts: IntSet - root, deleted, last, freeAvlNodes: PAvlNode - locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. + when not defined(gcDestructors): + root, deleted, last, freeAvlNodes: PAvlNode + lockActive, locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. nextChunkSize: int - bottomData: AvlNode + when not defined(gcDestructors): + bottomData: AvlNode heapLinks: HeapLinks when defined(nimTypeNames): allocCounter, deallocCounter: int +template smallChunkOverhead(): untyped = sizeof(SmallChunk) +template bigChunkOverhead(): untyped = sizeof(BigChunk) + +when hasThreadSupport: + template loada(x: untyped): untyped = atomicLoadN(unsafeAddr x, ATOMIC_RELAXED) + template storea(x, y: untyped) = atomicStoreN(unsafeAddr x, y, ATOMIC_RELAXED) + + when false: + # not yet required + template atomicStatDec(x, diff: untyped) = discard atomicSubFetch(unsafeAddr x, diff, ATOMIC_RELAXED) + template atomicStatInc(x, diff: untyped) = discard atomicAddFetch(unsafeAddr x, diff, ATOMIC_RELAXED) +else: + template loada(x: untyped): untyped = x + template storea(x, y: untyped) = x = y + +template atomicStatDec(x, diff: untyped) = dec x, diff +template atomicStatInc(x, diff: untyped) = inc x, diff + const fsLookupTable: array[byte, int8] = [ -1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, @@ -162,7 +235,7 @@ proc mappingSearch(r, fl, sl: var int) {.inline.} = let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1 r = r + t r = r and not t - r = min(r, MaxBigChunkSize) + r = min(r, MaxBigChunkSize).int fl = msbit(uint32 r) sl = (r shr (fl - MaxLog2Sli)) - MaxSli dec fl, FliOffset @@ -228,11 +301,11 @@ proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) = setBit(fl, a.flBitmap) proc incCurrMem(a: var MemRegion, bytes: int) {.inline.} = - inc(a.currMem, bytes) + atomicStatInc(a.currMem, bytes) proc decCurrMem(a: var MemRegion, bytes: int) {.inline.} = a.maxMem = max(a.maxMem, a.currMem) - dec(a.currMem, bytes) + atomicStatDec(a.currMem, bytes) proc getMaxMem(a: var MemRegion): int = # Since we update maxPagesCount only when freeing pages, @@ -240,6 +313,20 @@ proc getMaxMem(a: var MemRegion): int = # maximum of these both values here: result = max(a.currMem, a.maxMem) +const nimMaxHeap {.intdefine.} = 0 + +proc allocPages(a: var MemRegion, size: int): pointer = + when nimMaxHeap != 0: + if a.occ + size > nimMaxHeap * 1024 * 1024: + raiseOutOfMem() + osAllocPages(size) + +proc tryAllocPages(a: var MemRegion, size: int): pointer = + when nimMaxHeap != 0: + if a.occ + size > nimMaxHeap * 1024 * 1024: + raiseOutOfMem() + osTryAllocPages(size) + proc llAlloc(a: var MemRegion, size: int): pointer = # *low-level* alloc for the memory managers data structures. Deallocation # is done at the end of the allocator's life time. @@ -249,49 +336,50 @@ proc llAlloc(a: var MemRegion, size: int): pointer = # is one page: sysAssert roundup(size+sizeof(LLChunk), PageSize) == PageSize, "roundup 6" var old = a.llmem # can be nil and is correct with nil - a.llmem = cast[PLLChunk](osAllocPages(PageSize)) - when defined(avlcorruption): + a.llmem = cast[PLLChunk](allocPages(a, PageSize)) + when defined(nimAvlcorruption): trackLocation(a.llmem, PageSize) incCurrMem(a, PageSize) a.llmem.size = PageSize - sizeof(LLChunk) a.llmem.acc = sizeof(LLChunk) a.llmem.next = old - result = cast[pointer](cast[ByteAddress](a.llmem) + a.llmem.acc) + result = cast[pointer](cast[int](a.llmem) + a.llmem.acc) dec(a.llmem.size, size) inc(a.llmem.acc, size) zeroMem(result, size) -proc getBottom(a: var MemRegion): PAvlNode = - result = addr(a.bottomData) - if result.link[0] == nil: - result.link[0] = result - result.link[1] = result - -proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode = - if a.freeAvlNodes != nil: - result = a.freeAvlNodes - a.freeAvlNodes = a.freeAvlNodes.link[0] - else: - result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode))) - when defined(avlcorruption): - cprintf("tracking location: %p\n", result) - result.key = key - result.upperBound = upperBound - let bottom = getBottom(a) - result.link[0] = bottom - result.link[1] = bottom - result.level = 1 - #when defined(avlcorruption): - # track("allocAvlNode", result, sizeof(AvlNode)) - sysAssert(bottom == addr(a.bottomData), "bottom data") - sysAssert(bottom.link[0] == bottom, "bottom link[0]") - sysAssert(bottom.link[1] == bottom, "bottom link[1]") - -proc deallocAvlNode(a: var MemRegion, n: PAvlNode) {.inline.} = - n.link[0] = a.freeAvlNodes - a.freeAvlNodes = n - -proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int) = +when not defined(gcDestructors): + proc getBottom(a: var MemRegion): PAvlNode = + result = addr(a.bottomData) + if result.link[0] == nil: + result.link[0] = result + result.link[1] = result + + proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode = + if a.freeAvlNodes != nil: + result = a.freeAvlNodes + a.freeAvlNodes = a.freeAvlNodes.link[0] + else: + result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode))) + when defined(nimAvlcorruption): + cprintf("tracking location: %p\n", result) + result.key = key + result.upperBound = upperBound + let bottom = getBottom(a) + result.link[0] = bottom + result.link[1] = bottom + result.level = 1 + #when defined(nimAvlcorruption): + # track("allocAvlNode", result, sizeof(AvlNode)) + sysAssert(bottom == addr(a.bottomData), "bottom data") + sysAssert(bottom.link[0] == bottom, "bottom link[0]") + sysAssert(bottom.link[1] == bottom, "bottom link[1]") + + proc deallocAvlNode(a: var MemRegion, n: PAvlNode) {.inline.} = + n.link[0] = a.freeAvlNodes + a.freeAvlNodes = n + +proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int): ptr HeapLinks = var it = addr(a.heapLinks) while it != nil and it.len >= it.chunks.len: it = it.next if it == nil: @@ -300,12 +388,15 @@ proc addHeapLink(a: var MemRegion; p: PBigChunk, size: int) = a.heapLinks.next = n n.chunks[0] = (p, size) n.len = 1 + result = n else: let L = it.len it.chunks[L] = (p, size) inc it.len + result = it -include "system/avltree" +when not defined(gcDestructors): + include "system/avltree" proc llDeallocAll(a: var MemRegion) = var it = a.llmem @@ -370,7 +461,7 @@ iterator elements(t: IntSet): int {.inline.} = r = r.next proc isSmallChunk(c: PChunk): bool {.inline.} = - return c.size <= SmallChunkSize-smallChunkOverhead() + result = c.size <= SmallChunkSize-smallChunkOverhead() proc chunkUnused(c: PChunk): bool {.inline.} = result = (c.prevSize and 1) == 0 @@ -386,8 +477,8 @@ iterator allObjects(m: var MemRegion): pointer {.inline.} = var c = cast[PSmallChunk](c) let size = c.size - var a = cast[ByteAddress](addr(c.data)) - let limit = a + c.acc + var a = cast[int](addr(c.data)) + let limit = a + c.acc.int while a <% limit: yield cast[pointer](a) a = a +% size @@ -405,13 +496,13 @@ when not defined(gcDestructors): # ------------- chunk management ---------------------------------------------- proc pageIndex(c: PChunk): int {.inline.} = - result = cast[ByteAddress](c) shr PageShift + result = cast[int](c) shr PageShift proc pageIndex(p: pointer): int {.inline.} = - result = cast[ByteAddress](p) shr PageShift + result = cast[int](p) shr PageShift proc pageAddr(p: pointer): PChunk {.inline.} = - result = cast[PChunk](cast[ByteAddress](p) and not PageMask) + result = cast[PChunk](cast[int](p) and not PageMask) #sysAssert(Contains(allocator.chunkStarts, pageIndex(result))) when false: @@ -423,49 +514,44 @@ when false: it, it.next, it.prev, it.size) it = it.next -const nimMaxHeap {.intdefine.} = 0 - proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when not defined(emscripten): if not a.blockChunkSizeIncrease: let usedMem = a.occ #a.currMem # - a.freeMem - when nimMaxHeap != 0: - if usedMem > nimMaxHeap * 1024 * 1024: - raiseOutOfMem() if usedMem < 64 * 1024: a.nextChunkSize = PageSize*4 else: a.nextChunkSize = min(roundup(usedMem shr 2, PageSize), a.nextChunkSize * 2) - a.nextChunkSize = min(a.nextChunkSize, MaxBigChunkSize) + a.nextChunkSize = min(a.nextChunkSize, MaxBigChunkSize).int var size = size if size > a.nextChunkSize: - result = cast[PBigChunk](osAllocPages(size)) + result = cast[PBigChunk](allocPages(a, size)) else: - result = cast[PBigChunk](osTryAllocPages(a.nextChunkSize)) + result = cast[PBigChunk](tryAllocPages(a, a.nextChunkSize)) if result == nil: - result = cast[PBigChunk](osAllocPages(size)) + result = cast[PBigChunk](allocPages(a, size)) a.blockChunkSizeIncrease = true else: size = a.nextChunkSize incCurrMem(a, size) inc(a.freeMem, size) - a.addHeapLink(result, size) + let heapLink = a.addHeapLink(result, size) when defined(debugHeapLinks): cprintf("owner: %p; result: %p; next pointer %p; size: %ld\n", addr(a), - result, result.heapLink, result.size) + result, heapLink, size) when defined(memtracker): trackLocation(addr result.size, sizeof(int)) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "requestOsChunks 1") + sysAssert((cast[int](result) and PageMask) == 0, "requestOsChunks 1") #zeroMem(result, size) result.next = nil result.prev = nil result.size = size # update next.prevSize: - var nxt = cast[ByteAddress](result) +% size + var nxt = cast[int](result) +% size sysAssert((nxt and PageMask) == 0, "requestOsChunks 2") var next = cast[PChunk](nxt) if pageIndex(next) in a.chunkStarts: @@ -473,7 +559,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = next.prevSize = size or (next.prevSize and 1) # set result.prevSize: var lastSize = if a.lastSize != 0: a.lastSize else: PageSize - var prv = cast[ByteAddress](result) -% lastSize + var prv = cast[int](result) -% lastSize sysAssert((nxt and PageMask) == 0, "requestOsChunks 3") var prev = cast[PChunk](prv) if pageIndex(prev) in a.chunkStarts and prev.size == lastSize: @@ -519,21 +605,22 @@ proc listRemove[T](head: var T, c: T) {.inline.} = proc updatePrevSize(a: var MemRegion, c: PBigChunk, prevSize: int) {.inline.} = - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "updatePrevSize") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "updatePrevSize") if isAccessible(a, ri): ri.prevSize = prevSize or (ri.prevSize and 1) proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = - result = cast[PBigChunk](cast[ByteAddress](c) +% size) + result = cast[PBigChunk](cast[int](c) +% size) result.size = c.size - size track("result.size", addr result.size, sizeof(int)) - # XXX check if these two nil assignments are dead code given - # addChunkToMatrix's implementation: - result.next = nil - result.prev = nil + when not defined(nimOptimizedSplitChunk): + # still active because of weird codegen issue on some of our CIs: + result.next = nil + result.prev = nil # size and not used: result.prevSize = size + result.owner = addr a sysAssert((size and 1) == 0, "splitChunk 2") sysAssert((size and PageMask) == 0, "splitChunk: size is not a multiple of the PageSize") @@ -553,8 +640,8 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = when coalescLeft: let prevSize = c.prevSize if prevSize != 0: - var le = cast[PChunk](cast[ByteAddress](c) -% prevSize) - sysAssert((cast[ByteAddress](le) and PageMask) == 0, "freeBigChunk 4") + var le = cast[PChunk](cast[int](c) -% prevSize) + sysAssert((cast[int](le) and PageMask) == 0, "freeBigChunk 4") if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") if not isSmallChunk(le) and le.size < MaxBigChunkSize: @@ -564,11 +651,14 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = c = cast[PBigChunk](le) if c.size > MaxBigChunkSize: let rest = splitChunk2(a, c, MaxBigChunkSize) + when defined(nimOptimizedSplitChunk): + rest.next = nil + rest.prev = nil addChunkToMatrix(a, c) c = rest when coalescRight: - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "freeBigChunk 2") if isAccessible(a, ri) and chunkUnused(ri): sysAssert(not isSmallChunk(ri), "freeBigChunk 3") if not isSmallChunk(ri) and c.size < MaxBigChunkSize: @@ -583,10 +673,18 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = proc getBigChunk(a: var MemRegion, size: int): PBigChunk = sysAssert(size > 0, "getBigChunk 2") var size = size # roundup(size, PageSize) - var fl, sl: int + var fl = 0 + var sl = 0 mappingSearch(size, fl, sl) sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") result = findSuitableBlock(a, fl, sl) + + when RegionHasLock: + if not a.lockActive: + a.lockActive = true + initSysLock(a.lock) + acquireSys a.lock + if result == nil: if size < nimMinHeapPages * PageSize: result = requestOsChunks(a, nimMinHeapPages * PageSize) @@ -596,6 +694,7 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # if we over allocated split the chunk: if result.size > size: splitChunk(a, result, size) + result.owner = addr a else: removeChunkFromMatrix2(a, result, fl, sl) if result.size >= size + PageSize: @@ -603,22 +702,33 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # set 'used' to to true: result.prevSize = 1 track("setUsedToFalse", addr result.size, sizeof(int)) + sysAssert result.owner == addr a, "getBigChunk: No owner set!" incl(a, a.chunkStarts, pageIndex(result)) dec(a.freeMem, size) + when RegionHasLock: + releaseSys a.lock proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = - result = cast[PBigChunk](osAllocPages(size)) + result = cast[PBigChunk](allocPages(a, size)) + when RegionHasLock: + if not a.lockActive: + a.lockActive = true + initSysLock(a.lock) + acquireSys a.lock incCurrMem(a, size) # XXX add this to the heap links. But also remove it from it later. when false: a.addHeapLink(result, size) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "getHugeChunk") + sysAssert((cast[int](result) and PageMask) == 0, "getHugeChunk") result.next = nil result.prev = nil result.size = size # set 'used' to to true: result.prevSize = 1 + result.owner = addr a incl(a, a.chunkStarts, pageIndex(result)) + when RegionHasLock: + releaseSys a.lock proc freeHugeChunk(a: var MemRegion; c: PBigChunk) = let size = c.size @@ -682,119 +792,180 @@ else: template trackSize(x) = discard template untrackSize(x) = discard -when false: - # not yet used by the GCs - proc rawTryAlloc(a: var MemRegion; requestedSize: int): pointer = - sysAssert(allocInv(a), "rawAlloc: begin") - sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") - sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") - var size = roundup(requestedSize, MemAlign) - inc a.occ, size - trackSize(size) - sysAssert(size >= requestedSize, "insufficient allocated size!") - #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) - if size <= SmallChunkSize-smallChunkOverhead(): - # allocate a small block: for small chunks, we use only its next pointer - var s = size div MemAlign - var c = a.freeSmallChunks[s] - if c == nil: - result = nil - else: - sysAssert c.size == size, "rawAlloc 6" - if c.freeList == nil: - sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, - "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) - inc(c.acc, size) - else: - result = c.freeList - sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") - c.freeList = c.freeList.next - dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") - if c.free < size: - listRemove(a.freeSmallChunks[s], c) - sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% - size == 0, "rawAlloc 21") - sysAssert(allocInv(a), "rawAlloc: end small size") +proc deallocBigChunk(a: var MemRegion, c: PBigChunk) = + when RegionHasLock: + acquireSys a.lock + dec a.occ, c.size + untrackSize(c.size) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" + when not defined(gcDestructors): + a.deleted = getBottom(a) + del(a, a.root, cast[int](addr(c.data))) + if c.size >= HugeChunkSize: freeHugeChunk(a, c) + else: freeBigChunk(a, c) + when RegionHasLock: + releaseSys a.lock + +when defined(gcDestructors): + template atomicPrepend(head, elem: untyped) = + # see also https://en.cppreference.com/w/cpp/atomic/atomic_compare_exchange + when hasThreadSupport: + while true: + elem.next.storea head.loada + if atomicCompareExchangeN(addr head, addr elem.next, elem, weak = true, ATOMIC_RELEASE, ATOMIC_RELAXED): + break else: - inc size, bigChunkOverhead() - var fl, sl: int - mappingSearch(size, fl, sl) - sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") - let c = findSuitableBlock(a, fl, sl) - if c != nil: - removeChunkFromMatrix2(a, c, fl, sl) - if c.size >= size + PageSize: - splitChunk(a, c, size) - # set 'used' to to true: - c.prevSize = 1 - incl(a, a.chunkStarts, pageIndex(c)) - dec(a.freeMem, size) - result = addr(c.data) - sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") - sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") - if a.root == nil: a.root = getBottom(a) - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) - else: - result = nil + elem.next.storea head.loada + head.storea elem + + proc addToSharedFreeListBigChunks(a: var MemRegion; c: PBigChunk) {.inline.} = + sysAssert c.next == nil, "c.next pointer must be nil" + atomicPrepend a.sharedFreeListBigChunks, c + + proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} = + atomicPrepend c.owner.sharedFreeLists[size], f + + const MaxSteps = 20 + + proc compensateCounters(a: var MemRegion; c: PSmallChunk; size: int) = + # rawDealloc did NOT do the usual: + # `inc(c.free, size); dec(a.occ, size)` because it wasn't the owner of these + # memory locations. We have to compensate here for these for the entire list. + var it = c.freeList + var total = 0 + while it != nil: + inc total, size + let chunk = cast[PSmallChunk](pageAddr(it)) + if c != chunk: + # The cell is foreign, potentially even from a foreign thread. + # It must block the current chunk from being freed, as doing so would leak memory. + inc c.foreignCells + it = it.next + # By not adjusting the foreign chunk we reserve space in it to prevent deallocation + inc(c.free, total) + dec(a.occ, total) + + proc freeDeferredObjects(a: var MemRegion; root: PBigChunk) = + var it = root + var maxIters = MaxSteps # make it time-bounded + while true: + let rest = it.next.loada + it.next.storea nil + deallocBigChunk(a, cast[PBigChunk](it)) + if maxIters == 0: + if rest != nil: + addToSharedFreeListBigChunks(a, rest) + sysAssert a.sharedFreeListBigChunks != nil, "re-enqueing failed" + break + it = rest + dec maxIters + if it == nil: break proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = when defined(nimTypeNames): inc(a.allocCounter) sysAssert(allocInv(a), "rawAlloc: begin") sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") - sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") var size = roundup(requestedSize, MemAlign) + sysAssert(size >= sizeof(FreeCell), "rawAlloc: requested size too small") sysAssert(size >= requestedSize, "insufficient allocated size!") #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): + template fetchSharedCells(tc: PSmallChunk) = + # Consumes cells from (potentially) foreign threads from `a.sharedFreeLists[s]` + when defined(gcDestructors): + if tc.freeList == nil: + when hasThreadSupport: + # Steal the entire list from `sharedFreeList`: + tc.freeList = atomicExchangeN(addr a.sharedFreeLists[s], nil, ATOMIC_RELAXED) + else: + tc.freeList = a.sharedFreeLists[s] + a.sharedFreeLists[s] = nil + # if `tc.freeList` isn't nil, `tc` will gain capacity. + # We must calculate how much it gained and how many foreign cells are included. + compensateCounters(a, tc, size) + # allocate a small block: for small chunks, we use only its next pointer - var s = size div MemAlign + let s = size div MemAlign var c = a.freeSmallChunks[s] if c == nil: + # There is no free chunk of the requested size available, we need a new one. c = getSmallChunk(a) + # init all fields in case memory didn't get zeroed c.freeList = nil + c.foreignCells = 0 sysAssert c.size == PageSize, "rawAlloc 3" c.size = size - c.acc = size - c.free = SmallChunkSize - smallChunkOverhead() - size + c.acc = size.uint32 + c.free = SmallChunkSize - smallChunkOverhead() - size.int32 + sysAssert c.owner == addr(a), "rawAlloc: No owner set!" c.next = nil c.prev = nil - listAdd(a.freeSmallChunks[s], c) + # Shared cells are fetched here in case `c.size * 2 >= SmallChunkSize - smallChunkOverhead()`. + # For those single cell chunks, we would otherwise have to allocate a new one almost every time. + fetchSharedCells(c) + if c.free >= size: + # Because removals from `a.freeSmallChunks[s]` only happen in the other alloc branch and during dealloc, + # we must not add it to the list if it cannot be used the next time a pointer of `size` bytes is needed. + listAdd(a.freeSmallChunks[s], c) result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 4") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 4") else: + # There is a free chunk of the requested size available, use it. sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" #if c.size != size: # c_fprintf(stdout, "csize: %lld; size %lld\n", c.size, size) sysAssert c.size == size, "rawAlloc 6" if c.freeList == nil: - sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, + sysAssert(c.acc.int + smallChunkOverhead() + size <= SmallChunkSize, "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + result = cast[pointer](cast[int](addr(c.data)) +% c.acc.int) inc(c.acc, size) else: + # There are free cells available, prefer them over the accumulator result = c.freeList when not defined(gcDestructors): sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") c.freeList = c.freeList.next + if cast[PSmallChunk](pageAddr(result)) != c: + # This cell isn't a blocker for the current chunk's deallocation anymore + dec(c.foreignCells) + else: + sysAssert(c == cast[PSmallChunk](pageAddr(result)), "rawAlloc: Bad cell") + # Even if the cell we return is foreign, the local chunk's capacity decreases. + # The capacity was previously reserved in the source chunk (when it first got allocated), + # then added into the current chunk during dealloc, + # so the source chunk will not be freed or leak memory because of this. dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert(allocInv(a), "rawAlloc: end c != nil") - sysAssert(allocInv(a), "rawAlloc: before c.free < size") - if c.free < size: - sysAssert(allocInv(a), "rawAlloc: before listRemove test") - listRemove(a.freeSmallChunks[s], c) - sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + # We fetch deferred cells *after* advancing `c.freeList`/`acc` to adjust `c.free`. + # If after the adjustment it turns out there's free cells available, + # the chunk stays in `a.freeSmallChunks[s]` and the need for a new chunk is delayed. + fetchSharedCells(c) + sysAssert(allocInv(a), "rawAlloc: before c.free < size") + if c.free < size: + # Even after fetching shared cells the chunk has no usable memory left. It is no longer the active chunk + sysAssert(allocInv(a), "rawAlloc: before listRemove test") + listRemove(a.freeSmallChunks[s], c) + sysAssert(allocInv(a), "rawAlloc: end listRemove test") + sysAssert(((cast[int](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") inc a.occ, size trackSize(c.size) else: + when defined(gcDestructors): + when hasThreadSupport: + let deferredFrees = atomicExchangeN(addr a.sharedFreeListBigChunks, nil, ATOMIC_RELAXED) + else: + let deferredFrees = a.sharedFreeListBigChunks + a.sharedFreeListBigChunks = nil + if deferredFrees != nil: + freeDeferredObjects(a, deferredFrees) + size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block var c = if size >= HugeChunkSize: getHugeChunk(a, size) @@ -802,15 +973,16 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" result = addr(c.data) - sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") - sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") - if a.root == nil: a.root = getBottom(a) - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + sysAssert((cast[int](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[int](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + when not defined(gcDestructors): + if a.root == nil: a.root = getBottom(a) + add(a, a.root, cast[int](result), cast[int](result)+%size) inc a.occ, c.size trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") sysAssert(allocInv(a), "rawAlloc: end") - when logAlloc: cprintf("var pointer_%p = alloc(%ld)\n", result, requestedSize) + when logAlloc: cprintf("var pointer_%p = alloc(%ld) # %p\n", result, requestedSize, addr a) proc rawAlloc0(a: var MemRegion, requestedSize: int): pointer = result = rawAlloc(a, requestedSize) @@ -822,53 +994,77 @@ proc rawDealloc(a: var MemRegion, p: pointer) = #sysAssert(isAllocatedPtr(a, p), "rawDealloc: no allocated pointer") sysAssert(allocInv(a), "rawDealloc: begin") var c = pageAddr(p) + sysAssert(c != nil, "rawDealloc: begin") if isSmallChunk(c): # `p` is within a small chunk: var c = cast[PSmallChunk](c) - var s = c.size - dec a.occ, s - untrackSize(s) - sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% - s == 0, "rawDealloc 3") + let s = c.size + # ^ We might access thread foreign storage here. + # The other thread cannot possibly free this block as it's still alive. var f = cast[ptr FreeCell](p) - when not defined(gcDestructors): - #echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField))) - sysAssert(f.zeroField != 0, "rawDealloc 1") - f.zeroField = 0 - f.next = c.freeList - c.freeList = f - when overwriteFree: - # set to 0xff to check for usage after free bugs: - nimSetMem(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32, - s -% sizeof(FreeCell)) - # check if it is not in the freeSmallChunks[s] list: - if c.free < s: - # add it to the freeSmallChunks[s] array: - listAdd(a.freeSmallChunks[s div MemAlign], c) - inc(c.free, s) + if c.owner == addr(a): + # We own the block, there is no foreign thread involved. + dec a.occ, s + untrackSize(s) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% + s == 0, "rawDealloc 3") + when not defined(gcDestructors): + #echo("setting to nil: ", $cast[int](addr(f.zeroField))) + sysAssert(f.zeroField != 0, "rawDealloc 1") + f.zeroField = 0 + when overwriteFree: + # set to 0xff to check for usage after free bugs: + nimSetMem(cast[pointer](cast[int](p) +% sizeof(FreeCell)), -1'i32, + s -% sizeof(FreeCell)) + let activeChunk = a.freeSmallChunks[s div MemAlign] + if activeChunk != nil and c != activeChunk: + # This pointer is not part of the active chunk, lend it out + # and do not adjust the current chunk (same logic as compensateCounters.) + # Put the cell into the active chunk, + # may prevent a queue of available chunks from forming in a.freeSmallChunks[s div MemAlign]. + # This queue would otherwise waste memory in the form of free cells until we return to those chunks. + f.next = activeChunk.freeList + activeChunk.freeList = f # lend the cell + inc(activeChunk.free, s) # By not adjusting the current chunk's capacity it is prevented from being freed + inc(activeChunk.foreignCells) # The cell is now considered foreign from the perspective of the active chunk + else: + f.next = c.freeList + c.freeList = f + if c.free < s: + # The chunk could not have been active as it didn't have enough space to give + listAdd(a.freeSmallChunks[s div MemAlign], c) + inc(c.free, s) + else: + inc(c.free, s) + # Free only if the entire chunk is unused and there are no borrowed cells. + # If the chunk were to be freed while it references foreign cells, + # the foreign chunks will leak memory and can never be freed. + if c.free == SmallChunkSize-smallChunkOverhead() and c.foreignCells == 0: + listRemove(a.freeSmallChunks[s div MemAlign], c) + c.size = SmallChunkSize + freeBigChunk(a, cast[PBigChunk](c)) else: - inc(c.free, s) - if c.free == SmallChunkSize-smallChunkOverhead(): - listRemove(a.freeSmallChunks[s div MemAlign], c) - c.size = SmallChunkSize - freeBigChunk(a, cast[PBigChunk](c)) - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + when logAlloc: cprintf("dealloc(pointer_%p) # SMALL FROM %p CALLER %p\n", p, c.owner, addr(a)) + + when defined(gcDestructors): + addToSharedFreeList(c, f, s div MemAlign) + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 2") else: # set to 0xff to check for usage after free bugs: when overwriteFree: nimSetMem(p, -1'i32, c.size -% bigChunkOverhead()) - # free big chunk - var c = cast[PBigChunk](c) - dec a.occ, c.size - untrackSize(c.size) - sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" - a.deleted = getBottom(a) - del(a, a.root, cast[int](addr(c.data))) - if c.size >= HugeChunkSize: freeHugeChunk(a, c) - else: freeBigChunk(a, c) + when logAlloc: cprintf("dealloc(pointer_%p) # BIG %p\n", p, c.owner) + when defined(gcDestructors): + if c.owner == addr(a): + deallocBigChunk(a, cast[PBigChunk](c)) + else: + addToSharedFreeListBigChunks(c.owner[], cast[PBigChunk](c)) + else: + deallocBigChunk(a, cast[PBigChunk](c)) + sysAssert(allocInv(a), "rawDealloc: end") - when logAlloc: cprintf("dealloc(pointer_%p)\n", p) + #when logAlloc: cprintf("dealloc(pointer_%p)\n", p) when not defined(gcDestructors): proc isAllocatedPtr(a: MemRegion, p: pointer): bool = @@ -877,9 +1073,9 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() - result = (c.acc >% offset) and (offset %% c.size == 0) and + result = (c.acc.int >% offset) and (offset %% c.size == 0) and (cast[ptr FreeCell](p).zeroField >% 1) else: var c = cast[PBigChunk](c) @@ -895,12 +1091,12 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() - if c.acc >% offset: - sysAssert(cast[ByteAddress](addr(c.data)) +% offset == - cast[ByteAddress](p), "offset is not what you think it is") - var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +% + if c.acc.int >% offset: + sysAssert(cast[int](addr(c.data)) +% offset == + cast[int](p), "offset is not what you think it is") + var d = cast[ptr FreeCell](cast[int](addr(c.data)) +% offset -% (offset %% c.size)) if d.zeroField >% 1: result = d @@ -927,7 +1123,7 @@ when not defined(gcDestructors): proc ptrSize(p: pointer): int = when not defined(gcDestructors): - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) var c = pageAddr(p) sysAssert(not chunkUnused(c), "ptrSize") result = c.size -% sizeof(FreeCell) @@ -945,7 +1141,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} = result = rawAlloc(allocator, size+sizeof(FreeCell)) cast[ptr FreeCell](result).zeroField = 1 # mark it as used sysAssert(not isAllocatedPtr(allocator, result), "alloc") - result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) + result = cast[pointer](cast[int](result) +% sizeof(FreeCell)) track("alloc", result, size) else: result = rawAlloc(allocator, size) @@ -957,7 +1153,7 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer = proc dealloc(allocator: var MemRegion, p: pointer) = when not defined(gcDestructors): sysAssert(p != nil, "dealloc: p is nil") - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) sysAssert(x != nil, "dealloc: x is nil") sysAssert(isAccessible(allocator, x), "is not accessible") sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") @@ -990,7 +1186,7 @@ proc deallocOsPages(a: var MemRegion) = let (p, size) = it.chunks[i] when defined(debugHeapLinks): cprintf("owner %p; dealloc A: %p size: %ld; next: %p\n", addr(a), - it, it.size, next) + it, size, next) sysAssert size >= PageSize, "origSize too small" osDeallocPages(p, size) it = next @@ -1013,12 +1209,12 @@ when defined(nimTypeNames): template instantiateForRegion(allocator: untyped) {.dirty.} = {.push stackTrace: off.} - when defined(fulldebug): + when defined(nimFulldebug): proc interiorAllocatedPtr*(p: pointer): pointer = result = interiorAllocatedPtr(allocator, p) proc isAllocatedPtr*(p: pointer): bool = - let p = cast[pointer](cast[ByteAddress](p)-%ByteAddress(sizeof(Cell))) + let p = cast[pointer](cast[int](p)-%ByteAddress(sizeof(Cell))) result = isAllocatedPtr(allocator, p) proc deallocOsPages = deallocOsPages(allocator) @@ -1038,7 +1234,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = proc realloc0Impl(p: pointer, oldSize, newSize: Natural): pointer = result = realloc(allocator, p, newSize) if newSize > oldSize: - zeroMem(cast[pointer](cast[int](result) + oldSize), newSize - oldSize) + zeroMem(cast[pointer](cast[uint](result) + uint(oldSize)), newSize - oldSize) when false: proc countFreeMem(): int = @@ -1048,25 +1244,34 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = inc(result, it.size) it = it.next + when hasThreadSupport and not defined(gcDestructors): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "<stdlib.h>".} + + var sharedHeap: MemRegion + var heapLock: SysLock + initSysLock(heapLock) + addSysExitProc(proc() {.noconv.} = deinitSys(heapLock)) + proc getFreeMem(): int = - result = allocator.freeMem #sysAssert(result == countFreeMem()) + result = allocator.freeMem + + proc getTotalMem(): int = + result = allocator.currMem - proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() - proc getMaxMem*(): int = return getMaxMem(allocator) + proc getOccupiedMem(): int = + result = allocator.occ #getTotalMem() - getFreeMem() + + proc getMaxMem*(): int = + result = getMaxMem(allocator) when defined(nimTypeNames): proc getMemCounters*(): (int, int) = getMemCounters(allocator) # -------------------- shared heap region ---------------------------------- - when hasThreadSupport: - var sharedHeap: MemRegion - var heapLock: SysLock - initSysLock(heapLock) proc allocSharedImpl(size: Natural): pointer = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = alloc(sharedHeap, size) releaseSys(heapLock) @@ -1078,7 +1283,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = zeroMem(result, size) proc deallocSharedImpl(p: pointer) = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) dealloc(sharedHeap, p) releaseSys(heapLock) @@ -1086,7 +1291,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = deallocImpl(p) proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = realloc(sharedHeap, p, newSize) releaseSys(heapLock) @@ -1094,7 +1299,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = reallocImpl(p, newSize) proc reallocShared0Impl(p: pointer, oldSize, newSize: Natural): pointer = - when hasThreadSupport: + when hasThreadSupport and not defined(gcDestructors): acquireSys(heapLock) result = realloc0(sharedHeap, p, oldSize, newSize) releaseSys(heapLock) @@ -1102,20 +1307,31 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = realloc0Impl(p, oldSize, newSize) when hasThreadSupport: - template sharedMemStatsShared(v: int) = - acquireSys(heapLock) - result = v - releaseSys(heapLock) + when defined(gcDestructors): + proc getFreeSharedMem(): int = + allocator.freeMem + + proc getTotalSharedMem(): int = + allocator.currMem + + proc getOccupiedSharedMem(): int = + allocator.occ + + else: + template sharedMemStatsShared(v: int) = + acquireSys(heapLock) + result = v + releaseSys(heapLock) - proc getFreeSharedMem(): int = - sharedMemStatsShared(sharedHeap.freeMem) + proc getFreeSharedMem(): int = + sharedMemStatsShared(sharedHeap.freeMem) - proc getTotalSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem) + proc getTotalSharedMem(): int = + sharedMemStatsShared(sharedHeap.currMem) - proc getOccupiedSharedMem(): int = - sharedMemStatsShared(sharedHeap.occ) - #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + proc getOccupiedSharedMem(): int = + sharedMemStatsShared(sharedHeap.occ) + #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) {.pop.} {.pop.} diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index cb2303b61..3098e17d6 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -12,8 +12,6 @@ # All symbols are prefixed with 'c_' to avoid ambiguities {.push hints:off, stack_trace: off, profiler: off.} -when not defined(nimHasHotCodeReloading): - {.pragma: nonReloadable.} proc c_memchr*(s: pointer, c: cint, n: csize_t): pointer {. importc: "memchr", header: "<string.h>".} @@ -33,7 +31,10 @@ proc c_abort*() {. importc: "abort", header: "<stdlib.h>", noSideEffect, noreturn.} -when defined(linux) and defined(amd64): +when defined(nimBuiltinSetjmp): + type + C_JmpBuf* = array[5, pointer] +elif defined(linux) and defined(amd64): type C_JmpBuf* {.importc: "jmp_buf", header: "<setjmp.h>", bycopy.} = object abi: array[200 div sizeof(clong), clong] @@ -42,6 +43,7 @@ else: C_JmpBuf* {.importc: "jmp_buf", header: "<setjmp.h>".} = object +type CSighandlerT = proc (a: cint) {.noconv.} when defined(windows): const SIGABRT* = cint(22) @@ -50,6 +52,7 @@ when defined(windows): SIGINT* = cint(2) SIGSEGV* = cint(11) SIGTERM = cint(15) + SIG_DFL* = cast[CSighandlerT](0) elif defined(macosx) or defined(linux) or defined(freebsd) or defined(openbsd) or defined(netbsd) or defined(solaris) or defined(dragonfly) or defined(nintendoswitch) or defined(genode) or @@ -62,6 +65,7 @@ elif defined(macosx) or defined(linux) or defined(freebsd) or SIGSEGV* = cint(11) SIGTERM* = cint(15) SIGPIPE* = cint(13) + SIG_DFL* = CSighandlerT(nil) elif defined(haiku): const SIGABRT* = cint(6) @@ -71,8 +75,9 @@ elif defined(haiku): SIGSEGV* = cint(11) SIGTERM* = cint(15) SIGPIPE* = cint(7) + SIG_DFL* = CSighandlerT(nil) else: - when NoFakeVars: + when defined(nimscript): {.error: "SIGABRT not ported to your platform".} else: var @@ -81,6 +86,7 @@ else: SIGABRT* {.importc: "SIGABRT", nodecl.}: cint SIGFPE* {.importc: "SIGFPE", nodecl.}: cint SIGILL* {.importc: "SIGILL", nodecl.}: cint + SIG_DFL* {.importc: "SIG_DFL", nodecl.}: CSighandlerT when defined(macosx) or defined(linux): var SIGPIPE* {.importc: "SIGPIPE", nodecl.}: cint @@ -89,27 +95,67 @@ when defined(macosx): elif defined(haiku): const SIGBUS* = cint(30) -when defined(nimSigSetjmp) and not defined(nimStdSetjmp): +# "nimRawSetjmp" is defined by default for certain platforms, so we need the +# "nimStdSetjmp" escape hatch with it. +when defined(nimSigSetjmp): proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "<setjmp.h>", importc: "siglongjmp".} - template c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_sigsetjmp(jmpb: C_JmpBuf, savemask: cint): cint {. header: "<setjmp.h>", importc: "sigsetjmp".} c_sigsetjmp(jmpb, 0) +elif defined(nimBuiltinSetjmp): + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) = + # Apple's Clang++ has trouble converting array names to pointers, so we need + # to be very explicit here. + proc c_builtin_longjmp(jmpb: ptr pointer, retval: cint) {. + importc: "__builtin_longjmp", nodecl.} + # The second parameter needs to be 1 and sometimes the C/C++ compiler checks it. + c_builtin_longjmp(unsafeAddr jmpb[0], 1) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_builtin_setjmp(jmpb: ptr pointer): cint {. + importc: "__builtin_setjmp", nodecl.} + c_builtin_setjmp(unsafeAddr jmpb[0]) + elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. - header: "<setjmp.h>", importc: "_longjmp".} - proc c_setjmp*(jmpb: C_JmpBuf): cint {. - header: "<setjmp.h>", importc: "_setjmp".} + when defined(windows): + # No `_longjmp()` on Windows. + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "<setjmp.h>", importc: "longjmp".} + when defined(vcc) or defined(clangcl): + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "<setjmp.h>", importc: "setjmp".} + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} + + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "<setjmp.h>", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) + + when defined(nimHasStyleChecks): + {.pop.} + else: + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "<setjmp.h>", importc: "_longjmp".} + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "<setjmp.h>", importc: "_setjmp".} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "<setjmp.h>", importc: "longjmp".} proc c_setjmp*(jmpb: C_JmpBuf): cint {. header: "<setjmp.h>", importc: "setjmp".} -type CSighandlerT = proc (a: cint) {.noconv.} -proc c_signal*(sign: cint, handler: proc (a: cint) {.noconv.}): CSighandlerT {. +proc c_signal*(sign: cint, handler: CSighandlerT): CSighandlerT {. importc: "signal", header: "<signal.h>", discardable.} +proc c_raise*(sign: cint): cint {.importc: "raise", header: "<signal.h>".} type CFile {.importc: "FILE", header: "<stdio.h>", @@ -117,7 +163,7 @@ type CFilePtr* = ptr CFile ## The type representing a file handle. # duplicated between io and ansi_c -const stdioUsesMacros = defined(osx) and not defined(emscripten) +const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined(dragonfly)) and not defined(emscripten) const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" @@ -134,29 +180,54 @@ proc c_printf*(frmt: cstring): cint {. proc c_fputs*(c: cstring, f: CFilePtr): cint {. importc: "fputs", header: "<stdio.h>", discardable.} +proc c_fputc*(c: char, f: CFilePtr): cint {. + importc: "fputc", header: "<stdio.h>", discardable.} proc c_sprintf*(buf, frmt: cstring): cint {. importc: "sprintf", header: "<stdio.h>", varargs, noSideEffect.} # we use it only in a way that cannot lead to security issues -proc c_malloc*(size: csize_t): pointer {. - importc: "malloc", header: "<stdlib.h>".} -proc c_calloc*(nmemb, size: csize_t): pointer {. - importc: "calloc", header: "<stdlib.h>".} -proc c_free*(p: pointer) {. - importc: "free", header: "<stdlib.h>".} -proc c_realloc*(p: pointer, newsize: csize_t): pointer {. - importc: "realloc", header: "<stdlib.h>".} - -proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): cint {. +proc c_snprintf*(buf: cstring, n: csize_t, frmt: cstring): cint {. + importc: "snprintf", header: "<stdio.h>", varargs, noSideEffect.} + +when defined(zephyr) and not defined(zephyrUseLibcMalloc): + proc c_malloc*(size: csize_t): pointer {. + importc: "k_malloc", header: "<kernel.h>".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "k_calloc", header: "<kernel.h>".} + proc c_free*(p: pointer) {. + importc: "k_free", header: "<kernel.h>".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer = + # Zephyr's kernel malloc doesn't support realloc + result = c_malloc(newSize) + # match the ansi c behavior + if not result.isNil(): + copyMem(result, p, newSize) + c_free(p) +else: + proc c_malloc*(size: csize_t): pointer {. + importc: "malloc", header: "<stdlib.h>".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "calloc", header: "<stdlib.h>".} + proc c_free*(p: pointer) {. + importc: "free", header: "<stdlib.h>".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer {. + importc: "realloc", header: "<stdlib.h>".} + +proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): csize_t {. importc: "fwrite", header: "<stdio.h>".} -proc c_fflush(f: CFilePtr): cint {. +proc c_fflush*(f: CFilePtr): cint {. importc: "fflush", header: "<stdio.h>".} +proc rawWriteString*(f: CFilePtr, s: cstring, length: int) {.compilerproc, nonReloadable, inline.} = + # we cannot throw an exception here! + discard c_fwrite(s, 1, cast[csize_t](length), f) + discard c_fflush(f) + proc rawWrite*(f: CFilePtr, s: cstring) {.compilerproc, nonReloadable, inline.} = # we cannot throw an exception here! - discard c_fwrite(s, 1, cast[csize_t](s.len), f) + discard c_fwrite(s, 1, c_strlen(s), f) discard c_fflush(f) {.pop.} diff --git a/lib/system/arc.nim b/lib/system/arc.nim new file mode 100644 index 000000000..d001fcaa5 --- /dev/null +++ b/lib/system/arc.nim @@ -0,0 +1,267 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +#[ +In this new runtime we simplify the object layouts a bit: The runtime type +information is only accessed for the objects that have it and it's always +at offset 0 then. The ``ref`` object header is independent from the +runtime type and only contains a reference count. +]# + +when defined(gcOrc): + const + rcIncrement = 0b10000 # so that lowest 4 bits are not touched + rcMask = 0b1111 + rcShift = 4 # shift by rcShift to get the reference counter + +else: + const + rcIncrement = 0b1000 # so that lowest 3 bits are not touched + rcMask = 0b111 + rcShift = 3 # shift by rcShift to get the reference counter + +const + orcLeakDetector = defined(nimOrcLeakDetector) + +type + RefHeader = object + rc: int # the object header is now a single RC field. + # we could remove it in non-debug builds for the 'owned ref' + # design but this seems unwise. + when defined(gcOrc): + rootIdx: int # thanks to this we can delete potential cycle roots + # in O(1) without doubly linked lists + when defined(nimArcDebug) or defined(nimArcIds): + refId: int + when defined(gcOrc) and orcLeakDetector: + filename: cstring + line: int + + Cell = ptr RefHeader + +template setFrameInfo(c: Cell) = + when orcLeakDetector: + if framePtr != nil and framePtr.prev != nil: + c.filename = framePtr.prev.filename + c.line = framePtr.prev.line + else: + c.filename = nil + c.line = 0 + +template head(p: pointer): Cell = + cast[Cell](cast[int](p) -% sizeof(RefHeader)) + +const + traceCollector = defined(traceArc) + +when defined(nimArcDebug): + include cellsets + + const traceId = 20 # 1037 + + var gRefId: int + var freedCells: CellSet +elif defined(nimArcIds): + var gRefId: int + + const traceId = -1 + +when defined(gcAtomicArc) and hasThreadSupport: + template decrement(cell: Cell): untyped = + discard atomicDec(cell.rc, rcIncrement) + template increment(cell: Cell): untyped = + discard atomicInc(cell.rc, rcIncrement) + template count(x: Cell): untyped = + atomicLoadN(x.rc.addr, ATOMIC_ACQUIRE) shr rcShift +else: + template decrement(cell: Cell): untyped = + dec(cell.rc, rcIncrement) + template increment(cell: Cell): untyped = + inc(cell.rc, rcIncrement) + template count(x: Cell): untyped = + x.rc shr rcShift + +proc nimNewObj(size, alignment: int): pointer {.compilerRtl.} = + let hdrSize = align(sizeof(RefHeader), alignment) + let s = size + hdrSize + when defined(nimscript): + discard + else: + result = alignedAlloc0(s, alignment) +! hdrSize + when defined(nimArcDebug) or defined(nimArcIds): + head(result).refId = gRefId + atomicInc gRefId + if head(result).refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[nimNewObj] %p %ld\n", result, head(result).count) + when traceCollector: + cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result) + setFrameInfo head(result) + +proc nimNewObjUninit(size, alignment: int): pointer {.compilerRtl.} = + # Same as 'newNewObj' but do not initialize the memory to zero. + # The codegen proved for us that this is not necessary. + let hdrSize = align(sizeof(RefHeader), alignment) + let s = size + hdrSize + when defined(nimscript): + discard + else: + result = cast[ptr RefHeader](alignedAlloc(s, alignment) +! hdrSize) + head(result).rc = 0 + when defined(gcOrc): + head(result).rootIdx = 0 + when defined(nimArcDebug): + head(result).refId = gRefId + atomicInc gRefId + if head(result).refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[nimNewObjUninit] %p %ld\n", result, head(result).count) + + when traceCollector: + cprintf("[Allocated] %p result: %p\n", result -! sizeof(RefHeader), result) + setFrameInfo head(result) + +proc nimDecWeakRef(p: pointer) {.compilerRtl, inl.} = + decrement head(p) + +proc isUniqueRef*[T](x: ref T): bool {.inline.} = + ## Returns true if the object `x` points to is uniquely referenced. Such + ## an object can potentially be passed over to a different thread safely, + ## if great care is taken. This queries the internal reference count of + ## the object which is subject to lots of optimizations! In other words + ## the value of `isUniqueRef` can depend on the used compiler version and + ## optimizer setting. + ## Nevertheless it can be used as a very valuable debugging tool and can + ## be used to specify the constraints of a threading related API + ## via `assert isUniqueRef(x)`. + head(cast[pointer](x)).rc == 0 + +proc nimIncRef(p: pointer) {.compilerRtl, inl.} = + when defined(nimArcDebug): + if head(p).refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[IncRef] %p %ld\n", p, head(p).count) + + increment head(p) + when traceCollector: + cprintf("[INCREF] %p\n", head(p)) + +when not defined(gcOrc) or defined(nimThinout): + proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} = + # This is only used by the old RTTI mechanism and we know + # that 'dest[]' is nil and needs no destruction. Which is really handy + # as we cannot destroy the object reliably if it's an object of unknown + # compile-time type. + dest[] = src + if src != nil: nimIncRef src + +when not defined(nimscript) and defined(nimArcDebug): + proc deallocatedRefId*(p: pointer): int = + ## Returns the ref's ID if the ref was already deallocated. This + ## is a memory corruption check. Returns 0 if there is no error. + let c = head(p) + if freedCells.data != nil and freedCells.contains(c): + result = c.refId + else: + result = 0 + +proc nimRawDispose(p: pointer, alignment: int) {.compilerRtl.} = + when not defined(nimscript): + when traceCollector: + cprintf("[Freed] %p\n", p -! sizeof(RefHeader)) + when defined(nimOwnedEnabled): + if head(p).rc >= rcIncrement: + cstderr.rawWrite "[FATAL] dangling references exist\n" + rawQuit 1 + when defined(nimArcDebug): + # we do NOT really free the memory here in order to reliably detect use-after-frees + if freedCells.data == nil: init(freedCells) + freedCells.incl head(p) + else: + let hdrSize = align(sizeof(RefHeader), alignment) + alignedDealloc(p -! hdrSize, alignment) + +template `=dispose`*[T](x: owned(ref T)) = nimRawDispose(cast[pointer](x), T.alignOf) +#proc dispose*(x: pointer) = nimRawDispose(x) + +proc nimDestroyAndDispose(p: pointer) {.compilerRtl, raises: [].} = + let rti = cast[ptr PNimTypeV2](p) + if rti.destructor != nil: + cast[DestructorProc](rti.destructor)(p) + when false: + cstderr.rawWrite cast[ptr PNimTypeV2](p)[].name + cstderr.rawWrite "\n" + if d == nil: + cstderr.rawWrite "bah, nil\n" + else: + cstderr.rawWrite "has destructor!\n" + nimRawDispose(p, rti.align) + +when defined(gcOrc): + when defined(nimThinout): + include cyclebreaker + else: + include orc + #include cyclecollector + +proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + + when defined(nimArcDebug): + if cell.refId == traceId: + writeStackTrace() + cfprintf(cstderr, "[DecRef] %p %ld\n", p, cell.count) + + when defined(gcAtomicArc) and hasThreadSupport: + # `atomicDec` returns the new value + if atomicDec(cell.rc, rcIncrement) == -rcIncrement: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + else: + if cell.count == 0: + result = true + when traceCollector: + cprintf("[ABOUT TO DESTROY] %p\n", cell) + else: + decrement cell + # According to Lins it's correct to do nothing else here. + when traceCollector: + cprintf("[DECREF] %p\n", cell) + +proc GC_unref*[T](x: ref T) = + ## New runtime only supports this operation for 'ref T'. + var y {.cursor.} = x + `=destroy`(y) + +proc GC_ref*[T](x: ref T) = + ## New runtime only supports this operation for 'ref T'. + if x != nil: nimIncRef(cast[pointer](x)) + +when not defined(gcOrc): + template GC_fullCollect* = + ## Forces a full garbage collection pass. With `--mm:arc` a nop. + discard + +template setupForeignThreadGc* = + ## With `--mm:arc` a nop. + discard + +template tearDownForeignThreadGc* = + ## With `--mm:arc` a nop. + discard + +proc isObjDisplayCheck(source: PNimTypeV2, targetDepth: int16, token: uint32): bool {.compilerRtl, inl.} = + result = targetDepth <= source.depth and source.display[targetDepth] == token + +when defined(gcDestructors): + proc nimGetVTable(p: pointer, index: int): pointer + {.compilerRtl, inline, raises: [].} = + result = cast[ptr PNimTypeV2](p).vTable[index] diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim deleted file mode 100644 index 16ac8affe..000000000 --- a/lib/system/arithm.nim +++ /dev/null @@ -1,425 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# simple integer arithmetic with overflow checking - -proc raiseOverflow {.compilerproc, noinline.} = - # a single proc to reduce code size to a minimum - sysFatal(OverflowError, "over- or underflow") - -proc raiseDivByZero {.compilerproc, noinline.} = - sysFatal(DivByZeroError, "division by zero") - -when defined(builtinOverflow): - # Builtin compiler functions for improved performance - when sizeof(clong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddl_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubl_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smull_overflow", nodecl, nosideeffect.} - - elif sizeof(clonglong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddll_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubll_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smulll_overflow", nodecl, nosideeffect.} - - when sizeof(int) == 8: - proc addIntOverflow(a, b: int, c: var int): bool {.inline.} = - addInt64Overflow(a, b, c) - - proc subIntOverflow(a, b: int, c: var int): bool {.inline.} = - subInt64Overflow(a, b, c) - - proc mulIntOverflow(a, b: int, c: var int): bool {.inline.} = - mulInt64Overflow(a, b, c) - - elif sizeof(int) == 4 and sizeof(cint) == 4: - proc addIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_sadd_overflow", nodecl, nosideeffect.} - - proc subIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_ssub_overflow", nodecl, nosideeffect.} - - proc mulIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_smul_overflow", nodecl, nosideeffect.} - - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - if addInt64Overflow(a, b, result): - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - if subInt64Overflow(a, b, result): - raiseOverflow() - - proc mulInt64(a, b: int64): int64 {.compilerproc, inline.} = - if mulInt64Overflow(a, b, result): - raiseOverflow() -else: - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= int64(0) or (result xor b) >= int64(0): - return result - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= int64(0) or (result xor not b) >= int64(0): - return result - raiseOverflow() - - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if they're - # approximately the same. Else, because the native int product is the only - # one that can lose catastrophic amounts of information, it's the native int - # product that must have overflowed. - # - proc mulInt64(a, b: int64): int64 {.compilerproc.} = - var - resAsFloat, floatProd: float64 - result = a *% b - floatProd = toBiggestFloat(a) # conversion - floatProd = floatProd * toBiggestFloat(b) - resAsFloat = toBiggestFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -proc negInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): return -a - raiseOverflow() - -proc absInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): - if a >= 0: return a - else: return -a - raiseOverflow() - -proc divInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - if a == low(int64) and b == int64(-1): - raiseOverflow() - return a div b - -proc modInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - return a mod b - -proc absInt(a: int): int {.compilerproc, inline.} = - if a != low(int): - if a >= 0: return a - else: return -a - raiseOverflow() - -const - asmVersion = defined(I386) and (defined(vcc) or defined(wcc) or - defined(dmc) or defined(gcc) or defined(llvm_gcc)) - # my Version of Borland C++Builder does not have - # tasm32, which is needed for assembler blocks - # this is why Borland is not included in the 'when' - -when asmVersion and not defined(gcc) and not defined(llvm_gcc): - # assembler optimized versions for compilers that - # have an intel syntax assembler: - proc addInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - # a in eax, and b in edx - asm """ - mov eax, ecx - add eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc subInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - sub eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc negInt(a: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - neg eax - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc divInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - ret - """ - - proc modInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - mov eax, edx - ret - """ - - proc mulInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - imul ecx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - -elif false: # asmVersion and (defined(gcc) or defined(llvm_gcc)): - proc addInt(a, b: int): int {.compilerproc, inline.} = - # don't use a pure proc here! - asm """ - "addl %%ecx, %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - #".intel_syntax noprefix" - #/* Intel syntax here */ - #".att_syntax" - - proc subInt(a, b: int): int {.compilerproc, inline.} = - asm """ "subl %%ecx,%%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - - proc mulInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "imull %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc negInt(a: int): int {.compilerproc, inline.} = - asm """ "negl %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`) - """ - - proc divInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc modInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - "movl %%edx, %%eax" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - -when not declared(addInt) and defined(builtinOverflow): - proc addInt(a, b: int): int {.compilerproc, inline.} = - if addIntOverflow(a, b, result): - raiseOverflow() - -when not declared(subInt) and defined(builtinOverflow): - proc subInt(a, b: int): int {.compilerproc, inline.} = - if subIntOverflow(a, b, result): - raiseOverflow() - -when not declared(mulInt) and defined(builtinOverflow): - proc mulInt(a, b: int): int {.compilerproc, inline.} = - if mulIntOverflow(a, b, result): - raiseOverflow() - -# Platform independent versions of the above (slower!) -when not declared(addInt): - proc addInt(a, b: int): int {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= 0 or (result xor b) >= 0: - return result - raiseOverflow() - -when not declared(subInt): - proc subInt(a, b: int): int {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= 0 or (result xor not b) >= 0: - return result - raiseOverflow() - -when not declared(negInt): - proc negInt(a: int): int {.compilerproc, inline.} = - if a != low(int): return -a - raiseOverflow() - -when not declared(divInt): - proc divInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - if a == low(int) and b == -1: - raiseOverflow() - return a div b - -when not declared(modInt): - proc modInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - return a mod b - -when not declared(mulInt): - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if - # they're approximately the same. Else, because the native int product is - # the only one that can lose catastrophic amounts of information, it's the - # native int product that must have overflowed. - # - proc mulInt(a, b: int): int {.compilerproc.} = - var - resAsFloat, floatProd: float - - result = a *% b - floatProd = toFloat(a) * toFloat(b) - resAsFloat = toFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -# We avoid setting the FPU control word here for compatibility with libraries -# written in other languages. - -proc raiseFloatInvalidOp {.noinline.} = - sysFatal(FloatInvalidOpError, "FPU operation caused a NaN result") - -proc nanCheck(x: float64) {.compilerproc, inline.} = - if x != x: raiseFloatInvalidOp() - -proc raiseFloatOverflow(x: float64) {.noinline.} = - if x > 0.0: - sysFatal(FloatOverflowError, "FPU operation caused an overflow") - else: - sysFatal(FloatUnderflowError, "FPU operations caused an underflow") - -proc infCheck(x: float64) {.compilerproc, inline.} = - if x != 0.0 and x*0.5 == x: raiseFloatOverflow(x) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index ba9ade192..e229a0f4b 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -1,157 +1,50 @@ -proc succ*[T: Ordinal](x: T, y = 1): T {.magic: "Succ", noSideEffect.} - ## Returns the ``y``-th successor (default: 1) of the value ``x``. - ## ``T`` has to be an `ordinal type <#Ordinal>`_. +proc succ*[T, V: Ordinal](x: T, y: V = 1): T {.magic: "Succ", noSideEffect.} = + ## Returns the `y`-th successor (default: 1) of the value `x`. ## - ## If such a value does not exist, ``OverflowError`` is raised + ## If such a value does not exist, `OverflowDefect` is raised ## or a compile time error occurs. - ## - ## .. code-block:: Nim - ## let x = 5 - ## echo succ(5) # => 6 - ## echo succ(5, 3) # => 8 - -proc pred*[T: Ordinal](x: T, y = 1): T {.magic: "Pred", noSideEffect.} - ## Returns the ``y``-th predecessor (default: 1) of the value ``x``. - ## ``T`` has to be an `ordinal type <#Ordinal>`_. - ## - ## If such a value does not exist, ``OverflowError`` is raised - ## or a compile time error occurs. - ## - ## .. code-block:: Nim - ## let x = 5 - ## echo pred(5) # => 4 - ## echo pred(5, 3) # => 2 + runnableExamples: + assert succ(5) == 6 + assert succ(5, 3) == 8 -proc inc*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Inc", noSideEffect.} - ## Increments the ordinal ``x`` by ``y``. - ## - ## If such a value does not exist, ``OverflowError`` is raised or a compile - ## time error occurs. This is a short notation for: ``x = succ(x, y)``. +proc pred*[T, V: Ordinal](x: T, y: V = 1): T {.magic: "Pred", noSideEffect.} = + ## Returns the `y`-th predecessor (default: 1) of the value `x`. ## - ## .. code-block:: Nim - ## var i = 2 - ## inc(i) # i <- 3 - ## inc(i, 3) # i <- 6 + ## If such a value does not exist, `OverflowDefect` is raised + ## or a compile time error occurs. + runnableExamples: + assert pred(5) == 4 + assert pred(5, 3) == 2 -proc dec*[T: Ordinal|uint|uint64](x: var T, y = 1) {.magic: "Dec", noSideEffect.} - ## Decrements the ordinal ``x`` by ``y``. +proc inc*[T, V: Ordinal](x: var T, y: V = 1) {.magic: "Inc", noSideEffect.} = + ## Increments the ordinal `x` by `y`. ## - ## If such a value does not exist, ``OverflowError`` is raised or a compile - ## time error occurs. This is a short notation for: ``x = pred(x, y)``. + ## If such a value does not exist, `OverflowDefect` is raised or a compile + ## time error occurs. This is a short notation for: `x = succ(x, y)`. + runnableExamples: + var i = 2 + inc(i) + assert i == 3 + inc(i, 3) + assert i == 6 + +proc dec*[T, V: Ordinal](x: var T, y: V = 1) {.magic: "Dec", noSideEffect.} = + ## Decrements the ordinal `x` by `y`. ## - ## .. code-block:: Nim - ## var i = 2 - ## dec(i) # i <- 1 - ## dec(i, 3) # i <- -2 + ## If such a value does not exist, `OverflowDefect` is raised or a compile + ## time error occurs. This is a short notation for: `x = pred(x, y)`. + runnableExamples: + var i = 2 + dec(i) + assert i == 1 + dec(i, 3) + assert i == -2 # -------------------------------------------------------------------------- # built-in operators -when defined(nimNoZeroExtendMagic): - proc ze*(x: int8): int {.deprecated.} = - ## zero extends a smaller integer type to ``int``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint8](x))) - - proc ze*(x: int16): int {.deprecated.} = - ## zero extends a smaller integer type to ``int``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint16](x))) - - proc ze64*(x: int8): int64 {.deprecated.} = - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint8](x))) - - proc ze64*(x: int16): int64 {.deprecated.} = - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint16](x))) - - proc ze64*(x: int32): int64 {.deprecated.} = - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint32](x))) - - proc ze64*(x: int): int64 {.deprecated.} = - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. Does nothing if the size of an ``int`` is the same as ``int64``. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint](x))) - - proc toU8*(x: int): int8 {.deprecated.} = - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int8](x) - - proc toU16*(x: int): int16 {.deprecated.} = - ## treats `x` as unsigned and converts it to an ``int16`` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int16](x) - - proc toU32*(x: int64): int32 {.deprecated.} = - ## treats `x` as unsigned and converts it to an ``int32`` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int32](x) - -elif not defined(js): - proc ze*(x: int8): int {.magic: "Ze8ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to ``int``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze*(x: int16): int {.magic: "Ze16ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to ``int``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int8): int64 {.magic: "Ze8ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int16): int64 {.magic: "Ze16ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int32): int64 {.magic: "Ze32ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int): int64 {.magic: "ZeIToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to ``int64``. This treats `x` as - ## unsigned. Does nothing if the size of an ``int`` is the same as ``int64``. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU8*(x: int): int8 {.magic: "ToU8", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU16*(x: int): int16 {.magic: "ToU16", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an ``int16`` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU32*(x: int64): int32 {.magic: "ToU32", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an ``int32`` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - # integer calculations: proc `+`*(x: int): int {.magic: "UnaryPlusI", noSideEffect.} ## Unary `+` operator for an integer. Has no effect. @@ -167,20 +60,13 @@ proc `-`*(x: int16): int16 {.magic: "UnaryMinusI", noSideEffect.} proc `-`*(x: int32): int32 {.magic: "UnaryMinusI", noSideEffect.} proc `-`*(x: int64): int64 {.magic: "UnaryMinusI64", noSideEffect.} -proc `not`*(x: int): int {.magic: "BitnotI", noSideEffect.} +proc `not`*(x: int): int {.magic: "BitnotI", noSideEffect.} = ## Computes the `bitwise complement` of the integer `x`. - ## - ## .. code-block:: Nim - ## var - ## a = 0'u8 - ## b = 0'i8 - ## c = 1000'u16 - ## d = 1000'i16 - ## - ## echo not a # => 255 - ## echo not b # => -1 - ## echo not c # => 64535 - ## echo not d # => -1001 + runnableExamples: + assert not 0'u8 == 255 + assert not 0'i8 == -1 + assert not 1000'u16 == 64535 + assert not 1000'i16 == -1001 proc `not`*(x: int8): int8 {.magic: "BitnotI", noSideEffect.} proc `not`*(x: int16): int16 {.magic: "BitnotI", noSideEffect.} proc `not`*(x: int32): int32 {.magic: "BitnotI", noSideEffect.} @@ -207,40 +93,38 @@ proc `*`*(x, y: int16): int16 {.magic: "MulI", noSideEffect.} proc `*`*(x, y: int32): int32 {.magic: "MulI", noSideEffect.} proc `*`*(x, y: int64): int64 {.magic: "MulI", noSideEffect.} -proc `div`*(x, y: int): int {.magic: "DivI", noSideEffect.} +proc `div`*(x, y: int): int {.magic: "DivI", noSideEffect.} = ## Computes the integer division. ## - ## This is roughly the same as ``trunc(x/y)``. - ## - ## .. code-block:: Nim - ## ( 1 div 2) == 0 - ## ( 2 div 2) == 1 - ## ( 3 div 2) == 1 - ## ( 7 div 3) == 2 - ## (-7 div 3) == -2 - ## ( 7 div -3) == -2 - ## (-7 div -3) == 2 + ## This is roughly the same as `math.trunc(x/y).int`. + runnableExamples: + assert (1 div 2) == 0 + assert (2 div 2) == 1 + assert (3 div 2) == 1 + assert (7 div 3) == 2 + assert (-7 div 3) == -2 + assert (7 div -3) == -2 + assert (-7 div -3) == 2 proc `div`*(x, y: int8): int8 {.magic: "DivI", noSideEffect.} proc `div`*(x, y: int16): int16 {.magic: "DivI", noSideEffect.} proc `div`*(x, y: int32): int32 {.magic: "DivI", noSideEffect.} proc `div`*(x, y: int64): int64 {.magic: "DivI", noSideEffect.} -proc `mod`*(x, y: int): int {.magic: "ModI", noSideEffect.} +proc `mod`*(x, y: int): int {.magic: "ModI", noSideEffect.} = ## Computes the integer modulo operation (remainder). ## - ## This is the same as ``x - (x div y) * y``. - ## - ## .. code-block:: Nim - ## ( 7 mod 5) == 2 - ## (-7 mod 5) == -2 - ## ( 7 mod -5) == 2 - ## (-7 mod -5) == -2 + ## This is the same as `x - (x div y) * y`. + runnableExamples: + assert (7 mod 5) == 2 + assert (-7 mod 5) == -2 + assert (7 mod -5) == 2 + assert (-7 mod -5) == -2 proc `mod`*(x, y: int8): int8 {.magic: "ModI", noSideEffect.} proc `mod`*(x, y: int16): int16 {.magic: "ModI", noSideEffect.} proc `mod`*(x, y: int32): int32 {.magic: "ModI", noSideEffect.} proc `mod`*(x, y: int64): int64 {.magic: "ModI", noSideEffect.} -when defined(nimOldShiftRight) or not defined(nimAshr): +when defined(nimOldShiftRight): const shrDepMessage = "`shr` will become sign preserving." proc `shr`*(x: int, y: SomeInteger): int {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} proc `shr`*(x: int8, y: SomeInteger): int8 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} @@ -248,7 +132,7 @@ when defined(nimOldShiftRight) or not defined(nimAshr): proc `shr`*(x: int32, y: SomeInteger): int32 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} proc `shr`*(x: int64, y: SomeInteger): int64 {.magic: "ShrI", noSideEffect, deprecated: shrDepMessage.} else: - proc `shr`*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} + proc `shr`*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = ## Computes the `shift right` operation of `x` and `y`, filling ## vacant bit positions with the sign bit. ## @@ -256,131 +140,82 @@ else: ## is different than in *C*. ## ## See also: - ## * `ashr proc <#ashr,int,SomeInteger>`_ for arithmetic shift right - ## - ## .. code-block:: Nim - ## 0b0001_0000'i8 shr 2 == 0b0000_0100'i8 - ## 0b0000_0001'i8 shr 1 == 0b0000_0000'i8 - ## 0b1000_0000'i8 shr 4 == 0b1111_1000'i8 - ## -1 shr 5 == -1 - ## 1 shr 5 == 0 - ## 16 shr 2 == 4 - ## -16 shr 2 == -4 + ## * `ashr func<#ashr,int,SomeInteger>`_ for arithmetic shift right + runnableExamples: + assert 0b0001_0000'i8 shr 2 == 0b0000_0100'i8 + assert 0b0000_0001'i8 shr 1 == 0b0000_0000'i8 + assert 0b1000_0000'i8 shr 4 == 0b1111_1000'i8 + assert -1 shr 5 == -1 + assert 1 shr 5 == 0 + assert 16 shr 2 == 4 + assert -16 shr 2 == -4 proc `shr`*(x: int8, y: SomeInteger): int8 {.magic: "AshrI", noSideEffect.} proc `shr`*(x: int16, y: SomeInteger): int16 {.magic: "AshrI", noSideEffect.} proc `shr`*(x: int32, y: SomeInteger): int32 {.magic: "AshrI", noSideEffect.} proc `shr`*(x: int64, y: SomeInteger): int64 {.magic: "AshrI", noSideEffect.} -proc `shl`*(x: int, y: SomeInteger): int {.magic: "ShlI", noSideEffect.} +proc `shl`*(x: int, y: SomeInteger): int {.magic: "ShlI", noSideEffect.} = ## Computes the `shift left` operation of `x` and `y`. ## ## **Note**: `Operator precedence <manual.html#syntax-precedence>`_ ## is different than in *C*. - ## - ## .. code-block:: Nim - ## 1'i32 shl 4 == 0x0000_0010 - ## 1'i64 shl 4 == 0x0000_0000_0000_0010 + runnableExamples: + assert 1'i32 shl 4 == 0x0000_0010 + assert 1'i64 shl 4 == 0x0000_0000_0000_0010 proc `shl`*(x: int8, y: SomeInteger): int8 {.magic: "ShlI", noSideEffect.} proc `shl`*(x: int16, y: SomeInteger): int16 {.magic: "ShlI", noSideEffect.} proc `shl`*(x: int32, y: SomeInteger): int32 {.magic: "ShlI", noSideEffect.} proc `shl`*(x: int64, y: SomeInteger): int64 {.magic: "ShlI", noSideEffect.} -when defined(nimAshr): - proc ashr*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} - ## Shifts right by pushing copies of the leftmost bit in from the left, - ## and let the rightmost bits fall off. - ## - ## Note that `ashr` is not an operator so use the normal function - ## call syntax for it. - ## - ## See also: - ## * `shr proc <#shr,int,SomeInteger>`_ - ## - ## .. code-block:: Nim - ## ashr(0b0001_0000'i8, 2) == 0b0000_0100'i8 - ## ashr(0b1000_0000'i8, 8) == 0b1111_1111'i8 - ## ashr(0b1000_0000'i8, 1) == 0b1100_0000'i8 - proc ashr*(x: int8, y: SomeInteger): int8 {.magic: "AshrI", noSideEffect.} - proc ashr*(x: int16, y: SomeInteger): int16 {.magic: "AshrI", noSideEffect.} - proc ashr*(x: int32, y: SomeInteger): int32 {.magic: "AshrI", noSideEffect.} - proc ashr*(x: int64, y: SomeInteger): int64 {.magic: "AshrI", noSideEffect.} -else: - # used for bootstrapping the compiler - proc ashr*[T](x: T, y: SomeInteger): T = discard - -proc `and`*(x, y: int): int {.magic: "BitandI", noSideEffect.} - ## Computes the `bitwise and` of numbers `x` and `y`. +proc ashr*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = + ## Shifts right by pushing copies of the leftmost bit in from the left, + ## and let the rightmost bits fall off. + ## + ## Note that `ashr` is not an operator so use the normal function + ## call syntax for it. ## - ## .. code-block:: Nim - ## (0b0011 and 0b0101) == 0b0001 - ## (0b0111 and 0b1100) == 0b0100 + ## See also: + ## * `shr func<#shr,int,SomeInteger>`_ + runnableExamples: + assert ashr(0b0001_0000'i8, 2) == 0b0000_0100'i8 + assert ashr(0b1000_0000'i8, 8) == 0b1111_1111'i8 + assert ashr(0b1000_0000'i8, 1) == 0b1100_0000'i8 +proc ashr*(x: int8, y: SomeInteger): int8 {.magic: "AshrI", noSideEffect.} +proc ashr*(x: int16, y: SomeInteger): int16 {.magic: "AshrI", noSideEffect.} +proc ashr*(x: int32, y: SomeInteger): int32 {.magic: "AshrI", noSideEffect.} +proc ashr*(x: int64, y: SomeInteger): int64 {.magic: "AshrI", noSideEffect.} + +proc `and`*(x, y: int): int {.magic: "BitandI", noSideEffect.} = + ## Computes the `bitwise and` of numbers `x` and `y`. + runnableExamples: + assert (0b0011 and 0b0101) == 0b0001 + assert (0b0111 and 0b1100) == 0b0100 proc `and`*(x, y: int8): int8 {.magic: "BitandI", noSideEffect.} proc `and`*(x, y: int16): int16 {.magic: "BitandI", noSideEffect.} proc `and`*(x, y: int32): int32 {.magic: "BitandI", noSideEffect.} proc `and`*(x, y: int64): int64 {.magic: "BitandI", noSideEffect.} -proc `or`*(x, y: int): int {.magic: "BitorI", noSideEffect.} +proc `or`*(x, y: int): int {.magic: "BitorI", noSideEffect.} = ## Computes the `bitwise or` of numbers `x` and `y`. - ## - ## .. code-block:: Nim - ## (0b0011 or 0b0101) == 0b0111 - ## (0b0111 or 0b1100) == 0b1111 + runnableExamples: + assert (0b0011 or 0b0101) == 0b0111 + assert (0b0111 or 0b1100) == 0b1111 proc `or`*(x, y: int8): int8 {.magic: "BitorI", noSideEffect.} proc `or`*(x, y: int16): int16 {.magic: "BitorI", noSideEffect.} proc `or`*(x, y: int32): int32 {.magic: "BitorI", noSideEffect.} proc `or`*(x, y: int64): int64 {.magic: "BitorI", noSideEffect.} -proc `xor`*(x, y: int): int {.magic: "BitxorI", noSideEffect.} +proc `xor`*(x, y: int): int {.magic: "BitxorI", noSideEffect.} = ## Computes the `bitwise xor` of numbers `x` and `y`. - ## - ## .. code-block:: Nim - ## (0b0011 xor 0b0101) == 0b0110 - ## (0b0111 xor 0b1100) == 0b1011 + runnableExamples: + assert (0b0011 xor 0b0101) == 0b0110 + assert (0b0111 xor 0b1100) == 0b1011 proc `xor`*(x, y: int8): int8 {.magic: "BitxorI", noSideEffect.} proc `xor`*(x, y: int16): int16 {.magic: "BitxorI", noSideEffect.} proc `xor`*(x, y: int32): int32 {.magic: "BitxorI", noSideEffect.} proc `xor`*(x, y: int64): int64 {.magic: "BitxorI", noSideEffect.} -type - IntMax32 = int|int8|int16|int32 - -proc `+%`*(x, y: IntMax32): IntMax32 {.magic: "AddU", noSideEffect.} -proc `+%`*(x, y: int64): int64 {.magic: "AddU", noSideEffect.} - ## Treats `x` and `y` as unsigned and adds them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `-%`*(x, y: IntMax32): IntMax32 {.magic: "SubU", noSideEffect.} -proc `-%`*(x, y: int64): int64 {.magic: "SubU", noSideEffect.} - ## Treats `x` and `y` as unsigned and subtracts them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `*%`*(x, y: IntMax32): IntMax32 {.magic: "MulU", noSideEffect.} -proc `*%`*(x, y: int64): int64 {.magic: "MulU", noSideEffect.} - ## Treats `x` and `y` as unsigned and multiplies them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `/%`*(x, y: IntMax32): IntMax32 {.magic: "DivU", noSideEffect.} -proc `/%`*(x, y: int64): int64 {.magic: "DivU", noSideEffect.} - ## Treats `x` and `y` as unsigned and divides them. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - -proc `%%`*(x, y: IntMax32): IntMax32 {.magic: "ModU", noSideEffect.} -proc `%%`*(x, y: int64): int64 {.magic: "ModU", noSideEffect.} - ## Treats `x` and `y` as unsigned and compute the modulo of `x` and `y`. - ## - ## The result is truncated to fit into the result. - ## This implements modulo arithmetic. No overflow errors are possible. - - # unsigned integer operations: proc `not`*(x: uint): uint {.magic: "BitnotI", noSideEffect.} ## Computes the `bitwise complement` of the integer `x`. @@ -447,7 +282,7 @@ proc `*`*(x, y: uint64): uint64 {.magic: "MulU", noSideEffect.} proc `div`*(x, y: uint): uint {.magic: "DivU", noSideEffect.} ## Computes the integer division for unsigned integers. - ## This is roughly the same as ``trunc(x/y)``. + ## This is roughly the same as `trunc(x/y)`. proc `div`*(x, y: uint8): uint8 {.magic: "DivU", noSideEffect.} proc `div`*(x, y: uint16): uint16 {.magic: "DivU", noSideEffect.} proc `div`*(x, y: uint32): uint32 {.magic: "DivU", noSideEffect.} @@ -455,32 +290,116 @@ proc `div`*(x, y: uint64): uint64 {.magic: "DivU", noSideEffect.} proc `mod`*(x, y: uint): uint {.magic: "ModU", noSideEffect.} ## Computes the integer modulo operation (remainder) for unsigned integers. - ## This is the same as ``x - (x div y) * y``. + ## This is the same as `x - (x div y) * y`. proc `mod`*(x, y: uint8): uint8 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint16): uint16 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint32): uint32 {.magic: "ModU", noSideEffect.} proc `mod`*(x, y: uint64): uint64 {.magic: "ModU", noSideEffect.} - - - proc `+=`*[T: SomeInteger](x: var T, y: T) {. magic: "Inc", noSideEffect.} ## Increments an integer. -proc `+=`*[T: enum|bool](x: var T, y: T) {. - magic: "Inc", noSideEffect, deprecated: "use `inc` instead".} - ## **Deprecated since v0.20**: use `inc` instead. - proc `-=`*[T: SomeInteger](x: var T, y: T) {. magic: "Dec", noSideEffect.} ## Decrements an integer. -proc `-=`*[T: enum|bool](x: var T, y: T) {. - magic: "Dec", noSideEffect, deprecated: "0.20.0, use `dec` instead".} - ## **Deprecated since v0.20**: use `dec` instead. - proc `*=`*[T: SomeInteger](x: var T, y: T) {. inline, noSideEffect.} = ## Binary `*=` operator for integers. x = x * y + +# floating point operations: +proc `+`*(x: float32): float32 {.magic: "UnaryPlusF64", noSideEffect.} +proc `-`*(x: float32): float32 {.magic: "UnaryMinusF64", noSideEffect.} +proc `+`*(x, y: float32): float32 {.magic: "AddF64", noSideEffect.} +proc `-`*(x, y: float32): float32 {.magic: "SubF64", noSideEffect.} +proc `*`*(x, y: float32): float32 {.magic: "MulF64", noSideEffect.} +proc `/`*(x, y: float32): float32 {.magic: "DivF64", noSideEffect.} + +proc `+`*(x: float): float {.magic: "UnaryPlusF64", noSideEffect.} +proc `-`*(x: float): float {.magic: "UnaryMinusF64", noSideEffect.} +proc `+`*(x, y: float): float {.magic: "AddF64", noSideEffect.} +proc `-`*(x, y: float): float {.magic: "SubF64", noSideEffect.} +proc `*`*(x, y: float): float {.magic: "MulF64", noSideEffect.} +proc `/`*(x, y: float): float {.magic: "DivF64", noSideEffect.} + +proc `+=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Increments in place a floating point number. + x = x + y + +proc `-=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Decrements in place a floating point number. + x = x - y + +proc `*=`*[T: float|float32|float64] (x: var T, y: T) {. + inline, noSideEffect.} = + ## Multiplies in place a floating point number. + x = x * y + +proc `/=`*(x: var float64, y: float64) {.inline, noSideEffect.} = + ## Divides in place a floating point number. + x = x / y + +proc `/=`*[T: float|float32](x: var T, y: T) {.inline, noSideEffect.} = + ## Divides in place a floating point number. + x = x / y + +# the following have to be included in system, not imported for some reason: + +proc `+%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and adds them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) + cast[uint](y)) +proc `+%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) + cast[uint8](y)) +proc `+%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) + cast[uint16](y)) +proc `+%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) + cast[uint32](y)) +proc `+%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) + cast[uint64](y)) + +proc `-%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and subtracts them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) - cast[uint](y)) +proc `-%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) - cast[uint8](y)) +proc `-%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) - cast[uint16](y)) +proc `-%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) - cast[uint32](y)) +proc `-%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) - cast[uint64](y)) + +proc `*%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and multiplies them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) * cast[uint](y)) +proc `*%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) * cast[uint8](y)) +proc `*%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) * cast[uint16](y)) +proc `*%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) * cast[uint32](y)) +proc `*%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) * cast[uint64](y)) + +proc `/%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and divides them. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) div cast[uint](y)) +proc `/%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) div cast[uint8](y)) +proc `/%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) div cast[uint16](y)) +proc `/%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) div cast[uint32](y)) +proc `/%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) div cast[uint64](y)) + +proc `%%`*(x, y: int): int {.inline.} = + ## Treats `x` and `y` as unsigned and compute the modulo of `x` and `y`. + ## + ## The result is truncated to fit into the result. + ## This implements modulo arithmetic. No overflow errors are possible. + cast[int](cast[uint](x) mod cast[uint](y)) +proc `%%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) mod cast[uint8](y)) +proc `%%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) mod cast[uint16](y)) +proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast[uint32](y)) +proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) diff --git a/lib/system/assertions.nim b/lib/system/assertions.nim deleted file mode 100644 index cb1f3fd40..000000000 --- a/lib/system/assertions.nim +++ /dev/null @@ -1,101 +0,0 @@ -when not declared(sysFatal): - include "system/fatal" - -# --------------------------------------------------------------------------- -# helpers - -type InstantiationInfo = tuple[filename: string, line: int, column: int] - -proc `$`(x: int): string {.magic: "IntToStr", noSideEffect.} - -proc `$`(info: InstantiationInfo): string = - # The +1 is needed here - # instead of overriding `$` (and changing its meaning), consider explicit name. - info.filename & "(" & $info.line & ", " & $(info.column+1) & ")" - -# --------------------------------------------------------------------------- - - -proc raiseAssert*(msg: string) {.noinline, noreturn.} = - sysFatal(AssertionError, msg) - -proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = - # trick the compiler to not list ``AssertionError`` when called - # by ``assert``. - type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, - tags: [].} - Hide(raiseAssert)(msg) - -template assertImpl(cond: bool, msg: string, expr: string, enabled: static[bool]) = - when enabled: - const - loc = instantiationInfo(fullPaths = compileOption("excessiveStackTrace")) - ploc = $loc - bind instantiationInfo - mixin failedAssertImpl - {.line: loc.}: - if not cond: - failedAssertImpl(ploc & " `" & expr & "` " & msg) - -template assert*(cond: untyped, msg = "") = - ## Raises ``AssertionError`` with `msg` if `cond` is false. Note - ## that ``AssertionError`` is hidden from the effect system, so it doesn't - ## produce ``{.raises: [AssertionError].}``. This exception is only supposed - ## to be caught by unit testing frameworks. - ## - ## The compiler may not generate any code at all for ``assert`` if it is - ## advised to do so through the ``-d:danger`` or ``--assertions:off`` - ## `command line switches <nimc.html#compiler-usage-command-line-switches>`_. - const expr = astToStr(cond) - assertImpl(cond, msg, expr, compileOption("assertions")) - -template doAssert*(cond: untyped, msg = "") = - ## Similar to ``assert`` but is always turned on regardless of ``--assertions``. - const expr = astToStr(cond) - assertImpl(cond, msg, expr, true) - -template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = - ## Sets an assertion failure handler that will intercept any assert - ## statements following `onFailedAssert` in the current module scope. - ## - ## .. code-block:: nim - ## # module-wide policy to change the failed assert - ## # exception type in order to include a lineinfo - ## onFailedAssert(msg): - ## var e = new(TMyError) - ## e.msg = msg - ## e.lineinfo = instantiationInfo(-2) - ## raise e - ## - template failedAssertImpl(msgIMPL: string): untyped {.dirty.} = - let msg = msgIMPL - code - -template doAssertRaises*(exception: typedesc, code: untyped) = - ## Raises ``AssertionError`` if specified ``code`` does not raise the - ## specified exception. Example: - ## - ## .. code-block:: nim - ## doAssertRaises(ValueError): - ## raise newException(ValueError, "Hello World") - var wrong = false - when Exception is exception: - try: - if true: - code - wrong = true - except Exception: - discard - else: - try: - if true: - code - wrong = true - except exception: - discard - except Exception: - raiseAssert(astToStr(exception) & - " wasn't raised, another error was raised instead by:\n"& - astToStr(code)) - if wrong: - raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code)) diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 39ff9d743..9f4cbc0fe 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -7,14 +7,16 @@ # distribution, for details about the copyright. # +include seqs_v2_reimpl + proc genericResetAux(dest: pointer, n: ptr TNimNode) {.benign.} proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) {.benign.} proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, shallow: bool) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericAssignAux(cast[pointer](d +% n.offset), @@ -38,46 +40,67 @@ proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, # echo "ugh memory corruption! ", n.kind # quit 1 +template deepSeqAssignImpl(operation, additionalArg) {.dirty.} = + var d = cast[ptr NimSeqV2Reimpl](dest) + var s = cast[ptr NimSeqV2Reimpl](src) + d.len = s.len + let elem = mt.base + d.p = cast[ptr NimSeqPayloadReimpl](newSeqPayload(s.len, elem.size, elem.align)) + + let bs = elem.size + let ba = elem.align + let headerSize = align(sizeof(NimSeqPayloadBase), ba) + + for i in 0..d.len-1: + operation(d.p +! (headerSize+i*bs), s.p +! (headerSize+i*bs), mt.base, additionalArg) + proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericAssignAux 2") case mt.kind of tyString: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil or shallow or ( - cast[PGenericSeq](s2).reserved and seqShallowFlag) != 0: - unsureAsgnRef(x, s2) + when defined(nimSeqsV2): + var x = cast[ptr NimStringV2](dest) + var s2 = cast[ptr NimStringV2](s)[] + nimAsgnStrV2(x[], s2) else: - unsureAsgnRef(x, copyString(cast[NimString](s2))) + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil or shallow or ( + cast[PGenericSeq](s2).reserved and seqShallowFlag) != 0: + unsureAsgnRef(x, s2) + else: + unsureAsgnRef(x, copyString(cast[NimString](s2))) of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil or shallow or (seq.reserved and seqShallowFlag) != 0: - # this can happen! nil sequences are allowed - unsureAsgnRef(x, s2) - return - sysAssert(dest != nil, "genericAssignAux 3") - if ntfNoRefs in mt.base.flags: - var ss = nimNewSeqOfCap(mt, seq.len) - cast[PGenericSeq](ss).len = seq.len - unsureAsgnRef(x, ss) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - copyMem(cast[pointer](dst +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% GenericSeqSize), - seq.len * mt.base.size) + when defined(nimSeqsV2): + deepSeqAssignImpl(genericAssignAux, shallow) else: - unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - genericAssignAux( - cast[pointer](dst +% i *% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base, shallow) + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil or shallow or (seq.reserved and seqShallowFlag) != 0: + # this can happen! nil sequences are allowed + unsureAsgnRef(x, s2) + return + sysAssert(dest != nil, "genericAssignAux 3") + if ntfNoRefs in mt.base.flags: + var ss = nimNewSeqOfCap(mt, seq.len) + cast[PGenericSeq](ss).len = seq.len + unsureAsgnRef(x, ss) + var dst = cast[int](cast[PPointer](dest)[]) + copyMem(cast[pointer](dst +% align(GenericSeqSize, mt.base.align)), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align)), + seq.len *% mt.base.size) + else: + unsureAsgnRef(x, newSeq(mt, seq.len)) + var dst = cast[int](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericAssignAux( + cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + mt.base, shallow) of tyObject: var it = mt.base # don't use recursion here on the PNimType because the subtype @@ -88,14 +111,23 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = genericAssignAux(dest, src, mt.node, shallow) # we need to copy m_type field for tyObject, as it could be empty for # sequence reallocations: - var pint = cast[ptr PNimType](dest) - # We need to copy the *static* type not the dynamic type: - # if p of TB: - # var tbObj = TB(p) - # tbObj of TC # needs to be false! - #c_fprintf(stdout, "%s %s\n", pint[].name, mt.name) - chckObjAsgn(cast[ptr PNimType](src)[], mt) - pint[] = mt # cast[ptr PNimType](src)[] + when defined(nimSeqsV2): + var pint = cast[ptr PNimTypeV2](dest) + #chckObjAsgn(cast[ptr PNimTypeV2](src)[].typeInfoV2, mt) + pint[] = cast[PNimTypeV2](mt.typeInfoV2) + else: + var pint = cast[ptr PNimType](dest) + # We need to copy the *static* type not the dynamic type: + # if p of TB: + # var tbObj = TB(p) + # tbObj of TC # needs to be false! + #c_fprintf(stdout, "%s %s\n", pint[].name, mt.name) + let srcType = cast[ptr PNimType](src)[] + if srcType != nil: + # `!= nil` needed because of cases where object is not initialized properly (see bug #16706) + # note that you can have `srcType == nil` yet `src != nil` + chckObjAsgn(srcType, mt) + pint[] = mt # cast[ptr PNimType](src)[] of tyTuple: genericAssignAux(dest, src, mt.node, shallow) of tyArray, tyArrayConstr: @@ -135,7 +167,7 @@ when false: of tyPointer: k = "range" of tyOpenArray: k = "openarray" of tyString: k = "string" - of tyCString: k = "cstring" + of tyCstring: k = "cstring" of tyInt: k = "int" of tyInt32: k = "int32" else: k = "other" @@ -149,15 +181,15 @@ proc genericSeqAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericAssignOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericAssign(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) proc objectInit(dest: pointer, typ: PNimType) {.compilerproc, benign.} proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "objectInitAux") of nkSlot: objectInit(cast[pointer](d +% n.offset), n.typ) @@ -171,13 +203,17 @@ proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = proc objectInit(dest: pointer, typ: PNimType) = # the generic init proc that takes care of initialization of complex # objects on the stack or heap - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case typ.kind of tyObject: # iterate over any structural type # here we have to init the type field: - var pint = cast[ptr PNimType](dest) - pint[] = typ + when defined(nimSeqsV2): + var pint = cast[ptr PNimTypeV2](dest) + pint[] = cast[PNimTypeV2](typ.typeInfoV2) + else: + var pint = cast[ptr PNimType](dest) + pint[] = typ objectInitAux(dest, typ.node) of tyTuple: objectInitAux(dest, typ.node) @@ -190,7 +226,7 @@ proc objectInit(dest: pointer, typ: PNimType) = proc genericReset(dest: pointer, mt: PNimType) {.compilerproc, benign.} proc genericResetAux(dest: pointer, n: ptr TNimNode) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "genericResetAux") of nkSlot: genericReset(cast[pointer](d +% n.offset), n.typ) @@ -202,18 +238,35 @@ proc genericResetAux(dest: pointer, n: ptr TNimNode) = zeroMem(cast[pointer](d +% n.offset), n.typ.size) proc genericReset(dest: pointer, mt: PNimType) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) sysAssert(mt != nil, "genericReset 2") case mt.kind - of tyString, tyRef, tySequence: + of tyRef: unsureAsgnRef(cast[PPointer](dest), nil) + of tyString: + when defined(nimSeqsV2): + var s = cast[ptr NimStringV2](dest) + frees(s[]) + zeroMem(dest, mt.size) + else: + unsureAsgnRef(cast[PPointer](dest), nil) + of tySequence: + when defined(nimSeqsV2): + frees(cast[ptr NimSeqV2Reimpl](dest)[]) + zeroMem(dest, mt.size) + else: + unsureAsgnRef(cast[PPointer](dest), nil) of tyTuple: genericResetAux(dest, mt.node) of tyObject: genericResetAux(dest, mt.node) # also reset the type field for tyObject, for correct branch switching! - var pint = cast[ptr PNimType](dest) - pint[] = nil + when defined(nimSeqsV2): + var pint = cast[ptr PNimTypeV2](dest) + pint[] = nil + else: + var pint = cast[ptr PNimType](dest) + pint[] = nil of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: genericReset(cast[pointer](d +% i *% mt.base.size), mt.base) @@ -222,10 +275,12 @@ proc genericReset(dest: pointer, mt: PNimType) = proc selectBranch(discVal, L: int, a: ptr array[0x7fff, ptr TNimNode]): ptr TNimNode = - result = a[L] # a[L] contains the ``else`` part (but may be nil) if discVal <% L: - let x = a[discVal] - if x != nil: result = x + result = a[discVal] + if result == nil: + result = a[L] + else: + result = a[L] # a[L] contains the ``else`` part (but may be nil) proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, a: ptr array[0x7fff, ptr TNimNode], @@ -234,10 +289,10 @@ proc FieldDiscriminantCheck(oldDiscVal, newDiscVal: int, let newBranch = selectBranch(newDiscVal, L, a) when defined(nimOldCaseObjects): if newBranch != oldBranch and oldDiscVal != 0: - sysFatal(FieldError, "assignment to discriminant changes object branch") + sysFatal(FieldDefect, "assignment to discriminant changes object branch") else: if newBranch != oldBranch: if oldDiscVal != 0: - sysFatal(FieldError, "assignment to discriminant changes object branch") + sysFatal(FieldDefect, "assignment to discriminant changes object branch") else: - sysFatal(FieldError, "assignment to discriminant changes object branch; compile with -d:nimOldCaseObjects for a transition period") + sysFatal(FieldDefect, "assignment to discriminant changes object branch; compile with -d:nimOldCaseObjects for a transition period") diff --git a/lib/system/basic_types.nim b/lib/system/basic_types.nim index 39ad0a76c..bf81b9b6a 100644 --- a/lib/system/basic_types.nim +++ b/lib/system/basic_types.nim @@ -11,19 +11,45 @@ type uint32* {.magic: UInt32.} ## Unsigned 32 bit integer type. uint64* {.magic: UInt64.} ## Unsigned 64 bit integer type. +type + float* {.magic: Float.} ## Default floating point type. + float32* {.magic: Float32.} ## 32 bit floating point type. + float64* {.magic: Float.} ## 64 bit floating point type. + +# 'float64' is now an alias to 'float'; this solves many problems + +type + char* {.magic: Char.} ## Built-in 8 bit character type (unsigned). + string* {.magic: String.} ## Built-in string type. + cstring* {.magic: Cstring.} ## Built-in cstring (*compatible string*) type. + pointer* {.magic: Pointer.} ## Built-in pointer type, use the `addr` + ## operator to get a pointer to a variable. + + typedesc* {.magic: TypeDesc.} ## Meta type to denote a type description. + +type + `ptr`*[T] {.magic: Pointer.} ## Built-in generic untraced pointer type. + `ref`*[T] {.magic: Pointer.} ## Built-in generic traced pointer type. + + `nil` {.magic: "Nil".} + + void* {.magic: "VoidType".} ## Meta type to denote the absence of any type. + auto* {.magic: Expr.} ## Meta type for automatic type determination. + any* {.deprecated: "Deprecated since v1.5; Use auto instead.".} = distinct auto ## Deprecated; Use `auto` instead. See https://github.com/nim-lang/RFCs/issues/281 + untyped* {.magic: Expr.} ## Meta type to denote an expression that + ## is not resolved (for templates). + typed* {.magic: Stmt.} ## Meta type to denote an expression that + ## is resolved (for templates). + type # we need to start a new type section here, so that ``0`` can have a type - bool* {.magic: Bool.} = enum ## Built-in boolean type. + bool* {.magic: "Bool".} = enum ## Built-in boolean type. false = 0, true = 1 const - on* = true ## Alias for ``true``. - off* = false ## Alias for ``false``. + on* = true ## Alias for `true`. + off* = false ## Alias for `false`. type - Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, - ## bool, character, and enumeration types - ## as well as their subtypes. - SomeSignedInt* = int|int8|int16|int32|int64 ## Type class matching all signed integer types. @@ -33,34 +59,35 @@ type SomeInteger* = SomeSignedInt|SomeUnsignedInt ## Type class matching all integer types. + SomeFloat* = float|float32|float64 + ## Type class matching all floating point number types. + + SomeNumber* = SomeInteger|SomeFloat + ## Type class matching all number types. + SomeOrdinal* = int|int8|int16|int32|int64|bool|enum|uint|uint8|uint16|uint32|uint64 ## Type class matching all ordinal types; however this includes enums with - ## holes. - - BiggestInt* = int64 - ## is an alias for the biggest signed integer type the Nim compiler - ## supports. Currently this is ``int64``, but it is platform-dependent - ## in general. + ## holes. See also `Ordinal` {.push warning[GcMem]: off, warning[Uninit]: off.} {.push hints: off.} proc `not`*(x: bool): bool {.magic: "Not", noSideEffect.} - ## Boolean not; returns true if ``x == false``. + ## Boolean not; returns true if `x == false`. proc `and`*(x, y: bool): bool {.magic: "And", noSideEffect.} - ## Boolean ``and``; returns true if ``x == y == true`` (if both arguments + ## Boolean `and`; returns true if `x == y == true` (if both arguments ## are true). ## - ## Evaluation is lazy: if ``x`` is false, ``y`` will not even be evaluated. + ## Evaluation is lazy: if `x` is false, `y` will not even be evaluated. proc `or`*(x, y: bool): bool {.magic: "Or", noSideEffect.} - ## Boolean ``or``; returns true if ``not (not x and not y)`` (if any of + ## Boolean `or`; returns true if `not (not x and not y)` (if any of ## the arguments is true). ## - ## Evaluation is lazy: if ``x`` is true, ``y`` will not even be evaluated. + ## Evaluation is lazy: if `x` is true, `y` will not even be evaluated. proc `xor`*(x, y: bool): bool {.magic: "Xor", noSideEffect.} - ## Boolean `exclusive or`; returns true if ``x != y`` (if either argument + ## Boolean `exclusive or`; returns true if `x != y` (if either argument ## is true while the other is false). {.pop.} diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim new file mode 100644 index 000000000..0663247c2 --- /dev/null +++ b/lib/system/bitmasks.nim @@ -0,0 +1,39 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Page size of the system; in most cases 4096 bytes. For exotic OS or +# CPU this needs to be changed: +const + PageShift = when defined(nimPage256) or defined(cpu16): 3 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + + PageSize = 1 shl PageShift + PageMask = PageSize-1 + + + MemAlign = # also minimal allocatable memory block + when defined(nimMemAlignTiny): 4 + elif defined(useMalloc): + when defined(amd64): 16 + else: 8 + else: 16 + + BitsPerPage = PageSize div MemAlign + UnitsPerPage = BitsPerPage div (sizeof(int)*8) + # how many ints do we need to describe a page: + # on 32 bit systems this is only 16 (!) + + TrunkShift = 9 + BitsPerTrunk = 1 shl TrunkShift # needs to be power of 2 and divisible by 64 + TrunkMask = BitsPerTrunk - 1 + IntsPerTrunk = BitsPerTrunk div (sizeof(int)*8) + IntShift = 5 + ord(sizeof(int) == 8) # 5 or 6, depending on int width + IntMask = 1 shl IntShift - 1 diff --git a/lib/system/cellseqs_v1.nim b/lib/system/cellseqs_v1.nim new file mode 100644 index 000000000..1a305aa42 --- /dev/null +++ b/lib/system/cellseqs_v1.nim @@ -0,0 +1,46 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# ------------------- cell seq handling --------------------------------------- + +type + PCellArray = ptr UncheckedArray[PCell] + CellSeq {.final, pure.} = object + len, cap: int + d: PCellArray + +proc contains(s: CellSeq, c: PCell): bool {.inline.} = + for i in 0 ..< s.len: + if s.d[i] == c: + return true + return false + +proc resize(s: var CellSeq) = + s.cap = s.cap * 3 div 2 + let d = cast[PCellArray](alloc(s.cap * sizeof(PCell))) + copyMem(d, s.d, s.len * sizeof(PCell)) + dealloc(s.d) + s.d = d + +proc add(s: var CellSeq, c: PCell) {.inline.} = + if s.len >= s.cap: + resize(s) + s.d[s.len] = c + inc(s.len) + +proc init(s: var CellSeq, cap: int = 1024) = + s.len = 0 + s.cap = cap + s.d = cast[PCellArray](alloc0(cap * sizeof(PCell))) + +proc deinit(s: var CellSeq) = + dealloc(s.d) + s.d = nil + s.len = 0 + s.cap = 0 diff --git a/lib/system/cellseqs_v2.nim b/lib/system/cellseqs_v2.nim new file mode 100644 index 000000000..c6c7b1a8e --- /dev/null +++ b/lib/system/cellseqs_v2.nim @@ -0,0 +1,53 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2019 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Cell seqs for cyclebreaker and cyclicrefs_v2. + +type + CellTuple[T] = (T, PNimTypeV2) + CellArray[T] = ptr UncheckedArray[CellTuple[T]] + CellSeq[T] = object + len, cap: int + d: CellArray[T] + +proc resize[T](s: var CellSeq[T]) = + s.cap = s.cap * 3 div 2 + var newSize = s.cap * sizeof(CellTuple[T]) + when compileOption("threads"): + s.d = cast[CellArray[T]](reallocShared(s.d, newSize)) + else: + s.d = cast[CellArray[T]](realloc(s.d, newSize)) + +proc add[T](s: var CellSeq[T], c: T, t: PNimTypeV2) {.inline.} = + if s.len >= s.cap: + s.resize() + s.d[s.len] = (c, t) + inc(s.len) + +proc init[T](s: var CellSeq[T], cap: int = 1024) = + s.len = 0 + s.cap = cap + when compileOption("threads"): + s.d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) + else: + s.d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) + +proc deinit[T](s: var CellSeq[T]) = + if s.d != nil: + when compileOption("threads"): + deallocShared(s.d) + else: + dealloc(s.d) + s.d = nil + s.len = 0 + s.cap = 0 + +proc pop[T](s: var CellSeq[T]): (T, PNimTypeV2) = + result = s.d[s.len-1] + dec s.len diff --git a/lib/system/cellsets.nim b/lib/system/cellsets.nim index c73c84f52..92036c226 100644 --- a/lib/system/cellsets.nim +++ b/lib/system/cellsets.nim @@ -7,22 +7,64 @@ # distribution, for details about the copyright. # -# Efficient set of pointers for the GC (and repr) -type - RefCount = int +#[ + +Efficient set of pointers for the GC (and repr) +----------------------------------------------- + +The GC depends on an extremely efficient datastructure for storing a +set of pointers - this is called a `CellSet` in the source code. +Inserting, deleting and searching are done in constant time. However, +modifying a `CellSet` during traversal leads to undefined behaviour. + +All operations on a CellSet have to perform efficiently. Because a Cellset can +become huge a hash table alone is not suitable for this. + +We use a mixture of bitset and hash table for this. The hash table maps *pages* +to a page descriptor. The page descriptor contains a bit for any possible cell +address within this page. So including a cell is done as follows: - Cell {.pure.} = object - refcount: RefCount # the refcount and some flags - typ: PNimType - when trackAllocationSource: - filename: cstring - line: int - when useCellIds: - id: int +- Find the page descriptor for the page the cell belongs to. +- Set the appropriate bit in the page descriptor indicating that the + cell points to the start of a memory block. - PCell = ptr Cell +Removing a cell is analogous - the bit has to be set to zero. +Single page descriptors are never deleted from the hash table. This is not +needed as the data structures needs to be rebuilt periodically anyway. +Complete traversal is done in this way:: + + for each page descriptor d: + for each bit in d: + if bit == 1: + traverse the pointer belonging to this bit + +]# + +when defined(gcOrc) or defined(gcArc) or defined(gcAtomicArc): + type + PCell = Cell + + when not declaredInScope(PageShift): + include bitmasks + +else: + type + RefCount = int + + Cell {.pure.} = object + refcount: RefCount # the refcount and some flags + typ: PNimType + when trackAllocationSource: + filename: cstring + line: int + when useCellIds: + id: int + + PCell = ptr Cell + +type PPageDesc = ptr PageDesc BitIndex = range[0..UnitsPerPage-1] PageDesc {.final, pure.} = object @@ -35,39 +77,11 @@ type counter, max: int head: PPageDesc data: PPageDescArray - PCellArray = ptr UncheckedArray[PCell] - CellSeq {.final, pure.} = object - len, cap: int - d: PCellArray - -# ------------------- cell seq handling --------------------------------------- - -proc contains(s: CellSeq, c: PCell): bool {.inline.} = - for i in 0 .. s.len-1: - if s.d[i] == c: return true - return false - -proc add(s: var CellSeq, c: PCell) {.inline.} = - if s.len >= s.cap: - s.cap = s.cap * 3 div 2 - var d = cast[PCellArray](alloc(s.cap * sizeof(PCell))) - copyMem(d, s.d, s.len * sizeof(PCell)) - dealloc(s.d) - s.d = d - # XXX: realloc? - s.d[s.len] = c - inc(s.len) - -proc init(s: var CellSeq, cap: int = 1024) = - s.len = 0 - s.cap = cap - s.d = cast[PCellArray](alloc0(cap * sizeof(PCell))) - -proc deinit(s: var CellSeq) = - dealloc(s.d) - s.d = nil - s.len = 0 - s.cap = 0 + +when defined(gcOrc) or defined(gcArc) or defined(gcAtomicArc): + discard +else: + include cellseqs_v1 # ------------------- cell set handling --------------------------------------- diff --git a/lib/system/cgprocs.nim b/lib/system/cgprocs.nim index 9d0d248c3..9a7645f9b 100644 --- a/lib/system/cgprocs.nim +++ b/lib/system/cgprocs.nim @@ -8,13 +8,3 @@ # # Headers for procs that the code generator depends on ("compilerprocs") - -type - LibHandle = pointer # private type - ProcAddr = pointer # library loading and loading of procs: - -proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} -proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} -proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} - -proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} diff --git a/lib/system/channels.nim b/lib/system/channels_builtin.nim index 65398d65c..02b4d8cbf 100644 --- a/lib/system/channels.nim +++ b/lib/system/channels_builtin.nim @@ -10,10 +10,10 @@ ## Channel support for threads. ## ## **Note**: This is part of the system module. Do not import it directly. -## To activate thread support compile with the ``--threads:on`` command line switch. +## To activate thread support compile with the `--threads:on` command line switch. ## -## **Note:** Channels are designed for the ``Thread`` type. They are unstable when -## used with ``spawn`` +## **Note:** Channels are designed for the `Thread` type. They are unstable when +## used with `spawn` ## ## **Note:** The current implementation of message passing does ## not work with cyclic data structures. @@ -26,11 +26,11 @@ ## The following is a simple example of two different ways to use channels: ## blocking and non-blocking. ## -## .. code-block :: Nim +## ```Nim ## # Be sure to compile with --threads:on. ## # The channels and threads modules are part of system and should not be ## # imported. -## import os +## import std/os ## ## # Channels can either be: ## # - declared at the module level, or @@ -87,21 +87,23 @@ ## ## # Clean up the channel. ## chan.close() +## ``` ## ## Sample output ## ------------- ## The program should output something similar to this, but keep in mind that -## exact results may vary in the real world:: -## Hello World! -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Pretend I'm doing useful work... -## Another message +## exact results may vary in the real world: +## +## Hello World! +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Pretend I'm doing useful work... +## Another message ## ## Passing Channels Safely -## ---------------------- +## ----------------------- ## Note that when passing objects to procedures on another thread by pointer ## (for example through a thread's argument), objects created using the default ## allocator will use thread-local, GC-managed memory. Thus it is generally @@ -109,10 +111,10 @@ ## in which case they will use a process-wide (thread-safe) shared heap. ## ## However, it is possible to manually allocate shared memory for channels -## using e.g. ``system.allocShared0`` and pass these pointers through thread +## using e.g. `system.allocShared0` and pass these pointers through thread ## arguments: ## -## .. code-block :: Nim +## ```Nim ## proc worker(channel: ptr Channel[string]) = ## let greeting = channel[].recv() ## echo greeting @@ -134,10 +136,13 @@ ## deallocShared(channel) ## ## localChannelExample() # "Hello from the main thread!" +## ``` when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} +import std/private/syslocks + type pbytes = ptr UncheckedArray[byte] RawChannel {.pure, final.} = object ## msg queue for a thread @@ -151,7 +156,7 @@ type region: MemRegion PRawChannel = ptr RawChannel LoadStoreMode = enum mStore, mLoad - Channel* {.gcsafe.}[TMsg] = RawChannel ## a channel for thread communication + Channel*[TMsg] {.gcsafe.} = RawChannel ## a channel for thread communication const ChannelDeadMask = -2 @@ -182,8 +187,8 @@ when not usesDestructors: proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, mode: LoadStoreMode) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: storeAux(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), n.typ, t, mode) @@ -202,8 +207,8 @@ when not usesDestructors: cast[pointer](cast[int](p) +% x) var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "mt == nil") case mt.kind of tyString: @@ -214,7 +219,7 @@ when not usesDestructors: x[] = nil else: var ss = cast[NimString](s2) - var ns = cast[NimString](alloc(t.region, ss.len+1 + GenericSeqSize)) + var ns = cast[NimString](alloc(t.region, GenericSeqSize + ss.len+1)) copyMem(ns, ss, ss.len+1 + GenericSeqSize) x[] = ns else: @@ -239,18 +244,18 @@ when not usesDestructors: else: sysAssert(dest != nil, "dest == nil") if mode == mStore: - x[] = alloc0(t.region, seq.len *% mt.base.size +% GenericSeqSize) + x[] = alloc0(t.region, align(GenericSeqSize, mt.base.align) +% seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) var dstseq = cast[PGenericSeq](dst) dstseq.len = seq.len dstseq.reserved = seq.len for i in 0..seq.len-1: storeAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), + cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% + i *% mt.base.size), mt.base, t, mode) if mode != mStore: dealloc(t.region, s2) of tyObject: @@ -265,8 +270,8 @@ when not usesDestructors: storeAux(dest, src, mt.node, t, mode) of tyArray, tyArrayConstr: for i in 0..(mt.size div mt.base.size)-1: - storeAux(cast[pointer](d +% i*% mt.base.size), - cast[pointer](s +% i*% mt.base.size), mt.base, t, mode) + storeAux(cast[pointer](d +% i *% mt.base.size), + cast[pointer](s +% i *% mt.base.size), mt.base, t, mode) of tyRef: var s = cast[PPointer](src)[] var x = cast[PPointer](dest) @@ -347,7 +352,7 @@ template lockChannel(q, action): untyped = proc sendImpl(q: PRawChannel, typ: PNimType, msg: pointer, noBlock: bool): bool = if q.mask == ChannelDeadMask: - sysFatal(DeadThreadError, "cannot send message; thread died") + sysFatal(DeadThreadDefect, "cannot send message; thread died") acquireSys(q.lock) if q.maxItems > 0: # Wait until count is less than maxItems @@ -360,8 +365,8 @@ proc sendImpl(q: PRawChannel, typ: PNimType, msg: pointer, noBlock: bool): bool rawSend(q, msg, typ) q.elemType = typ - releaseSys(q.lock) signalSysCond(q.cond) + releaseSys(q.lock) result = true proc send*[TMsg](c: var Channel[TMsg], msg: sink TMsg) {.inline.} = @@ -389,7 +394,7 @@ proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) = q.ready = false if typ != q.elemType: releaseSys(q.lock) - sysFatal(ValueError, "cannot receive message of wrong type") + raise newException(ValueError, "cannot receive message of wrong type") rawRecv(q, res, typ) if q.maxItems > 0 and q.count == q.maxItems - 1: # Parent thread is awaiting in send. Wake it up. @@ -410,8 +415,8 @@ proc tryRecv*[TMsg](c: var Channel[TMsg]): tuple[dataAvailable: bool, ## Tries to receive a message from the channel `c`, but this can fail ## for all sort of reasons, including contention. ## - ## If it fails, it returns ``(false, default(msg))`` otherwise it - ## returns ``(true, msg)``. + ## If it fails, it returns `(false, default(msg))` otherwise it + ## returns `(true, msg)`. var q = cast[PRawChannel](addr(c)) if q.mask != ChannelDeadMask: if tryAcquireSys(q.lock): @@ -448,8 +453,7 @@ proc close*[TMsg](c: var Channel[TMsg]) = deinitRawChannel(addr(c)) proc ready*[TMsg](c: var Channel[TMsg]): bool = - ## Returns true iff some thread is waiting on the channel `c` for + ## Returns true if some thread is waiting on the channel `c` for ## new messages. var q = cast[PRawChannel](addr(c)) result = q.ready - diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 52642bbf9..b48855964 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -9,24 +9,65 @@ # Implementation of some runtime checks. include system/indexerrors +when defined(nimPreviewSlimSystem): + import std/formatfloat proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = when hostOS == "standalone": - sysFatal(RangeError, "value out of range") + sysFatal(RangeDefect, "value out of range") else: - sysFatal(RangeError, "value out of range: ", $val) + sysFatal(RangeDefect, "value out of range: ", $val) + +proc raiseIndexError4(l1, h1, h2: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, "index out of bounds: " & $l1 & ".." & $h1 & " notin 0.." & $(h2 - 1)) proc raiseIndexError3(i, a, b: int) {.compilerproc, noinline.} = - sysFatal(IndexError, formatErrorIndexBound(i, a, b)) + sysFatal(IndexDefect, formatErrorIndexBound(i, a, b)) proc raiseIndexError2(i, n: int) {.compilerproc, noinline.} = - sysFatal(IndexError, formatErrorIndexBound(i, n)) + sysFatal(IndexDefect, formatErrorIndexBound(i, n)) proc raiseIndexError() {.compilerproc, noinline.} = - sysFatal(IndexError, "index out of bounds") + sysFatal(IndexDefect, "index out of bounds") proc raiseFieldError(f: string) {.compilerproc, noinline.} = - sysFatal(FieldError, f) + ## remove after bootstrap > 1.5.1 + sysFatal(FieldDefect, f) + +when defined(nimV2): + proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, f & $discVal & "'") + + proc raiseFieldErrorStr(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, formatFieldDefect(f, discVal)) +else: + proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldDefect, formatFieldDefect(f, discVal)) + +proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} = + when defined(standalone): + sysFatal(RangeDefect, "value out of range") + else: + sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) + +proc raiseRangeErrorF(i, a, b: float) {.compilerproc, noinline.} = + when defined(standalone): + sysFatal(RangeDefect, "value out of range") + else: + sysFatal(RangeDefect, "value out of range: " & $i & " notin " & $a & " .. " & $b) + +proc raiseRangeErrorU(i, a, b: uint64) {.compilerproc, noinline.} = + # todo: better error reporting + sysFatal(RangeDefect, "value out of range") + +proc raiseRangeErrorNoArgs() {.compilerproc, noinline.} = + sysFatal(RangeDefect, "value out of range") + +proc raiseObjectConversionError() {.compilerproc, noinline.} = + sysFatal(ObjectConversionDefect, "invalid object conversion") proc chckIndx(i, a, b: int): int = if i >= a and i <= b: @@ -50,24 +91,24 @@ proc chckRangeU(i, a, b: uint64): uint64 {.compilerproc.} = if i >= a and i <= b: return i else: - sysFatal(RangeError, "value out of range") + sysFatal(RangeDefect, "value out of range") proc chckRangeF(x, a, b: float): float = if x >= a and x <= b: return x else: when hostOS == "standalone": - sysFatal(RangeError, "value out of range") + sysFatal(RangeDefect, "value out of range") else: - sysFatal(RangeError, "value out of range: ", $x) + sysFatal(RangeDefect, "value out of range: ", $x) proc chckNil(p: pointer) = if p == nil: - sysFatal(NilAccessError, "attempt to write to a nil address") + sysFatal(NilAccessDefect, "attempt to write to a nil address") proc chckNilDisp(p: pointer) {.compilerproc.} = if p == nil: - sysFatal(NilAccessError, "cannot dispatch; dispatcher is nil") + sysFatal(NilAccessDefect, "cannot dispatch; dispatcher is nil") when not defined(nimV2): @@ -77,12 +118,12 @@ when not defined(nimV2): if x == subclass: return # optimized fast path while x != subclass: if x == nil: - sysFatal(ObjectConversionError, "invalid object conversion") + sysFatal(ObjectConversionDefect, "invalid object conversion") x = x.base proc chckObjAsgn(a, b: PNimType) {.compilerproc, inline.} = if a != b: - sysFatal(ObjectAssignmentError, "invalid object assignment") + sysFatal(ObjectAssignmentDefect, "invalid object assignment") type ObjCheckCache = array[0..1, PNimType] @@ -116,6 +157,5 @@ when not defined(nimV2): return true when defined(nimV2): - proc nimFieldDiscriminantCheckV2(oldDiscVal, newDiscVal: uint8) {.compilerproc.} = - if oldDiscVal != newDiscVal: - sysFatal(FieldError, "assignment to discriminant changes object branch") + proc raiseObjectCaseTransition() {.compilerproc.} = + sysFatal(FieldDefect, "assignment to discriminant changes object branch") diff --git a/lib/system/comparisons.nim b/lib/system/comparisons.nim index 787820989..a8d78bb93 100644 --- a/lib/system/comparisons.nim +++ b/lib/system/comparisons.nim @@ -1,24 +1,24 @@ # comparison operators: -proc `==`*[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} +proc `==`*[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} = ## Checks whether values within the *same enum* have the same underlying value. - ## - ## .. code-block:: Nim - ## type - ## Enum1 = enum - ## Field1 = 3, Field2 - ## Enum2 = enum - ## Place1, Place2 = 3 - ## var - ## e1 = Field1 - ## e2 = Enum1(Place2) - ## echo (e1 == e2) # true - ## echo (e1 == Place2) # raises error -proc `==`*(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} - ## .. code-block:: Nim - ## var # this is a wildly dangerous example - ## a = cast[pointer](0) - ## b = cast[pointer](nil) - ## echo (a == b) # true due to the special meaning of `nil`/0 as a pointer + runnableExamples: + type + Enum1 = enum + field1 = 3, field2 + Enum2 = enum + place1, place2 = 3 + var + e1 = field1 + e2 = place2.ord.Enum1 + assert e1 == e2 + assert not compiles(e1 == place2) # raises error +proc `==`*(x, y: pointer): bool {.magic: "EqRef", noSideEffect.} = + ## Checks for equality between two `pointer` variables. + runnableExamples: + var # this is a wildly dangerous example + a = cast[pointer](0) + b = cast[pointer](nil) + assert a == b # true due to the special meaning of `nil`/0 as a pointer proc `==`*(x, y: string): bool {.magic: "EqStr", noSideEffect.} ## Checks for equality between two `string` variables. @@ -26,117 +26,118 @@ proc `==`*(x, y: char): bool {.magic: "EqCh", noSideEffect.} ## Checks for equality between two `char` variables. proc `==`*(x, y: bool): bool {.magic: "EqB", noSideEffect.} ## Checks for equality between two `bool` variables. -proc `==`*[T](x, y: set[T]): bool {.magic: "EqSet", noSideEffect.} +proc `==`*[T](x, y: set[T]): bool {.magic: "EqSet", noSideEffect.} = ## Checks for equality between two variables of type `set`. - ## - ## .. code-block:: Nim - ## var a = {1, 2, 2, 3} # duplication in sets is ignored - ## var b = {1, 2, 3} - ## echo (a == b) # true + runnableExamples: + assert {1, 2, 2, 3} == {1, 2, 3} # duplication in sets is ignored + proc `==`*[T](x, y: ref T): bool {.magic: "EqRef", noSideEffect.} ## Checks that two `ref` variables refer to the same item. proc `==`*[T](x, y: ptr T): bool {.magic: "EqRef", noSideEffect.} ## Checks that two `ptr` variables refer to the same item. -proc `==`*[T: proc](x, y: T): bool {.magic: "EqProc", noSideEffect.} +proc `==`*[T: proc | iterator](x, y: T): bool {.magic: "EqProc", noSideEffect.} ## Checks that two `proc` variables refer to the same procedure. proc `<=`*[Enum: enum](x, y: Enum): bool {.magic: "LeEnum", noSideEffect.} -proc `<=`*(x, y: string): bool {.magic: "LeStr", noSideEffect.} +proc `<=`*(x, y: string): bool {.magic: "LeStr", noSideEffect.} = ## Compares two strings and returns true if `x` is lexicographically ## before `y` (uppercase letters come before lowercase letters). - ## - ## .. code-block:: Nim - ## let - ## a = "abc" - ## b = "abd" - ## c = "ZZZ" - ## assert a <= b - ## assert a <= a - ## assert (a <= c) == false -proc `<=`*(x, y: char): bool {.magic: "LeCh", noSideEffect.} + runnableExamples: + let + a = "abc" + b = "abd" + c = "ZZZ" + assert a <= b + assert a <= a + assert not (a <= c) + +proc `<=`*(x, y: char): bool {.magic: "LeCh", noSideEffect.} = ## Compares two chars and returns true if `x` is lexicographically ## before `y` (uppercase letters come before lowercase letters). - ## - ## .. code-block:: Nim - ## let - ## a = 'a' - ## b = 'b' - ## c = 'Z' - ## assert a <= b - ## assert a <= a - ## assert (a <= c) == false -proc `<=`*[T](x, y: set[T]): bool {.magic: "LeSet", noSideEffect.} + runnableExamples: + let + a = 'a' + b = 'b' + c = 'Z' + assert a <= b + assert a <= a + assert not (a <= c) + +proc `<=`*[T](x, y: set[T]): bool {.magic: "LeSet", noSideEffect.} = ## Returns true if `x` is a subset of `y`. ## ## A subset `x` has all of its members in `y` and `y` doesn't necessarily ## have more members than `x`. That is, `x` can be equal to `y`. - ## - ## .. code-block:: Nim - ## let - ## a = {3, 5} - ## b = {1, 3, 5, 7} - ## c = {2} - ## assert a <= b - ## assert a <= a - ## assert (a <= c) == false + runnableExamples: + let + a = {3, 5} + b = {1, 3, 5, 7} + c = {2} + assert a <= b + assert a <= a + assert not (a <= c) + proc `<=`*(x, y: bool): bool {.magic: "LeB", noSideEffect.} proc `<=`*[T](x, y: ref T): bool {.magic: "LePtr", noSideEffect.} proc `<=`*(x, y: pointer): bool {.magic: "LePtr", noSideEffect.} proc `<`*[Enum: enum](x, y: Enum): bool {.magic: "LtEnum", noSideEffect.} -proc `<`*(x, y: string): bool {.magic: "LtStr", noSideEffect.} +proc `<`*(x, y: string): bool {.magic: "LtStr", noSideEffect.} = ## Compares two strings and returns true if `x` is lexicographically ## before `y` (uppercase letters come before lowercase letters). - ## - ## .. code-block:: Nim - ## let - ## a = "abc" - ## b = "abd" - ## c = "ZZZ" - ## assert a < b - ## assert (a < a) == false - ## assert (a < c) == false -proc `<`*(x, y: char): bool {.magic: "LtCh", noSideEffect.} + runnableExamples: + let + a = "abc" + b = "abd" + c = "ZZZ" + assert a < b + assert not (a < a) + assert not (a < c) + +proc `<`*(x, y: char): bool {.magic: "LtCh", noSideEffect.} = ## Compares two chars and returns true if `x` is lexicographically ## before `y` (uppercase letters come before lowercase letters). - ## - ## .. code-block:: Nim - ## let - ## a = 'a' - ## b = 'b' - ## c = 'Z' - ## assert a < b - ## assert (a < a) == false - ## assert (a < c) == false -proc `<`*[T](x, y: set[T]): bool {.magic: "LtSet", noSideEffect.} + runnableExamples: + let + a = 'a' + b = 'b' + c = 'Z' + assert a < b + assert not (a < a) + assert not (a < c) + +proc `<`*[T](x, y: set[T]): bool {.magic: "LtSet", noSideEffect.} = ## Returns true if `x` is a strict or proper subset of `y`. ## ## A strict or proper subset `x` has all of its members in `y` but `y` has ## more elements than `y`. - ## - ## .. code-block:: Nim - ## let - ## a = {3, 5} - ## b = {1, 3, 5, 7} - ## c = {2} - ## assert a < b - ## assert (a < a) == false - ## assert (a < c) == false + runnableExamples: + let + a = {3, 5} + b = {1, 3, 5, 7} + c = {2} + assert a < b + assert not (a < a) + assert not (a < c) + proc `<`*(x, y: bool): bool {.magic: "LtB", noSideEffect.} proc `<`*[T](x, y: ref T): bool {.magic: "LtPtr", noSideEffect.} proc `<`*[T](x, y: ptr T): bool {.magic: "LtPtr", noSideEffect.} proc `<`*(x, y: pointer): bool {.magic: "LtPtr", noSideEffect.} -template `!=`*(x, y: untyped): untyped = - ## Unequals operator. This is a shorthand for ``not (x == y)``. +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template `!=`*(x, y: untyped): untyped {.callsite.} = + ## Unequals operator. This is a shorthand for `not (x == y)`. not (x == y) -template `>=`*(x, y: untyped): untyped = - ## "is greater or equals" operator. This is the same as ``y <= x``. +template `>=`*(x, y: untyped): untyped {.callsite.} = + ## "is greater or equals" operator. This is the same as `y <= x`. y <= x -template `>`*(x, y: untyped): untyped = - ## "is greater" operator. This is the same as ``y < x``. +template `>`*(x, y: untyped): untyped {.callsite.} = + ## "is greater" operator. This is the same as `y < x`. y < x @@ -161,25 +162,45 @@ proc `<`*(x, y: int16): bool {.magic: "LtI", noSideEffect.} proc `<`*(x, y: int32): bool {.magic: "LtI", noSideEffect.} proc `<`*(x, y: int64): bool {.magic: "LtI", noSideEffect.} +proc `<=`*(x, y: uint): bool {.magic: "LeU", noSideEffect.} + ## Returns true if `x <= y`. +proc `<=`*(x, y: uint8): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint16): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint32): bool {.magic: "LeU", noSideEffect.} +proc `<=`*(x, y: uint64): bool {.magic: "LeU", noSideEffect.} -proc `<=%`*(x, y: IntMax32): bool {.magic: "LeU", noSideEffect.} -proc `<=%`*(x, y: int64): bool {.magic: "LeU64", noSideEffect.} - ## Treats `x` and `y` as unsigned and compares them. - ## Returns true if ``unsigned(x) <= unsigned(y)``. +proc `<`*(x, y: uint): bool {.magic: "LtU", noSideEffect.} + ## Returns true if `x < y`. +proc `<`*(x, y: uint8): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint16): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint32): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: uint64): bool {.magic: "LtU", noSideEffect.} -proc `<%`*(x, y: IntMax32): bool {.magic: "LtU", noSideEffect.} -proc `<%`*(x, y: int64): bool {.magic: "LtU64", noSideEffect.} +proc `<=%`*(x, y: int): bool {.inline.} = + ## Treats `x` and `y` as unsigned and compares them. + ## Returns true if `unsigned(x) <= unsigned(y)`. + cast[uint](x) <= cast[uint](y) +proc `<=%`*(x, y: int8): bool {.inline.} = cast[uint8](x) <= cast[uint8](y) +proc `<=%`*(x, y: int16): bool {.inline.} = cast[uint16](x) <= cast[uint16](y) +proc `<=%`*(x, y: int32): bool {.inline.} = cast[uint32](x) <= cast[uint32](y) +proc `<=%`*(x, y: int64): bool {.inline.} = cast[uint64](x) <= cast[uint64](y) + +proc `<%`*(x, y: int): bool {.inline.} = ## Treats `x` and `y` as unsigned and compares them. - ## Returns true if ``unsigned(x) < unsigned(y)``. + ## Returns true if `unsigned(x) < unsigned(y)`. + cast[uint](x) < cast[uint](y) +proc `<%`*(x, y: int8): bool {.inline.} = cast[uint8](x) < cast[uint8](y) +proc `<%`*(x, y: int16): bool {.inline.} = cast[uint16](x) < cast[uint16](y) +proc `<%`*(x, y: int32): bool {.inline.} = cast[uint32](x) < cast[uint32](y) +proc `<%`*(x, y: int64): bool {.inline.} = cast[uint64](x) < cast[uint64](y) template `>=%`*(x, y: untyped): untyped = y <=% x ## Treats `x` and `y` as unsigned and compares them. - ## Returns true if ``unsigned(x) >= unsigned(y)``. + ## Returns true if `unsigned(x) >= unsigned(y)`. template `>%`*(x, y: untyped): untyped = y <% x ## Treats `x` and `y` as unsigned and compares them. - ## Returns true if ``unsigned(x) > unsigned(y)``. - + ## Returns true if `unsigned(x) > unsigned(y)`. proc `==`*(x, y: uint): bool {.magic: "EqI", noSideEffect.} ## Compares two unsigned integers for equality. @@ -188,21 +209,14 @@ proc `==`*(x, y: uint16): bool {.magic: "EqI", noSideEffect.} proc `==`*(x, y: uint32): bool {.magic: "EqI", noSideEffect.} proc `==`*(x, y: uint64): bool {.magic: "EqI", noSideEffect.} +proc `<=`*(x, y: float32): bool {.magic: "LeF64", noSideEffect.} +proc `<=`*(x, y: float): bool {.magic: "LeF64", noSideEffect.} -proc `<=`*(x, y: uint): bool {.magic: "LeU", noSideEffect.} - ## Returns true if ``x <= y``. -proc `<=`*(x, y: uint8): bool {.magic: "LeU", noSideEffect.} -proc `<=`*(x, y: uint16): bool {.magic: "LeU", noSideEffect.} -proc `<=`*(x, y: uint32): bool {.magic: "LeU", noSideEffect.} -proc `<=`*(x, y: uint64): bool {.magic: "LeU", noSideEffect.} - -proc `<`*(x, y: uint): bool {.magic: "LtU", noSideEffect.} - ## Returns true if ``unsigned(x) < unsigned(y)``. -proc `<`*(x, y: uint8): bool {.magic: "LtU", noSideEffect.} -proc `<`*(x, y: uint16): bool {.magic: "LtU", noSideEffect.} -proc `<`*(x, y: uint32): bool {.magic: "LtU", noSideEffect.} -proc `<`*(x, y: uint64): bool {.magic: "LtU", noSideEffect.} +proc `<`*(x, y: float32): bool {.magic: "LtF64", noSideEffect.} +proc `<`*(x, y: float): bool {.magic: "LtF64", noSideEffect.} +proc `==`*(x, y: float32): bool {.magic: "EqF64", noSideEffect.} +proc `==`*(x, y: float): bool {.magic: "EqF64", noSideEffect.} {.push stackTrace: off.} @@ -217,6 +231,13 @@ proc min*(x, y: int32): int32 {.magic: "MinI", noSideEffect.} = proc min*(x, y: int64): int64 {.magic: "MinI", noSideEffect.} = ## The minimum value of two integers. if x <= y: x else: y +proc min*(x, y: float32): float32 {.noSideEffect, inline.} = + if x <= y or y != y: x else: y +proc min*(x, y: float64): float64 {.noSideEffect, inline.} = + if x <= y or y != y: x else: y +proc min*[T: not SomeFloat](x, y: T): T {.inline.} = + ## Generic minimum operator of 2 values based on `<=`. + if x <= y: x else: y proc max*(x, y: int): int {.magic: "MaxI", noSideEffect.} = if y <= x: x else: y @@ -229,16 +250,23 @@ proc max*(x, y: int32): int32 {.magic: "MaxI", noSideEffect.} = proc max*(x, y: int64): int64 {.magic: "MaxI", noSideEffect.} = ## The maximum value of two integers. if y <= x: x else: y +proc max*(x, y: float32): float32 {.noSideEffect, inline.} = + if y <= x or y != y: x else: y +proc max*(x, y: float64): float64 {.noSideEffect, inline.} = + if y <= x or y != y: x else: y +proc max*[T: not SomeFloat](x, y: T): T {.inline.} = + ## Generic maximum operator of 2 values based on `<=`. + if y <= x: x else: y proc min*[T](x: openArray[T]): T = - ## The minimum value of `x`. ``T`` needs to have a ``<`` operator. + ## The minimum value of `x`. `T` needs to have a `<` operator. result = x[0] for i in 1..high(x): if x[i] < result: result = x[i] proc max*[T](x: openArray[T]): T = - ## The maximum value of `x`. ``T`` needs to have a ``<`` operator. + ## The maximum value of `x`. `T` needs to have a `<` operator. result = x[0] for i in 1..high(x): if result < x[i]: result = x[i] @@ -247,11 +275,17 @@ proc max*[T](x: openArray[T]): T = proc clamp*[T](x, a, b: T): T = - ## Limits the value ``x`` within the interval [a, b]. + ## Limits the value `x` within the interval \[a, b]. + ## This proc is equivalent to but faster than `max(a, min(b, x))`. + ## + ## .. warning:: `a <= b` is assumed and will not be checked (currently). ## - ## .. code-block:: Nim - ## assert((1.4).clamp(0.0, 1.0) == 1.0) - ## assert((0.5).clamp(0.0, 1.0) == 0.5) + ## **See also:** + ## `math.clamp` for a version that takes a `Slice[T]` instead. + runnableExamples: + assert (1.4).clamp(0.0, 1.0) == 1.0 + assert (0.5).clamp(0.0, 1.0) == 0.5 + assert 4.clamp(1, 3) == max(1, min(3, 4)) if x < a: return a if x > b: return b return x @@ -276,12 +310,8 @@ proc `==`*[T](x, y: seq[T]): bool {.noSideEffect.} = ## Generic equals operator for sequences: relies on a equals operator for ## the element type `T`. when nimvm: - when not defined(nimNoNil): - if x.isNil and y.isNil: - return true - else: - if x.len == 0 and y.len == 0: - return true + if x.len == 0 and y.len == 0: + return true else: when not defined(js): proc seqToPtr[T](x: seq[T]): pointer {.inline, noSideEffect.} = @@ -294,13 +324,9 @@ proc `==`*[T](x, y: seq[T]): bool {.noSideEffect.} = return true else: var sameObject = false - asm """`sameObject` = `x` === `y`""" + {.emit: """`sameObject` = `x` === `y`;""".} if sameObject: return true - when not defined(nimNoNil): - if x.isNil or y.isNil: - return false - if x.len != y.len: return false diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim new file mode 100644 index 000000000..cdb976ed5 --- /dev/null +++ b/lib/system/compilation.nim @@ -0,0 +1,209 @@ +const + NimMajor* {.intdefine.}: int = 2 + ## is the major number of Nim's version. Example: + ## ```nim + ## when (NimMajor, NimMinor, NimPatch) >= (1, 3, 1): discard + ## ``` + # see also std/private/since + + NimMinor* {.intdefine.}: int = 2 + ## is the minor number of Nim's version. + ## Odd for devel, even for releases. + + NimPatch* {.intdefine.}: int = 1 + ## is the patch number of Nim's version. + ## Odd for devel, even for releases. + +{.push profiler: off.} +let nimvm* {.magic: "Nimvm", compileTime.}: bool = false + ## May be used only in `when` expression. + ## It is true in Nim VM context and false otherwise. +{.pop.} + +const + isMainModule* {.magic: "IsMainModule".}: bool = false + ## True only when accessed in the main module. This works thanks to + ## compiler magic. It is useful to embed testing code in a module. + + CompileDate* {.magic: "CompileDate".}: string = "0000-00-00" + ## The date (in UTC) of compilation as a string of the form + ## `YYYY-MM-DD`. This works thanks to compiler magic. + + CompileTime* {.magic: "CompileTime".}: string = "00:00:00" + ## The time (in UTC) of compilation as a string of the form + ## `HH:MM:SS`. This works thanks to compiler magic. + +proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## defined. + ## + ## `x` is an external symbol introduced through the compiler's + ## `-d:x switch <nimc.html#compiler-usage-compileminustime-symbols>`_ to enable + ## build time conditionals: + ## ```nim + ## when not defined(release): + ## # Do here programmer friendly expensive sanity checks. + ## # Put here the normal code + ## ``` + ## + ## See also: + ## * `compileOption <#compileOption,string>`_ for `on|off` options + ## * `compileOption <#compileOption,string,string>`_ for enum options + ## * `define pragmas <manual.html#implementation-specific-pragmas-compileminustime-define-pragmas>`_ + +proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared. `x` has to be an identifier or a qualified identifier. + ## + ## This can be used to check whether a library provides a certain + ## feature or not: + ## ```nim + ## when not declared(strutils.toUpper): + ## # provide our own toUpper proc here, because strutils is + ## # missing it. + ## ``` + ## + ## See also: + ## * `declaredInScope <#declaredInScope,untyped>`_ + +proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared in the current scope. `x` has to be an identifier. + +proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = + ## Special compile-time procedure that checks whether `x` can be compiled + ## without any semantic error. + ## This can be used to check whether a type supports some operation: + ## ```nim + ## when compiles(3 + 4): + ## echo "'+' for integers is available" + ## ``` + discard + +proc astToStr*[T](x: T): string {.magic: "AstToStr", noSideEffect.} + ## Converts the AST of `x` into a string representation. This is very useful + ## for debugging. + +proc runnableExamples*(rdoccmd = "", body: untyped) {.magic: "RunnableExamples".} = + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a `runnableExamples` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the `##` doc comment. As the last step of documentation + ## generation each runnableExample is put in its own file `$file_examples$i.nim`, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. + runnableExamples: + proc timesTwo*(x: int): int = + ## This proc doubles a number. + runnableExamples: + # at module scope + const exported* = 123 + assert timesTwo(5) == 10 + block: # at block scope + defer: echo "done" + runnableExamples "-d:foo -b:cpp": + import std/compilesettings + assert querySetting(backend) == "cpp" + assert defined(foo) + runnableExamples "-r:off": ## this one is only compiled + import std/browsers + openDefaultBrowser "https://forum.nim-lang.org/" + 2 * x + +proc compileOption*(option: string): bool {. + magic: "CompileOption", noSideEffect.} = + ## Can be used to determine an `on|off` compile-time option. + ## + ## See also: + ## * `compileOption <#compileOption,string,string>`_ for enum options + ## * `defined <#defined,untyped>`_ + ## * `std/compilesettings module <compilesettings.html>`_ + runnableExamples("--floatChecks:off"): + static: doAssert not compileOption("floatchecks") + {.push floatChecks: on.} + static: doAssert compileOption("floatchecks") + # floating point NaN and Inf checks enabled in this scope + {.pop.} + +proc compileOption*(option, arg: string): bool {. + magic: "CompileOptionArg", noSideEffect.} = + ## Can be used to determine an enum compile-time option. + ## + ## See also: + ## * `compileOption <#compileOption,string>`_ for `on|off` options + ## * `defined <#defined,untyped>`_ + ## * `std/compilesettings module <compilesettings.html>`_ + runnableExamples: + when compileOption("opt", "size") and compileOption("gc", "boehm"): + discard "compiled with optimization for size and uses Boehm's GC" + +template currentSourcePath*: string = instantiationInfo(-1, true).filename + ## Returns the full file-system path of the current source. + ## + ## To get the directory containing the current source, use it with + ## `ospaths2.parentDir() <ospaths2.html#parentDir%2Cstring>`_ as + ## `currentSourcePath.parentDir()`. + ## + ## The path returned by this template is set at compile time. + ## + ## See the docstring of `macros.getProjectPath() <macros.html#getProjectPath>`_ + ## for an example to see the distinction between the `currentSourcePath()` + ## and `getProjectPath()`. + ## + ## See also: + ## * `ospaths2.getCurrentDir() proc <ospaths2.html#getCurrentDir>`_ + +proc slurp*(filename: string): string {.magic: "Slurp".} + ## This is an alias for `staticRead <#staticRead,string>`_. + +proc staticRead*(filename: string): string {.magic: "Slurp".} + ## Compile-time `readFile <syncio.html#readFile,string>`_ proc for easy + ## `resource`:idx: embedding: + ## + ## The maximum file size limit that `staticRead` and `slurp` can read is + ## near or equal to the *free* memory of the device you are using to compile. + ## ```nim + ## const myResource = staticRead"mydatafile.bin" + ## ``` + ## + ## `slurp <#slurp,string>`_ is an alias for `staticRead`. + +proc gorge*(command: string, input = "", cache = ""): string {. + magic: "StaticExec".} = discard + ## This is an alias for `staticExec <#staticExec,string,string,string>`_. + +proc staticExec*(command: string, input = "", cache = ""): string {. + magic: "StaticExec".} = discard + ## Executes an external process at compile-time and returns its text output + ## (stdout + stderr). + ## + ## If `input` is not an empty string, it will be passed as a standard input + ## to the executed program. + ## ```nim + ## const buildInfo = "Revision " & staticExec("git rev-parse HEAD") & + ## "\nCompiled on " & staticExec("uname -v") + ## ``` + ## + ## `gorge <#gorge,string,string,string>`_ is an alias for `staticExec`. + ## + ## Note that you can use this proc inside a pragma like + ## `passc <manual.html#implementation-specific-pragmas-passc-pragma>`_ or + ## `passl <manual.html#implementation-specific-pragmas-passl-pragma>`_. + ## + ## If `cache` is not empty, the results of `staticExec` are cached within + ## the `nimcache` directory. Use `--forceBuild` to get rid of this caching + ## behaviour then. `command & input & cache` (the concatenated string) is + ## used to determine whether the entry in the cache is still valid. You can + ## use versioning information for `cache`: + ## ```nim + ## const stateMachine = staticExec("dfaoptimizer", "input", "0.8.0") + ## ``` + +proc gorgeEx*(command: string, input = "", cache = ""): tuple[output: string, + exitCode: int] = + ## Similar to `gorge <#gorge,string,string,string>`_ but also returns the + ## precious exit code. + discard diff --git a/lib/system/coro_detection.nim b/lib/system/coro_detection.nim new file mode 100644 index 000000000..f6c1b5c15 --- /dev/null +++ b/lib/system/coro_detection.nim @@ -0,0 +1,20 @@ +## Coroutine detection logic + +template coroutinesSupportedPlatform(): bool = + when defined(sparc) or defined(ELATE) or defined(boehmgc) or defined(gogc) or + defined(nogc) or defined(gcRegions) or defined(gcMarkAndSweep): + false + else: + true + +when defined(nimCoroutines): + # Explicit opt-in. + when not coroutinesSupportedPlatform(): + {.error: "Coroutines are not supported on this architecture and/or garbage collector.".} + const nimCoroutines* = true +elif defined(noNimCoroutines): + # Explicit opt-out. + const nimCoroutines* = false +else: + # Autodetect coroutine support. + const nimCoroutines* = false diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim new file mode 100644 index 000000000..34969cb32 --- /dev/null +++ b/lib/system/countbits_impl.nim @@ -0,0 +1,93 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Contains the used algorithms for counting bits. + +from std/private/bitops_utils import forwardImpl, castToUnsigned + +const useBuiltins* = not defined(noIntrinsicsBitOpts) +const noUndefined* = defined(noUndefinedBitOpts) +const useGCC_builtins* = (defined(gcc) or defined(llvm_gcc) or + defined(clang)) and useBuiltins +const useICC_builtins* = defined(icc) and useBuiltins +const useVCC_builtins* = defined(vcc) and useBuiltins +const arch64* = sizeof(int) == 8 + +template countBitsImpl(n: uint32): int = + # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + var v = uint32(n) + v = v - ((v shr 1'u32) and 0x55555555'u32) + v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) + (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int + +template countBitsImpl(n: uint64): int = + # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + var v = uint64(n) + v = v - ((v shr 1'u64) and 0x5555555555555555'u64) + v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) + v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) + ((v * 0x0101010101010101'u64) shr 56'u64).int + + +when useGCC_builtins: + # Returns the number of set 1-bits in value. + proc builtin_popcount(x: cuint): cint {.importc: "__builtin_popcount", cdecl.} + proc builtin_popcountll(x: culonglong): cint {. + importc: "__builtin_popcountll", cdecl.} + +elif useVCC_builtins: + # Counts the number of one bits (population count) in a 16-, 32-, or 64-byte unsigned integer. + func builtin_popcnt16(a2: uint16): uint16 {. + importc: "__popcnt16", header: "<intrin.h>".} + func builtin_popcnt32(a2: uint32): uint32 {. + importc: "__popcnt", header: "<intrin.h>".} + func builtin_popcnt64(a2: uint64): uint64 {. + importc: "__popcnt64", header: "<intrin.h>".} + +elif useICC_builtins: + # Intel compiler intrinsics: http://fulla.fnal.gov/intel/compiler_c/main_cls/intref_cls/common/intref_allia_misc.htm + # see also: https://software.intel.com/en-us/node/523362 + # Count the number of bits set to 1 in an integer a, and return that count in dst. + func builtin_popcnt32(a: cint): cint {. + importc: "_popcnt", header: "<immintrin.h>".} + func builtin_popcnt64(a: uint64): cint {. + importc: "_popcnt64", header: "<immintrin.h>".} + + +func countSetBitsImpl*(x: SomeInteger): int {.inline.} = + ## Counts the set bits in an integer (also called `Hamming weight`:idx:). + # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. + # like GCC and MSVC + let x = x.castToUnsigned + when nimvm: + result = forwardImpl(countBitsImpl, x) + else: + when useGCC_builtins: + when sizeof(x) <= 4: result = builtin_popcount(x.cuint).int + else: result = builtin_popcountll(x.culonglong).int + elif useVCC_builtins: + when sizeof(x) <= 2: result = builtin_popcnt16(x.uint16).int + elif sizeof(x) <= 4: result = builtin_popcnt32(x.uint32).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).uint32).int + + builtin_popcnt32((x.uint64 shr 32'u64).uint32).int + elif useICC_builtins: + when sizeof(x) <= 4: result = builtin_popcnt32(x.cint).int + elif arch64: result = builtin_popcnt64(x.uint64).int + else: result = builtin_popcnt32((x.uint64 and 0xFFFFFFFF'u64).cint).int + + builtin_popcnt32((x.uint64 shr 32'u64).cint).int + else: + when sizeof(x) <= 4: result = countBitsImpl(x.uint32) + else: result = countBitsImpl(x.uint64) + +proc countBits32*(n: uint32): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) + +proc countBits64*(n: uint64): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) diff --git a/lib/system/ctypes.nim b/lib/system/ctypes.nim new file mode 100644 index 000000000..b788274bd --- /dev/null +++ b/lib/system/ctypes.nim @@ -0,0 +1,84 @@ +## Some type definitions for compatibility between different +## backends and platforms. + +type + BiggestInt* = int64 + ## is an alias for the biggest signed integer type the Nim compiler + ## supports. Currently this is `int64`, but it is platform-dependent + ## in general. + + BiggestFloat* = float64 + ## is an alias for the biggest floating point type the Nim + ## compiler supports. Currently this is `float64`, but it is + ## platform-dependent in general. + + BiggestUInt* = uint64 + ## is an alias for the biggest unsigned integer type the Nim compiler + ## supports. Currently this is `uint64`, but it is platform-dependent + ## in general. + +when defined(windows): + type + clong* {.importc: "long", nodecl.} = int32 + ## This is the same as the type `long` in *C*. + culong* {.importc: "unsigned long", nodecl.} = uint32 + ## This is the same as the type `unsigned long` in *C*. +else: + type + clong* {.importc: "long", nodecl.} = int + ## This is the same as the type `long` in *C*. + culong* {.importc: "unsigned long", nodecl.} = uint + ## This is the same as the type `unsigned long` in *C*. + +type # these work for most platforms: + cchar* {.importc: "char", nodecl.} = char + ## This is the same as the type `char` in *C*. + cschar* {.importc: "signed char", nodecl.} = int8 + ## This is the same as the type `signed char` in *C*. + cshort* {.importc: "short", nodecl.} = int16 + ## This is the same as the type `short` in *C*. + cint* {.importc: "int", nodecl.} = int32 + ## This is the same as the type `int` in *C*. + csize_t* {.importc: "size_t", nodecl.} = uint + ## This is the same as the type `size_t` in *C*. + clonglong* {.importc: "long long", nodecl.} = int64 + ## This is the same as the type `long long` in *C*. + cfloat* {.importc: "float", nodecl.} = float32 + ## This is the same as the type `float` in *C*. + cdouble* {.importc: "double", nodecl.} = float64 + ## This is the same as the type `double` in *C*. + clongdouble* {.importc: "long double", nodecl.} = BiggestFloat + ## This is the same as the type `long double` in *C*. + ## This C type is not supported by Nim's code generator. + + cuchar* {.importc: "unsigned char", nodecl, deprecated: "use `char` or `uint8` instead".} = char + ## Deprecated: Use `uint8` instead. + cushort* {.importc: "unsigned short", nodecl.} = uint16 + ## This is the same as the type `unsigned short` in *C*. + cuint* {.importc: "unsigned int", nodecl.} = uint32 + ## This is the same as the type `unsigned int` in *C*. + culonglong* {.importc: "unsigned long long", nodecl.} = uint64 + ## This is the same as the type `unsigned long long` in *C*. + +type + ByteAddress* {.deprecated: "use `uint`".} = int + ## is the signed integer type that should be used for converting + ## pointers to integer addresses for readability. + + cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring] + ## This is binary compatible to the type `char**` in *C*. The array's + ## high value is large enough to disable bounds checking in practice. + ## Use `cstringArrayToSeq proc <#cstringArrayToSeq,cstringArray,Natural>`_ + ## to convert it into a `seq[string]`. + +when not defined(nimPreviewSlimSystem): + # pollutes namespace + type + PFloat32* {.deprecated: "use `ptr float32`".} = ptr float32 + ## An alias for `ptr float32`. + PFloat64* {.deprecated: "use `ptr float64`".} = ptr float64 + ## An alias for `ptr float64`. + PInt64* {.deprecated: "use `ptr int64`".} = ptr int64 + ## An alias for `ptr int64`. + PInt32* {.deprecated: "use `ptr int32`".} = ptr int32 + ## An alias for `ptr int32`. diff --git a/lib/system/cyclebreaker.nim b/lib/system/cyclebreaker.nim new file mode 100644 index 000000000..45b0a5a65 --- /dev/null +++ b/lib/system/cyclebreaker.nim @@ -0,0 +1,184 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +#[ +A Cycle breaker for Nim +----------------------- + +Instead of "collecting" cycles with all of its pitfalls we will break cycles. +We exploit that every 'ref' can be 'nil' for this and so get away without +a distinction between weak and strong pointers. The required runtime +mechanisms are the same though: We need to be able to traverse the graph. +This design has the tremendous benefit that it doesn't require a dedicated +'rawDispose' operation and that it plays well with Nim's cost model. +The cost of freeing a subgraph with cycles is 2 * N rather than N, that's all. + +Cycles do not have to be prepared via .acyclic, there are not multiple +pointless traversals, only a single proc, `breakCycles` is exposed as a +separate module. + +Algorithm +--------- + +We traverse the graph and notice the nodes we've already traversed. If we +marked the node already, we set the pointer that leads to this node to 'nil' +and decrement the reference count of the cell we pointed at. + +We notice that multiple paths to the same object do not mean +we found a cycle, it only means the node is shared. + + + a -------> b <----- c + | ^ ^ + +----------+ | + | | + +-------------------+ + +If we simply remove all links to already processed nodes we end up with: + + a -------> b c + | ^ + + | + | | + +-------------------+ + +That seems acceptable, no leak is produced. This implies that the standard +depth-first traversal suffices. + +]# + +include cellseqs_v2 + +const + colGreen = 0b000 + colYellow = 0b001 + colRed = 0b010 + colorMask = 0b011 + +type + TraceProc = proc (p, env: pointer) {.nimcall, benign.} + DisposeProc = proc (p: pointer) {.nimcall, benign.} + +template color(c): untyped = c.rc and colorMask +template setColor(c, col) = + c.rc = c.rc and not colorMask or col + +proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = + let h = head(p) + inc h.rc, rcIncrement + +proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = discard + +type + GcEnv = object + traceStack: CellSeq[ptr pointer] + +proc trace(p: pointer; desc: PNimTypeV2; j: var GcEnv) {.inline.} = + when false: + cprintf("[Trace] desc: %p %p\n", desc, p) + cprintf("[Trace] trace: %p\n", desc.traceImpl) + if desc.traceImpl != nil: + cast[TraceProc](desc.traceImpl)(p, addr(j)) + +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl.} = + let p = cast[ptr pointer](q) + when traceCollector: + cprintf("[Trace] raw: %p\n", p) + cprintf("[Trace] deref: %p\n", p[]) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, desc) + +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl.} = + let p = cast[ptr pointer](q) + when traceCollector: + cprintf("[TraceDyn] raw: %p\n", p) + cprintf("[TraceDyn] deref: %p\n", p[]) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) + +var markerGeneration: int + +proc breakCycles(s: Cell; desc: PNimTypeV2) = + let markerColor = if (markerGeneration and 1) == 0: colRed + else: colYellow + atomicInc markerGeneration + when traceCollector: + cprintf("[BreakCycles] starting: %p %s RC %ld trace proc %p\n", + s, desc.name, s.rc shr rcShift, desc.traceImpl) + + var j: GcEnv + init j.traceStack + s.setColor markerColor + trace(s +! sizeof(RefHeader), desc, j) + + while j.traceStack.len > 0: + let (u, desc) = j.traceStack.pop() + let p = u[] + let t = head(p) + if t.color != markerColor: + t.setColor markerColor + trace(p, desc, j) + when traceCollector: + cprintf("[BreakCycles] followed: %p RC %ld\n", t, t.rc shr rcShift) + else: + if (t.rc shr rcShift) > 0: + dec t.rc, rcIncrement + # mark as a link that the produced destructor does not have to follow: + u[] = nil + when traceCollector: + cprintf("[BreakCycles] niled out: %p RC %ld\n", t, t.rc shr rcShift) + else: + # anyhow as a link that the produced destructor does not have to follow: + u[] = nil + when traceCollector: + cprintf("[Bug] %p %s RC %ld\n", t, desc.name, t.rc shr rcShift) + deinit j.traceStack + +proc thinout*[T](x: ref T) {.inline.} = + ## turn the subgraph starting with `x` into its spanning tree by + ## `nil`'ing out any pointers that would harm the spanning tree + ## structure. Any back pointers that introduced cycles + ## and thus would keep the graph from being freed are `nil`'ed. + ## This is a form of cycle collection that works well with Nim's ARC + ## and its associated cost model. + proc getDynamicTypeInfo[T](x: T): PNimTypeV2 {.magic: "GetTypeInfoV2", noSideEffect.} + + breakCycles(head(cast[pointer](x)), getDynamicTypeInfo(x[])) + +proc thinout*[T: proc](x: T) {.inline.} = + proc rawEnv[T: proc](x: T): pointer {.noSideEffect, inline.} = + {.emit: """ + `result` = `x`.ClE_0; + """.} + + let p = rawEnv(x) + breakCycles(head(p), cast[ptr PNimTypeV2](p)[]) + +proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + # According to Lins it's correct to do nothing else here. + #cprintf("[DeCREF] %p\n", p) + +proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p %s\n", p, desc.name) + else: + dec cell.rc, rcIncrement + #cprintf("[DeCREF] %p %s %ld\n", p, desc.name, cell.rc) diff --git a/lib/system/cyclicrefs_v2.nim b/lib/system/cyclicrefs_v2.nim deleted file mode 100644 index 4fbb4fc94..000000000 --- a/lib/system/cyclicrefs_v2.nim +++ /dev/null @@ -1,232 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2019 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Cycle collector based on Lins' Jump Stack and other ideas, -# see for example: -# https://pdfs.semanticscholar.org/f2b2/0d168acf38ff86305809a55ef2c5d6ebc787.pdf -# Further refinement in 2008 by the notion of "critical links", see -# "Cyclic reference counting" by Rafael Dueire Lins -# R.D. Lins / Information Processing Letters 109 (2008) 71–78 - -const - colGreen = 0b000 - colYellow = 0b001 - colRed = 0b010 - jumpStackFlag = 0b100 # stored in jumpstack - rcShift = 3 # shift by rcShift to get the reference counter - colorMask = 0b011 - -type - TraceProc = proc (p, env: pointer) {.nimcall, benign.} - DisposeProc = proc (p: pointer) {.nimcall, benign.} - -template color(c): untyped = c.rc and colorMask -template setColor(c, col) = - when col == colGreen: - c.rc = c.rc and not colorMask - else: - c.rc = c.rc and not colorMask or col - -proc nimIncRefCyclic(p: pointer) {.compilerRtl, inl.} = - let h = head(p) - inc h.rc, rcIncrement - h.setColor colYellow # mark as potential cycle! - -proc markCyclic*[T](x: ref T) {.inline.} = - ## Mark the underlying object as a candidate for cycle collections. - ## Experimental API. Do not use! - let h = head(cast[pointer](x)) - h.setColor colYellow -type - CellTuple = (Cell, PNimType) - CellArray = ptr UncheckedArray[CellTuple] - CellSeq = object - len, cap: int - d: CellArray - - GcEnv = object - traceStack: CellSeq - jumpStack: CellSeq - -# ------------------- cell seq handling -------------------------------------- - -proc add(s: var CellSeq, c: Cell; t: PNimType) {.inline.} = - if s.len >= s.cap: - s.cap = s.cap * 3 div 2 - when defined(useMalloc): - var d = cast[CellArray](c_malloc(uint(s.cap * sizeof(CellTuple)))) - else: - var d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) - copyMem(d, s.d, s.len * sizeof(CellTuple)) - when defined(useMalloc): - c_free(s.d) - else: - dealloc(s.d) - s.d = d - # XXX: realloc? - s.d[s.len] = (c, t) - inc(s.len) - -proc init(s: var CellSeq, cap: int = 1024) = - s.len = 0 - s.cap = cap - when defined(useMalloc): - s.d = cast[CellArray](c_malloc(uint(s.cap * sizeof(CellTuple)))) - else: - s.d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) - -proc deinit(s: var CellSeq) = - when defined(useMalloc): - c_free(s.d) - else: - dealloc(s.d) - s.d = nil - s.len = 0 - s.cap = 0 - -proc pop(s: var CellSeq): (Cell, PNimType) = - result = s.d[s.len-1] - dec s.len - -# ---------------------------------------------------------------------------- - -proc trace(s: Cell; desc: PNimType; j: var GcEnv) {.inline.} = - if desc.traceImpl != nil: - var p = s +! sizeof(RefHeader) - cast[TraceProc](desc.traceImpl)(p, addr(j)) - -proc free(s: Cell; desc: PNimType) {.inline.} = - when traceCollector: - cprintf("[From ] %p rc %ld color %ld in jumpstack %ld\n", s, s.rc shr rcShift, - s.color, s.rc and jumpStackFlag) - var p = s +! sizeof(RefHeader) - if desc.disposeImpl != nil: - cast[DisposeProc](desc.disposeImpl)(p) - nimRawDispose(p) - -proc collect(s: Cell; desc: PNimType; j: var GcEnv) = - if s.color == colRed: - s.setColor colGreen - trace(s, desc, j) - while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() - if t.color == colRed: - t.setColor colGreen - trace(t, desc, j) - free(t, desc) - free(s, desc) - #cprintf("[Cycle free] %p %ld\n", s, s.rc shr rcShift) - -proc markRed(s: Cell; desc: PNimType; j: var GcEnv) = - if s.color != colRed: - s.setColor colRed - trace(s, desc, j) - while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() - when traceCollector: - cprintf("[Cycle dec] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) - dec t.rc, rcIncrement - if (t.rc and not rcMask) >= 0 and (t.rc and jumpStackFlag) == 0: - t.rc = t.rc or jumpStackFlag - when traceCollector: - cprintf("[Now in jumpstack] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) - j.jumpStack.add(t, desc) - if t.color != colRed: - t.setColor colRed - trace(t, desc, j) - -proc scanGreen(s: Cell; desc: PNimType; j: var GcEnv) = - s.setColor colGreen - trace(s, desc, j) - while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() - if t.color != colGreen: - t.setColor colGreen - trace(t, desc, j) - inc t.rc, rcIncrement - when traceCollector: - cprintf("[Cycle inc] %p %ld color %ld\n", t, t.rc shr rcShift, t.color) - -proc nimTraceRef(p: pointer; desc: PNimType; env: pointer) {.compilerRtl.} = - if p != nil: - var t = head(p) - var j = cast[ptr GcEnv](env) - j.traceStack.add(t, desc) - -proc nimTraceRefDyn(p: pointer; env: pointer) {.compilerRtl.} = - if p != nil: - let desc = cast[ptr PNimType](p)[] - var t = head(p) - var j = cast[ptr GcEnv](env) - j.traceStack.add(t, desc) - -proc scan(s: Cell; desc: PNimType; j: var GcEnv) = - when traceCollector: - cprintf("[doScanGreen] %p %ld\n", s, s.rc shr rcShift) - # even after trial deletion, `s` is still alive, so undo - # the decrefs by calling `scanGreen`: - if (s.rc and not rcMask) >= 0: - scanGreen(s, desc, j) - s.setColor colYellow - else: - # first we have to repair all the nodes we have seen - # that are still alive; we also need to mark what they - # refer to as alive: - while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop - # not in jump stack anymore! - t.rc = t.rc and not jumpStackFlag - if t.color == colRed and (t.rc and not rcMask) >= 0: - scanGreen(t, desc, j) - t.setColor colYellow - when traceCollector: - cprintf("[jump stack] %p %ld\n", t, t.rc shr rcShift) - # we have proven that `s` and its subgraph are dead, so we can - # collect these nodes: - collect(s, desc, j) - -proc traceCycle(s: Cell; desc: PNimType) {.noinline.} = - when traceCollector: - cprintf("[traceCycle] %p %ld\n", s, s.rc shr rcShift) - var j: GcEnv - init j.jumpStack - init j.traceStack - markRed(s, desc, j) - scan(s, desc, j) - while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop - # not in jump stack anymore! - t.rc = t.rc and not jumpStackFlag - deinit j.jumpStack - deinit j.traceStack - -proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = - if p != nil: - var cell = head(p) - if (cell.rc and not rcMask) == 0: - result = true - #cprintf("[DESTROY] %p\n", p) - else: - dec cell.rc, rcIncrement - if cell.color == colYellow: - let desc = cast[ptr PNimType](p)[] - traceCycle(cell, desc) - # According to Lins it's correct to do nothing else here. - #cprintf("[DeCREF] %p\n", p) - -proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimType): bool {.compilerRtl, inl.} = - if p != nil: - var cell = head(p) - if (cell.rc and not rcMask) == 0: - result = true - #cprintf("[DESTROY] %p %s\n", p, desc.name) - else: - dec cell.rc, rcIncrement - if cell.color == colYellow: traceCycle(cell, desc) - #cprintf("[DeCREF] %p %s %ld\n", p, desc.name, cell.rc) diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 66a65cd34..72d35f518 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -61,8 +61,8 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; tab: var PtrTable) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericDeepCopyAux(cast[pointer](d +% n.offset), @@ -85,33 +85,40 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericDeepCopyAux 2") case mt.kind of tyString: - var x = cast[PPointer](dest) - var s2 = cast[PPointer](s)[] - if s2 == nil: - unsureAsgnRef(x, s2) + when defined(nimSeqsV2): + var x = cast[ptr NimStringV2](dest) + var s2 = cast[ptr NimStringV2](s)[] + nimAsgnStrV2(x[], s2) else: - unsureAsgnRef(x, copyDeepString(cast[NimString](s2))) + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + unsureAsgnRef(x, s2) + else: + unsureAsgnRef(x, copyDeepString(cast[NimString](s2))) of tySequence: - var s2 = cast[PPointer](src)[] - var seq = cast[PGenericSeq](s2) - var x = cast[PPointer](dest) - if s2 == nil: - unsureAsgnRef(x, s2) - return - sysAssert(dest != nil, "genericDeepCopyAux 3") - unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) - for i in 0..seq.len-1: - genericDeepCopyAux( - cast[pointer](dst +% i *% mt.base.size +% GenericSeqSize), - cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% - GenericSeqSize), - mt.base, tab) + when defined(nimSeqsV2): + deepSeqAssignImpl(genericDeepCopyAux, tab) + else: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + unsureAsgnRef(x, s2) + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + unsureAsgnRef(x, newSeq(mt, seq.len)) + var dst = cast[int](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux( + cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + mt.base, tab) of tyObject: # we need to copy m_type field for tyObject, as it could be empty for # sequence reallocations: @@ -133,7 +140,10 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = unsureAsgnRef(cast[PPointer](dest), s2) elif mt.base.deepcopy != nil: let z = mt.base.deepcopy(s2) - unsureAsgnRef(cast[PPointer](dest), z) + when defined(nimSeqsV2): + cast[PPointer](dest)[] = z + else: + unsureAsgnRef(cast[PPointer](dest), z) else: let z = tab.get(s2) if z == nil: @@ -150,10 +160,16 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = let x = usrToCell(s2) let realType = x.typ sysAssert realType == mt, " types do differ" - # this version should work for any possible GC: - let typ = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[] else: mt.base - let z = newObj(mt, typ.size) - unsureAsgnRef(cast[PPointer](dest), z) + when defined(nimSeqsV2): + let typ = if mt.base.kind == tyObject: cast[PNimType](cast[ptr PNimTypeV2](s2)[].typeInfoV1) + else: mt.base + let z = nimNewObj(typ.size, typ.align) + cast[PPointer](dest)[] = z + else: + # this version should work for any other GC: + let typ = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[] else: mt.base + let z = newObj(mt, typ.size) + unsureAsgnRef(cast[PPointer](dest), z) tab.put(s2, z) genericDeepCopyAux(z, s2, typ, tab) else: @@ -169,11 +185,11 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = copyMem(dest, src, mt.size) proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = - GC_disable() + when not defined(nimSeqsV2): GC_disable() var tab = initPtrTable() genericDeepCopyAux(dest, src, mt, tab) deinit tab - GC_enable() + when not defined(nimSeqsV2): GC_enable() proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = # also invoked for 'string' @@ -183,8 +199,8 @@ proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericDeepCopyOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericDeepCopy(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 64860ef39..89a739d5a 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -1,15 +1,38 @@ -proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. ``$`` is Nim's general way of - ## spelling `toString`:idx:. +## `$` is Nim's general way of spelling `toString`:idx:. +runnableExamples: + assert $0.1 == "0.1" + assert $(-2*3) == "-6" -proc `$`*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. +import std/private/[digitsutils, miscdollars] + +when not defined(nimPreviewSlimSystem): + import std/formatfloat + export addFloat + + func `$`*(x: float | float32): string = + ## Outplace version of `addFloat`. + result.addFloat(x) + +proc `$`*(x: int): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) + +proc `$`*(x: int64): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) + +proc `$`*(x: uint64): string {.raises: [].} = + ## Outplace version of `addInt`. + addInt(result, x) + +# same as old `ctfeWhitelist` behavior, whether or not this is a good idea. +template gen(T) = + # xxx simplify this by supporting this in compiler: int{lit} | uint64{lit} | int64{lit} + func `$`*(x: T{lit}): string {.compileTime.} = result.addInt(x) +gen(int) +gen(uint64) +gen(int64) -proc `$`*(x: float): string {.magic: "FloatToStr", noSideEffect.} - ## The stringify operator for a float argument. Returns `x` - ## converted to a decimal string. proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## The stringify operator for a boolean argument. Returns `x` @@ -18,9 +41,9 @@ proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} proc `$`*(x: char): string {.magic: "CharToStr", noSideEffect.} ## The stringify operator for a character argument. Returns `x` ## converted to a string. - ## - ## .. code-block:: Nim + ## ```Nim ## assert $'c' == "c" + ## ``` proc `$`*(x: cstring): string {.magic: "CStrToStr", noSideEffect.} ## The stringify operator for a CString argument. Returns `x` @@ -29,74 +52,40 @@ proc `$`*(x: cstring): string {.magic: "CStrToStr", noSideEffect.} proc `$`*(x: string): string {.magic: "StrToStr", noSideEffect.} ## The stringify operator for a string argument. Returns `x` ## as it is. This operator is useful for generic code, so - ## that ``$expr`` also works if ``expr`` is already a string. + ## that `$expr` also works if `expr` is already a string. proc `$`*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} ## The stringify operator for an enumeration argument. This works for ## any enumeration type thanks to compiler magic. ## - ## If a ``$`` operator for a concrete enumeration is provided, this is + ## If a `$` operator for a concrete enumeration is provided, this is ## used instead. (In other words: *Overwriting* is possible.) proc `$`*(t: typedesc): string {.magic: "TypeTrait".} ## Returns the name of the given type. ## - ## For more procedures dealing with ``typedesc``, see + ## For more procedures dealing with `typedesc`, see ## `typetraits module <typetraits.html>`_. ## - ## .. code-block:: Nim - ## doAssert $(type(42)) == "int" - ## doAssert $(type("Foo")) == "string" - ## static: doAssert $(type(@['A', 'B'])) == "seq[char]" - - -proc isNamedTuple(T: typedesc): bool = - # Taken from typetraits. - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": - return compiles(t.Field0) - else: - return true - return false - -proc `$`*[T: tuple|object](x: T): string = - ## Generic ``$`` operator for tuples that is lifted from the components + ## ```Nim + ## doAssert $(typeof(42)) == "int" + ## doAssert $(typeof("Foo")) == "string" + ## static: doAssert $(typeof(@['A', 'B'])) == "seq[char]" + ## ``` + +proc `$`*[T: tuple](x: T): string = + ## Generic `$` operator for tuples that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(23, 45) == "(23, 45)" ## $(a: 23, b: 45) == "(a: 23, b: 45)" ## $() == "()" - result = "(" - var firstElement = true - const isNamed = T is object or isNamedTuple(T) - when not isNamed: - var count = 0 - for name, value in fieldPairs(x): - if not firstElement: result.add(", ") - when isNamed: - result.add(name) - result.add(": ") - else: - count.inc - when compiles($value): - when value isnot string and value isnot seq and compiles(value.isNil): - if value.isNil: result.add "nil" - else: result.addQuoted(value) - else: - result.addQuoted(value) - firstElement = false - else: - result.add("...") - firstElement = false - when not isNamed: - if count == 1: - result.add(",") # $(1,) should print as the semantically legal (1,) - result.add(")") + ## ``` + tupleObjectDollar(result, x) +when not defined(nimPreviewSlimSystem): + import std/objectdollar + export objectdollar proc collectionToString[T](x: T, prefix, separator, suffix: string): string = result = prefix @@ -118,27 +107,27 @@ proc collectionToString[T](x: T, prefix, separator, suffix: string): string = result.add(suffix) proc `$`*[T](x: set[T]): string = - ## Generic ``$`` operator for sets that is lifted from the components + ## Generic `$` operator for sets that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## ${23, 45} == "{23, 45}" + ## ``` collectionToString(x, "{", ", ", "}") proc `$`*[T](x: seq[T]): string = - ## Generic ``$`` operator for seqs that is lifted from the components + ## Generic `$` operator for seqs that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45]) == "@[23, 45]" + ## ``` collectionToString(x, "@[", ", ", "]") proc `$`*[T, U](x: HSlice[T, U]): string = - ## Generic ``$`` operator for slices that is lifted from the components + ## Generic `$` operator for slices that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(1 .. 5) == "1 .. 5" + ## ``` result = $x.a result.add(" .. ") result.add($x.b) @@ -146,13 +135,13 @@ proc `$`*[T, U](x: HSlice[T, U]): string = when not defined(nimNoArrayToString): proc `$`*[T, IDX](x: array[IDX, T]): string = - ## Generic ``$`` operator for arrays that is lifted from the components. + ## Generic `$` operator for arrays that is lifted from the components. collectionToString(x, "[", ", ", "]") proc `$`*[T](x: openArray[T]): string = - ## Generic ``$`` operator for openarrays that is lifted from the components + ## Generic `$` operator for openarrays that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" + ## ``` collectionToString(x, "[", ", ", "]") diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index d16dbdf92..2162b234f 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -19,26 +19,42 @@ const proc nimLoadLibraryError(path: string) = # carefully written to avoid memory allocation: - cstderr.rawWrite("could not load: ") + const prefix = "could not load: " + cstderr.rawWrite(prefix) cstderr.rawWrite(path) - cstderr.rawWrite("\n") when not defined(nimDebugDlOpen) and not defined(windows): - cstderr.rawWrite("compile with -d:nimDebugDlOpen for more information\n") - when defined(windows) and defined(guiapp): - # Because console output is not shown in GUI apps, display error as message box: - const prefix = "could not load: " - var msg: array[1000, char] - copyMem(msg[0].addr, prefix.cstring, prefix.len) - copyMem(msg[prefix.len].addr, path.cstring, min(path.len + 1, 1000 - prefix.len)) - discard MessageBoxA(nil, msg[0].addr, nil, 0) - quit(1) + cstderr.rawWrite("\n(compile with -d:nimDebugDlOpen for more information)") + when defined(windows): + const badExe = "\n(bad format; library may be wrong architecture)" + let loadError = GetLastError() + if loadError == ERROR_BAD_EXE_FORMAT: + cstderr.rawWrite(badExe) + when defined(guiapp): + # Because console output is not shown in GUI apps, display the error as a + # message box instead: + var + msg: array[1000, char] + msgLeft = msg.len - 1 # leave (at least) one for nullchar + msgIdx = 0 + copyMem(msg[msgIdx].addr, prefix.cstring, prefix.len) + msgLeft -= prefix.len + msgIdx += prefix.len + let pathLen = min(path.len, msgLeft) + copyMem(msg[msgIdx].addr, path.cstring, pathLen) + msgLeft -= pathLen + msgIdx += pathLen + if loadError == ERROR_BAD_EXE_FORMAT and msgLeft >= badExe.len: + copyMem(msg[msgIdx].addr, badExe.cstring, badExe.len) + discard MessageBoxA(nil, msg[0].addr, nil, 0) + cstderr.rawWrite("\n") + rawQuit(1) proc procAddrError(name: cstring) {.compilerproc, nonReloadable, hcrInline.} = # carefully written to avoid memory allocation: cstderr.rawWrite("could not import: ") cstderr.rawWrite(name) cstderr.rawWrite("\n") - quit(1) + rawQuit(1) # this code was inspired from Lua's source code: # Lua - An Extensible Extension Language @@ -75,7 +91,10 @@ when defined(posix): dlclose(lib) proc nimLoadLibrary(path: string): LibHandle = - result = dlopen(path, RTLD_NOW) + let flags = + when defined(globalSymbols): RTLD_NOW or RTLD_GLOBAL + else: RTLD_NOW + result = dlopen(path, flags) when defined(nimDebugDlOpen): let error = dlerror() if error != nil: @@ -142,41 +161,38 @@ elif defined(windows) or defined(dos): dec(m) k = k div 10 if k == 0: break - when defined(nimNoArrayToCstringConversion): - result = getProcAddress(cast[THINSTANCE](lib), addr decorated) - else: - result = getProcAddress(cast[THINSTANCE](lib), decorated) + result = getProcAddress(cast[THINSTANCE](lib), cast[cstring](addr decorated)) if result != nil: return procAddrError(name) elif defined(genode): - proc nimUnloadLibrary(lib: LibHandle) {. - error: "nimUnloadLibrary not implemented".} + proc nimUnloadLibrary(lib: LibHandle) = + raiseAssert("nimUnloadLibrary not implemented") - proc nimLoadLibrary(path: string): LibHandle {. - error: "nimLoadLibrary not implemented".} + proc nimLoadLibrary(path: string): LibHandle = + raiseAssert("nimLoadLibrary not implemented") - proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {. - error: "nimGetProcAddr not implemented".} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = + raiseAssert("nimGetProcAddr not implemented") -elif defined(nintendoswitch): +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr) or defined(nuttx): proc nimUnloadLibrary(lib: LibHandle) = cstderr.rawWrite("nimUnLoadLibrary not implemented") cstderr.rawWrite("\n") - quit(1) + rawQuit(1) proc nimLoadLibrary(path: string): LibHandle = cstderr.rawWrite("nimLoadLibrary not implemented") cstderr.rawWrite("\n") - quit(1) + rawQuit(1) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = cstderr.rawWrite("nimGetProAddr not implemented") cstderr.rawWrite(name) cstderr.rawWrite("\n") - quit(1) + rawQuit(1) else: {.error: "no implementation for dyncalls".} diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim index d1e05dad5..ea6776f58 100644 --- a/lib/system/embedded.nim +++ b/lib/system/embedded.nim @@ -19,8 +19,9 @@ proc nimFrame(s: PFrame) {.compilerRtl, inl, exportc: "nimFrame".} = discard proc popFrame {.compilerRtl, inl.} = discard proc setFrame(s: PFrame) {.compilerRtl, inl.} = discard -proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = discard -proc popSafePoint {.compilerRtl, inl.} = discard +when not gotoBasedExceptions: + proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = discard + proc popSafePoint {.compilerRtl, inl.} = discard proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} = discard proc popCurrentException {.compilerRtl, inl.} = discard @@ -32,10 +33,14 @@ const proc quitOrDebug() {.noreturn, importc: "abort", header: "<stdlib.h>", nodecl.} proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} = - sysFatal(ReraiseError, "exception handling is not available") + sysFatal(ReraiseDefect, "exception handling is not available") + +proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, + line: int) {.compilerRtl.} = + sysFatal(ReraiseDefect, "exception handling is not available") proc reraiseException() {.compilerRtl.} = - sysFatal(ReraiseError, "no exception to reraise") + sysFatal(ReraiseDefect, "no exception to reraise") proc writeStackTrace() = discard @@ -43,4 +48,14 @@ proc unsetControlCHook() = discard proc setControlCHook(hook: proc () {.noconv.}) = discard proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = - sysFatal(ReraiseError, "exception handling is not available") + sysFatal(ReraiseDefect, "exception handling is not available") + +when gotoBasedExceptions: + var nimInErrorMode {.threadvar.}: bool + + proc nimErrorFlag(): ptr bool {.compilerRtl, inl.} = + result = addr(nimInErrorMode) + + proc nimTestErrorFlag() {.compilerRtl.} = + if nimInErrorMode: + sysFatal(ReraiseDefect, "exception handling is not available") diff --git a/lib/system/exceptions.nim b/lib/system/exceptions.nim index f575f9b9f..63588f858 100644 --- a/lib/system/exceptions.nim +++ b/lib/system/exceptions.nim @@ -1,53 +1,13 @@ +## Exception and effect types used in Nim code. + type - RootEffect* {.compilerproc.} = object of RootObj ## \ - ## Base effect class. - ## - ## Each effect should inherit from `RootEffect` unless you know what - ## you're doing. TimeEffect* = object of RootEffect ## Time effect. IOEffect* = object of RootEffect ## IO effect. ReadIOEffect* = object of IOEffect ## Effect describing a read IO operation. WriteIOEffect* = object of IOEffect ## Effect describing a write IO operation. ExecIOEffect* = object of IOEffect ## Effect describing an executing IO operation. - StackTraceEntry* = object ## In debug mode exceptions store the stack trace that led - ## to them. A `StackTraceEntry` is a single entry of the - ## stack trace. - procname*: cstring ## Name of the proc that is currently executing. - line*: int ## Line number of the proc that is currently executing. - filename*: cstring ## Filename of the proc that is currently executing. - - Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ - ## Base exception class. - ## - ## Each exception has to inherit from `Exception`. See the full `exception - ## hierarchy <manual.html#exception-handling-exception-hierarchy>`_. - parent*: ref Exception ## Parent exception (can be used as a stack). - name*: cstring ## The exception's name is its Nim identifier. - ## This field is filled automatically in the - ## ``raise`` statement. - msg* {.exportc: "message".}: string ## The exception's message. Not - ## providing an exception message - ## is bad style. - when defined(js): - trace: string - else: - trace: seq[StackTraceEntry] - when defined(nimBoostrapCsources0_19_0): - # see #10315, bootstrap with `nim cpp` from csources gave error: - # error: no member named 'raise_id' in 'Exception' - raise_id: uint # set when exception is raised - else: - raiseId: uint # set when exception is raised - up: ref Exception # used for stacking exceptions. Not exported! - - Defect* = object of Exception ## \ - ## Abstract base class for all exceptions that Nim's runtime raises - ## but that are strictly uncatchable as they can also be mapped to - ## a ``quit`` / ``trap`` / ``exit`` operation. - - CatchableError* = object of Exception ## \ - ## Abstract class for all exceptions that are catchable. +type IOError* = object of CatchableError ## \ ## Raised if an IO error occurred. EOFError* = object of IOError ## \ @@ -59,19 +19,19 @@ type ## Raised if a dynamic library could not be loaded. ResourceExhaustedError* = object of CatchableError ## \ ## Raised if a resource request could not be fulfilled. - ArithmeticError* = object of Defect ## \ + ArithmeticDefect* = object of Defect ## \ ## Raised if any kind of arithmetic error occurred. - DivByZeroError* = object of ArithmeticError ## \ + DivByZeroDefect* = object of ArithmeticDefect ## \ ## Raised for runtime integer divide-by-zero errors. - OverflowError* = object of ArithmeticError ## \ + OverflowDefect* = object of ArithmeticDefect ## \ ## Raised for runtime integer overflows. ## ## This happens for calculations whose results are too large to fit in the ## provided bits. - AccessViolationError* = object of Defect ## \ + AccessViolationDefect* = object of Defect ## \ ## Raised for invalid memory access errors - AssertionError* = object of Defect ## \ + AssertionDefect* = object of Defect ## \ ## Raised when assertion is proved wrong. ## ## Usually the result of using the `assert() template @@ -84,54 +44,79 @@ type ## Mostly used by the `tables <tables.html>`_ module, it can also be raised ## by other collection modules like `sets <sets.html>`_ or `strtabs ## <strtabs.html>`_. - OutOfMemError* = object of Defect ## \ + OutOfMemDefect* = object of Defect ## \ ## Raised for unsuccessful attempts to allocate memory. - IndexError* = object of Defect ## \ + IndexDefect* = object of Defect ## \ ## Raised if an array index is out of bounds. - FieldError* = object of Defect ## \ + FieldDefect* = object of Defect ## \ ## Raised if a record field is not accessible because its discriminant's ## value does not fit. - RangeError* = object of Defect ## \ + RangeDefect* = object of Defect ## \ ## Raised if a range check error occurred. - StackOverflowError* = object of Defect ## \ + StackOverflowDefect* = object of Defect ## \ ## Raised if the hardware stack used for subroutine calls overflowed. - ReraiseError* = object of Defect ## \ + ReraiseDefect* = object of Defect ## \ ## Raised if there is no exception to reraise. - ObjectAssignmentError* = object of Defect ## \ + ObjectAssignmentDefect* = object of Defect ## \ ## Raised if an object gets assigned to its parent's object. - ObjectConversionError* = object of Defect ## \ + ObjectConversionDefect* = object of Defect ## \ ## Raised if an object is converted to an incompatible object type. - ## You can use ``of`` operator to check if conversion will succeed. - FloatingPointError* = object of Defect ## \ + ## You can use `of` operator to check if conversion will succeed. + FloatingPointDefect* = object of Defect ## \ ## Base class for floating point exceptions. - FloatInvalidOpError* = object of FloatingPointError ## \ + FloatInvalidOpDefect* = object of FloatingPointDefect ## \ ## Raised by invalid operations according to IEEE. ## - ## Raised by ``0.0/0.0``, for example. - FloatDivByZeroError* = object of FloatingPointError ## \ + ## Raised by `0.0/0.0`, for example. + FloatDivByZeroDefect* = object of FloatingPointDefect ## \ ## Raised by division by zero. ## ## Divisor is zero and dividend is a finite nonzero number. - FloatOverflowError* = object of FloatingPointError ## \ + FloatOverflowDefect* = object of FloatingPointDefect ## \ ## Raised for overflows. ## ## The operation produced a result that exceeds the range of the exponent. - FloatUnderflowError* = object of FloatingPointError ## \ + FloatUnderflowDefect* = object of FloatingPointDefect ## \ ## Raised for underflows. ## ## The operation produced a result that is too small to be represented as a ## normal number. - FloatInexactError* = object of FloatingPointError ## \ + FloatInexactDefect* = object of FloatingPointDefect ## \ ## Raised for inexact results. ## ## The operation produced a result that cannot be represented with infinite - ## precision -- for example: ``2.0 / 3.0, log(1.1)`` + ## precision -- for example: `2.0 / 3.0, log(1.1)` ## ## **Note**: Nim currently does not detect these! - DeadThreadError* = object of Defect ## \ + DeadThreadDefect* = object of Defect ## \ ## Raised if it is attempted to send a message to a dead thread. - NilAccessError* = object of Defect ## \ - ## Raised on dereferences of ``nil`` pointers. + NilAccessDefect* = object of Defect ## \ + ## Raised on dereferences of `nil` pointers. ## ## This is only raised if the `segfaults module <segfaults.html>`_ was imported! + +when not defined(nimPreviewSlimSystem): + type + ArithmeticError* {.deprecated: "See corresponding Defect".} = ArithmeticDefect + DivByZeroError* {.deprecated: "See corresponding Defect".} = DivByZeroDefect + OverflowError* {.deprecated: "See corresponding Defect".} = OverflowDefect + AccessViolationError* {.deprecated: "See corresponding Defect".} = AccessViolationDefect + AssertionError* {.deprecated: "See corresponding Defect".} = AssertionDefect + OutOfMemError* {.deprecated: "See corresponding Defect".} = OutOfMemDefect + IndexError* {.deprecated: "See corresponding Defect".} = IndexDefect + + FieldError* {.deprecated: "See corresponding Defect".} = FieldDefect + RangeError* {.deprecated: "See corresponding Defect".} = RangeDefect + StackOverflowError* {.deprecated: "See corresponding Defect".} = StackOverflowDefect + ReraiseError* {.deprecated: "See corresponding Defect".} = ReraiseDefect + ObjectAssignmentError* {.deprecated: "See corresponding Defect".} = ObjectAssignmentDefect + ObjectConversionError* {.deprecated: "See corresponding Defect".} = ObjectConversionDefect + FloatingPointError* {.deprecated: "See corresponding Defect".} = FloatingPointDefect + FloatInvalidOpError* {.deprecated: "See corresponding Defect".} = FloatInvalidOpDefect + FloatDivByZeroError* {.deprecated: "See corresponding Defect".} = FloatDivByZeroDefect + FloatOverflowError* {.deprecated: "See corresponding Defect".} = FloatOverflowDefect + FloatUnderflowError* {.deprecated: "See corresponding Defect".} = FloatUnderflowDefect + FloatInexactError* {.deprecated: "See corresponding Defect".} = FloatInexactDefect + DeadThreadError* {.deprecated: "See corresponding Defect".} = DeadThreadDefect + NilAccessError* {.deprecated: "See corresponding Defect".} = NilAccessDefect diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index a9c7ba02a..dae5c4a4a 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -10,6 +10,11 @@ # Exception handling code. Carefully coded so that tiny programs which do not # use the heap (and nor exceptions) do not include the GC or memory allocator. +import std/private/miscdollars +import stacktraces + +const noStacktraceAvailable = "No stack traceback available\n" + var errorMessageWriter*: (proc(msg: string) {.tags: [WriteIOEffect], benign, nimcall.}) @@ -17,17 +22,27 @@ var ## instead of `stdmsg.write` when printing stacktrace. ## Unstable API. +when defined(windows): + proc GetLastError(): int32 {.header: "<windows.h>", nodecl.} + const ERROR_BAD_EXE_FORMAT = 193 + when not defined(windows) or not defined(guiapp): proc writeToStdErr(msg: cstring) = rawWrite(cstderr, msg) - + proc writeToStdErr(msg: cstring, length: int) = + rawWriteString(cstderr, msg, length) else: proc MessageBoxA(hWnd: pointer, lpText, lpCaption: cstring, uType: int): int32 {. header: "<windows.h>", nodecl.} - proc writeToStdErr(msg: cstring) = discard MessageBoxA(nil, msg, nil, 0) + proc writeToStdErr(msg: cstring, length: int) = + discard MessageBoxA(nil, msg, nil, 0) + +proc writeToStdErr(msg: string) {.inline.} = + # fix bug #13115: handles correctly '\0' unlike default implicit conversion to cstring + writeToStdErr(msg.cstring, msg.len) -proc showErrorMessage(data: cstring) {.gcsafe, raises: [].} = +proc showErrorMessage(data: cstring, length: int) {.gcsafe, raises: [].} = var toWrite = true if errorMessageWriter != nil: try: @@ -40,7 +55,10 @@ proc showErrorMessage(data: cstring) {.gcsafe, raises: [].} = # stderr not available by default, use the LOG session echo data else: - writeToStdErr(data) + writeToStdErr(data, length) + +proc showErrorMessage2(data: string) {.inline.} = + showErrorMessage(data.cstring, data.len) proc chckIndx(i, a, b: int): int {.inline, compilerproc, benign.} proc chckRange(i, a, b: int): int {.inline, compilerproc, benign.} @@ -53,30 +71,47 @@ type len: int prev: ptr GcFrameHeader +when NimStackTraceMsgs: + var frameMsgBuf* {.threadvar.}: string + +when not defined(nimV2): + var + framePtr {.threadvar.}: PFrame + var - framePtr {.threadvar.}: PFrame - excHandler {.threadvar.}: PSafePoint - # list of exception handlers - # a global variable for the root of all try blocks currException {.threadvar.}: ref Exception - gcFramePtr {.threadvar.}: GcFrame -when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: +when not gotoBasedExceptions: var - raiseCounter {.threadvar.}: uint + excHandler {.threadvar.}: PSafePoint + # list of exception handlers + # a global variable for the root of all try blocks + gcFramePtr {.threadvar.}: GcFrame -type - FrameState = tuple[gcFramePtr: GcFrame, framePtr: PFrame, - excHandler: PSafePoint, currException: ref Exception] +when gotoBasedExceptions: + type + FrameState = tuple[framePtr: PFrame, + currException: ref Exception] +else: + type + FrameState = tuple[gcFramePtr: GcFrame, framePtr: PFrame, + excHandler: PSafePoint, currException: ref Exception] proc getFrameState*(): FrameState {.compilerRtl, inl.} = - return (gcFramePtr, framePtr, excHandler, currException) + when gotoBasedExceptions: + return (framePtr, currException) + else: + return (gcFramePtr, framePtr, excHandler, currException) proc setFrameState*(state: FrameState) {.compilerRtl, inl.} = - gcFramePtr = state.gcFramePtr - framePtr = state.framePtr - excHandler = state.excHandler - currException = state.currException + when gotoBasedExceptions: + framePtr = state.framePtr + currException = state.currException + else: + gcFramePtr = state.gcFramePtr + framePtr = state.framePtr + excHandler = state.excHandler + currException = state.currException proc getFrame*(): PFrame {.compilerRtl, inl.} = framePtr @@ -98,69 +133,44 @@ when false: proc setFrame*(s: PFrame) {.compilerRtl, inl.} = framePtr = s -proc getGcFrame*(): GcFrame {.compilerRtl, inl.} = gcFramePtr -proc popGcFrame*() {.compilerRtl, inl.} = gcFramePtr = gcFramePtr.prev -proc setGcFrame*(s: GcFrame) {.compilerRtl, inl.} = gcFramePtr = s -proc pushGcFrame*(s: GcFrame) {.compilerRtl, inl.} = - s.prev = gcFramePtr - zeroMem(cast[pointer](cast[int](s)+%sizeof(GcFrameHeader)), s.len*sizeof(pointer)) - gcFramePtr = s +when not gotoBasedExceptions: + proc getGcFrame*(): GcFrame {.compilerRtl, inl.} = gcFramePtr + proc popGcFrame*() {.compilerRtl, inl.} = gcFramePtr = gcFramePtr.prev + proc setGcFrame*(s: GcFrame) {.compilerRtl, inl.} = gcFramePtr = s + proc pushGcFrame*(s: GcFrame) {.compilerRtl, inl.} = + s.prev = gcFramePtr + zeroMem(cast[pointer](cast[int](s)+%sizeof(GcFrameHeader)), s.len*sizeof(pointer)) + gcFramePtr = s -proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = - s.prev = excHandler - excHandler = s + proc pushSafePoint(s: PSafePoint) {.compilerRtl, inl.} = + s.prev = excHandler + excHandler = s -proc popSafePoint {.compilerRtl, inl.} = - excHandler = excHandler.prev + proc popSafePoint {.compilerRtl, inl.} = + excHandler = excHandler.prev proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = e.up = currException currException = e - #showErrorMessage "A" + #showErrorMessage2 "A" proc popCurrentException {.compilerRtl, inl.} = currException = currException.up - #showErrorMessage "B" + #showErrorMessage2 "B" proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = - # in cpp backend exceptions can pop-up in the different order they were raised, example #5628 - if currException.raiseId == id: - currException = currException.up - else: - var cur = currException.up - var prev = currException - while cur != nil and cur.raiseId != id: - prev = cur - cur = cur.up - if cur == nil: - showErrorMessage("popCurrentExceptionEx() exception was not found in the exception stack. Aborting...") - quit(1) - prev.up = cur.up + discard "only for bootstrapping compatbility" proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = currException = e # some platforms have native support for stack traces: const - nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and + nativeStackTraceSupported = (defined(macosx) or defined(linux)) and not NimStackTrace hasSomeStackTrace = NimStackTrace or defined(nimStackTraceOverride) or (defined(nativeStackTrace) and nativeStackTraceSupported) -when defined(nimStackTraceOverride): - type StackTraceOverrideProc* = proc (): string {.nimcall, noinline, benign, raises: [], tags: [].} - ## Procedure type for overriding the default stack trace. - - var stackTraceOverrideGetTraceback: StackTraceOverrideProc = proc(): string {.noinline.} = - result = "Stack trace override procedure not registered.\n" - - proc registerStackTraceOverride*(overrideProc: StackTraceOverrideProc) = - ## Override the default stack trace inside rawWriteStackTrace() with your - ## own procedure. - stackTraceOverrideGetTraceback = overrideProc - - proc auxWriteStackTraceWithOverride(s: var string) = - add(s, stackTraceOverrideGetTraceback()) when defined(nativeStacktrace) and nativeStackTraceSupported: type @@ -178,13 +188,13 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when not hasThreadSupport: var - tempAddresses: array[0..127, pointer] # should not be alloc'd on stack + tempAddresses: array[maxStackTraceLines, pointer] # should not be alloc'd on stack tempDlInfo: TDl_info proc auxWriteStackTraceWithBacktrace(s: var string) = when hasThreadSupport: var - tempAddresses: array[0..127, pointer] # but better than a threadvar + tempAddresses: array[maxStackTraceLines, pointer] # but better than a threadvar tempDlInfo: TDl_info # This is allowed to be expensive since it only happens during crashes # (but this way you don't need manual stack tracing) @@ -212,11 +222,7 @@ when defined(nativeStacktrace) and nativeStackTraceSupported: when hasSomeStackTrace and not hasThreadSupport: var - tempFrames: array[0..127, PFrame] # should not be alloc'd on stack - -const - reraisedFromBegin = -10 - reraisedFromEnd = -100 + tempFrames: array[maxStackTraceLines, PFrame] # should not be alloc'd on stack template reraisedFrom(z): untyped = StackTraceEntry(procname: nil, line: z, filename: nil) @@ -240,21 +246,35 @@ proc auxWriteStackTrace(f: PFrame; s: var seq[StackTraceEntry]) = s[last] = StackTraceEntry(procname: it.procname, line: it.line, filename: it.filename) + when NimStackTraceMsgs: + let first = if it.prev == nil: 0 else: it.prev.frameMsgLen + if it.frameMsgLen > first: + s[last].frameMsg.setLen(it.frameMsgLen - first) + # somehow string slicing not available here + for i in first .. it.frameMsgLen-1: + s[last].frameMsg[i-first] = frameMsgBuf[i] it = it.prev dec last -template addFrameEntry(s, f: untyped) = +template addFrameEntry(s: var string, f: StackTraceEntry|PFrame) = var oldLen = s.len - add(s, f.filename) - if f.line > 0: - add(s, '(') - add(s, $f.line) - add(s, ')') + s.toLocation(f.filename, f.line, 0) for k in 1..max(1, 25-(s.len-oldLen)): add(s, ' ') add(s, f.procname) + when NimStackTraceMsgs: + when typeof(f) is StackTraceEntry: + add(s, f.frameMsg) + else: + var first = if f.prev == nil: 0 else: f.prev.frameMsgLen + for i in first..<f.frameMsgLen: add(s, frameMsgBuf[i]) add(s, "\n") -proc `$`(s: seq[StackTraceEntry]): string = +proc `$`(stackTraceEntries: seq[StackTraceEntry]): string = + when defined(nimStackTraceOverride): + let s = addDebuggingInfo(stackTraceEntries) + else: + let s = stackTraceEntries + result = newStringOfCap(2000) for i in 0 .. s.len-1: if s[i].line == reraisedFromBegin: result.add "[[reraised from:\n" @@ -266,7 +286,7 @@ when hasSomeStackTrace: proc auxWriteStackTrace(f: PFrame, s: var string) = when hasThreadSupport: var - tempFrames: array[0..127, PFrame] # but better than a threadvar + tempFrames: array[maxStackTraceLines, PFrame] # but better than a threadvar const firstCalls = 32 var @@ -314,7 +334,7 @@ when hasSomeStackTrace: auxWriteStackTraceWithOverride(s) elif NimStackTrace: if framePtr == nil: - add(s, "No stack traceback available\n") + add(s, noStacktraceAvailable) else: add(s, "Traceback (most recent call last)\n") auxWriteStackTrace(framePtr, s) @@ -322,10 +342,12 @@ when hasSomeStackTrace: add(s, "Traceback from system (most recent call last)\n") auxWriteStackTraceWithBacktrace(s) else: - add(s, "No stack traceback available\n") + add(s, noStacktraceAvailable) proc rawWriteStackTrace(s: var seq[StackTraceEntry]) = - when NimStackTrace: + when defined(nimStackTraceOverride): + auxWriteStackTraceWithOverride(s) + elif NimStackTrace: auxWriteStackTrace(framePtr, s) else: s = @[] @@ -346,13 +368,13 @@ else: proc stackTraceAvailable*(): bool = result = false var onUnhandledException*: (proc (errorMsg: string) {. - nimcall.}) ## Set this error \ + nimcall, gcsafe.}) ## Set this error \ ## handler to override the existing behaviour on an unhandled exception. ## - ## The default is to write a stacktrace to ``stderr`` and then call ``quit(1)``. + ## The default is to write a stacktrace to `stderr` and then call `quit(1)`. ## Unstable API. -proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = +proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy, gcsafe.} = when hasSomeStackTrace: var buf = newStringOfCap(2000) if e.trace.len == 0: @@ -360,7 +382,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = else: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") @@ -370,13 +393,14 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = if onUnhandledException != nil: onUnhandledException(buf) else: - showErrorMessage(buf) - `=destroy`(buf) + showErrorMessage2(buf) + {.gcsafe.}: + `=destroy`(buf) else: # ugly, but avoids heap allocations :-) template xadd(buf, s, slen) = if L + slen < high(buf): - copyMem(addr(buf[L]), cstring(s), slen) + copyMem(addr(buf[L]), (when s is cstring: s else: cstring(s)), slen) inc L, slen template add(buf, s) = xadd(buf, s, s.len) @@ -385,56 +409,56 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = if e.trace.len != 0: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") xadd(buf, e.name, e.name.len) add(buf, "]\n") - when defined(nimNoArrayToCstringConversion): - template tbuf(): untyped = addr buf - else: - template tbuf(): untyped = buf - if onUnhandledException != nil: - onUnhandledException($tbuf()) + onUnhandledException($cast[cstring](buf.addr)) else: - showErrorMessage(tbuf()) + showErrorMessage(cast[cstring](buf.addr), L) -proc reportUnhandledError(e: ref Exception) {.nodestroy.} = +proc reportUnhandledError(e: ref Exception) {.nodestroy, gcsafe.} = if unhandledExceptionHook != nil: unhandledExceptionHook(e) when hostOS != "any": reportUnhandledErrorAux(e) - else: - discard() -proc nimLeaveFinally() {.compilerRtl.} = - when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: - {.emit: "throw;".} - else: - if excHandler != nil: - c_longjmp(excHandler.context, 1) +when not gotoBasedExceptions: + proc nimLeaveFinally() {.compilerRtl.} = + when defined(cpp) and not defined(noCppExceptions) and not gotoBasedExceptions: + {.emit: "throw;".} else: - reportUnhandledError(currException) - quit(1) + if excHandler != nil: + c_longjmp(excHandler.context, 1) + else: + reportUnhandledError(currException) + rawQuit(1) when gotoBasedExceptions: - var nimInErrorMode {.threadvar.}: int + var nimInErrorMode {.threadvar.}: bool - proc nimErrorFlag(): ptr int {.compilerRtl, inl.} = + proc nimErrorFlag(): ptr bool {.compilerRtl, inl.} = result = addr(nimInErrorMode) proc nimTestErrorFlag() {.compilerRtl.} = - ## This proc must be called before ``currException`` is destroyed. + ## This proc must be called before `currException` is destroyed. ## It also must be called at the end of every thread to ensure no ## error is swallowed. - if currException != nil: + if nimInErrorMode and currException != nil: reportUnhandledError(currException) currException = nil - quit(1) + rawQuit(1) proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = + when defined(nimPanics): + if e of Defect: + reportUnhandledError(e) + rawQuit(1) + if localRaiseHook != nil: if not localRaiseHook(e): return if globalRaiseHook != nil: @@ -444,15 +468,9 @@ proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = {.emit: "throw;".} else: pushCurrentException(e) - raiseCounter.inc - if raiseCounter == 0: - raiseCounter.inc # skip zero at overflow - e.raiseId = raiseCounter - {.emit: "`e`->raise();".} - elif defined(nimQuirky) or gotoBasedExceptions: - # XXX This check should likely also be done in the setjmp case below. - if e != currException: - pushCurrentException(e) + {.emit: "throw `e`;".} + elif quirkyExceptions or gotoBasedExceptions: + pushCurrentException(e) when gotoBasedExceptions: inc nimInErrorMode else: @@ -461,14 +479,19 @@ proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = c_longjmp(excHandler.context, 1) else: reportUnhandledError(e) - quit(1) + rawQuit(1) proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring, line: int) {.compilerRtl, nodestroy.} = if e.name.isNil: e.name = ename when hasSomeStackTrace: when defined(nimStackTraceOverride): - e.trace = @[] + if e.trace.len == 0: + rawWriteStackTrace(e.trace) + else: + e.trace.add reraisedFrom(reraisedFromBegin) + auxWriteStackTraceWithOverride(e.trace) + e.trace.add reraisedFrom(reraisedFromEnd) elif NimStackTrace: if e.trace.len == 0: rawWriteStackTrace(e.trace) @@ -486,27 +509,35 @@ proc raiseException(e: sink(ref Exception), ename: cstring) {.compilerRtl.} = proc reraiseException() {.compilerRtl.} = if currException == nil: - sysFatal(ReraiseError, "no exception to reraise") + sysFatal(ReraiseDefect, "no exception to reraise") else: when gotoBasedExceptions: inc nimInErrorMode else: raiseExceptionAux(currException) +proc threadTrouble() = + # also forward declared, it is 'raises: []' hence the try-except. + try: + if currException != nil: reportUnhandledError(currException) + except: + discard + rawQuit 1 + proc writeStackTrace() = when hasSomeStackTrace: var s = "" rawWriteStackTrace(s) - cast[proc (s: cstring) {.noSideEffect, tags: [], nimcall, raises: [].}](showErrorMessage)(s) else: - cast[proc (s: cstring) {.noSideEffect, tags: [], nimcall, raises: [].}](showErrorMessage)("No stack traceback available\n") + let s = noStacktraceAvailable + cast[proc (s: string) {.noSideEffect, tags: [], nimcall, raises: [].}](showErrorMessage2)(s) proc getStackTrace(): string = when hasSomeStackTrace: result = "" rawWriteStackTrace(result) else: - result = "No stack traceback available\n" + result = noStacktraceAvailable proc getStackTrace(e: ref Exception): string = if not isNil(e): @@ -514,13 +545,10 @@ proc getStackTrace(e: ref Exception): string = else: result = "" -proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = - ## Returns the attached stack trace to the exception ``e`` as - ## a ``seq``. This is not yet available for the JS backend. - when not defined(nimSeqsV2): - shallowCopy(result, e.trace) - else: - result = move(e.trace) +proc getStackTraceEntries*(e: ref Exception): lent seq[StackTraceEntry] = + ## Returns the attached stack trace to the exception `e` as + ## a `seq`. This is not yet available for the JS backend. + e.trace proc getStackTraceEntries*(): seq[StackTraceEntry] = ## Returns the stack trace entries for the current stack trace. @@ -532,26 +560,33 @@ const nimCallDepthLimit {.intdefine.} = 2000 proc callDepthLimitReached() {.noinline.} = writeStackTrace() - showErrorMessage("Error: call depth limit reached in a debug build (" & + let msg = "Error: call depth limit reached in a debug build (" & $nimCallDepthLimit & " function calls). You can change it with " & "-d:nimCallDepthLimit=<int> but really try to avoid deep " & - "recursions instead.\n") - quit(1) + "recursions instead.\n" + showErrorMessage2(msg) + rawQuit(1) proc nimFrame(s: PFrame) {.compilerRtl, inl, raises: [].} = - s.calldepth = if framePtr == nil: 0 else: framePtr.calldepth+1 + if framePtr == nil: + s.calldepth = 0 + when NimStackTraceMsgs: s.frameMsgLen = 0 + else: + s.calldepth = framePtr.calldepth+1 + when NimStackTraceMsgs: s.frameMsgLen = framePtr.frameMsgLen s.prev = framePtr framePtr = s if s.calldepth == nimCallDepthLimit: callDepthLimitReached() when defined(cpp) and appType != "lib" and not gotoBasedExceptions and not defined(js) and not defined(nimscript) and - hostOS != "standalone" and not defined(noCppExceptions): + hostOS != "standalone" and hostOS != "any" and not defined(noCppExceptions) and + not quirkyExceptions: type StdException {.importcpp: "std::exception", header: "<exception>".} = object - proc what(ex: StdException): cstring {.importcpp: "((char *)#.what())".} + proc what(ex: StdException): cstring {.importcpp: "((char *)#.what())", nodecl.} proc setTerminate(handler: proc() {.noconv.}) {.importc: "std::set_terminate", header: "<exception>".} @@ -562,9 +597,9 @@ when defined(cpp) and appType != "lib" and not gotoBasedExceptions and var msg = "Unknown error in unexpected exception handler" try: - {.emit"#if !defined(_MSC_VER) || (_MSC_VER >= 1923)".} + {.emit: "#if !defined(_MSC_VER) || (_MSC_VER >= 1923)".} raise - {.emit"#endif".} + {.emit: "#endif".} except Exception: msg = currException.getStackTrace() & "Error: unhandled exception: " & currException.msg & " [" & $currException.name & "]" @@ -573,9 +608,9 @@ when defined(cpp) and appType != "lib" and not gotoBasedExceptions and except: msg = "Error: unhandled unknown cpp exception" - {.emit"#if defined(_MSC_VER) && (_MSC_VER < 1923)".} + {.emit: "#if defined(_MSC_VER) && (_MSC_VER < 1923)".} msg = "Error: unhandled unknown cpp exception" - {.emit"#endif".} + {.emit: "#endif".} when defined(genode): # stderr not available by default, use the LOG session @@ -583,9 +618,12 @@ when defined(cpp) and appType != "lib" and not gotoBasedExceptions and else: writeToStdErr msg & "\n" - quit 1 + rawQuit 1 when not defined(noSignalHandler) and not defined(useNimRtl): + type Sighandler = proc (a: cint) {.noconv, benign.} + # xxx factor with ansi_c.CSighandlerT, posix.Sighandler + proc signalHandler(sign: cint) {.exportc: "signalHandler", noconv.} = template processSignal(s, action: untyped) {.dirty.} = if s == SIGINT: action("SIGINT: Interrupted by Ctrl-C.\n") @@ -613,17 +651,34 @@ when not defined(noSignalHandler) and not defined(useNimRtl): var buf = newStringOfCap(2000) rawWriteStackTrace(buf) processSignal(sign, buf.add) # nice hu? currying a la Nim :-) - showErrorMessage(buf) + showErrorMessage2(buf) when not usesDestructors: GC_enable() else: var msg: cstring template asgn(y) = msg = y processSignal(sign, asgn) - showErrorMessage(msg) - quit(1) # always quit when SIGABRT + # xxx use string for msg instead of cstring, and here use showErrorMessage2(msg) + # unless there's a good reason to use cstring in signal handler to avoid + # using gc? + showErrorMessage(msg, msg.len) + + when defined(posix): + # reset the signal handler to OS default + c_signal(sign, SIG_DFL) + + # re-raise the signal, which will arrive once this handler exit. + # this lets the OS perform actions like core dumping and will + # also return the correct exit code to the shell. + discard c_raise(sign) + else: + rawQuit(1) + + var SIG_IGN {.importc: "SIG_IGN", header: "<signal.h>".}: Sighandler proc registerSignalHandler() = + # xxx `signal` is deprecated and has many caveats, we should use `sigaction` instead, e.g. + # https://stackoverflow.com/questions/231912/what-is-the-difference-between-sigaction-and-signal c_signal(SIGINT, signalHandler) c_signal(SIGSEGV, signalHandler) c_signal(SIGABRT, signalHandler) @@ -632,14 +687,17 @@ when not defined(noSignalHandler) and not defined(useNimRtl): when declared(SIGBUS): c_signal(SIGBUS, signalHandler) when declared(SIGPIPE): - c_signal(SIGPIPE, signalHandler) + when defined(nimLegacySigpipeHandler): + c_signal(SIGPIPE, signalHandler) + else: + c_signal(SIGPIPE, SIG_IGN) registerSignalHandler() # call it in initialization section proc setControlCHook(hook: proc () {.noconv.}) = # ugly cast, but should work on all architectures: - type SignalHandler = proc (sign: cint) {.noconv, benign.} - c_signal(SIGINT, cast[SignalHandler](hook)) + when declared(Sighandler): + c_signal(SIGINT, cast[Sighandler](hook)) when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook() = diff --git a/lib/system/fatal.nim b/lib/system/fatal.nim index d68d06712..25c05e52d 100644 --- a/lib/system/fatal.nim +++ b/lib/system/fatal.nim @@ -9,57 +9,50 @@ {.push profiler: off.} -when defined(nimHasExceptionsQuery): - const gotoBasedExceptions = compileOption("exceptions", "goto") -else: - const gotoBasedExceptions = false +const + gotoBasedExceptions = compileOption("exceptions", "goto") + quirkyExceptions = compileOption("exceptions", "quirky") when hostOS == "standalone": include "$projectpath/panicoverride" - proc sysFatal(exceptn: typedesc, message: string) {.inline.} = + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline.} = panic(message) - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline.} = + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline.} = rawoutput(message) panic(arg) -elif (defined(nimQuirky) or gotoBasedExceptions) and not defined(nimscript): +elif quirkyExceptions and not defined(nimscript): import ansi_c - proc name(t: typedesc): string {.magic: "TypeTrait".} - - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noreturn.} = - writeStackTrace() - var buf = newStringOfCap(200) - add(buf, "Error: unhandled exception: ") - add(buf, message) - add(buf, arg) - add(buf, " [") - add(buf, name exceptn) - add(buf, "]\n") - cstderr.rawWrite buf - quit 1 + func name(t: typedesc): string {.magic: "TypeTrait".} - proc sysFatal(exceptn: typedesc, message: string) {.inline, noreturn.} = + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline, noreturn.} = + when nimvm: + # TODO when doAssertRaises works in CT, add a test for it + raise (ref exceptn)(msg: message & arg) + else: + {.noSideEffect.}: + writeStackTrace() + var buf = newStringOfCap(200) + add(buf, "Error: unhandled exception: ") + add(buf, message) + add(buf, arg) + add(buf, " [") + add(buf, name exceptn) + add(buf, "]\n") + cstderr.rawWrite buf + rawQuit 1 + + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline, noreturn.} = sysFatal(exceptn, message, "") else: - proc sysFatal(exceptn: typedesc, message: string) {.inline, noreturn.} = - when declared(owned): - var e: owned(ref exceptn) - else: - var e: ref exceptn - new(e) - e.msg = message - raise e + func sysFatal(exceptn: typedesc[Defect], message: string) {.inline, noreturn.} = + raise (ref exceptn)(msg: message) + + func sysFatal(exceptn: typedesc[Defect], message, arg: string) {.inline, noreturn.} = + raise (ref exceptn)(msg: message & arg) - proc sysFatal(exceptn: typedesc, message, arg: string) {.inline, noreturn.} = - when declared(owned): - var e: owned(ref exceptn) - else: - var e: ref exceptn - new(e) - e.msg = message & arg - raise e {.pop.} diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim index cb46c8c36..70dd857d5 100644 --- a/lib/system/formatfloat.nim +++ b/lib/system/formatfloat.nim @@ -1,59 +1,6 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2019 Nim contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -proc c_sprintf(buf, frmt: cstring): cint {.header: "<stdio.h>", - importc: "sprintf", varargs, noSideEffect.} - -proc writeToBuffer(buf: var array[65, char]; value: cstring) = - var i = 0 - while value[i] != '\0': - buf[i] = value[i] - inc i - -proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int = - ## This is the implementation to format floats in the Nim - ## programming language. The specific format for floating point - ## numbers is not specified in the Nim programming language and - ## might change slightly in the future, but at least wherever you - ## format a float, it should be consistent. - ## - ## returns the amount of bytes written to `buf` not counting the - ## terminating '\0' character. - ## - ## * `buf` - A buffer to write into. The buffer does not need to be - ## initialized and it will be overridden. - ## - var n: int = c_sprintf(addr buf, "%.16g", value) - var hasDot = false - for i in 0..n-1: - if buf[i] == ',': - buf[i] = '.' - hasDot = true - elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: - hasDot = true - if not hasDot: - buf[n] = '.' - buf[n+1] = '0' - buf[n+2] = '\0' - result = n + 2 - else: - result = n - # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' - # of '-1.#IND' are produced. - # We want to get rid of these here: - if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: - writeToBuffer(buf, "nan") - result = 3 - elif buf[n-1] == 'F': - if buf[0] == '-': - writeToBuffer(buf, "-inf") - result = 4 - else: - writeToBuffer(buf, "inf") - result = 3 +when not defined(nimPreviewSlimSystem): + import std/formatfloat + export formatfloat + {.deprecated: "use `std/formatfloat`".} +else: + {.error: "use `std/formatfloat`".} diff --git a/lib/system/gc.nim b/lib/system/gc.nim index 7be9f4b1f..9289c7f55 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -12,6 +12,55 @@ # Refcounting + Mark&Sweep. Complex algorithms avoided. # Been there, done that, didn't work. +#[ + +A *cell* is anything that is traced by the GC +(sequences, refs, strings, closures). + +The basic algorithm is *Deferrent Reference Counting* with cycle detection. +References on the stack are not counted for better performance and easier C +code generation. + +Each cell has a header consisting of a RC and a pointer to its type +descriptor. However the program does not know about these, so they are placed at +negative offsets. In the GC code the type `PCell` denotes a pointer +decremented by the right offset, so that the header can be accessed easily. It +is extremely important that `pointer` is not confused with a `PCell`. + +In Nim the compiler cannot always know if a reference +is stored on the stack or not. This is caused by var parameters. +Consider this example: + + ```Nim + proc setRef(r: var ref TNode) = + new(r) + + proc usage = + var + r: ref TNode + setRef(r) # here we should not update the reference counts, because + # r is on the stack + setRef(r.left) # here we should update the refcounts! + ``` + +We have to decide at runtime whether the reference is on the stack or not. +The generated code looks roughly like this: + + ```C + void setref(TNode** ref) { + unsureAsgnRef(ref, newObj(TNode_TI, sizeof(TNode))) + } + void usage(void) { + setRef(&r) + setRef(&r->left) + } + ``` + +Note that for systems with a continuous stack (which most systems have) +the check whether the ref is on the stack is very cheap (only two +comparisons). +]# + {.push profiler:off.} const @@ -29,7 +78,7 @@ when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} when hasThreadSupport: - import sharedlist + import std/sharedlist const rcIncrement = 0b1000 # so that lowest 3 bits are not touched @@ -47,7 +96,7 @@ type waZctDecRef, waPush #, waDebug - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -114,7 +163,7 @@ template gcAssert(cond: bool, msg: string) = writeStackTrace() #var x: ptr int #echo x[] - quit 1 + rawQuit 1 proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = if (c.refcount and ZctFlag) == 0: @@ -123,11 +172,11 @@ proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -172,11 +221,11 @@ template gcTrace(cell, state: untyped) = when traceGC: traceCell(cell, state) # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap) {.benign, raises: [].} +proc isOnStack(p: pointer): bool {.noinline, benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes proc incRef(c: PCell) {.inline.} = @@ -289,7 +338,7 @@ proc cellsetReset(s: var CellSet) = {.push stacktrace:off.} proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -309,7 +358,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -335,12 +384,11 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) + forAllChildrenAux(cast[pointer](d +% align(GenericSeqSize, cell.typ.base.align) +% i *% cell.typ.base.size), cell.typ.base, op) else: discard proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} = @@ -411,7 +459,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) #gcAssert typ.kind in {tyString, tySequence} or size >= typ.base.size, "size too small" - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -436,20 +484,22 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(typ, size, gch) when defined(memProfiler): nimProfile(size) -proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = result = rawNewObj(typ, size, gch) zeroMem(result, size) when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) +{.pop.} -proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = # generates a new object and sets its reference counter to 1 incTypeSize typ, size sysAssert(allocInv(gch.region), "newObjRC1 begin") @@ -459,7 +509,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -476,12 +526,14 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = sysAssert(allocInv(gch.region), "newObjRC1 end") when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size result = newObjRC1(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) +{.pop.} proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = collectCT(gch) @@ -491,15 +543,17 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = sysAssert(allocInv(gch.region), "growObj begin") var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size + var elemSize,elemAlign = 1 + if ol.typ.kind != tyString: + elemSize = ol.typ.base.size + elemAlign = ol.typ.base.align incTypeSize ol.typ, newsize - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len * elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") # This can be wrong for intermediate temps that are nevertheless on the # heap because of lambda lifting: #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") @@ -509,35 +563,8 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = gcTrace(res, csAllocated) track("growObj old", ol, 0) track("growObj new", res, newsize) - when defined(nimIncrSeqV3): - # since we steal the old seq's contents, we set the old length to 0. - cast[PGenericSeq](old).len = 0 - elif reallyDealloc: - sysAssert(allocInv(gch.region), "growObj before dealloc") - if ol.refcount shr rcShift <=% 1: - # free immediately to save space: - if (ol.refcount and ZctFlag) != 0: - var j = gch.zct.len-1 - var d = gch.zct.d - while j >= 0: - if d[j] == ol: - d[j] = res - break - dec(j) - beforeDealloc(gch, ol, "growObj stack trash") - decTypeSize(ol, ol.typ) - rawDealloc(gch.region, ol) - else: - # we split the old refcount in 2 parts. XXX This is still not entirely - # correct if the pointer that receives growObj's result is on the stack. - # A better fix would be to emit the location specific write barrier for - # 'growObj', but this is lots of more work and who knows what new problems - # this would create. - res.refcount = rcIncrement - decRef(ol) - else: - sysAssert(ol.typ != nil, "growObj: 5") - zeroMem(ol, sizeof(Cell)) + # since we steal the old seq's contents, we set the old length to 0. + cast[PGenericSeq](old).len = 0 when useCellIds: inc gch.idGenerator res.id = gch.idGenerator * 1000_000 + gch.gcThreadId @@ -584,7 +611,7 @@ proc markS(gch: var GcHeap, c: PCell) = if not containsOrIncl(gch.marked, d): forAllChildren(d, waMarkPrecise) -proc markGlobals(gch: var GcHeap) = +proc markGlobals(gch: var GcHeap) {.raises: [].} = if gch.gcThreadId == 0: for i in 0 .. globalMarkersLen-1: globalMarkers[i]() for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() @@ -601,7 +628,7 @@ when logGC: if cycleCheckA[i] == c: return true if cycleCheckALen == len(cycleCheckA): gcAssert(false, "cycle detection overflow") - quit 1 + rawQuit 1 cycleCheckA[cycleCheckALen] = c inc cycleCheckALen @@ -639,9 +666,9 @@ proc doOperation(p: pointer, op: WalkOp) = proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = doOperation(d, WalkOp(op)) -proc collectZCT(gch: var GcHeap): bool {.benign.} +proc collectZCT(gch: var GcHeap): bool {.benign, raises: [].} -proc collectCycles(gch: var GcHeap) = +proc collectCycles(gch: var GcHeap) {.raises: [].} = when hasThreadSupport: for c in gch.toDispose: nimGCunref(c) @@ -658,16 +685,16 @@ proc collectCycles(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: # mark the cell: incRef(objStart) add(gch.decStack, objStart) when false: + let cell = usrToCell(p) if isAllocatedPtr(gch.region, cell): sysAssert false, "allocated pointer but not interior?" # mark the cell: @@ -748,7 +775,7 @@ proc unmarkStackAndRegisters(gch: var GcHeap) = decRef(d[i]) gch.decStack.len = 0 -proc collectCTBody(gch: var GcHeap) = +proc collectCTBody(gch: var GcHeap) {.raises: [].} = when withRealTime: let t0 = getticks() sysAssert(allocInv(gch.region), "collectCT: begin") @@ -823,10 +850,10 @@ when withRealTime: stack.bottomSaved = stack.bottom when stackIncreases: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) else: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) GC_step(gch, us, strongAdvice) @@ -838,9 +865,10 @@ when not defined(useNimRtl): proc GC_disable() = inc(gch.recGcLock) proc GC_enable() = - if gch.recGcLock <= 0: - raise newException(AssertionError, - "API usage error: GC_enable called but GC is already enabled") + when defined(nimDoesntTrackDefects): + if gch.recGcLock <= 0: + raise newException(AssertionDefect, + "API usage error: GC_enable called but GC is already enabled") dec(gch.recGcLock) proc GC_setStrategy(strategy: GC_Strategy) = @@ -850,7 +878,7 @@ when not defined(useNimRtl): gch.cycleThreshold = InitialCycleThreshold proc GC_disableMarkAndSweep() = - gch.cycleThreshold = high(gch.cycleThreshold)-1 + gch.cycleThreshold = high(typeof(gch.cycleThreshold))-1 # set to the max value to suppress the cycle detector proc GC_fullCollect() = diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim deleted file mode 100644 index 4da9b4f94..000000000 --- a/lib/system/gc2.nim +++ /dev/null @@ -1,745 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2017 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Garbage Collector -# -# The basic algorithm is an incremental mark -# and sweep GC to free cycles. It is hard realtime in that if you play -# according to its rules, no deadline will ever be missed. -# Since this kind of collector is very bad at recycling dead objects -# early, Nim's codegen emits ``nimEscape`` calls at strategic -# places. For this to work even 'unsureAsgnRef' needs to mark things -# so that only return values need to be considered in ``nimEscape``. - -{.push profiler:off.} - -const - CycleIncrease = 2 # is a multiplicative increase - InitialCycleThreshold = 512*1024 # start collecting after 500KB - ZctThreshold = 500 # we collect garbage if the ZCT's size - # reaches this threshold - # this seems to be a good value - withRealTime = defined(useRealtimeGC) - -when withRealTime and not declared(getTicks): - include "system/timers" -when defined(memProfiler): - proc nimProfile(requestedSize: int) {.benign.} - -when hasThreadSupport: - include sharedlist - -type - ObjectSpaceIter = object - state: range[-1..0] - -iterToProc(allObjects, ptr ObjectSpaceIter, allObjectsAsProc) - -const - escapedBit = 0b1000 # so that lowest 3 bits are not touched - rcBlackOrig = 0b000 - rcWhiteOrig = 0b001 - rcGrey = 0b010 # traditional color for incremental mark&sweep - rcUnused = 0b011 - colorMask = 0b011 -type - WalkOp = enum - waMarkGlobal, # part of the backup mark&sweep - waMarkGrey, - waZctDecRef, - waDebug - - Phase {.pure.} = enum - None, Marking, Sweeping - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} - # A ref type can have a finalizer that is called before the object's - # storage is freed. - - GcStat = object - stackScans: int # number of performed stack scans (for statistics) - completedCollections: int # number of performed full collections - maxThreshold: int # max threshold that has been set - maxStackSize: int # max stack size - maxStackCells: int # max stack cells in ``decStack`` - cycleTableSize: int # max entries in cycle table - maxPause: int64 # max measured GC pause in nanoseconds - - GcStack {.final, pure.} = object - when nimCoroutines: - prev: ptr GcStack - next: ptr GcStack - maxStackSize: int # Used to track statistics because we can not use - # GcStat.maxStackSize when multiple stacks exist. - bottom: pointer - - when withRealTime or nimCoroutines: - pos: pointer # Used with `withRealTime` only for code clarity, see GC_Step(). - when withRealTime: - bottomSaved: pointer - - GcHeap = object # this contains the zero count and - # non-zero count table - black, red: int # either 0 or 1. - stack: GcStack - when nimCoroutines: - activeStack: ptr GcStack # current executing coroutine stack. - phase: Phase - cycleThreshold: int - when useCellIds: - idGenerator: int - greyStack: CellSeq - recGcLock: int # prevent recursion via finalizers; no thread lock - when withRealTime: - maxPause: Nanos # max allowed pause in nanoseconds; active if > 0 - region: MemRegion # garbage collected region - stat: GcStat - additionalRoots: CellSeq # explicit roots for GC_ref/unref - spaceIter: ObjectSpaceIter - pDumpHeapFile: pointer # File that is used for GC_dumpHeap - when hasThreadSupport: - toDispose: SharedList[pointer] - gcThreadId: int - -var - gch {.rtlThreadVar.}: GcHeap - -when not defined(useNimRtl): - instantiateForRegion(gch.region) - -# Which color to use for new objects is tricky: When we're marking, -# they have to be *white* so that everything is marked that is only -# reachable from them. However, when we are sweeping, they have to -# be black, so that we don't free them prematuredly. In order to save -# a comparison gch.phase == Phase.Marking, we use the pseudo-color -# 'red' for new objects. -template allocColor(): untyped = gch.red - -template gcAssert(cond: bool, msg: string) = - when defined(useGcAssert): - if not cond: - echo "[GCASSERT] ", msg - GC_disable() - writeStackTrace() - quit 1 - -proc cellToUsr(cell: PCell): pointer {.inline.} = - # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) - -proc usrToCell(usr: pointer): PCell {.inline.} = - # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) - -proc extGetCellType(c: pointer): PNimType {.compilerproc.} = - # used for code generation concerning debugging - result = usrToCell(c).typ - -proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = - result = 0 - -# this that has to equals zero, otherwise we have to round up UnitsPerPage: -when BitsPerPage mod (sizeof(int)*8) != 0: - {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} - -template color(c): untyped = c.refCount and colorMask -template setColor(c, col) = - c.refcount = c.refcount and not colorMask or col - -template markAsEscaped(c: PCell) = - c.refcount = c.refcount or escapedBit - -template didEscape(c: PCell): bool = - (c.refCount and escapedBit) != 0 - -proc writeCell(file: File; msg: cstring, c: PCell) = - var kind = -1 - if c.typ != nil: kind = ord(c.typ.kind) - let col = if c.color == rcGrey: 'g' - elif c.color == gch.black: 'b' - else: 'w' - when useCellIds: - let id = c.id - else: - let id = c - when defined(nimTypeNames): - c_fprintf(file, "%s %p %d escaped=%ld color=%c of type %s\n", - msg, id, kind, didEscape(c), col, c.typ.name) - elif leakDetector: - c_fprintf(file, "%s %p %d escaped=%ld color=%c from %s(%ld)\n", - msg, id, kind, didEscape(c), col, c.filename, c.line) - else: - c_fprintf(file, "%s %p %d escaped=%ld color=%c\n", - msg, id, kind, didEscape(c), col) - -proc writeCell(msg: cstring, c: PCell) = - stdout.writeCell(msg, c) - -proc myastToStr[T](x: T): string {.magic: "AstToStr", noSideEffect.} - -template gcTrace(cell, state: untyped) = - when traceGC: writeCell(myastToStr(state), cell) - -# forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} -proc isOnStack(p: pointer): bool {.noinline, benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} -# we need the prototype here for debugging purposes - -proc nimGCref(p: pointer) {.compilerproc.} = - let cell = usrToCell(p) - markAsEscaped(cell) - add(gch.additionalRoots, cell) - -proc nimGCunref(p: pointer) {.compilerproc.} = - let cell = usrToCell(p) - var L = gch.additionalRoots.len-1 - var i = L - let d = gch.additionalRoots.d - while i >= 0: - if d[i] == cell: - d[i] = d[L] - dec gch.additionalRoots.len - break - dec(i) - -proc nimGCunrefNoCycle(p: pointer) {.compilerproc, inline.} = - discard "can we do some freeing here?" - -proc nimGCunrefRC1(p: pointer) {.compilerproc, inline.} = - discard "can we do some freeing here?" - -template markGrey(x: PCell) = - if x.color != 1-gch.black and gch.phase == Phase.Marking: - if not isAllocatedPtr(gch.region, x): - c_fprintf(stdout, "[GC] markGrey proc: %p\n", x) - #GC_dumpHeap() - sysAssert(false, "wtf") - x.setColor(rcGrey) - add(gch.greyStack, x) - -proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - # the code generator calls this proc! - gcAssert(not isOnStack(dest), "asgnRef") - # BUGFIX: first incRef then decRef! - if src != nil: - let s = usrToCell(src) - markAsEscaped(s) - markGrey(s) - dest[] = src - -proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, - deprecated: "old compiler compat".} = asgnRef(dest, src) - -proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc.} = - # unsureAsgnRef marks 'src' as grey only if dest is not on the - # stack. It is used by the code generator if it cannot decide whether a - # reference is in the stack or not (this can happen for var parameters). - if src != nil: - let s = usrToCell(src) - markAsEscaped(s) - if not isOnStack(dest): markGrey(s) - dest[] = src - -proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) - case n.kind - of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) - of nkList: - for i in 0..n.len-1: - forAllSlotsAux(dest, n.sons[i], op) - of nkCase: - var m = selectBranch(dest, n) - if m != nil: forAllSlotsAux(dest, m, op) - of nkNone: sysAssert(false, "forAllSlotsAux") - -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) - if dest == nil: return # nothing to do - if ntfNoRefs notin mt.flags: - case mt.kind - of tyRef, tyString, tySequence: # leaf: - doOperation(cast[PPointer](d)[], op) - of tyObject, tyTuple: - forAllSlotsAux(dest, mt.node, op) - of tyArray, tyArrayConstr, tyOpenArray: - for i in 0..(mt.size div mt.base.size)-1: - forAllChildrenAux(cast[pointer](d +% i *% mt.base.size), mt.base, op) - else: discard - -proc forAllChildren(cell: PCell, op: WalkOp) = - gcAssert(cell != nil, "forAllChildren: 1") - gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2") - gcAssert(cell.typ != nil, "forAllChildren: 3") - gcAssert cell.typ.kind in {tyRef, tySequence, tyString}, "forAllChildren: 4" - let marker = cell.typ.marker - if marker != nil: - marker(cellToUsr(cell), op.int) - else: - case cell.typ.kind - of tyRef: # common case - forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) - of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) - var s = cast[PGenericSeq](d) - if s != nil: - for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) - else: discard - -{.push stackTrace: off, profiler:off.} -proc gcInvariant*() = - sysAssert(allocInv(gch.region), "injected") - when declared(markForDebug): - markForDebug(gch) -{.pop.} - -include gc_common - -proc initGC() = - when not defined(useNimRtl): - gch.red = (1-gch.black) - gch.cycleThreshold = InitialCycleThreshold - gch.stat.stackScans = 0 - gch.stat.completedCollections = 0 - gch.stat.maxThreshold = 0 - gch.stat.maxStackSize = 0 - gch.stat.maxStackCells = 0 - gch.stat.cycleTableSize = 0 - # init the rt - init(gch.additionalRoots) - init(gch.greyStack) - when hasThreadSupport: - init(gch.toDispose) - gch.gcThreadId = atomicInc(gHeapidGenerator) - 1 - gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") - -proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = - # generates a new object and sets its reference counter to 0 - sysAssert(allocInv(gch.region), "rawNewObj begin") - gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) - var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") - # now it is buffered in the ZCT - res.typ = typ - when leakDetector and not hasThreadSupport: - if framePtr != nil and framePtr.prev != nil: - res.filename = framePtr.prev.filename - res.line = framePtr.prev.line - # refcount is zero, color is black, but mark it to be in the ZCT - res.refcount = allocColor() - sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") - when logGC: writeCell("new cell", res) - gcTrace(res, csAllocated) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - sysAssert(allocInv(gch.region), "rawNewObj end") - -{.pop.} - -proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) - when defined(memProfiler): nimProfile(size) - -proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = rawNewObj(typ, size, gch) - zeroMem(result, size) - when defined(memProfiler): nimProfile(size) - -proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) - result = newObj(typ, size) - cast[PGenericSeq](result).len = len - cast[PGenericSeq](result).reserved = len - when defined(memProfiler): nimProfile(size) - -proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = - result = newObj(typ, size) - -proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - result = newSeq(typ, len) - -proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = - collectCT(gch) - var ol = usrToCell(old) - sysAssert(ol.typ != nil, "growObj: 1") - gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") - - var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size - incTypeSize ol.typ, newsize - - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize - copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), - newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") - when false: - # this is wrong since seqs can be shared via 'shallow': - when reallyDealloc: rawDealloc(gch.region, ol) - else: - zeroMem(ol, sizeof(Cell)) - when useCellIds: - inc gch.idGenerator - res.id = gch.idGenerator - result = cellToUsr(res) - when defined(memProfiler): nimProfile(newsize-oldsize) - -proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - result = growObj(old, newsize, gch) - -{.push profiler:off.} - - -template takeStartTime(workPackageSize) {.dirty.} = - const workPackage = workPackageSize - var debugticker = 1000 - when withRealTime: - var steps = workPackage - var t0: Ticks - if gch.maxPause > 0: t0 = getticks() - -template takeTime {.dirty.} = - when withRealTime: dec steps - dec debugticker - -template checkTime {.dirty.} = - if debugticker <= 0: - #echo "in loop" - debugticker = 1000 - when withRealTime: - if steps == 0: - steps = workPackage - if gch.maxPause > 0: - let duration = getticks() - t0 - # the GC's measuring is not accurate and needs some cleanup actions - # (stack unmarking), so subtract some short amount of time in - # order to miss deadlines less often: - if duration >= gch.maxPause - 50_000: - return false - -# ---------------- dump heap ---------------- - -template dumpHeapFile(gch: var GcHeap): File = - cast[File](gch.pDumpHeapFile) - -proc debugGraph(s: PCell) = - c_fprintf(gch.dumpHeapFile, "child %p\n", s) - -proc dumpRoot(gch: var GcHeap; s: PCell) = - if isAllocatedPtr(gch.region, s): - c_fprintf(gch.dumpHeapFile, "global_root %p\n", s) - else: - c_fprintf(gch.dumpHeapFile, "global_root_invalid %p\n", s) - -proc GC_dumpHeap*(file: File) = - ## Dumps the GCed heap's content to a file. Can be useful for - ## debugging. Produces an undocumented text file format that - ## can be translated into "dot" syntax via the "heapdump2dot" tool. - gch.pDumpHeapFile = file - var spaceIter: ObjectSpaceIter - when false: - var d = gch.decStack.d - for i in 0 .. gch.decStack.len-1: - if isAllocatedPtr(gch.region, d[i]): - c_fprintf(file, "onstack %p\n", d[i]) - else: - c_fprintf(file, "onstack_invalid %p\n", d[i]) - if gch.gcThreadId == 0: - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() - for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() - while true: - let x = allObjectsAsProc(gch.region, addr spaceIter) - if spaceIter.state < 0: break - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - writeCell(file, "cell ", c) - forAllChildren(c, waDebug) - c_fprintf(file, "end\n") - gch.pDumpHeapFile = nil - -proc GC_dumpHeap() = - var f: File - if open(f, "heap.txt", fmWrite): - GC_dumpHeap(f) - f.close() - else: - c_fprintf(stdout, "cannot write heap.txt") - -# ---------------- cycle collector ------------------------------------------- - -proc freeCyclicCell(gch: var GcHeap, c: PCell) = - gcAssert(isAllocatedPtr(gch.region, c), "freeCyclicCell: freed pointer?") - prepareDealloc(c) - gcTrace(c, csCycFreed) - when logGC: writeCell("cycle collector dealloc cell", c) - when reallyDealloc: - sysAssert(allocInv(gch.region), "free cyclic cell") - rawDealloc(gch.region, c) - else: - gcAssert(c.typ != nil, "freeCyclicCell") - zeroMem(c, sizeof(Cell)) - -proc sweep(gch: var GcHeap): bool = - takeStartTime(100) - #echo "loop start" - let white = 1-gch.black - #c_fprintf(stdout, "black is %d\n", black) - while true: - let x = allObjectsAsProc(gch.region, addr gch.spaceIter) - if gch.spaceIter.state < 0: break - takeTime() - if isCell(x): - # cast to PCell is correct here: - var c = cast[PCell](x) - gcAssert c.color != rcGrey, "cell is still grey?" - if c.color == white: freeCyclicCell(gch, c) - # Since this is incremental, we MUST not set the object to 'white' here. - # We could set all the remaining objects to white after the 'sweep' - # completed but instead we flip the meaning of black/white to save one - # traversal over the heap! - checkTime() - # prepare for next iteration: - #echo "loop end" - gch.spaceIter = ObjectSpaceIter() - result = true - -proc markRoot(gch: var GcHeap, c: PCell) {.inline.} = - if c.color == 1-gch.black: - c.setColor(rcGrey) - add(gch.greyStack, c) - -proc markIncremental(gch: var GcHeap): bool = - var L = addr(gch.greyStack.len) - takeStartTime(100) - while L[] > 0: - var c = gch.greyStack.d[0] - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") - - #sysAssert(isAllocatedPtr(gch.region, c), "markIncremental: isAllocatedPtr") - gch.greyStack.d[0] = gch.greyStack.d[L[] - 1] - dec(L[]) - takeTime() - if c.color == rcGrey: - c.setColor(gch.black) - forAllChildren(c, waMarkGrey) - elif c.color == (1-gch.black): - gcAssert false, "wtf why are there white objects in the greystack?" - checkTime() - gcAssert gch.greyStack.len == 0, "markIncremental: greystack not empty " - result = true - -proc markGlobals(gch: var GcHeap) = - if gch.gcThreadId == 0: - for i in 0 .. globalMarkersLen-1: globalMarkers[i]() - for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]() - -proc doOperation(p: pointer, op: WalkOp) = - if p == nil: return - var c: PCell = usrToCell(p) - gcAssert(c != nil, "doOperation: 1") - # the 'case' should be faster than function pointers because of easy - # prediction: - case op - of waZctDecRef: - #if not isAllocatedPtr(gch.region, c): - # c_fprintf(stdout, "[GC] decref bug: %p", c) - gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef") - discard "use me for nimEscape?" - of waMarkGlobal: - template handleRoot = - if gch.dumpHeapFile.isNil: - markRoot(gch, c) - else: - dumpRoot(gch, c) - handleRoot() - discard allocInv(gch.region) - of waMarkGrey: - when false: - if not isAllocatedPtr(gch.region, c): - c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c) - #GC_dumpHeap() - sysAssert(false, "wtf") - if c.color == 1-gch.black: - c.setColor(rcGrey) - add(gch.greyStack, c) - of waDebug: debugGraph(c) - -proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = - doOperation(d, WalkOp(op)) - -proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = - # the addresses are not as cells on the stack, so turn them to cells: - sysAssert(allocInv(gch.region), "gcMark begin") - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) - if c >% PageSize: - # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) - if objStart != nil: - # mark the cell: - markRoot(gch, objStart) - sysAssert(allocInv(gch.region), "gcMark end") - -proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} = - forEachStackSlot(gch, gcMark) - -proc collectALittle(gch: var GcHeap): bool = - case gch.phase - of Phase.None: - if getOccupiedMem(gch.region) >= gch.cycleThreshold: - gch.phase = Phase.Marking - markGlobals(gch) - result = collectALittle(gch) - #when false: c_fprintf(stdout, "collectALittle: introduced bug E %ld\n", gch.phase) - #discard allocInv(gch.region) - of Phase.Marking: - when hasThreadSupport: - for c in gch.toDispose: - nimGCunref(c) - prepareForInteriorPointerChecking(gch.region) - markStackAndRegisters(gch) - inc(gch.stat.stackScans) - if markIncremental(gch): - gch.phase = Phase.Sweeping - gch.red = 1 - gch.red - of Phase.Sweeping: - gcAssert gch.greyStack.len == 0, "greystack not empty" - when hasThreadSupport: - for c in gch.toDispose: - nimGCunref(c) - if sweep(gch): - gch.phase = Phase.None - # flip black/white meanings: - gch.black = 1 - gch.black - gcAssert gch.red == 1 - gch.black, "red color is wrong" - inc(gch.stat.completedCollections) - result = true - -proc collectCTBody(gch: var GcHeap) = - when withRealTime: - let t0 = getticks() - sysAssert(allocInv(gch.region), "collectCT: begin") - - when not nimCoroutines: - gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) - #gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len) - if collectALittle(gch): - gch.cycleThreshold = max(InitialCycleThreshold, getOccupiedMem() * - CycleIncrease) - gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) - sysAssert(allocInv(gch.region), "collectCT: end") - when withRealTime: - let duration = getticks() - t0 - gch.stat.maxPause = max(gch.stat.maxPause, duration) - when defined(reportMissedDeadlines): - if gch.maxPause > 0 and duration > gch.maxPause: - c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) - -when nimCoroutines: - proc currentStackSizes(): int = - for stack in items(gch.stack): - result = result + stack.stackSize() - -proc collectCT(gch: var GcHeap) = - # stackMarkCosts prevents some pathological behaviour: Stack marking - # becomes more expensive with large stacks and large stacks mean that - # cells with RC=0 are more likely to be kept alive by the stack. - when nimCoroutines: - let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold) - else: - let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold) - if (gch.greyStack.len >= stackMarkCosts or (cycleGC and - getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and - gch.recGcLock == 0: - collectCTBody(gch) - -when withRealTime: - proc toNano(x: int): Nanos {.inline.} = - result = x * 1000 - - proc GC_setMaxPause*(MaxPauseInUs: int) = - gch.maxPause = MaxPauseInUs.toNano - - proc GC_step(gch: var GcHeap, us: int, strongAdvice: bool) = - gch.maxPause = us.toNano - #if (getOccupiedMem(gch.region)>=gch.cycleThreshold) or - # alwaysGC or strongAdvice: - collectCTBody(gch) - - proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} = - if stackSize >= 0: - var stackTop {.volatile.}: pointer - gch.getActiveStack().pos = addr(stackTop) - - for stack in gch.stack.items(): - stack.bottomSaved = stack.bottom - when stackIncreases: - stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) - else: - stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) - - GC_step(gch, us, strongAdvice) - - if stackSize >= 0: - for stack in gch.stack.items(): - stack.bottom = stack.bottomSaved - -when not defined(useNimRtl): - proc GC_disable() = - inc(gch.recGcLock) - proc GC_enable() = - if gch.recGcLock > 0: - dec(gch.recGcLock) - - proc GC_setStrategy(strategy: GC_Strategy) = - discard - - proc GC_enableMarkAndSweep() = discard - proc GC_disableMarkAndSweep() = discard - - proc GC_fullCollect() = - var oldThreshold = gch.cycleThreshold - gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) - gch.cycleThreshold = oldThreshold - - proc GC_getStatistics(): string = - GC_disable() - result = "[GC] total memory: " & $(getTotalMem()) & "\n" & - "[GC] occupied memory: " & $(getOccupiedMem()) & "\n" & - "[GC] stack scans: " & $gch.stat.stackScans & "\n" & - "[GC] stack cells: " & $gch.stat.maxStackCells & "\n" & - "[GC] completed collections: " & $gch.stat.completedCollections & "\n" & - "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" & - "[GC] grey stack capacity: " & $gch.greyStack.cap & "\n" & - "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" & - "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n" - when nimCoroutines: - result.add "[GC] number of stacks: " & $gch.stack.len & "\n" - for stack in items(gch.stack): - result.add "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" - else: - result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" - GC_enable() - -{.pop.} diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index fe07766d9..eb0884560 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -78,8 +78,11 @@ template decTypeSize(cell, t) = when defined(nimTypeNames): if t.kind in {tyString, tySequence}: let cap = cast[PGenericSeq](cellToUsr(cell)).space - let size = if t.kind == tyString: cap+1+GenericSeqSize - else: addInt(mulInt(cap, t.base.size), GenericSeqSize) + let size = + if t.kind == tyString: + cap + 1 + GenericSeqSize + else: + align(GenericSeqSize, t.base.align) + cap * t.base.size atomicDec t.sizes, size+sizeof(Cell) else: atomicDec t.sizes, t.base.size+sizeof(Cell) @@ -163,16 +166,16 @@ when defined(nimdoc): ## this thread will only be initialized once per thread, no matter how often ## it is called. ## - ## This function is available only when ``--threads:on`` and ``--tlsEmulation:off`` + ## This function is available only when `--threads:on` and `--tlsEmulation:off` ## switches are used discard proc tearDownForeignThreadGc*() {.gcsafe.} = - ## Call this to tear down the GC, previously initialized by ``setupForeignThreadGc``. + ## Call this to tear down the GC, previously initialized by `setupForeignThreadGc`. ## If GC has not been previously initialized, or has already been torn down, the ## call does nothing. ## - ## This function is available only when ``--threads:on`` and ``--tlsEmulation:off`` + ## This function is available only when `--threads:on` and `--tlsEmulation:off` ## switches are used discard elif declared(threadType): @@ -214,18 +217,19 @@ proc stackSize(stack: ptr GcStack): int {.noinline.} = when nimCoroutines: var pos = stack.pos else: - var pos {.volatile.}: pointer + var pos {.volatile, noinit.}: pointer pos = addr(pos) if pos != nil: when stackIncreases: - result = cast[ByteAddress](pos) -% cast[ByteAddress](stack.bottom) + result = cast[int](pos) -% cast[int](stack.bottom) else: - result = cast[ByteAddress](stack.bottom) -% cast[ByteAddress](pos) + result = cast[int](stack.bottom) -% cast[int](pos) else: result = 0 proc stackSize(): int {.noinline.} = + result = 0 for stack in gch.stack.items(): result = result + stack.stackSize() @@ -269,6 +273,9 @@ when nimCoroutines: gch.activeStack = gch.stack.find(bottom) gch.activeStack.setPosition(addr(sp)) + proc GC_getActiveStack() : pointer {.cdecl, exportc.} = + return gch.activeStack.bottom + when not defined(useNimRtl): proc nimGC_setStackBottom(theStackBottom: pointer) = # Initializes main stack of the thread. @@ -288,25 +295,28 @@ when not defined(useNimRtl): # the first init must be the one that defines the stack bottom: gch.stack.bottom = theStackBottom elif theStackBottom != gch.stack.bottom: - var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 - var b = cast[ByteAddress](gch.stack.bottom) + var a = cast[int](theStackBottom) # and not PageMask - PageSize*2 + var b = cast[int](gch.stack.bottom) #c_fprintf(stdout, "old: %p new: %p;\n",gch.stack.bottom,theStackBottom) when stackIncreases: gch.stack.bottom = cast[pointer](min(a, b)) else: gch.stack.bottom = cast[pointer](max(a, b)) + when nimCoroutines: + if theStackBottom != nil: gch.stack.bottom = theStackBottom + gch.stack.setPosition(theStackBottom) {.pop.} proc isOnStack(p: pointer): bool = - var stackTop {.volatile.}: pointer + var stackTop {.volatile, noinit.}: pointer stackTop = addr(stackTop) - var a = cast[ByteAddress](gch.getActiveStack().bottom) - var b = cast[ByteAddress](stackTop) + var a = cast[int](gch.getActiveStack().bottom) + var b = cast[int](stackTop) when not stackIncreases: swap(a, b) - var x = cast[ByteAddress](p) + var x = cast[int](p) result = a <=% x and x <=% b when defined(sparc): # For SPARC architecture. @@ -327,7 +337,7 @@ when defined(sparc): # For SPARC architecture. # Addresses decrease as the stack grows. while sp <= max: gcMark(gch, sp[]) - sp = cast[PPointer](cast[ByteAddress](sp) +% sizeof(pointer)) + sp = cast[PPointer](cast[int](sp) +% sizeof(pointer)) elif defined(ELATE): {.error: "stack marking code is to be written for this architecture".} @@ -344,8 +354,8 @@ elif stackIncreases: template forEachStackSlotAux(gch, gcMark: untyped) {.dirty.} = for stack in gch.stack.items(): - var max = cast[ByteAddress](gch.stack.bottom) - var sp = cast[ByteAddress](addr(registers)) -% sizeof(pointer) + var max = cast[int](gch.stack.bottom) + var sp = cast[int](addr(registers)) -% sizeof(pointer) while sp >=% max: gcMark(gch, cast[PPointer](sp)[]) sp = sp -% sizeof(pointer) @@ -373,15 +383,14 @@ else: gch.getActiveStack().setPosition(addr(registers)) if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. for stack in gch.stack.items(): - var max = cast[ByteAddress](stack.bottom) - var sp = cast[ByteAddress](addr(registers)) + var max = cast[int](stack.bottom) + var sp = cast[int](addr(registers)) when defined(amd64): if stack.isActiveStack(): # words within the jmp_buf structure may not be properly aligned. let regEnd = sp +% sizeof(registers) while sp <% regEnd: gcMark(gch, cast[PPointer](sp)[]) - gcMark(gch, cast[PPointer](sp +% sizeof(pointer) div 2)[]) sp = sp +% sizeof(pointer) # Make sure sp is word-aligned sp = sp and not (sizeof(pointer) - 1) @@ -405,7 +414,7 @@ else: # end of non-portable code # ---------------------------------------------------------------------------- -proc prepareDealloc(cell: PCell) = +proc prepareDealloc(cell: PCell) {.raises: [].} = when declared(useMarkForDebug): when useMarkForDebug: gcAssert(cell notin gch.marked, "Cell still alive!") @@ -422,10 +431,10 @@ proc prepareDealloc(cell: PCell) = decTypeSize(cell, t) proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = - ## Frees the thread local heap. Runs every finalizer if ``runFinalizers`` - ## is true. If ``allowGcAfterwards`` is true, a minimal amount of allocation + ## Frees the thread local heap. Runs every finalizer if `runFinalizers` + ## is true. If `allowGcAfterwards` is true, a minimal amount of allocation ## happens to ensure the GC can continue to work after the call - ## to ``deallocHeap``. + ## to `deallocHeap`. template deallocCell(x) = if isCell(x): # cast to PCell is correct here: @@ -448,7 +457,7 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = initGC() type - GlobalMarkerProc = proc () {.nimcall, benign.} + GlobalMarkerProc = proc () {.nimcall, benign, raises: [].} var globalMarkersLen {.exportc.}: int globalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] @@ -462,7 +471,7 @@ proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = inc globalMarkersLen else: cstderr.rawWrite("[GC] cannot register global variable; too many global variables") - quit 1 + rawQuit 1 proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = if threadLocalMarkersLen <= high(threadLocalMarkers): @@ -470,4 +479,4 @@ proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} inc threadLocalMarkersLen else: cstderr.rawWrite("[GC] cannot register thread local variable; too many thread local variables") - quit 1 + rawQuit 1 diff --git a/lib/system/gc_hooks.nim b/lib/system/gc_hooks.nim index 70f02e657..ace62eea0 100644 --- a/lib/system/gc_hooks.nim +++ b/lib/system/gc_hooks.nim @@ -24,7 +24,7 @@ proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = inc globalMarkersLen else: cstderr.rawWrite("[GC] cannot register global variable; too many global variables") - quit 1 + rawQuit 1 proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} = if threadLocalMarkersLen <= high(threadLocalMarkers): @@ -32,7 +32,7 @@ proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerproc.} inc threadLocalMarkersLen else: cstderr.rawWrite("[GC] cannot register thread local variable; too many thread local variables") - quit 1 + rawQuit 1 proc traverseGlobals*() = for i in 0..globalMarkersLen-1: diff --git a/lib/system/gc_interface.nim b/lib/system/gc_interface.nim index 9183e6d48..84145f33a 100644 --- a/lib/system/gc_interface.nim +++ b/lib/system/gc_interface.nim @@ -14,7 +14,7 @@ when hasAlloc: gcOptimizeSpace ## optimize for memory footprint when hasAlloc and not defined(js) and not usesDestructors: - proc GC_disable*() {.rtl, inl, benign.} + proc GC_disable*() {.rtl, inl, benign, raises: [].} ## Disables the GC. If called `n` times, `n` calls to `GC_enable` ## are needed to reactivate the GC. ## @@ -22,7 +22,7 @@ when hasAlloc and not defined(js) and not usesDestructors: ## the mark and sweep phase with ## `GC_disableMarkAndSweep <#GC_disableMarkAndSweep>`_. - proc GC_enable*() {.rtl, inl, benign.} + proc GC_enable*() {.rtl, inl, benign, raises: [].} ## Enables the GC again. proc GC_fullCollect*() {.rtl, benign.} @@ -54,7 +54,7 @@ when hasAlloc and not defined(js) and not usesDestructors: proc GC_unref*(x: string) {.magic: "GCunref", benign.} ## See the documentation of `GC_ref <#GC_ref,string>`_. - proc nimGC_setStackBottom*(theStackBottom: pointer) {.compilerRtl, noinline, benign.} + proc nimGC_setStackBottom*(theStackBottom: pointer) {.compilerRtl, noinline, benign, raises: [].} ## Expands operating GC stack range to `theStackBottom`. Does nothing ## if current stack bottom is already lower than `theStackBottom`. diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 271543445..c885a6893 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -27,7 +27,7 @@ when defined(memProfiler): proc nimProfile(requestedSize: int) when hasThreadSupport: - import sharedlist + import std/sharedlist type WalkOp = enum @@ -36,7 +36,7 @@ type # local waMarkPrecise # fast precise marking - Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign, raises: [].} # A ref type can have a finalizer that is called before the object's # storage is freed. @@ -90,21 +90,21 @@ template gcAssert(cond: bool, msg: string) = if not cond: cstderr.rawWrite "[GCASSERT] " cstderr.rawWrite msg - quit 1 + rawQuit 1 proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging result = usrToCell(c).typ -proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} = +proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline, compilerproc.} = dest[] = src proc internRefcount(p: pointer): int {.exportc: "getRefcount".} = @@ -115,10 +115,10 @@ when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap; size: int) {.benign.} -proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} -proc doOperation(p: pointer, op: WalkOp) {.benign.} -proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign, raises: [].} +proc forAllChildren(cell: PCell, op: WalkOp) {.benign, raises: [].} +proc doOperation(p: pointer, op: WalkOp) {.benign, raises: [].} +proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign, raises: [].} # we need the prototype here for debugging purposes when defined(nimGcRefLeak): @@ -217,7 +217,7 @@ proc initGC() = gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -229,7 +229,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -255,12 +255,11 @@ proc forAllChildren(cell: PCell, op: WalkOp) = forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: when not defined(nimSeqsV2): - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: - forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +% - GenericSeqSize), cell.typ.base, op) + forAllChildrenAux(cast[pointer](d +% align(GenericSeqSize, cell.typ.base.align) +% i *% cell.typ.base.size), cell.typ.base, op) else: discard proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = @@ -269,7 +268,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -305,20 +304,22 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = when defined(memProfiler): nimProfile(size) when not defined(nimSeqsV2): + {.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = # `newObj` already uses locks, so no need for them here. - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + let size = align(GenericSeqSize, typ.base.align) + len * typ.base.size result = newObj(typ, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len when defined(memProfiler): nimProfile(size) + {.pop.} proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = collectCT(gch, newsize + sizeof(Cell)) @@ -327,15 +328,17 @@ when not defined(nimSeqsV2): gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) - var elemSize = 1 - if ol.typ.kind != tyString: elemSize = ol.typ.base.size + var elemSize, elemAlign = 1 + if ol.typ.kind != tyString: + elemSize = ol.typ.base.size + elemAlign = ol.typ.base.align incTypeSize ol.typ, newsize - var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") when withBitvectors: incl(gch.allocated, res) when useCellIds: inc gch.idGenerator @@ -425,15 +428,6 @@ proc sweep(gch: var GcHeap) = if c.refcount == rcBlack: c.refcount = rcWhite else: freeCyclicCell(gch, c) -when false: - proc newGcInvariant*() = - for x in allObjects(gch.region): - if isCell(x): - var c = cast[PCell](x) - if c.typ == nil: - writeStackTrace() - quit 1 - proc markGlobals(gch: var GcHeap) = if gch.gcThreadId == 0: when defined(nimTracing): @@ -452,11 +446,10 @@ proc markGlobals(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: - var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? - var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) + var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) if objStart != nil: mark(gch, objStart) @@ -492,9 +485,10 @@ when not defined(useNimRtl): proc GC_disable() = inc(gch.recGcLock) proc GC_enable() = - if gch.recGcLock <= 0: - raise newException(AssertionError, - "API usage error: GC_enable called but GC is already enabled") + when defined(nimDoesntTrackDefects): + if gch.recGcLock <= 0: + raise newException(AssertionDefect, + "API usage error: GC_enable called but GC is already enabled") dec(gch.recGcLock) proc GC_setStrategy(strategy: GC_Strategy) = discard @@ -503,7 +497,7 @@ when not defined(useNimRtl): gch.cycleThreshold = InitialThreshold proc GC_disableMarkAndSweep() = - gch.cycleThreshold = high(gch.cycleThreshold)-1 + gch.cycleThreshold = high(typeof(gch.cycleThreshold))-1 # set to the max value to suppress the cycle detector when defined(nimTracing): diff --git a/lib/system/gc_regions.nim b/lib/system/gc_regions.nim index 6b1640ab1..d96de7eac 100644 --- a/lib/system/gc_regions.nim +++ b/lib/system/gc_regions.nim @@ -7,23 +7,12 @@ # # "Stack GC" for embedded devices or ultra performance requirements. +import std/private/syslocks -when defined(nimphpext): - proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - proc emalloc(size: int): pointer {.importc: "_emalloc".} - proc efree(mem: pointer) {.importc: "_efree".} - - proc osAllocPages(size: int): pointer {.inline.} = - emalloc(size) - - proc osTryAllocPages(size: int): pointer {.inline.} = - emalloc(size) - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - efree(p) +when defined(memProfiler): + proc nimProfile(requestedSize: int) {.benign.} -elif defined(useMalloc): +when defined(useMalloc): proc roundup(x, v: int): int {.inline.} = result = (x + (v-1)) and not (v-1) proc emalloc(size: int): pointer {.importc: "malloc", header: "<stdlib.h>".} @@ -99,16 +88,16 @@ type region: ptr MemRegion var - tlRegion {.threadVar.}: MemRegion -# tempStrRegion {.threadVar.}: MemRegion # not yet used + tlRegion {.threadvar.}: MemRegion +# tempStrRegion {.threadvar.}: MemRegion # not yet used -template withRegion*(r: MemRegion; body: untyped) = +template withRegion*(r: var MemRegion; body: untyped) = let oldRegion = tlRegion tlRegion = r try: body finally: - #r = tlRegion + r = tlRegion tlRegion = oldRegion template inc(p: pointer, s: int) = @@ -277,14 +266,13 @@ when false: setObstackPtr(obs) template withScratchRegion*(body: untyped) = - var scratch: MemRegion let oldRegion = tlRegion - tlRegion = scratch + tlRegion = MemRegion() try: body finally: + deallocAll() tlRegion = oldRegion - deallocAll(scratch) when false: proc joinRegion*(dest: var MemRegion; src: MemRegion) = @@ -334,20 +322,21 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(tlRegion, typ, size) when defined(memProfiler): nimProfile(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = - let size = roundup(addInt(mulInt(len, typ.base.size), GenericSeqSize), - MemAlign) + let size = roundup(align(GenericSeqSize, typ.base.align) + len * typ.base.size, MemAlign) result = rawNewSeq(tlRegion, typ, size) zeroMem(result, size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len proc newStr(typ: PNimType, len: int; init: bool): pointer {.compilerRtl.} = - let size = roundup(addInt(len, GenericSeqSize), MemAlign) + let size = roundup(len + GenericSeqSize, MemAlign) result = rawNewSeq(tlRegion, typ, size) if init: zeroMem(result, size) cast[PGenericSeq](result).len = 0 cast[PGenericSeq](result).reserved = len +{.pop.} proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = result = rawNewObj(tlRegion, typ, size) @@ -362,7 +351,8 @@ proc growObj(regionUnused: var MemRegion; old: pointer, newsize: int): pointer = result = rawNewSeq(sh.region[], typ, roundup(newsize, MemAlign)) let elemSize = if typ.kind == tyString: 1 else: typ.base.size - let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + let elemAlign = if typ.kind == tyString: 1 else: typ.base.align + let oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize zeroMem(result +! oldsize, newsize-oldsize) copyMem(result, old, oldsize) dealloc(sh.region[], old, roundup(oldsize, MemAlign)) @@ -448,5 +438,5 @@ proc getTotalMem*(r: MemRegion): int = proc nimGC_setStackBottom(theStackBottom: pointer) = discard -proc nimGCref(x: pointer) {.compilerProc.} = discard -proc nimGCunref(x: pointer) {.compilerProc.} = discard +proc nimGCref(x: pointer) {.compilerproc.} = discard +proc nimGCunref(x: pointer) {.compilerproc.} = discard diff --git a/lib/system/hti.nim b/lib/system/hti.nim index c20f132fb..a26aff982 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -40,7 +40,7 @@ type tyPointer, tyOpenArray, tyString, - tyCString, + tyCstring, tyForward, tyInt, tyInt8, @@ -59,7 +59,7 @@ type tyOwned, tyUnused1, tyUnused2, tyVarargsHidden, tyUncheckedArray, - tyProxyHidden, + tyErrorHidden, tyBuiltInTypeClassHidden, tyUserTypeClassHidden, tyUserTypeClassInstHidden, @@ -69,7 +69,7 @@ type tyAnythingHidden, tyStaticHidden, tyFromExprHidden, - tyOpt, + tyOptDeprecated, tyVoidHidden TNimNodeKind = enum nkNone, nkSlot, nkList, nkCase @@ -90,6 +90,7 @@ type when defined(gcHooks): head*: pointer size*: int + align*: int kind: TNimKind flags: set[TNimTypeFlag] base*: ptr TNimType @@ -97,6 +98,8 @@ type finalizer*: pointer # the finalizer for the type marker*: proc (p: pointer, op: int) {.nimcall, benign, tags: [], raises: [].} # marker proc for GC deepcopy: proc (p: pointer): pointer {.nimcall, benign, tags: [], raises: [].} + when defined(nimSeqsV2): + typeInfoV2*: pointer when defined(nimTypeNames): name: cstring nextType: ptr TNimType diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index adbda9fd6..3bf0b9893 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -30,12 +30,13 @@ when defined(createNimRtl): {.pragma: inl.} {.pragma: compilerRtl, compilerproc, exportc: "nimrtl_$1", dynlib.} elif defined(useNimRtl): - when defined(windows): - const nimrtl* = "nimrtl.dll" - elif defined(macosx): - const nimrtl* = "libnimrtl.dylib" - else: - const nimrtl* = "libnimrtl.so" + #[ + `{.rtl.}` should only be used for non-generic procs. + ]# + const nimrtl* = + when defined(windows): "nimrtl.dll" + elif defined(macosx): "libnimrtl.dylib" + else: "libnimrtl.so" {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, importc: "nimrtl_$1", dynlib: nimrtl.} @@ -44,13 +45,6 @@ else: {.pragma: inl, inline.} {.pragma: compilerRtl, compilerproc.} -when defined(nimlocks): - {.pragma: benign, gcsafe, locks: 0.} -else: - {.pragma: benign, gcsafe.} +{.pragma: benign, gcsafe.} -template since(version, body: untyped) {.dirty.} = - ## limitation: can't be used to annotate a template (eg typetraits.get), would - ## error: cannot attach a custom pragma. - when (NimMajor, NimMinor) >= version: - body +{.push sinkInference: on.} diff --git a/lib/system/indexerrors.nim b/lib/system/indexerrors.nim index 1b91789bd..6a8cb8a0a 100644 --- a/lib/system/indexerrors.nim +++ b/lib/system/indexerrors.nim @@ -1,4 +1,5 @@ # imported by other modules, unlike helpers.nim which is included +# xxx this is now included instead of imported, we should import instead template formatErrorIndexBound*[T](i, a, b: T): string = when defined(standalone): @@ -9,3 +10,6 @@ template formatErrorIndexBound*[T](i, a, b: T): string = template formatErrorIndexBound*[T](i, n: T): string = formatErrorIndexBound(i, 0, n) + +template formatFieldDefect*(f, discVal): string = + f & discVal & "'" diff --git a/lib/system/indices.nim b/lib/system/indices.nim new file mode 100644 index 000000000..f2bad2528 --- /dev/null +++ b/lib/system/indices.nim @@ -0,0 +1,164 @@ +when not defined(nimHasSystemRaisesDefect): + {.pragma: systemRaisesDefect.} + +type + BackwardsIndex* = distinct int ## Type that is constructed by `^` for + ## reversed array accesses. + ## (See `^ template <#^.t,int>`_) + +template `^`*(x: int): BackwardsIndex = BackwardsIndex(x) + ## Builtin `roof`:idx: operator that can be used for convenient array access. + ## `a[^x]` is a shortcut for `a[a.len-x]`. + ## + ## ```nim + ## let + ## a = [1, 3, 5, 7, 9] + ## b = "abcdefgh" + ## + ## echo a[^1] # => 9 + ## echo b[^2] # => g + ## ``` + +proc `[]`*[T](s: openArray[T]; i: BackwardsIndex): T {.inline, systemRaisesDefect.} = + system.`[]`(s, s.len - int(i)) + +proc `[]`*[Idx, T](a: array[Idx, T]; i: BackwardsIndex): T {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] +proc `[]`*(s: string; i: BackwardsIndex): char {.inline, systemRaisesDefect.} = s[s.len - int(i)] + +proc `[]`*[T](s: var openArray[T]; i: BackwardsIndex): var T {.inline, systemRaisesDefect.} = + system.`[]`(s, s.len - int(i)) +proc `[]`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex): var T {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] +proc `[]`*(s: var string; i: BackwardsIndex): var char {.inline, systemRaisesDefect.} = s[s.len - int(i)] + +proc `[]=`*[T](s: var openArray[T]; i: BackwardsIndex; x: T) {.inline, systemRaisesDefect.} = + system.`[]=`(s, s.len - int(i), x) +proc `[]=`*[Idx, T](a: var array[Idx, T]; i: BackwardsIndex; x: T) {.inline, systemRaisesDefect.} = + a[Idx(a.len - int(i) + int low(a))] = x +proc `[]=`*(s: var string; i: BackwardsIndex; x: char) {.inline, systemRaisesDefect.} = + s[s.len - int(i)] = x + +template `..^`*(a, b: untyped): untyped = + ## A shortcut for `.. ^` to avoid the common gotcha that a space between + ## '..' and '^' is required. + a .. ^b + +template `..<`*(a, b: untyped): untyped = + ## A shortcut for `a .. pred(b)`. + ## ```nim + ## for i in 5 ..< 9: + ## echo i # => 5; 6; 7; 8 + ## ``` + a .. (when b is BackwardsIndex: succ(b) else: pred(b)) + +template `[]`*(s: string; i: int): char = arrGet(s, i) +template `[]=`*(s: string; i: int; val: char) = arrPut(s, i, val) + +template `^^`(s, i: untyped): untyped = + (when i is BackwardsIndex: s.len - int(i) else: int(i)) + +template spliceImpl(s, a, L, b: typed): untyped = + # make room for additional elements or cut: + var shift = b.len - max(0,L) # ignore negative slice size + var newLen = s.len + shift + if shift > 0: + # enlarge: + setLen(s, newLen) + for i in countdown(newLen-1, a+b.len): movingCopy(s[i], s[i-shift]) + else: + for i in countup(a+b.len, newLen-1): movingCopy(s[i], s[i-shift]) + # cut down: + setLen(s, newLen) + # fill the hole: + for i in 0 ..< b.len: s[a+i] = b[i] + +proc `[]`*[T, U: Ordinal](s: string, x: HSlice[T, U]): string {.inline, systemRaisesDefect.} = + ## Slice operation for strings. + ## Returns the inclusive range `[s[x.a], s[x.b]]`: + ## ```nim + ## var s = "abcdef" + ## assert s[1..3] == "bcd" + ## ``` + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + result = newString(L) + for i in 0 ..< L: result[i] = s[i + a] + +proc `[]=`*[T, U: Ordinal](s: var string, x: HSlice[T, U], b: string) {.systemRaisesDefect.} = + ## Slice assignment for strings. + ## + ## If `b.len` is not exactly the number of elements that are referred to + ## by `x`, a `splice`:idx: is performed: + ## + runnableExamples: + var s = "abcdefgh" + s[1 .. ^2] = "xyz" + assert s == "axyzh" + + var a = s ^^ x.a + var L = (s ^^ x.b) - a + 1 + if L == b.len: + for i in 0..<L: s[i+a] = b[i] + else: + spliceImpl(s, a, L, b) + +proc `[]`*[Idx, T; U, V: Ordinal](a: array[Idx, T], x: HSlice[U, V]): seq[T] {.systemRaisesDefect.} = + ## Slice operation for arrays. + ## Returns the inclusive range `[a[x.a], a[x.b]]`: + ## ```nim + ## var a = [1, 2, 3, 4] + ## assert a[0..2] == @[1, 2, 3] + ## ``` + ## + ## See also: + ## * `toOpenArray(array[I, T];I,I) <#toOpenArray,array[I,T],I,I>`_ + let xa = a ^^ x.a + let L = (a ^^ x.b) - xa + 1 + result = newSeq[T](L) + for i in 0..<L: result[i] = a[Idx(i + xa)] + +proc `[]=`*[Idx, T; U, V: Ordinal](a: var array[Idx, T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} = + ## Slice assignment for arrays. + ## ```nim + ## var a = [10, 20, 30, 40, 50] + ## a[1..2] = @[99, 88] + ## assert a == [10, 99, 88, 40, 50] + ## ``` + let xa = a ^^ x.a + let L = (a ^^ x.b) - xa + 1 + if L == b.len: + for i in 0..<L: a[Idx(i + xa)] = b[i] + else: + sysFatal(RangeDefect, "different lengths for slice assignment") + +proc `[]`*[T; U, V: Ordinal](s: openArray[T], x: HSlice[U, V]): seq[T] {.systemRaisesDefect.} = + ## Slice operation for sequences. + ## Returns the inclusive range `[s[x.a], s[x.b]]`: + ## ```nim + ## var s = @[1, 2, 3, 4] + ## assert s[0..2] == @[1, 2, 3] + ## ``` + ## + ## See also: + ## * `toOpenArray(openArray[T];int,int) <#toOpenArray,openArray[T],int,int>`_ + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + newSeq(result, L) + for i in 0 ..< L: result[i] = s[i + a] + +proc `[]=`*[T; U, V: Ordinal](s: var seq[T], x: HSlice[U, V], b: openArray[T]) {.systemRaisesDefect.} = + ## Slice assignment for sequences. + ## + ## If `b.len` is not exactly the number of elements that are referred to + ## by `x`, a `splice`:idx: is performed. + runnableExamples: + var s = @"abcdefgh" + s[1 .. ^2] = @"xyz" + assert s == @"axyzh" + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + if L == b.len: + for i in 0 ..< L: s[i+a] = b[i] + else: + spliceImpl(s, a, L, b) diff --git a/lib/system/integerops.nim b/lib/system/integerops.nim new file mode 100644 index 000000000..4ef3594f1 --- /dev/null +++ b/lib/system/integerops.nim @@ -0,0 +1,132 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Integer arithmetic with overflow checking. Uses +# intrinsics or inline assembler. + +proc raiseOverflow {.compilerproc, noinline.} = + # a single proc to reduce code size to a minimum + sysFatal(OverflowDefect, "over- or underflow") + +proc raiseDivByZero {.compilerproc, noinline.} = + sysFatal(DivByZeroDefect, "division by zero") + +{.pragma: nimbaseH, importc, nodecl, noSideEffect, compilerproc.} + +when not defined(nimEmulateOverflowChecks): + # take the #define from nimbase.h + + proc nimAddInt(a, b: int, res: ptr int): bool {.nimbaseH.} + proc nimSubInt(a, b: int, res: ptr int): bool {.nimbaseH.} + proc nimMulInt(a, b: int, res: ptr int): bool {.nimbaseH.} + + proc nimAddInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + proc nimSubInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + proc nimMulInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + +# unary minus and 'abs' not required here anymore and are directly handled +# in the code generator. +# 'nimModInt' does exist in nimbase.h without check as we moved the +# check for 0 to the codgen. +proc nimModInt(a, b: int; res: ptr int): bool {.nimbaseH.} + +proc nimModInt64(a, b: int64; res: ptr int64): bool {.nimbaseH.} + +# Platform independent versions. + +template addImplFallback(name, T, U) {.dirty.} = + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) + cast[U](b)) + if (r xor a) >= T(0) or (r xor b) >= T(0): + res[] = r + else: + result = true + +addImplFallback(nimAddInt, int, uint) +addImplFallback(nimAddInt64, int64, uint64) + +template subImplFallback(name, T, U) {.dirty.} = + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) - cast[U](b)) + if (r xor a) >= 0 or (r xor not b) >= 0: + res[] = r + else: + result = true + +subImplFallback(nimSubInt, int, uint) +subImplFallback(nimSubInt64, int64, uint64) + +template mulImplFallback(name, T, U, conv) {.dirty.} = + # + # This code has been inspired by Python's source code. + # The native int product x*y is either exactly right or *way* off, being + # just the last n bits of the true product, where n is the number of bits + # in an int (the delivered product is the true product plus i*2**n for + # some integer i). + # + # The native float64 product x*y is subject to three + # rounding errors: on a sizeof(int)==8 box, each cast to double can lose + # info, and even on a sizeof(int)==4 box, the multiplication can lose info. + # But, unlike the native int product, it's not in *range* trouble: even + # if sizeof(int)==32 (256-bit ints), the product easily fits in the + # dynamic range of a float64. So the leading 50 (or so) bits of the float64 + # product are correct. + # + # We check these two ways against each other, and declare victory if + # they're approximately the same. Else, because the native int product is + # the only one that can lose catastrophic amounts of information, it's the + # native int product that must have overflowed. + # + when not declared(name): + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + let r = cast[T](cast[U](a) * cast[U](b)) + let floatProd = conv(a) * conv(b) + let resAsFloat = conv(r) + # Fast path for normal case: small multiplicands, and no info + # is lost in either method. + if resAsFloat == floatProd: + res[] = r + else: + # Somebody somewhere lost info. Close enough, or way off? Note + # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). + # The difference either is or isn't significant compared to the + # true value (of which floatProd is a good approximation). + + # abs(diff)/abs(prod) <= 1/32 iff + # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" + if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): + res[] = r + else: + result = true + +mulImplFallback(nimMulInt, int, uint, toFloat) +mulImplFallback(nimMulInt64, int64, uint64, toBiggestFloat) + + +template divImplFallback(name, T) {.dirty.} = + proc name(a, b: T; res: ptr T): bool {.compilerproc, inline.} = + # we moved the b == 0 case out into the codegen. + if a == low(T) and b == T(-1): + result = true + else: + res[] = a div b + +divImplFallback(nimDivInt, int) +divImplFallback(nimDivInt64, int64) + +proc raiseFloatInvalidOp {.compilerproc, noinline.} = + sysFatal(FloatInvalidOpDefect, "FPU operation caused a NaN result") + +proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = + if x > 0.0: + sysFatal(FloatOverflowDefect, "FPU operation caused an overflow") + else: + sysFatal(FloatUnderflowDefect, "FPU operations caused an underflow") diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index 53b287db0..125bee98f 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -1,34 +1,59 @@ -iterator items*[T](a: openArray[T]): T {.inline.} = +## Default iterators for some Nim types. + +when defined(nimPreviewSlimSystem): + import std/assertions + +when not defined(nimNoLentIterators): + template lent2(T): untyped = lent T +else: + template lent2(T): untyped = T + +template unCheckedInc(x) = + {.push overflowChecks: off.} + inc(x) + {.pop.} + +iterator items*[T: not char](a: openArray[T]): lent2 T {.inline.} = ## Iterates over each item of `a`. var i = 0 while i < len(a): yield a[i] - inc(i) + unCheckedInc(i) + +iterator items*[T: char](a: openArray[T]): T {.inline.} = + ## Iterates over each item of `a`. + # a VM bug currently prevents taking address of openArray[char] + # elements converted from a string (would fail in `tests/misc/thallo.nim`) + # in any case there's no performance advantage of returning char by address. + var i = 0 + while i < len(a): + yield a[i] + unCheckedInc(i) iterator mitems*[T](a: var openArray[T]): var T {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. var i = 0 while i < len(a): yield a[i] - inc(i) + unCheckedInc(i) iterator items*[IX, T](a: array[IX, T]): T {.inline.} = ## Iterates over each item of `a`. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield a[i] if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator mitems*[IX, T](a: var array[IX, T]): var T {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. - var i = low(IX) - if i <= high(IX): + when a.len > 0: + var i = low(IX) while true: yield a[i] if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator items*[T](a: set[T]): T {.inline.} = ## Iterates over each element of `a`. `items` iterates only over the @@ -36,156 +61,196 @@ iterator items*[T](a: set[T]): T {.inline.} = ## able to hold). var i = low(T).int while i <= high(T).int: - if T(i) in a: yield T(i) - inc(i) + when T is enum and not defined(js): + if cast[T](i) in a: yield cast[T](i) + else: + if T(i) in a: yield T(i) + unCheckedInc(i) iterator items*(a: cstring): char {.inline.} = ## Iterates over each item of `a`. - when defined(js): + runnableExamples: + from std/sequtils import toSeq + assert toSeq("abc\0def".cstring) == @['a', 'b', 'c'] + assert toSeq("abc".cstring) == @['a', 'b', 'c'] + #[ + assert toSeq(nil.cstring) == @[] # xxx fails with SIGSEGV + this fails with SIGSEGV; unclear whether we want to instead yield nothing + or pay a small price to check for `nil`, a benchmark is needed. Note that + other procs support `nil`. + ]# + template impl() = var i = 0 - var L = len(a) - while i < L: + let n = len(a) + while i < n: yield a[i] - inc(i) + unCheckedInc(i) + when defined(js): impl() else: - var i = 0 - while a[i] != '\0': - yield a[i] - inc(i) + when nimvm: + # xxx `cstring` should behave like c backend instead. + impl() + else: + var i = 0 + while a[i] != '\0': + yield a[i] + unCheckedInc(i) iterator mitems*(a: var cstring): var char {.inline.} = ## Iterates over each item of `a` so that you can modify the yielded value. - when defined(js): + # xxx this should give CT error in js RT. + runnableExamples: + from std/sugar import collect + var a = "abc\0def" + prepareMutation(a) + var b = a.cstring + let s = collect: + for bi in mitems(b): + if bi == 'b': bi = 'B' + bi + assert s == @['a', 'B', 'c'] + assert b == "aBc" + assert a == "aBc\0def" + + template impl() = var i = 0 - var L = len(a) - while i < L: + let n = len(a) + while i < n: yield a[i] - inc(i) + unCheckedInc(i) + when defined(js): impl() else: - var i = 0 - while a[i] != '\0': - yield a[i] - inc(i) + when nimvm: impl() + else: + var i = 0 + while a[i] != '\0': + yield a[i] + unCheckedInc(i) -iterator items*(E: typedesc[enum]): E = - ## Iterates over the values of the enum ``E``. +iterator items*[T: enum and Ordinal](E: typedesc[T]): T = + ## Iterates over the values of `E`. + ## See also `enumutils.items` for enums with holes. + runnableExamples: + type Goo = enum g0 = 2, g1, g2 + from std/sequtils import toSeq + assert Goo.toSeq == [g0, g1, g2] for v in low(E) .. high(E): yield v -iterator items*[T](s: HSlice[T, T]): T = +iterator items*[T: Ordinal](s: Slice[T]): T = ## Iterates over the slice `s`, yielding each value between `s.a` and `s.b` ## (inclusively). for x in s.a .. s.b: yield x iterator pairs*[T](a: openArray[T]): tuple[key: int, val: T] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. var i = 0 while i < len(a): yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator mpairs*[T](a: var openArray[T]): tuple[key: int, val: var T]{.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. var i = 0 while i < len(a): yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator pairs*[IX, T](a: array[IX, T]): tuple[key: IX, val: T] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - var i = low(IX) - if i <= high(IX): + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + when a.len > 0: + var i = low(IX) while true: yield (i, a[i]) if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator mpairs*[IX, T](a: var array[IX, T]): tuple[key: IX, val: var T] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. - var i = low(IX) - if i <= high(IX): + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. + when a.len > 0: + var i = low(IX) while true: yield (i, a[i]) if i >= high(IX): break - inc(i) + unCheckedInc(i) iterator pairs*[T](a: seq[T]): tuple[key: int, val: T] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. var i = 0 let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator mpairs*[T](a: var seq[T]): tuple[key: int, val: var T] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. var i = 0 let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator pairs*(a: string): tuple[key: int, val: char] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. var i = 0 let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") iterator mpairs*(a: var string): tuple[key: int, val: var char] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. var i = 0 let L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") iterator pairs*(a: cstring): tuple[key: int, val: char] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. when defined(js): var i = 0 var L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) else: var i = 0 while a[i] != '\0': yield (i, a[i]) - inc(i) + unCheckedInc(i) iterator mpairs*(a: var cstring): tuple[key: int, val: var char] {.inline.} = - ## Iterates over each item of `a`. Yields ``(index, a[index])`` pairs. - ## ``a[index]`` can be modified. + ## Iterates over each item of `a`. Yields `(index, a[index])` pairs. + ## `a[index]` can be modified. when defined(js): var i = 0 var L = len(a) while i < L: yield (i, a[i]) - inc(i) + unCheckedInc(i) else: var i = 0 while a[i] != '\0': yield (i, a[i]) - inc(i) + unCheckedInc(i) -iterator items*[T](a: seq[T]): T {.inline.} = +iterator items*[T](a: seq[T]): lent2 T {.inline.} = ## Iterates over each item of `a`. var i = 0 let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator mitems*[T](a: var seq[T]): var T {.inline.} = @@ -194,7 +259,7 @@ iterator mitems*[T](a: var seq[T]): var T {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the seq changed while iterating over it") iterator items*(a: string): char {.inline.} = @@ -203,7 +268,7 @@ iterator items*(a: string): char {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") iterator mitems*(a: var string): var char {.inline.} = @@ -212,7 +277,7 @@ iterator mitems*(a: var string): var char {.inline.} = let L = len(a) while i < L: yield a[i] - inc(i) + unCheckedInc(i) assert(len(a) == L, "the length of the string changed while iterating over it") @@ -220,24 +285,24 @@ iterator fields*[T: tuple|object](x: T): RootObj {. magic: "Fields", noSideEffect.} = ## Iterates over every field of `x`. ## - ## **Warning**: This really transforms the 'for' and unrolls the loop. - ## The current implementation also has a bug - ## that affects symbol binding in the loop body. + ## .. warning:: This really transforms the 'for' and unrolls the loop. + ## The current implementation also has a bug + ## that affects symbol binding in the loop body. runnableExamples: var t = (1, "foo") - for v in fields(t): v = default(type(v)) + for v in fields(t): v = default(typeof(v)) doAssert t == (0, "") iterator fields*[S:tuple|object, T:tuple|object](x: S, y: T): tuple[key: string, val: RootObj] {. magic: "Fields", noSideEffect.} = ## Iterates over every field of `x` and `y`. ## - ## **Warning**: This really transforms the 'for' and unrolls the loop. - ## The current implementation also has a bug that affects symbol binding - ## in the loop body. + ## .. warning:: This really transforms the 'for' and unrolls the loop. + ## The current implementation also has a bug that affects symbol binding + ## in the loop body. runnableExamples: var t1 = (1, "foo") - var t2 = default(type(t1)) + var t2 = default(typeof(t1)) for v1, v2 in fields(t1, t2): v2 = v1 doAssert t1 == t2 @@ -246,16 +311,16 @@ iterator fieldPairs*[T: tuple|object](x: T): tuple[key: string, val: RootObj] {. ## Iterates over every field of `x` returning their name and value. ## ## When you iterate over objects with different field types you have to use - ## the compile time ``when`` instead of a runtime ``if`` to select the code + ## the compile time `when` instead of a runtime `if` to select the code ## you want to run for each type. To perform the comparison use the `is ## operator <manual.html#generics-is-operator>`_. - ## Another way to do the same without ``when`` is to leave the task of + ## Another way to do the same without `when` is to leave the task of ## picking the appropriate code to a secondary proc which you overload for ## each field type and pass the `value` to. ## - ## **Warning**: This really transforms the 'for' and unrolls the loop. The - ## current implementation also has a bug that affects symbol binding in the - ## loop body. + ## .. warning:: This really transforms the 'for' and unrolls the loop. The + ## current implementation also has a bug that affects symbol binding in the + ## loop body. runnableExamples: type Custom = object @@ -274,9 +339,9 @@ iterator fieldPairs*[S: tuple|object, T: tuple|object](x: S, y: T): tuple[ magic: "FieldPairs", noSideEffect.} = ## Iterates over every field of `x` and `y`. ## - ## **Warning**: This really transforms the 'for' and unrolls the loop. - ## The current implementation also has a bug that affects symbol binding - ## in the loop body. + ## .. warning:: This really transforms the 'for' and unrolls the loop. + ## The current implementation also has a bug that affects symbol binding + ## in the loop body. runnableExamples: type Foo = object x1: int diff --git a/lib/system/iterators_1.nim b/lib/system/iterators_1.nim index 07c731c3b..d00e3f823 100644 --- a/lib/system/iterators_1.nim +++ b/lib/system/iterators_1.nim @@ -9,25 +9,33 @@ iterator countdown*[T](a, b: T, step: Positive = 1): T {.inline.} = ## ## `T` may be any ordinal type, `step` may only be positive. ## - ## **Note**: This fails to count to ``low(int)`` if T = int for + ## **Note**: This fails to count to `low(int)` if T = int for ## efficiency reasons. - ## - ## .. code-block:: Nim - ## for i in countdown(7, 3): - ## echo i # => 7; 6; 5; 4; 3 - ## - ## for i in countdown(9, 2, 3): - ## echo i # => 9; 6; 3 + runnableExamples: + import std/sugar + let x = collect(newSeq): + for i in countdown(7, 3): + i + + assert x == @[7, 6, 5, 4, 3] + + let y = collect(newseq): + for i in countdown(9, 2, 3): + i + assert y == @[9, 6, 3] when T is (uint|uint64): var res = a while res >= b: yield res if res == b: break dec(res, step) - elif T is IntLikeForCount: + elif T is IntLikeForCount and T is Ordinal: var res = int(a) while res >= int(b): - yield T(res) + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: + yield T(res) dec(res, step) else: var res = a @@ -35,148 +43,106 @@ iterator countdown*[T](a, b: T, step: Positive = 1): T {.inline.} = yield res dec(res, step) -when defined(nimNewRoof): - iterator countup*[T](a, b: T, step: Positive = 1): T {.inline.} = - ## Counts from ordinal value `a` to `b` (inclusive) with the given - ## step count. - ## - ## `T` may be any ordinal type, `step` may only be positive. - ## - ## **Note**: This fails to count to ``high(int)`` if T = int for - ## efficiency reasons. - ## - ## .. code-block:: Nim - ## for i in countup(3, 7): - ## echo i # => 3; 4; 5; 6; 7 - ## - ## for i in countup(2, 9, 3): - ## echo i # => 2; 5; 8 - mixin inc - when T is IntLikeForCount: - var res = int(a) - while res <= int(b): - yield T(res) - inc(res, step) - else: - var res = a - while res <= b: - yield res - inc(res, step) +iterator countup*[T](a, b: T, step: Positive = 1): T {.inline.} = + ## Counts from ordinal value `a` to `b` (inclusive) with the given + ## step count. + ## + ## `T` may be any ordinal type, `step` may only be positive. + ## + ## **Note**: This fails to count to `high(int)` if T = int for + ## efficiency reasons. + runnableExamples: + import std/sugar + let x = collect(newSeq): + for i in countup(3, 7): + i + + assert x == @[3, 4, 5, 6, 7] - iterator `..`*[T](a, b: T): T {.inline.} = - ## An alias for `countup(a, b, 1)`. - ## - ## See also: - ## * [..<](#..<.i,T,T) - ## - ## .. code-block:: Nim - ## for i in 3 .. 7: - ## echo i # => 3; 4; 5; 6; 7 - mixin inc - when T is IntLikeForCount: - var res = int(a) - while res <= int(b): + let y = collect(newseq): + for i in countup(2, 9, 3): + i + assert y == @[2, 5, 8] + mixin inc + when T is IntLikeForCount and T is Ordinal: + var res = int(a) + while res <= int(b): + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: yield T(res) - inc(res) - else: - var res = a - while res <= b: - yield res - inc(res) - - template dotdotImpl(t) {.dirty.} = - iterator `..`*(a, b: t): t {.inline.} = - ## A type specialized version of ``..`` for convenience so that - ## mixing integer types works better. - ## - ## See also: - ## * [..<](#..<.i,T,T) - var res = a - while res <= b: - yield res - inc(res) - - dotdotImpl(int64) - dotdotImpl(int32) - dotdotImpl(uint64) - dotdotImpl(uint32) - - iterator `..<`*[T](a, b: T): T {.inline.} = - mixin inc - var i = a - while i < b: - yield i - inc i + inc(res, step) + else: + var res = a + while res <= b: + yield res + inc(res, step) - template dotdotLessImpl(t) {.dirty.} = - iterator `..<`*(a, b: t): t {.inline.} = - ## A type specialized version of ``..<`` for convenience so that - ## mixing integer types works better. - var res = a - while res < b: - yield res - inc(res) +iterator `..`*[T](a, b: T): T {.inline.} = + ## An alias for `countup(a, b, 1)`. + ## + ## See also: + ## * [..<](#..<.i,T,T) + runnableExamples: + import std/sugar - dotdotLessImpl(int64) - dotdotLessImpl(int32) - dotdotLessImpl(uint64) - dotdotLessImpl(uint32) + let x = collect(newSeq): + for i in 3 .. 7: + i -else: # not defined(nimNewRoof) - iterator countup*[S, T](a: S, b: T, step = 1): T {.inline.} = - ## Counts from ordinal value `a` up to `b` (inclusive) with the given - ## step count. - ## - ## `S`, `T` may be any ordinal type, `step` may only be positive. - ## - ## **Note**: This fails to count to ``high(int)`` if T = int for - ## efficiency reasons. - ## - ## .. code-block:: Nim - ## for i in countup(3, 7): - ## echo i # => 3; 4; 5; 6; 7 - ## - ## for i in countup(2, 9, 3): - ## echo i # => 2; 5; 8 - when T is IntLikeForCount: - var res = int(a) - while res <= int(b): + assert x == @[3, 4, 5, 6, 7] + mixin inc + when T is IntLikeForCount and T is Ordinal: + var res = int(a) + while res <= int(b): + when defined(nimHasCastExtendedVm): + yield cast[T](res) + else: yield T(res) - inc(res, step) - else: - var res = T(a) - while res <= b: - yield res - inc(res, step) + inc(res) + else: + var res = a + while res <= b: + yield res + inc(res) - iterator `..`*[S, T](a: S, b: T): T {.inline.} = - ## An alias for `countup(a, b, 1)`. +template dotdotImpl(t) {.dirty.} = + iterator `..`*(a, b: t): t {.inline.} = + ## A type specialized version of `..` for convenience so that + ## mixing integer types works better. ## ## See also: ## * [..<](#..<.i,T,T) - ## - ## .. code-block:: Nim - ## for i in 3 .. 7: - ## echo i # => 3; 4; 5; 6; 7 - mixin inc - when T is IntLikeForCount: - var res = int(a) - while res <= int(b): - yield T(res) - inc(res) - else: - var res = T(a) - while res <= b: - yield res - inc(res) + var res = a + while res <= b: + yield res + inc(res) + +dotdotImpl(int64) +dotdotImpl(int32) +dotdotImpl(uint64) +dotdotImpl(uint32) + +iterator `..<`*[T](a, b: T): T {.inline.} = + mixin inc + var i = a + while i < b: + yield i + inc i - iterator `..<`*[S, T](a: S, b: T): T {.inline.} = - mixin inc - var i = T(a) - while i < b: - yield i - inc i +template dotdotLessImpl(t) {.dirty.} = + iterator `..<`*(a, b: t): t {.inline.} = + ## A type specialized version of `..<` for convenience so that + ## mixing integer types works better. + var res = a + while res < b: + yield res + inc(res) +dotdotLessImpl(int64) +dotdotLessImpl(int32) +dotdotLessImpl(uint64) +dotdotLessImpl(uint32) iterator `||`*[S, T](a: S, b: T, annotation: static string = "parallel for"): T {. inline, magic: "OmpParFor", sideEffect.} = @@ -189,9 +155,9 @@ iterator `||`*[S, T](a: S, b: T, annotation: static string = "parallel for"): T ## for further information. ## ## Note that the compiler maps that to - ## the ``#pragma omp parallel for`` construct of `OpenMP`:idx: and as + ## the `#pragma omp parallel for` construct of `OpenMP`:idx: and as ## such isn't aware of the parallelism in your code! Be careful! Later - ## versions of ``||`` will get proper support by Nim's code generator + ## versions of `||` will get proper support by Nim's code generator ## and GC. discard @@ -207,8 +173,8 @@ iterator `||`*[S, T](a: S, b: T, step: Positive, annotation: static string = "pa ## for further information. ## ## Note that the compiler maps that to - ## the ``#pragma omp parallel for`` construct of `OpenMP`:idx: and as + ## the `#pragma omp parallel for` construct of `OpenMP`:idx: and as ## such isn't aware of the parallelism in your code! Be careful! Later - ## versions of ``||`` will get proper support by Nim's code generator + ## versions of `||` will get proper support by Nim's code generator ## and GC. discard diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 2f3512cfc..5599240fd 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -7,7 +7,8 @@ # distribution, for details about the copyright. # -import system/indexerrors +include system/indexerrors +import std/private/miscdollars proc log*(s: cstring) {.importc: "console.log", varargs, nodecl.} @@ -48,7 +49,7 @@ proc nimCharToStr(x: char): string {.compilerproc.} = result[0] = x proc isNimException(): bool {.asmNoStackFrame.} = - asm "return `lastJSError` && `lastJSError`.m_type;" + {.emit: "return `lastJSError` && `lastJSError`.m_type;".} proc getCurrentException*(): ref Exception {.compilerRtl, benign.} = if isNimException(): result = cast[ref Exception](lastJSError) @@ -68,9 +69,16 @@ proc getCurrentExceptionMsg*(): string = return $msg return "" +proc setCurrentException*(exc: ref Exception) = + lastJSError = cast[PJSError](exc) + +proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} = + ## Used to set up exception handling for closure iterators + setCurrentException(e) + proc auxWriteStackTrace(f: PCallFrame): string = type - TempFrame = tuple[procname: cstring, line: int] + TempFrame = tuple[procname: cstring, line: int, filename: cstring] var it = f i = 0 @@ -79,6 +87,7 @@ proc auxWriteStackTrace(f: PCallFrame): string = while it != nil and i <= high(tempFrames): tempFrames[i].procname = it.procname tempFrames[i].line = it.line + tempFrames[i].filename = it.filename inc(i) inc(total) it = it.prev @@ -92,10 +101,9 @@ proc auxWriteStackTrace(f: PCallFrame): string = add(result, $(total-i)) add(result, " calls omitted) ...\n") for j in countdown(i-1, 0): + result.toLocation($tempFrames[j].filename, tempFrames[j].line, 0) + add(result, " at ") add(result, tempFrames[j].procname) - if tempFrames[j].line > 0: - add(result, ", line: ") - add(result, $tempFrames[j].line) add(result, "\n") proc rawWriteStackTrace(): string = @@ -104,6 +112,11 @@ proc rawWriteStackTrace(): string = else: result = "No stack traceback available\n" +proc writeStackTrace() = + var trace = rawWriteStackTrace() + trace.setLen(trace.len - 1) + echo trace + proc getStackTrace*(): string = rawWriteStackTrace() proc getStackTrace*(e: ref Exception): string = e.trace @@ -121,7 +134,8 @@ proc unhandledException(e: ref Exception) {. when NimStackTrace: add(buf, rawWriteStackTrace()) let cbuf = cstring(buf) - framePtr = nil + when NimStackTrace: + framePtr = nil {.emit: """ if (typeof(Error) !== "undefined") { throw new Error(`cbuf`); @@ -138,35 +152,35 @@ proc raiseException(e: ref Exception, ename: cstring) {. unhandledException(e) when NimStackTrace: e.trace = rawWriteStackTrace() - asm "throw `e`;" + {.emit: "throw `e`;".} proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: - raise newException(ReraiseError, "no exception to reraise") + raise newException(ReraiseDefect, "no exception to reraise") else: if excHandler == 0: if isNimException(): unhandledException(cast[ref Exception](lastJSError)) - asm "throw lastJSError;" + {.emit: "throw lastJSError;".} proc raiseOverflow {.exportc: "raiseOverflow", noreturn, compilerproc.} = - raise newException(OverflowError, "over- or underflow") + raise newException(OverflowDefect, "over- or underflow") proc raiseDivByZero {.exportc: "raiseDivByZero", noreturn, compilerproc.} = - raise newException(DivByZeroError, "division by zero") + raise newException(DivByZeroDefect, "division by zero") proc raiseRangeError() {.compilerproc, noreturn.} = - raise newException(RangeError, "value out of range") + raise newException(RangeDefect, "value out of range") proc raiseIndexError(i, a, b: int) {.compilerproc, noreturn.} = - raise newException(IndexError, formatErrorIndexBound(int(i), int(a), int(b))) + raise newException(IndexDefect, formatErrorIndexBound(int(i), int(a), int(b))) -proc raiseFieldError(f: string) {.compilerproc, noreturn.} = - raise newException(FieldError, f) +proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noreturn.} = + raise newException(FieldDefect, formatFieldDefect(f, discVal)) proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = {}; for (var i = 0; i < arguments.length; ++i) { var x = arguments[i]; @@ -179,13 +193,12 @@ proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = } } return result; - """ + """.} proc makeNimstrLit(c: cstring): string {.asmNoStackFrame, compilerproc.} = {.emit: """ - var ln = `c`.length; - var result = new Array(ln); - for (var i = 0; i < ln; ++i) { + var result = []; + for (var i = 0; i < `c`.length; ++i) { result[i] = `c`.charCodeAt(i); } return result; @@ -268,62 +281,64 @@ proc toJSStr(s: string): cstring {.compilerproc.} = result = join(res) proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} = - asm """ - return new Array(`len`); - """ + {.emit: """ + var result = new Array(`len`); + for (var i = 0; i < `len`; i++) {result[i] = 0;} + return result; + """.} proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} = # argument type is a fake - asm """ + {.emit: """ var result = 0; for (var elem in `a`) { ++result; } return result; - """ + """.} proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ for (var elem in `a`) { if (!`b`[elem]) return false; } for (var elem in `b`) { if (!`a`[elem]) return false; } return true; - """ + """.} proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ for (var elem in `a`) { if (!`b`[elem]) return false; } return true; - """ + """.} proc SetLt(a, b: int): bool {.compilerproc.} = result = SetLe(a, b) and not SetEq(a, b) proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { if (`b`[elem]) { result[elem] = true; } } return result; - """ + """.} proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { result[elem] = true; } for (var elem in `b`) { result[elem] = true; } return result; - """ + """.} proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var result = {}; for (var elem in `a`) { if (!`b`[elem]) { result[elem] = true; } } return result; - """ + """.} proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`a` == `b`) return 0; if (!`a`) return -1; if (!`b`) return 1; @@ -332,13 +347,18 @@ proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerproc.} = if (result != 0) return result; } return `a`.length - `b`.length; - """ + """.} proc cmp(x, y: string): int = - return cmpStrings(x, y) + when nimvm: + if x == y: result = 0 + elif x < y: result = -1 + else: result = 1 + else: + result = cmpStrings(x, y) proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`a` == `b`) return true; if (`a` === null && `b`.length == 0) return true; if (`b` === null && `a`.length == 0) return true; @@ -348,29 +368,29 @@ proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerproc.} = for (var i = 0; i < alen; ++i) if (`a`[i] != `b`[i]) return false; return true; - """ + """.} when defined(kwin): proc rawEcho {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { buf += `toJSStr`(arguments[i]); } print(buf); - """ + """.} elif not defined(nimOldEcho): proc ewriteln(x: cstring) = log(x) proc rawEcho {.compilerproc, asmNoStackFrame.} = - asm """ + {.emit: """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { buf += `toJSStr`(arguments[i]); } console.log(buf); - """ + """.} else: proc ewriteln(x: cstring) = @@ -397,78 +417,85 @@ else: """.} # Arithmetic: +proc checkOverflowInt(a: int) {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`a` > 2147483647 || `a` < -2147483648) `raiseOverflow`(); + """.} + proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` + `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + `checkOverflowInt`(result); return result; - """ + """.} proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` - `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + `checkOverflowInt`(result); return result; - """ + """.} proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ var result = `a` * `b`; - if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); + `checkOverflowInt`(result); return result; - """ + """.} proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); return Math.trunc(`a` / `b`); - """ + """.} proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ + {.emit: """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); return Math.trunc(`a` % `b`); - """ + """.} + +proc checkOverflowInt64(a: int64) {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`a` > 9223372036854775807n || `a` < -9223372036854775808n) `raiseOverflow`(); + """.} -proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc addInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` + `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); + `checkOverflowInt64`(result); return result; - """ + """.} -proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc subInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` - `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); + `checkOverflowInt64`(result); return result; - """ + """.} -proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ +proc mulInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ var result = `a` * `b`; - if (result > 9223372036854775807 - || result < -9223372036854775808) `raiseOverflow`(); + `checkOverflowInt64`(result); return result; - """ + """.} -proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.trunc(`a` / `b`); - """ +proc divInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`b` == 0n) `raiseDivByZero`(); + if (`b` == -1n && `a` == 9223372036854775807n) `raiseOverflow`(); + return `a` / `b`; + """.} -proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = - asm """ - if (`b` == 0) `raiseDivByZero`(); - if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); - return Math.trunc(`a` % `b`); - """ +proc modInt64(a, b: int64): int64 {.asmNoStackFrame, compilerproc.} = + {.emit: """ + if (`b` == 0n) `raiseDivByZero`(); + if (`b` == -1n && `a` == 9223372036854775807n) `raiseOverflow`(); + return `a` % `b`; + """.} proc negInt(a: int): int {.compilerproc.} = result = a*(-1) @@ -482,34 +509,12 @@ proc absInt(a: int): int {.compilerproc.} = proc absInt64(a: int64): int64 {.compilerproc.} = result = if a < 0: a*(-1) else: a -when not defined(nimNoZeroExtendMagic): - proc ze*(a: int): int {.compilerproc.} = - result = a - - proc ze64*(a: int64): int64 {.compilerproc.} = - result = a - - proc toU8*(a: int): int8 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU16*(a: int): int16 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU32*(a: int64): int32 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b -proc chckNilDisp(p: pointer) {.compilerproc.} = +proc chckNilDisp(p: JSRef) {.compilerproc.} = if p == nil: - sysFatal(NilAccessError, "cannot dispatch; dispatcher is nil") + sysFatal(NilAccessDefect, "cannot dispatch; dispatcher is nil") include "system/hti" @@ -525,22 +530,22 @@ proc nimCopyAux(dest, src: JSRef, n: ptr TNimNode) {.compilerproc.} = case n.kind of nkNone: sysAssert(false, "nimCopyAux") of nkSlot: - asm """ + {.emit: """ `dest`[`n`.offset] = nimCopy(`dest`[`n`.offset], `src`[`n`.offset], `n`.typ); - """ + """.} of nkList: - asm """ + {.emit: """ for (var i = 0; i < `n`.sons.length; i++) { nimCopyAux(`dest`, `src`, `n`.sons[i]); } - """ + """.} of nkCase: - asm """ + {.emit: """ `dest`[`n`.offset] = nimCopy(`dest`[`n`.offset], `src`[`n`.offset], `n`.typ); for (var i = 0; i < `n`.sons.length; ++i) { nimCopyAux(`dest`, `src`, `n`.sons[i][1]); } - """ + """.} proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = case ti.kind @@ -548,9 +553,9 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = if not isFatPointer(ti): result = src else: - asm "`result` = [`src`[0], `src`[1]];" + {.emit: "`result` = [`src`[0], `src`[1]];".} of tySet: - asm """ + {.emit: """ if (`dest` === null || `dest` === undefined) { `dest` = {}; } @@ -559,80 +564,72 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = } for (var key in `src`) { `dest`[key] = `src`[key]; } `result` = `dest`; - """ + """.} of tyTuple, tyObject: if ti.base != nil: result = nimCopy(dest, src, ti.base) elif ti.kind == tyObject: - asm "`result` = (`dest` === null || `dest` === undefined) ? {m_type: `ti`} : `dest`;" + {.emit: "`result` = (`dest` === null || `dest` === undefined) ? {m_type: `ti`} : `dest`;".} else: - asm "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;" + {.emit: "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;".} nimCopyAux(result, src, ti.node) - of tySequence, tyArrayConstr, tyOpenArray, tyArray: - asm """ + of tyArrayConstr, tyArray: + # In order to prevent a type change (TypedArray -> Array) and to have better copying performance, + # arrays constructors are considered separately + {.emit: """ + if(ArrayBuffer.isView(`src`)) { + if(`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new `src`.constructor(`src`); + } else { + `dest`.set(`src`, 0); + } + `result` = `dest`; + } else { + if (`src` === null) { + `result` = null; + } + else { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new Array(`src`.length); + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } + } + } + """.} + of tySequence, tyOpenArray: + {.emit: """ if (`src` === null) { `result` = null; } else { - if (`dest` === null || `dest` === undefined) { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { `dest` = new Array(`src`.length); } - else { - `dest`.length = `src`.length; - } `result` = `dest`; for (var i = 0; i < `src`.length; ++i) { `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); } } - """ + """.} of tyString: - asm """ + {.emit: """ if (`src` !== null) { `result` = `src`.slice(0); } - """ + """.} else: result = src -proc genericReset(x: JSRef, ti: PNimType): JSRef {.compilerproc.} = - asm "`result` = null;" - case ti.kind - of tyPtr, tyRef, tyVar, tyNil: - if isFatPointer(ti): - asm """ - `result` = [null, 0]; - """ - of tySet: - asm """ - `result` = {}; - """ - of tyTuple, tyObject: - if ti.kind == tyObject: - asm "`result` = {m_type: `ti`};" - else: - asm "`result` = {};" - of tySequence, tyOpenArray: - asm """ - `result` = []; - """ - of tyArrayConstr, tyArray: - asm """ - `result` = new Array(`x`.length); - for (var i = 0; i < `x`.length; ++i) { - `result`[i] = genericReset(`x`[i], `ti`.base); - } - """ - else: - discard - proc arrayConstr(len: int, value: JSRef, typ: PNimType): JSRef {. asmNoStackFrame, compilerproc.} = # types are fake - asm """ + {.emit: """ var result = new Array(`len`); for (var i = 0; i < `len`; ++i) result[i] = nimCopy(null, `value`, `typ`); return result; - """ + """.} proc chckIndx(i, a, b: int): int {.compilerproc.} = if i >= a and i <= b: return i @@ -648,7 +645,7 @@ proc chckObj(obj, subclass: PNimType) {.compilerproc.} = if x == subclass: return # optimized fast path while x != subclass: if x == nil: - raise newException(ObjectConversionError, "invalid object conversion") + raise newException(ObjectConversionDefect, "invalid object conversion") x = x.base proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = @@ -661,11 +658,12 @@ proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = return true proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} = - asm "`x`.push(`c`);" + {.emit: "`x`.push(`c`);".} {.pop.} proc tenToThePowerOf(b: int): BiggestFloat = + # xxx deadcode var b = b var a = 10.0 result = 1.0 @@ -679,86 +677,75 @@ proc tenToThePowerOf(b: int): BiggestFloat = const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} -# XXX use JS's native way here -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {. - compilerproc.} = - var - esign = 1.0 - sign = 1.0 - i = start - exponent: int - flags: int - number = 0.0 + +proc parseFloatNative(a: openarray[char]): float = + var str = "" + for x in a: + str.add x + + let cstr = cstring str + + {.emit: """ + `result` = Number(`cstr`); + """.} + +proc nimParseBiggestFloat(s: openarray[char], number: var BiggestFloat): int {.compilerproc.} = + var sign: bool + var i = 0 if s[i] == '+': inc(i) elif s[i] == '-': - sign = -1.0 + sign = true inc(i) if s[i] == 'N' or s[i] == 'n': if s[i+1] == 'A' or s[i+1] == 'a': if s[i+2] == 'N' or s[i+2] == 'n': if s[i+3] notin IdentChars: number = NaN - return i+3 - start + return i+3 return 0 if s[i] == 'I' or s[i] == 'i': if s[i+1] == 'N' or s[i+1] == 'n': if s[i+2] == 'F' or s[i+2] == 'f': if s[i+3] notin IdentChars: - number = Inf*sign - return i+3 - start + number = if sign: -Inf else: Inf + return i+3 return 0 - while s[i] in {'0'..'9'}: - # Read integer part - flags = flags or 1 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) + + var buf: string + # we could also use an `array[char, N]` buffer to avoid reallocs, or + # use a 2-pass algorithm that first computes the length. + if sign: buf.add '-' + template addInc = + buf.add s[i] inc(i) + template eatUnderscores = while s[i] == '_': inc(i) - # Decimal? - if s[i] == '.': - var hd = 1.0 + while s[i] in {'0'..'9'}: # Read integer part + buf.add s[i] inc(i) - while s[i] in {'0'..'9'}: - # Read fractional part - flags = flags or 2 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - hd = hd * 10.0 - inc(i) - while s[i] == '_': inc(i) - number = number / hd # this complicated way preserves precision + eatUnderscores() + if s[i] == '.': # Decimal? + addInc() + while s[i] in {'0'..'9'}: # Read fractional part + addInc() + eatUnderscores() # Again, read integer and fractional part - if flags == 0: return 0 - # Exponent? - if s[i] in {'e', 'E'}: - inc(i) - if s[i] == '+': - inc(i) - elif s[i] == '-': - esign = -1.0 - inc(i) - if s[i] notin {'0'..'9'}: - return 0 + if buf.len == ord(sign): return 0 + if s[i] in {'e', 'E'}: # Exponent? + addInc() + if s[i] == '+': inc(i) + elif s[i] == '-': addInc() + if s[i] notin {'0'..'9'}: return 0 while s[i] in {'0'..'9'}: - exponent = exponent * 10 + ord(s[i]) - ord('0') - inc(i) - while s[i] == '_': inc(i) - # Calculate Exponent - let hd = tenToThePowerOf(exponent) - if esign > 0.0: number = number * hd - else: number = number / hd - # evaluate sign - number = number * sign - result = i - start - -when defined(nodejs): - # Deprecated. Use `alert` defined in dom.nim - proc alert*(s: cstring) {.importc: "console.log", nodecl, deprecated.} -else: - # Deprecated. Use `alert` defined in dom.nim - proc alert*(s: cstring) {.importc, nodecl, deprecated.} + addInc() + eatUnderscores() + number = parseFloatNative(buf) + result = i # Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce # 'Math.trunc' for Nim's ``div`` and ``mod`` operators: -const jsMathTrunc = """ +when defined(nimJsMathTruncPolyfill): + {.emit: """ if (!Math.trunc) { Math.trunc = function(v) { v = +v; @@ -766,5 +753,16 @@ if (!Math.trunc) { return (v - v % 1) || (v < 0 ? -0 : v === 0 ? v : 0); }; } -""" -when not defined(nodejs): {.emit: jsMathTrunc .} +""".} + +proc cmpClosures(a, b: JSRef): bool {.compilerproc, asmNoStackFrame.} = + # Both `a` and `b` need to be a closure + {.emit: """ + if (`a` !== null && `a`.ClP_0 !== undefined && + `b` !== null && `b`.ClP_0 !== undefined) { + return `a`.ClP_0 == `b`.ClP_0 && `a`.ClE_0 == `b`.ClE_0; + } else { + return `a` == `b`; + } + """ + .} diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim index df4df2c15..a94d0995c 100644 --- a/lib/system/memalloc.nim +++ b/lib/system/memalloc.nim @@ -1,38 +1,51 @@ when notJSnotNims: proc zeroMem*(p: pointer, size: Natural) {.inline, noSideEffect, - tags: [], locks: 0, raises: [].} - ## Overwrites the contents of the memory at ``p`` with the value 0. + tags: [], raises: [].} + ## Overwrites the contents of the memory at `p` with the value 0. ## - ## Exactly ``size`` bytes will be overwritten. Like any procedure + ## Exactly `size` bytes will be overwritten. Like any procedure ## dealing with raw memory this is **unsafe**. proc copyMem*(dest, source: pointer, size: Natural) {.inline, benign, - tags: [], locks: 0, raises: [].} - ## Copies the contents from the memory at ``source`` to the memory - ## at ``dest``. - ## Exactly ``size`` bytes will be copied. The memory + tags: [], raises: [].} + ## Copies the contents from the memory at `source` to the memory + ## at `dest`. + ## Exactly `size` bytes will be copied. The memory ## regions may not overlap. Like any procedure dealing with raw ## memory this is **unsafe**. proc moveMem*(dest, source: pointer, size: Natural) {.inline, benign, - tags: [], locks: 0, raises: [].} - ## Copies the contents from the memory at ``source`` to the memory - ## at ``dest``. + tags: [], raises: [].} + ## Copies the contents from the memory at `source` to the memory + ## at `dest`. ## - ## Exactly ``size`` bytes will be copied. The memory - ## regions may overlap, ``moveMem`` handles this case appropriately - ## and is thus somewhat more safe than ``copyMem``. Like any procedure + ## Exactly `size` bytes will be copied. The memory + ## regions may overlap, `moveMem` handles this case appropriately + ## and is thus somewhat more safe than `copyMem`. Like any procedure ## dealing with raw memory this is still **unsafe**, though. proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect, - tags: [], locks: 0, raises: [].} - ## Compares the memory blocks ``a`` and ``b``. ``size`` bytes will + tags: [], raises: [].} + ## Compares the memory blocks `a` and `b`. `size` bytes will ## be compared. ## ## If the blocks are equal, `true` is returned, `false` ## otherwise. Like any procedure dealing with raw memory this is ## **unsafe**. + proc cmpMem*(a, b: pointer, size: Natural): int {.inline, noSideEffect, + tags: [], raises: [].} + ## Compares the memory blocks `a` and `b`. `size` bytes will + ## be compared. + ## + ## Returns: + ## * a value less than zero, if `a < b` + ## * a value greater than zero, if `a > b` + ## * zero, if `a == b` + ## + ## Like any procedure dealing with raw memory this is + ## **unsafe**. + when hasAlloc and not defined(js): proc allocImpl*(size: Natural): pointer {.noconv, rtl, tags: [], benign, raises: [].} @@ -67,7 +80,7 @@ when hasAlloc and not defined(js): when defined(nimAllocStats): var stats: AllocStats - template incStat(what: untyped) = inc stats.what + template incStat(what: untyped) = atomicInc stats.what proc getAllocStats*(): AllocStats = stats else: @@ -75,23 +88,23 @@ when hasAlloc and not defined(js): proc getAllocStats*(): AllocStats = discard template alloc*(size: Natural): pointer = - ## Allocates a new memory block with at least ``size`` bytes. + ## Allocates a new memory block with at least `size` bytes. ## - ## The block has to be freed with `realloc(block, 0) <#realloc,pointer,Natural>`_ + ## The block has to be freed with `realloc(block, 0) <#realloc.t,pointer,Natural>`_ ## or `dealloc(block) <#dealloc,pointer>`_. ## The block is not initialized, so reading ## from it before writing to it is undefined behaviour! ## ## The allocated memory belongs to its allocating thread! - ## Use `allocShared <#allocShared,Natural>`_ to allocate from a shared heap. + ## Use `allocShared <#allocShared.t,Natural>`_ to allocate from a shared heap. ## ## See also: - ## * `alloc0 <#alloc0,Natural>`_ + ## * `alloc0 <#alloc0.t,Natural>`_ incStat(allocCount) allocImpl(size) proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline, benign, raises: [].} = - ## Allocates a new memory block with at least ``T.sizeof * size`` bytes. + ## Allocates a new memory block with at least `T.sizeof * size` bytes. ## ## The block has to be freed with `resize(block, 0) <#resize,ptr.T,Natural>`_ ## or `dealloc(block) <#dealloc,pointer>`_. @@ -106,20 +119,20 @@ when hasAlloc and not defined(js): cast[ptr T](alloc(T.sizeof * size)) template alloc0*(size: Natural): pointer = - ## Allocates a new memory block with at least ``size`` bytes. + ## Allocates a new memory block with at least `size` bytes. ## - ## The block has to be freed with `realloc(block, 0) <#realloc,pointer,Natural>`_ + ## The block has to be freed with `realloc(block, 0) <#realloc.t,pointer,Natural>`_ ## or `dealloc(block) <#dealloc,pointer>`_. ## The block is initialized with all bytes containing zero, so it is - ## somewhat safer than `alloc <#alloc,Natural>`_. + ## somewhat safer than `alloc <#alloc.t,Natural>`_. ## ## The allocated memory belongs to its allocating thread! - ## Use `allocShared0 <#allocShared0,Natural>`_ to allocate from a shared heap. + ## Use `allocShared0 <#allocShared0.t,Natural>`_ to allocate from a shared heap. incStat(allocCount) alloc0Impl(size) proc create*(T: typedesc, size = 1.Positive): ptr T {.inline, benign, raises: [].} = - ## Allocates a new memory block with at least ``T.sizeof * size`` bytes. + ## Allocates a new memory block with at least `T.sizeof * size` bytes. ## ## The block has to be freed with `resize(block, 0) <#resize,ptr.T,Natural>`_ ## or `dealloc(block) <#dealloc,pointer>`_. @@ -134,13 +147,13 @@ when hasAlloc and not defined(js): ## Grows or shrinks a given memory block. ## ## If `p` is **nil** then a new memory block is returned. - ## In either way the block has at least ``newSize`` bytes. - ## If ``newSize == 0`` and `p` is not **nil** ``realloc`` calls ``dealloc(p)``. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `realloc` calls `dealloc(p)`. ## In other cases the block has to be freed with ## `dealloc(block) <#dealloc,pointer>`_. ## ## The allocated memory belongs to its allocating thread! - ## Use `reallocShared <#reallocShared,pointer,Natural>`_ to reallocate + ## Use `reallocShared <#reallocShared.t,pointer,Natural>`_ to reallocate ## from a shared heap. reallocImpl(p, newSize) @@ -148,8 +161,8 @@ when hasAlloc and not defined(js): ## Grows or shrinks a given memory block. ## ## If `p` is **nil** then a new memory block is returned. - ## In either way the block has at least ``newSize`` bytes. - ## If ``newSize == 0`` and `p` is not **nil** ``realloc`` calls ``dealloc(p)``. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `realloc` calls `dealloc(p)`. ## In other cases the block has to be freed with ## `dealloc(block) <#dealloc,pointer>`_. ## @@ -157,7 +170,7 @@ when hasAlloc and not defined(js): ## somewhat safer then realloc ## ## The allocated memory belongs to its allocating thread! - ## Use `reallocShared <#reallocShared,pointer,Natural>`_ to reallocate + ## Use `reallocShared <#reallocShared.t,pointer,Natural>`_ to reallocate ## from a shared heap. realloc0Impl(p, oldSize, newSize) @@ -165,18 +178,18 @@ when hasAlloc and not defined(js): ## Grows or shrinks a given memory block. ## ## If `p` is **nil** then a new memory block is returned. - ## In either way the block has at least ``T.sizeof * newSize`` bytes. - ## If ``newSize == 0`` and `p` is not **nil** ``resize`` calls ``dealloc(p)``. - ## In other cases the block has to be freed with ``free``. + ## In either way the block has at least `T.sizeof * newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `resize` calls `dealloc(p)`. + ## In other cases the block has to be freed with `free`. ## ## The allocated memory belongs to its allocating thread! ## Use `resizeShared <#resizeShared,ptr.T,Natural>`_ to reallocate ## from a shared heap. cast[ptr T](realloc(p, T.sizeof * newSize)) - template dealloc*(p: pointer) = - ## Frees the memory allocated with ``alloc``, ``alloc0`` or - ## ``realloc``. + proc dealloc*(p: pointer) {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} = + ## Frees the memory allocated with `alloc`, `alloc0`, + ## `realloc`, `create` or `createU`. ## ## **This procedure is dangerous!** ## If one forgets to free the memory a leak occurs; if one tries to @@ -190,24 +203,24 @@ when hasAlloc and not defined(js): template allocShared*(size: Natural): pointer = ## Allocates a new memory block on the shared heap with at - ## least ``size`` bytes. + ## least `size` bytes. ## ## The block has to be freed with - ## `reallocShared(block, 0) <#reallocShared,pointer,Natural>`_ + ## `reallocShared(block, 0) <#reallocShared.t,pointer,Natural>`_ ## or `deallocShared(block) <#deallocShared,pointer>`_. ## ## The block is not initialized, so reading from it before writing ## to it is undefined behaviour! ## ## See also: - ## `allocShared0 <#allocShared0,Natural>`_. + ## * `allocShared0 <#allocShared0.t,Natural>`_. incStat(allocCount) allocSharedImpl(size) proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline, tags: [], benign, raises: [].} = ## Allocates a new memory block on the shared heap with at - ## least ``T.sizeof * size`` bytes. + ## least `T.sizeof * size` bytes. ## ## The block has to be freed with ## `resizeShared(block, 0) <#resizeShared,ptr.T,Natural>`_ or @@ -222,21 +235,21 @@ when hasAlloc and not defined(js): template allocShared0*(size: Natural): pointer = ## Allocates a new memory block on the shared heap with at - ## least ``size`` bytes. + ## least `size` bytes. ## ## The block has to be freed with - ## `reallocShared(block, 0) <#reallocShared,pointer,Natural>`_ + ## `reallocShared(block, 0) <#reallocShared.t,pointer,Natural>`_ ## or `deallocShared(block) <#deallocShared,pointer>`_. ## ## The block is initialized with all bytes ## containing zero, so it is somewhat safer than - ## `allocShared <#allocShared,Natural>`_. + ## `allocShared <#allocShared.t,Natural>`_. incStat(allocCount) allocShared0Impl(size) proc createShared*(T: typedesc, size = 1.Positive): ptr T {.inline.} = ## Allocates a new memory block on the shared heap with at - ## least ``T.sizeof * size`` bytes. + ## least `T.sizeof * size` bytes. ## ## The block has to be freed with ## `resizeShared(block, 0) <#resizeShared,ptr.T,Natural>`_ or @@ -251,9 +264,9 @@ when hasAlloc and not defined(js): ## Grows or shrinks a given memory block on the heap. ## ## If `p` is **nil** then a new memory block is returned. - ## In either way the block has at least ``newSize`` bytes. - ## If ``newSize == 0`` and `p` is not **nil** ``reallocShared`` calls - ## ``deallocShared(p)``. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `reallocShared` calls + ## `deallocShared(p)`. ## In other cases the block has to be freed with ## `deallocShared <#deallocShared,pointer>`_. reallocSharedImpl(p, newSize) @@ -265,9 +278,9 @@ when hasAlloc and not defined(js): ## containing zero, so it is somewhat safer then reallocShared ## ## If `p` is **nil** then a new memory block is returned. - ## In either way the block has at least ``newSize`` bytes. - ## If ``newSize == 0`` and `p` is not **nil** ``reallocShared`` calls - ## ``deallocShared(p)``. + ## In either way the block has at least `newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `reallocShared` calls + ## `deallocShared(p)`. ## In other cases the block has to be freed with ## `deallocShared <#deallocShared,pointer>`_. reallocShared0Impl(p, oldSize, newSize) @@ -276,16 +289,16 @@ when hasAlloc and not defined(js): ## Grows or shrinks a given memory block on the heap. ## ## If `p` is **nil** then a new memory block is returned. - ## In either way the block has at least ``T.sizeof * newSize`` bytes. - ## If ``newSize == 0`` and `p` is not **nil** ``resizeShared`` calls - ## ``freeShared(p)``. + ## In either way the block has at least `T.sizeof * newSize` bytes. + ## If `newSize == 0` and `p` is not **nil** `resizeShared` calls + ## `freeShared(p)`. ## In other cases the block has to be freed with ## `freeShared <#freeShared,ptr.T>`_. cast[ptr T](reallocShared(p, T.sizeof * newSize)) proc deallocShared*(p: pointer) {.noconv, compilerproc, rtl, benign, raises: [], tags: [].} = - ## Frees the memory allocated with ``allocShared``, ``allocShared0`` or - ## ``reallocShared``. + ## Frees the memory allocated with `allocShared`, `allocShared0` or + ## `reallocShared`. ## ## **This procedure is dangerous!** ## If one forgets to free the memory a leak occurs; if one tries to @@ -295,8 +308,8 @@ when hasAlloc and not defined(js): deallocSharedImpl(p) proc freeShared*[T](p: ptr T) {.inline, benign, raises: [].} = - ## Frees the memory allocated with ``createShared``, ``createSharedU`` or - ## ``resizeShared``. + ## Frees the memory allocated with `createShared`, `createSharedU` or + ## `resizeShared`. ## ## **This procedure is dangerous!** ## If one forgets to free the memory a leak occurs; if one tries to @@ -304,6 +317,87 @@ when hasAlloc and not defined(js): ## or other memory may be corrupted. deallocShared(p) + include bitmasks + + template `+!`(p: pointer, s: SomeInteger): pointer = + cast[pointer](cast[int](p) +% int(s)) + + template `-!`(p: pointer, s: SomeInteger): pointer = + cast[pointer](cast[int](p) -% int(s)) + + proc alignedAlloc(size, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = allocShared(size) + else: + result = alloc(size) + else: + # allocate (size + align - 1) necessary for alignment, + # plus 2 bytes to store offset + when compileOption("threads"): + let base = allocShared(size + align - 1 + sizeof(uint16)) + else: + let base = alloc(size + align - 1 + sizeof(uint16)) + # memory layout: padding + offset (2 bytes) + user_data + # in order to deallocate: read offset at user_data - 2 bytes, + # then deallocate user_data - offset + let offset = align - (cast[int](base) and (align - 1)) + cast[ptr uint16](base +! (offset - sizeof(uint16)))[] = uint16(offset) + result = base +! offset + + proc alignedAlloc0(size, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = allocShared0(size) + else: + result = alloc0(size) + else: + # see comments for alignedAlloc + when compileOption("threads"): + let base = allocShared0(size + align - 1 + sizeof(uint16)) + else: + let base = alloc0(size + align - 1 + sizeof(uint16)) + let offset = align - (cast[int](base) and (align - 1)) + cast[ptr uint16](base +! (offset - sizeof(uint16)))[] = uint16(offset) + result = base +! offset + + proc alignedDealloc(p: pointer, align: int) {.compilerproc.} = + if align <= MemAlign: + when compileOption("threads"): + deallocShared(p) + else: + dealloc(p) + else: + # read offset at p - 2 bytes, then deallocate (p - offset) pointer + let offset = cast[ptr uint16](p -! sizeof(uint16))[] + when compileOption("threads"): + deallocShared(p -! offset) + else: + dealloc(p -! offset) + + proc alignedRealloc(p: pointer, oldSize, newSize, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = reallocShared(p, newSize) + else: + result = realloc(p, newSize) + else: + result = alignedAlloc(newSize, align) + copyMem(result, p, oldSize) + alignedDealloc(p, align) + + proc alignedRealloc0(p: pointer, oldSize, newSize, align: Natural): pointer = + if align <= MemAlign: + when compileOption("threads"): + result = reallocShared0(p, oldSize, newSize) + else: + result = realloc0(p, oldSize, newSize) + else: + result = alignedAlloc(newSize, align) + copyMem(result, p, oldSize) + zeroMem(result +! oldSize, newSize - oldSize) + alignedDealloc(p, align) + {.pop.} # GC interface: @@ -339,7 +433,7 @@ when defined(js): proc reallocShared0(p: pointer, oldsize, newsize: Natural): pointer = discard -when hasAlloc and hasThreadSupport: +when hasAlloc and hasThreadSupport and not defined(useMalloc): proc getOccupiedSharedMem*(): int {.rtl.} ## Returns the number of bytes that are owned by the process ## on the shared heap and hold data. This is only available when diff --git a/lib/system/memory.nim b/lib/system/memory.nim index 8d190e5f9..156773c48 100644 --- a/lib/system/memory.nim +++ b/lib/system/memory.nim @@ -2,9 +2,6 @@ const useLibC = not defined(nimNoLibc) -when not defined(nimHasHotCodeReloading): - {.pragma: nonReloadable.} - when useLibC: import ansi_c @@ -45,9 +42,10 @@ proc nimCmpMem*(a, b: pointer, size: Natural): cint {.compilerproc, nonReloadabl if d != 0: return d inc i -proc nimCStrLen*(a: cstring): csize_t {.compilerproc, nonReloadable, inline.} = +proc nimCStrLen*(a: cstring): int {.compilerproc, nonReloadable, inline.} = + if a.isNil: return 0 when useLibC: - c_strlen(a) + cast[int](c_strlen(a)) else: var a = cast[ptr byte](a) while a[] != 0: diff --git a/lib/system/memtracker.nim b/lib/system/memtracker.nim index 115d4a973..289f4e024 100644 --- a/lib/system/memtracker.nim +++ b/lib/system/memtracker.nim @@ -35,7 +35,7 @@ type count*: int disabled: bool data*: array[400, LogEntry] - TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], locks: 0, gcsafe.} + TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], gcsafe.} var gLog*: TrackLog @@ -70,9 +70,9 @@ proc addEntry(entry: LogEntry) = if interesting: gLog.disabled = true cprintf("interesting %s:%ld %s\n", entry.file, entry.line, entry.op) - let x = cast[proc() {.nimcall, tags: [], gcsafe, locks: 0, raises: [].}](writeStackTrace) + let x = cast[proc() {.nimcall, tags: [], gcsafe, raises: [].}](writeStackTrace) x() - quit 1 + rawQuit 1 #if gLog.count > high(gLog.data): # gLogger(gLog) # gLog.count = 0 @@ -85,7 +85,7 @@ proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.co size: size, file: file, line: line, thread: myThreadId()) proc memTrackerOp*(op: cstring; address: pointer; size: int) {.tags: [], - locks: 0, gcsafe.} = + gcsafe.} = addEntry LogEntry(op: op, address: address, size: size, file: "", line: 0, thread: myThreadId()) @@ -100,6 +100,7 @@ proc logPendingOps() {.noconv.} = gLogger(gLog) gLog.count = 0 -addQuitProc logPendingOps +import std/exitprocs +addExitProc logPendingOps {.pop.} diff --git a/lib/system/mm/boehm.nim b/lib/system/mm/boehm.nim index d02d52b52..362d2d470 100644 --- a/lib/system/mm/boehm.nim +++ b/lib/system/mm/boehm.nim @@ -1,4 +1,6 @@ + + proc boehmGCinit {.importc: "GC_init", boehmGC.} proc boehmGC_disable {.importc: "GC_disable", boehmGC.} proc boehmGC_enable {.importc: "GC_enable", boehmGC.} @@ -51,14 +53,14 @@ when not defined(useNimRtl): proc realloc0Impl(p: pointer, oldSize, newSize: Natural): pointer = result = boehmRealloc(p, newSize) if result == nil: raiseOutOfMem() - if newsize > oldsize: - zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) + if newSize > oldSize: + zeroMem(cast[pointer](cast[int](result) + oldSize), newSize - oldSize) proc deallocImpl(p: pointer) = boehmDealloc(p) proc allocSharedImpl(size: Natural): pointer = allocImpl(size) proc allocShared0Impl(size: Natural): pointer = alloc0Impl(size) - proc reallocSharedImpl(p: pointer, newsize: Natural): pointer = reallocImpl(p, newsize) - proc reallocShared0Impl(p: pointer, oldsize, newsize: Natural): pointer = realloc0Impl(p, oldsize, newsize) + proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = reallocImpl(p, newSize) + proc reallocShared0Impl(p: pointer, oldSize, newSize: Natural): pointer = realloc0Impl(p, oldSize, newSize) proc deallocSharedImpl(p: pointer) = deallocImpl(p) when hasThreadSupport: @@ -96,15 +98,18 @@ proc initGC() = proc boehmgc_finalizer(obj: pointer, typedFinalizer: (proc(x: pointer) {.cdecl.})) = typedFinalizer(obj) + proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = if ntfNoRefs in typ.flags: result = allocAtomic(size) else: result = alloc(size) if typ.finalizer != nil: boehmRegisterFinalizer(result, boehmgc_finalizer, typ.finalizer, nil, nil) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) + result = newObj(typ, align(GenericSeqSize, typ.base.align) + len * typ.base.size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len +{.pop.} proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) @@ -133,4 +138,3 @@ proc deallocOsPages(r: var MemRegion) {.inline.} = discard proc deallocOsPages() {.inline.} = discard include "system/cellsets" - diff --git a/lib/system/mm/go.nim b/lib/system/mm/go.nim index 8e8558bea..8f3aeb964 100644 --- a/lib/system/mm/go.nim +++ b/lib/system/mm/go.nim @@ -35,8 +35,6 @@ proc goMalloc(size: uint): pointer {.importc: "go_malloc", dynlib: goLib.} proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.} proc writebarrierptr(dest: PPointer, src: pointer) {.importc: "writebarrierptr", codegenDecl:"$1 $2$3 __asm__ (\"main.Atomic_store_pointer\");\n$1 $2$3", dynlib: goLib.} -proc `$`*(x: uint64): string {.noSideEffect, raises: [].} - proc GC_getStatistics(): string = var mstats = goMemStats() result = "[GC] total allocated memory: " & $(mstats.total_alloc) & "\n" & @@ -101,35 +99,39 @@ proc newObjNoInit(typ: PNimType, size: int): pointer = writebarrierptr(addr(result), newObj(typ, size)) proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - writebarrierptr(addr(result), newObj(typ, len * typ.base.size + GenericSeqSize)) + writebarrierptr(addr(result), newObj(typ, align(GenericSeqSize, typ.base.align) + len * typ.base.size)) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len cast[PGenericSeq](result).elemSize = typ.base.size + cast[PGenericSeq](result).elemAlign = typ.base.align proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = writebarrierptr(addr(result), newSeq(typ, len)) proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = - result = newObj(typ, cap * typ.base.size + GenericSeqSize) + result = newObjNoInit(typ, align(GenericSeqSize, typ.base.align) + cap * typ.base.size) cast[PGenericSeq](result).len = 0 cast[PGenericSeq](result).reserved = cap cast[PGenericSeq](result).elemSize = typ.base.size + cast[PGenericSeq](result).elemAlign = typ.base.align proc typedMemMove(dest: pointer, src: pointer, size: uint) {.importc: "typedmemmove", dynlib: goLib.} proc growObj(old: pointer, newsize: int): pointer = # the Go GC doesn't have a realloc + let old = cast[PGenericSeq](old) var metadataOld = cast[PGenericSeq](old) if metadataOld.elemSize == 0: metadataOld.elemSize = 1 - let oldsize = cast[PGenericSeq](old).len * cast[PGenericSeq](old).elemSize + GenericSeqSize + + let oldsize = align(GenericSeqSize, old.elemAlign) + old.len * old.elemSize writebarrierptr(addr(result), goMalloc(newsize.uint)) typedMemMove(result, old, oldsize.uint) proc nimGCref(p: pointer) {.compilerproc, inline.} = discard proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard -proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} = discard -proc nimGCunrefRC1(p: pointer) {.compilerProc, inline.} = discard +proc nimGCunrefNoCycle(p: pointer) {.compilerproc, inline.} = discard +proc nimGCunrefRC1(p: pointer) {.compilerproc, inline.} = discard proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = discard proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = @@ -149,4 +151,3 @@ proc alloc0(r: var MemRegion, size: int): pointer = proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) proc deallocOsPages(r: var MemRegion) {.inline.} = discard proc deallocOsPages() {.inline.} = discard - diff --git a/lib/system/mm/malloc.nim b/lib/system/mm/malloc.nim index 27e32f87d..47f1a95ae 100644 --- a/lib/system/mm/malloc.nim +++ b/lib/system/mm/malloc.nim @@ -1,17 +1,28 @@ +{.push stackTrace: off.} + proc allocImpl(size: Natural): pointer = - c_malloc(size.csize_t) + result = c_malloc(size.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc alloc0Impl(size: Natural): pointer = - c_calloc(size.csize_t, 1) - -proc reallocImpl(p: pointer, newsize: Natural): pointer = - c_realloc(p, newSize.csize_t) - -proc realloc0Impl(p: pointer, oldsize, newsize: Natural): pointer = - result = realloc(p, newsize.csize_t) - if newsize > oldsize: - zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) + result = c_calloc(size.csize_t, 1) + when defined(zephyr): + if result == nil: + raiseOutOfMem() + +proc reallocImpl(p: pointer, newSize: Natural): pointer = + result = c_realloc(p, newSize.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() + +proc realloc0Impl(p: pointer, oldsize, newSize: Natural): pointer = + result = realloc(p, newSize.csize_t) + if newSize > oldSize: + zeroMem(cast[pointer](cast[uint](result) + uint(oldSize)), newSize - oldSize) proc deallocImpl(p: pointer) = c_free(p) @@ -25,11 +36,11 @@ proc allocSharedImpl(size: Natural): pointer = proc allocShared0Impl(size: Natural): pointer = alloc0Impl(size) -proc reallocSharedImpl(p: pointer, newsize: Natural): pointer = - reallocImpl(p, newsize) +proc reallocSharedImpl(p: pointer, newSize: Natural): pointer = + reallocImpl(p, newSize) -proc reallocShared0Impl(p: pointer, oldsize, newsize: Natural): pointer = - realloc0Impl(p, oldsize, newsize) +proc reallocShared0Impl(p: pointer, oldsize, newSize: Natural): pointer = + realloc0Impl(p, oldSize, newSize) proc deallocSharedImpl(p: pointer) = deallocImpl(p) @@ -38,10 +49,13 @@ proc deallocSharedImpl(p: pointer) = deallocImpl(p) proc GC_disable() = discard proc GC_enable() = discard -proc GC_fullCollect() = discard + +when not defined(gcOrc): + proc GC_fullCollect() = discard + proc GC_enableMarkAndSweep() = discard + proc GC_disableMarkAndSweep() = discard + proc GC_setStrategy(strategy: GC_Strategy) = discard -proc GC_enableMarkAndSweep() = discard -proc GC_disableMarkAndSweep() = discard proc getOccupiedMem(): int = discard proc getFreeMem(): int = discard @@ -60,8 +74,10 @@ proc growObj(old: pointer, newsize: int): pointer = proc nimGCref(p: pointer) {.compilerproc, inline.} = discard proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard -proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = - dest[] = src +when not defined(gcDestructors): + proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = + dest[] = src + proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} = dest[] = src proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline, @@ -72,9 +88,10 @@ type proc alloc(r: var MemRegion, size: int): pointer = result = alloc(size) -proc alloc0Impl(r: var MemRegion, size: int): pointer = +proc alloc0(r: var MemRegion, size: int): pointer = result = alloc0Impl(size) proc dealloc(r: var MemRegion, p: pointer) = dealloc(p) -proc deallocOsPages(r: var MemRegion) {.inline.} = discard -proc deallocOsPages() {.inline.} = discard +proc deallocOsPages(r: var MemRegion) = discard +proc deallocOsPages() = discard +{.pop.} diff --git a/lib/system/mm/none.nim b/lib/system/mm/none.nim index 3572b062c..7818a0805 100644 --- a/lib/system/mm/none.nim +++ b/lib/system/mm/none.nim @@ -19,10 +19,12 @@ proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = proc newObjNoInit(typ: PNimType, size: int): pointer = result = alloc(size) +{.push overflowChecks: on.} proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = - result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize)) + result = newObj(typ, align(GenericSeqSize, typ.align) + len * typ.base.size) cast[PGenericSeq](result).len = len cast[PGenericSeq](result).reserved = len +{.pop.} proc growObj(old: pointer, newsize: int): pointer = result = realloc(old, newsize) diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index d2d0d576f..26f2f0bbf 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -17,10 +17,10 @@ const debugGC = false # we wish to debug the GC... logGC = false traceGC = false # extensive debugging - alwaysCycleGC = defined(smokeCycles) - alwaysGC = defined(fulldebug) # collect after every memory + alwaysCycleGC = defined(nimSmokeCycles) + alwaysGC = defined(nimFulldebug) # collect after every memory # allocation (for debugging) - leakDetector = defined(leakDetector) + leakDetector = defined(nimLeakDetector) overwriteFree = defined(nimBurnFree) # overwrite memory with 0xFF before free trackAllocationSource = leakDetector @@ -30,7 +30,7 @@ const coalescRight = true coalescLeft = true logAlloc = false - useCellIds = defined(corruption) + useCellIds = defined(nimCorruption) type PPointer = ptr pointer @@ -38,32 +38,15 @@ type PByte = ptr ByteArray PString = ptr string -# Page size of the system; in most cases 4096 bytes. For exotic OS or -# CPU this needs to be changed: -const - PageShift = when defined(cpu16): 8 else: 12 # \ - # my tests showed no improvements for using larger page sizes. - PageSize = 1 shl PageShift - PageMask = PageSize-1 - - MemAlign = 8 # also minimal allocatable memory block - - BitsPerPage = PageSize div MemAlign - UnitsPerPage = BitsPerPage div (sizeof(int)*8) - # how many ints do we need to describe a page: - # on 32 bit systems this is only 16 (!) - - TrunkShift = 9 - BitsPerTrunk = 1 shl TrunkShift # needs to be power of 2 and divisible by 64 - TrunkMask = BitsPerTrunk - 1 - IntsPerTrunk = BitsPerTrunk div (sizeof(int)*8) - IntShift = 5 + ord(sizeof(int) == 8) # 5 or 6, depending on int width - IntMask = 1 shl IntShift - 1 +when declared(IntsPerTrunk): + discard +else: + include bitmasks proc raiseOutOfMem() {.noinline.} = if outOfMemHook != nil: outOfMemHook() cstderr.rawWrite("out of memory\n") - quit(1) + rawQuit(1) when defined(boehmgc): include system / mm / boehm @@ -74,6 +57,16 @@ elif defined(gogc): elif (defined(nogc) or defined(gcDestructors)) and defined(useMalloc): include system / mm / malloc + when defined(nogc): + proc GC_getStatistics(): string = "" + proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} = + result = alloc0(size) + + proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} = + result = newObj(typ, align(GenericSeqSize, typ.align) + len * typ.base.size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + elif defined(nogc): include system / mm / none @@ -83,16 +76,15 @@ else: when not usesDestructors: include "system/cellsets" - when not leakDetector and not useCellIds: + when not leakDetector and not useCellIds and not defined(nimV2): sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") - when compileOption("gc", "v2"): - include "system/gc2" - elif defined(gcRegions): + when defined(gcRegions): # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here include "system/gc_regions" elif defined(nimV2) or usesDestructors: - var allocator {.rtlThreadVar.}: MemRegion - instantiateForRegion(allocator) + when not defined(useNimRtl): + var allocator {.rtlThreadVar.}: MemRegion + instantiateForRegion(allocator) when defined(gcHooks): include "system/gc_hooks" elif defined(gcMarkAndSweep): @@ -102,18 +94,20 @@ else: include "system/gc" when not declared(nimNewSeqOfCap) and not defined(nimSeqsV2): + {.push overflowChecks: on.} proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} = when defined(gcRegions): - let s = mulInt(cap, typ.base.size) # newStr already adds GenericSeqSize + let s = cap * typ.base.size # newStr already adds GenericSeqSize result = newStr(typ, s, ntfNoRefs notin typ.base.flags) else: - let s = addInt(mulInt(cap, typ.base.size), GenericSeqSize) + let s = align(GenericSeqSize, typ.base.align) + cap * typ.base.size when declared(newObjNoInit): result = if ntfNoRefs in typ.base.flags: newObjNoInit(typ, s) else: newObj(typ, s) else: result = newObj(typ, s) cast[PGenericSeq](result).len = 0 cast[PGenericSeq](result).reserved = cap + {.pop.} {.pop.} diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 719e92c4a..cf81f6d86 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -14,11 +14,11 @@ const buildOS* {.magic: "BuildOS".}: string = "" - ## The OS this build is running on. Can be different from ``system.hostOS`` + ## The OS this build is running on. Can be different from `system.hostOS` ## for cross compilations. buildCPU* {.magic: "BuildCPU".}: string = "" - ## The CPU this build is running on. Can be different from ``system.hostCPU`` + ## The CPU this build is running on. Can be different from `system.hostCPU` ## for cross compilations. template builtin = discard @@ -26,14 +26,11 @@ template builtin = discard # We know the effects better than the compiler: {.push hint[XDeclaredButNotUsed]: off.} -proc listDirs*(dir: string): seq[string] = - ## Lists all the subdirectories (non-recursively) in the directory `dir`. - builtin -proc listFiles*(dir: string): seq[string] = - ## Lists all the files (non-recursively) in the directory `dir`. - builtin - -proc removeDir(dir: string) {. +proc listDirsImpl(dir: string): seq[string] {. + tags: [ReadIOEffect], raises: [OSError].} = builtin +proc listFilesImpl(dir: string): seq[string] {. + tags: [ReadIOEffect], raises: [OSError].} = builtin +proc removeDir(dir: string, checkDir = true) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc removeFile(dir: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin @@ -47,6 +44,7 @@ proc copyDir(src, dest: string) {. tags: [ReadIOEffect, WriteIOEffect], raises: [OSError].} = builtin proc createDir(dir: string) {.tags: [WriteIOEffect], raises: [OSError].} = builtin + proc getError: string = builtin proc setCurrentDir(dir: string) = builtin proc getCurrentDir*(): string = @@ -59,7 +57,7 @@ proc warningImpl(arg, orig: string) = discard proc hintImpl(arg, orig: string) = discard proc paramStr*(i: int): string = - ## Retrieves the ``i``'th command line parameter. + ## Retrieves the `i`'th command line parameter. builtin proc paramCount*(): int = @@ -68,7 +66,7 @@ proc paramCount*(): int = proc switch*(key: string, val="") = ## Sets a Nim compiler command line switch, for - ## example ``switch("checks", "on")``. + ## example `switch("checks", "on")`. builtin proc warning*(name: string; val: bool) = @@ -84,16 +82,16 @@ proc hint*(name: string; val: bool) = proc patchFile*(package, filename, replacement: string) = ## Overrides the location of a given file belonging to the ## passed package. - ## If the ``replacement`` is not an absolute path, the path + ## If the `replacement` is not an absolute path, the path ## is interpreted to be local to the Nimscript file that contains - ## the call to ``patchFile``, Nim's ``--path`` is not used at all + ## the call to `patchFile`, Nim's `--path` is not used at all ## to resolve the filename! + ## The compiler also performs `path substitution <nimc.html#compiler-usage-commandminusline-switches>`_ on `replacement`. ## ## Example: - ## - ## .. code-block:: nim - ## + ## ```nim ## patchFile("stdlib", "asyncdispatch", "patches/replacement") + ## ``` discard proc getCommand*(): string = @@ -114,19 +112,19 @@ proc cmpic*(a, b: string): int = cmpIgnoreCase(a, b) proc getEnv*(key: string; default = ""): string {.tags: [ReadIOEffect].} = - ## Retrieves the environment variable of name ``key``. + ## Retrieves the environment variable of name `key`. builtin proc existsEnv*(key: string): bool {.tags: [ReadIOEffect].} = - ## Checks for the existence of an environment variable named ``key``. + ## Checks for the existence of an environment variable named `key`. builtin proc putEnv*(key, val: string) {.tags: [WriteIOEffect].} = - ## Sets the value of the environment variable named ``key`` to ``val``. + ## Sets the value of the environment variable named `key` to `val`. builtin proc delEnv*(key: string) {.tags: [WriteIOEffect].} = - ## Deletes the environment variable named ``key``. + ## Deletes the environment variable named `key`. builtin proc fileExists*(filename: string): bool {.tags: [ReadIOEffect].} = @@ -138,17 +136,8 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -proc existsFile*(filename: string): bool = - ## An alias for ``fileExists``. - fileExists(filename) - -proc existsDir*(dir: string): bool = - ## An alias for ``dirExists``. - dirExists(dir) - -proc selfExe*(): string = +proc selfExe*(): string {.deprecated: "Deprecated since v1.7; Use getCurrentCompilerExe".} = ## Returns the currently running nim or nimble executable. - # TODO: consider making this as deprecated alias of `getCurrentCompilerExe` builtin proc toExe*(filename: string): string = @@ -161,16 +150,28 @@ proc toDll*(filename: string): string = proc strip(s: string): string = var i = 0 - while s[i] in {' ', '\c', '\L'}: inc i + while s[i] in {' ', '\c', '\n'}: inc i result = s.substr(i) + if result[0] == '"' and result[^1] == '"': + result = result[1..^2] template `--`*(key, val: untyped) = - ## A shortcut for ``switch(astToStr(key), astToStr(val))``. - switch(astToStr(key), strip astToStr(val)) + ## A shortcut for `switch <#switch,string,string>`_ + ## Example: + ## ```nim + ## --path:somePath # same as switch("path", "somePath") + ## --path:"someOtherPath" # same as switch("path", "someOtherPath") + ## --hint:"[Conf]:off" # same as switch("hint", "[Conf]:off") + ## ``` + switch(strip(astToStr(key)), strip(astToStr(val))) template `--`*(key: untyped) = - ## A shortcut for ``switch(astToStr(key)``. - switch(astToStr(key), "") + ## A shortcut for `switch <#switch,string,string>`_ + ## Example: + ## ```nim + ## --listCmd # same as switch("listCmd") + ## ``` + switch(strip(astToStr(key))) type ScriptMode* {.pure.} = enum ## Controls the behaviour of the script. @@ -193,13 +194,23 @@ template checkOsError = template log(msg: string, body: untyped) = if mode in {ScriptMode.Verbose, ScriptMode.Whatif}: echo "[NimScript] ", msg - if mode != ScriptMode.WhatIf: + if mode != ScriptMode.Whatif: body -proc rmDir*(dir: string) {.raises: [OSError].} = +proc listDirs*(dir: string): seq[string] = + ## Lists all the subdirectories (non-recursively) in the directory `dir`. + result = listDirsImpl(dir) + checkOsError() + +proc listFiles*(dir: string): seq[string] = + ## Lists all the files (non-recursively) in the directory `dir`. + result = listFilesImpl(dir) + checkOsError() + +proc rmDir*(dir: string, checkDir = false) {.raises: [OSError].} = ## Removes the directory `dir`. log "rmDir: " & dir: - removeDir dir + removeDir(dir, checkDir = checkDir) checkOsError() proc rmFile*(file: string) {.raises: [OSError].} = @@ -240,32 +251,39 @@ proc cpDir*(`from`, to: string) {.raises: [OSError].} = checkOsError() proc exec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with - ## a non-zero exit code, an OSError exception is raised. + ## a non-zero exit code, an OSError exception is raised. The command is + ## executed relative to the current source path. ## - ## **Note:** If you need a version of ``exec`` that returns the exit code - ## and text output of the command, you can use `system.gorgeEx - ## <system.html#gorgeEx,string,string,string>`_. + ## .. note:: If you need a version of `exec` that returns the exit code + ## and text output of the command, you can use `system.gorgeEx + ## <system.html#gorgeEx,string,string,string>`_. log "exec: " & command: if rawExec(command) != 0: raise newException(OSError, "FAILED: " & command) checkOsError() proc exec*(command: string, input: string, cache = "") {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. + ## + ## .. warning:: This version of `exec` is executed relative to the nimscript + ## module path, which affects how the command resolves relative paths. Thus + ## it is generally better to use `gorgeEx` directly when you need more + ## control over the execution environment or when working with commands + ## that deal with relative paths. log "exec: " & command: let (output, exitCode) = gorgeEx(command, input, cache) + echo output if exitCode != 0: raise newException(OSError, "FAILED: " & command) - echo output proc selfExec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external command with the current nim/nimble executable. - ## ``Command`` must not contain the "nim " part. + ## `Command` must not contain the "nim " part. let c = selfExe() & " " & command log "exec: " & c: if rawExec(c) != 0: @@ -302,9 +320,9 @@ proc projectPath*(): string = builtin proc thisDir*(): string = - ## Retrieves the directory of the current ``nims`` script file. Its path is - ## obtained via ``currentSourcePath`` (although, currently, - ## ``currentSourcePath`` resolves symlinks, unlike ``thisDir``). + ## Retrieves the directory of the current `nims` script file. Its path is + ## obtained via `currentSourcePath` (although, currently, + ## `currentSourcePath` resolves symlinks, unlike `thisDir`). builtin proc cd*(dir: string) {.raises: [OSError].} = @@ -328,19 +346,19 @@ template withDir*(dir: string; body: untyped): untyped = ## ## If you need a permanent change, use the `cd() <#cd,string>`_ proc. ## Usage example: - ## - ## .. code-block:: nim + ## ```nim + ## # inside /some/path/ ## withDir "foo": - ## # inside foo - ## #back to last dir - var curDir = getCurrentDir() + ## # move to /some/path/foo/ + ## # back in /some/path/ + ## ``` + let curDir = getCurrentDir() try: cd(dir) body finally: cd(curDir) - proc writeTask(name, desc: string) = if desc.len > 0: var spaces = " " @@ -348,25 +366,25 @@ proc writeTask(name, desc: string) = echo name, spaces, desc proc cppDefine*(define: string) = - ## tell Nim that ``define`` is a C preprocessor ``#define`` and so always + ## tell Nim that `define` is a C preprocessor `#define` and so always ## needs to be mangled. builtin -proc stdinReadLine(): TaintedString {. +proc stdinReadLine(): string {. tags: [ReadIOEffect], raises: [IOError].} = builtin -proc stdinReadAll(): TaintedString {. +proc stdinReadAll(): string {. tags: [ReadIOEffect], raises: [IOError].} = builtin -proc readLineFromStdin*(): TaintedString {.raises: [IOError].} = +proc readLineFromStdin*(): string {.raises: [IOError].} = ## Reads a line of data from stdin - blocks until \n or EOF which happens when stdin is closed log "readLineFromStdin": result = stdinReadLine() checkError(EOFError) -proc readAllFromStdin*(): TaintedString {.raises: [IOError].} = +proc readAllFromStdin*(): string {.raises: [IOError].} = ## Reads all data from stdin - blocks until EOF which happens when stdin is closed log "readAllFromStdin": result = stdinReadAll() @@ -376,11 +394,27 @@ when not defined(nimble): template `==?`(a, b: string): bool = cmpIgnoreStyle(a, b) == 0 template task*(name: untyped; description: string; body: untyped): untyped = ## Defines a task. Hidden tasks are supported via an empty description. + ## ## Example: + ## ```nim + ## task build, "default build is via the C backend": + ## setCommand "c" + ## ``` + ## + ## For a task named `foo`, this template generates a `proc` named + ## `fooTask`. This is useful if you need to call one task in + ## another in your Nimscript. + ## + ## Example: + ## + ## ```nim + ## task foo, "foo": # > nim foo + ## echo "Running foo" # Running foo ## - ## .. code-block:: nim - ## task build, "default build is via the C backend": - ## setCommand "c" + ## task bar, "bar": # > nim bar + ## echo "Running bar" # Running bar + ## fooTask() # Running foo + ## ``` proc `name Task`*() = setCommand "nop" body diff --git a/lib/system/orc.nim b/lib/system/orc.nim new file mode 100644 index 000000000..c02a24989 --- /dev/null +++ b/lib/system/orc.nim @@ -0,0 +1,543 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Cycle collector based on +# https://www.cs.purdue.edu/homes/hosking/690M/Bacon01Concurrent.pdf +# And ideas from Lins' in 2008 by the notion of "critical links", see +# "Cyclic reference counting" by Rafael Dueire Lins +# R.D. Lins / Information Processing Letters 109 (2008) 71–78 +# + +include cellseqs_v2 + +const + colBlack = 0b000 + colGray = 0b001 + colWhite = 0b010 + maybeCycle = 0b100 # possibly part of a cycle; this has to be a "sticky" bit + jumpStackFlag = 0b1000 + colorMask = 0b011 + + logOrc = defined(nimArcIds) + +type + TraceProc = proc (p, env: pointer) {.nimcall, benign.} + DisposeProc = proc (p: pointer) {.nimcall, benign.} + +template color(c): untyped = c.rc and colorMask +template setColor(c, col) = + when col == colBlack: + c.rc = c.rc and not colorMask + else: + c.rc = c.rc and not colorMask or col + +const + optimizedOrc = false # not defined(nimOldOrc) +# XXX Still incorrect, see tests/arc/tdestroy_in_loopcond + +proc nimIncRefCyclic(p: pointer; cyclic: bool) {.compilerRtl, inl.} = + let h = head(p) + inc h.rc, rcIncrement + when optimizedOrc: + if cyclic: + h.rc = h.rc or maybeCycle + +proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = + when optimizedOrc: + if p != nil: + let h = head(p) + h.rc = h.rc or maybeCycle + +proc unsureAsgnRef(dest: ptr pointer, src: pointer) {.inline.} = + # This is only used by the old RTTI mechanism and we know + # that 'dest[]' is nil and needs no destruction. Which is really handy + # as we cannot destroy the object reliably if it's an object of unknown + # compile-time type. + dest[] = src + if src != nil: nimIncRefCyclic(src, true) + +const + useJumpStack = false # for thavlak the jump stack doesn't improve the performance at all + +type + GcEnv = object + traceStack: CellSeq[ptr pointer] + when useJumpStack: + jumpStack: CellSeq[ptr pointer] # Lins' jump stack in order to speed up traversals + toFree: CellSeq[Cell] + freed, touched, edges, rcSum: int + keepThreshold: bool + +proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = + if desc.traceImpl != nil: + var p = s +! sizeof(RefHeader) + cast[TraceProc](desc.traceImpl)(p, addr(j)) + +include threadids + +when logOrc or orcLeakDetector: + proc writeCell(msg: cstring; s: Cell; desc: PNimTypeV2) = + when orcLeakDetector: + cfprintf(cstderr, "%s %s file: %s:%ld; color: %ld; thread: %ld\n", + msg, desc.name, s.filename, s.line, s.color, getThreadId()) + else: + cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld; thread: %ld\n", + msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color, getThreadId()) + +proc free(s: Cell; desc: PNimTypeV2) {.inline.} = + when traceCollector: + cprintf("[From ] %p rc %ld color %ld\n", s, s.rc shr rcShift, s.color) + let p = s +! sizeof(RefHeader) + + when logOrc: writeCell("free", s, desc) + + if desc.destructor != nil: + cast[DestructorProc](desc.destructor)(p) + + when false: + cstderr.rawWrite desc.name + cstderr.rawWrite " " + if desc.destructor == nil: + cstderr.rawWrite "lacks dispose" + if desc.traceImpl != nil: + cstderr.rawWrite ", but has trace\n" + else: + cstderr.rawWrite ", and lacks trace\n" + else: + cstderr.rawWrite "has dispose!\n" + + nimRawDispose(p, desc.align) + +template orcAssert(cond, msg) = + when logOrc: + if not cond: + cfprintf(cstderr, "[Bug!] %s\n", msg) + rawQuit 1 + +when logOrc: + proc strstr(s, sub: cstring): cstring {.header: "<string.h>", importc.} + +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + + orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!" + + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, desc) + +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = + let p = cast[ptr pointer](q) + if p[] != nil: + var j = cast[ptr GcEnv](env) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) + +var + roots {.threadvar.}: CellSeq[Cell] + +proc unregisterCycle(s: Cell) = + # swap with the last element. O(1) + let idx = s.rootIdx-1 + when false: + if idx >= roots.len or idx < 0: + cprintf("[Bug!] %ld %ld\n", idx, roots.len) + rawQuit 1 + roots.d[idx] = roots.d[roots.len-1] + roots.d[idx][0].rootIdx = idx+1 + dec roots.len + s.rootIdx = 0 + +proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + #[ + proc scanBlack(s: Cell) = + setColor(s, colBlack) + for t in sons(s): + t.rc = t.rc + rcIncrement + if t.color != colBlack: + scanBlack(t) + ]# + s.setColor colBlack + let until = j.traceStack.len + trace(s, desc, j) + when logOrc: writeCell("root still alive", s, desc) + while j.traceStack.len > until: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + inc t.rc, rcIncrement + if t.color != colBlack: + t.setColor colBlack + trace(t, desc, j) + when logOrc: writeCell("child still alive", t, desc) + +proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + #[ + proc markGray(s: Cell) = + if s.color != colGray: + setColor(s, colGray) + for t in sons(s): + t.rc = t.rc - rcIncrement + if t.color != colGray: + markGray(t) + ]# + if s.color != colGray: + s.setColor colGray + inc j.touched + # keep in mind that refcounts are zero based so add 1 here: + inc j.rcSum, (s.rc shr rcShift) + 1 + orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + dec t.rc, rcIncrement + inc j.edges + when useJumpStack: + if (t.rc shr rcShift) >= 0 and (t.rc and jumpStackFlag) == 0: + t.rc = t.rc or jumpStackFlag + when traceCollector: + cprintf("[Now in jumpstack] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) + j.jumpStack.add(entry, desc) + if t.color != colGray: + t.setColor colGray + inc j.touched + # we already decremented its refcount so account for that: + inc j.rcSum, (t.rc shr rcShift) + 2 + trace(t, desc, j) + +proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = + #[ + proc scan(s: Cell) = + if s.color == colGray: + if s.rc > 0: + scanBlack(s) + else: + s.setColor(colWhite) + for t in sons(s): scan(t) + ]# + if s.color == colGray: + if (s.rc shr rcShift) >= 0: + scanBlack(s, desc, j) + # XXX this should be done according to Lins' paper but currently breaks + #when useJumpStack: + # s.setColor colPurple + else: + when useJumpStack: + # first we have to repair all the nodes we have seen + # that are still alive; we also need to mark what they + # refer to as alive: + while j.jumpStack.len > 0: + let (entry, desc) = j.jumpStack.pop + let t = head entry[] + # not in jump stack anymore! + t.rc = t.rc and not jumpStackFlag + if t.color == colGray and (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + # XXX this should be done according to Lins' paper but currently breaks + #t.setColor colPurple + when traceCollector: + cprintf("[jump stack] %p %ld\n", t, t.rc shr rcShift) + + orcAssert(j.traceStack.len == 0, "scan: trace stack not empty") + s.setColor(colWhite) + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + if t.color == colGray: + if (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + else: + when useJumpStack: + # first we have to repair all the nodes we have seen + # that are still alive; we also need to mark what they + # refer to as alive: + while j.jumpStack.len > 0: + let (entry, desc) = j.jumpStack.pop + let t = head entry[] + # not in jump stack anymore! + t.rc = t.rc and not jumpStackFlag + if t.color == colGray and (t.rc shr rcShift) >= 0: + scanBlack(t, desc, j) + # XXX this should be done according to Lins' paper but currently breaks + #t.setColor colPurple + when traceCollector: + cprintf("[jump stack] %p %ld\n", t, t.rc shr rcShift) + + t.setColor(colWhite) + trace(t, desc, j) + +when false: + proc writeCell(msg: cstring; s: Cell) = + cfprintf(cstderr, "%s %p root index: %ld; RC: %ld; color: %ld\n", + msg, s, s.rootIdx, s.rc shr rcShift, s.color) + +proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = + #[ + was: 'collectWhite'. + + proc collectWhite(s: Cell) = + if s.color == colWhite and not buffered(s): + s.setColor(colBlack) + for t in sons(s): + collectWhite(t) + free(s) # watch out, a bug here! + ]# + if s.color == col and s.rootIdx == 0: + orcAssert(j.traceStack.len == 0, "collectWhite: trace stack not empty") + + s.setColor(colBlack) + j.toFree.add(s, desc) + trace(s, desc, j) + while j.traceStack.len > 0: + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + entry[] = nil # ensure that the destructor does touch moribund objects! + if t.color == col and t.rootIdx == 0: + j.toFree.add(t, desc) + t.setColor(colBlack) + trace(t, desc, j) + +const + defaultThreshold = when defined(nimFixedOrc): 10_000 else: 128 + +when defined(nimStressOrc): + const rootsThreshold = 10 # broken with -d:nimStressOrc: 10 and for havlak iterations 1..8 +else: + var rootsThreshold {.threadvar.}: int + +proc collectCyclesBacon(j: var GcEnv; lowMark: int) = + # pretty direct translation from + # https://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf + # Fig. 2. Synchronous Cycle Collection + #[ + for s in roots: + markGray(s) + for s in roots: + scan(s) + for s in roots: + remove s from roots + s.buffered = false + collectWhite(s) + ]# + let last = roots.len - 1 + when logOrc: + for i in countdown(last, lowMark): + writeCell("root", roots.d[i][0], roots.d[i][1]) + + for i in countdown(last, lowMark): + markGray(roots.d[i][0], roots.d[i][1], j) + + var colToCollect = colWhite + if j.rcSum == j.edges: + # short-cut: we know everything is garbage: + colToCollect = colGray + # remember the fact that we got so lucky: + j.keepThreshold = true + else: + for i in countdown(last, lowMark): + scan(roots.d[i][0], roots.d[i][1], j) + + init j.toFree + for i in 0 ..< roots.len: + let s = roots.d[i][0] + s.rootIdx = 0 + collectColor(s, roots.d[i][1], colToCollect, j) + + # Bug #22927: `free` calls destructors which can append to `roots`. + # We protect against this here by setting `roots.len` to 0 and also + # setting the threshold so high that no cycle collection can be triggered + # until we are out of this critical section: + when not defined(nimStressOrc): + let oldThreshold = rootsThreshold + rootsThreshold = high(int) + roots.len = 0 + + for i in 0 ..< j.toFree.len: + when orcLeakDetector: + writeCell("CYCLIC OBJECT FREED", j.toFree.d[i][0], j.toFree.d[i][1]) + free(j.toFree.d[i][0], j.toFree.d[i][1]) + + when not defined(nimStressOrc): + rootsThreshold = oldThreshold + + inc j.freed, j.toFree.len + deinit j.toFree + +when defined(nimOrcStats): + var freedCyclicObjects {.threadvar.}: int + +proc partialCollect(lowMark: int) = + when false: + if roots.len < 10 + lowMark: return + when logOrc: + cfprintf(cstderr, "[partialCollect] begin\n") + var j: GcEnv + init j.traceStack + collectCyclesBacon(j, lowMark) + when logOrc: + cfprintf(cstderr, "[partialCollect] end; freed %ld touched: %ld work: %ld\n", j.freed, j.touched, + roots.len - lowMark) + roots.len = lowMark + deinit j.traceStack + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed + +proc collectCycles() = + ## Collect cycles. + when logOrc: + cfprintf(cstderr, "[collectCycles] begin\n") + + var j: GcEnv + init j.traceStack + when useJumpStack: + init j.jumpStack + collectCyclesBacon(j, 0) + while j.jumpStack.len > 0: + let (t, desc) = j.jumpStack.pop + # not in jump stack anymore! + t.rc = t.rc and not jumpStackFlag + deinit j.jumpStack + else: + collectCyclesBacon(j, 0) + + deinit j.traceStack + if roots.len == 0: + deinit roots + + when not defined(nimStressOrc): + # compute the threshold based on the previous history + # of the cycle collector's effectiveness: + # we're effective when we collected 50% or more of the nodes + # we touched. If we're effective, we can reset the threshold: + if j.keepThreshold: + discard + elif j.freed * 2 >= j.touched: + when not defined(nimFixedOrc): + rootsThreshold = max(rootsThreshold div 3 * 2, 16) + else: + rootsThreshold = 0 + #cfprintf(cstderr, "[collectCycles] freed %ld, touched %ld new threshold %ld\n", j.freed, j.touched, rootsThreshold) + elif rootsThreshold < high(int) div 4: + rootsThreshold = (if rootsThreshold <= 0: defaultThreshold else: rootsThreshold) * 3 div 2 + when logOrc: + cfprintf(cstderr, "[collectCycles] end; freed %ld new threshold %ld touched: %ld mem: %ld rcSum: %ld edges: %ld\n", j.freed, rootsThreshold, j.touched, + getOccupiedMem(), j.rcSum, j.edges) + when defined(nimOrcStats): + inc freedCyclicObjects, j.freed + +when defined(nimOrcStats): + type + OrcStats* = object ## Statistics of the cycle collector subsystem. + freedCyclicObjects*: int ## Number of freed cyclic objects. + proc GC_orcStats*(): OrcStats = + ## Returns the statistics of the cycle collector subsystem. + result = OrcStats(freedCyclicObjects: freedCyclicObjects) + +proc registerCycle(s: Cell; desc: PNimTypeV2) = + s.rootIdx = roots.len+1 + if roots.d == nil: init(roots) + add(roots, s, desc) + + if roots.len - defaultThreshold >= rootsThreshold: + collectCycles() + when logOrc: + writeCell("[added root]", s, desc) + + orcAssert strstr(desc.name, "TType") == nil, "added a TType as a root!" + +proc GC_runOrc* = + ## Forces a cycle collection pass. + collectCycles() + orcAssert roots.len == 0, "roots not empty!" + +proc GC_enableOrc*() = + ## Enables the cycle collector subsystem of `--mm:orc`. This is a `--mm:orc` + ## specific API. Check with `when defined(gcOrc)` for its existence. + when not defined(nimStressOrc): + rootsThreshold = 0 + +proc GC_disableOrc*() = + ## Disables the cycle collector subsystem of `--mm:orc`. This is a `--mm:orc` + ## specific API. Check with `when defined(gcOrc)` for its existence. + when not defined(nimStressOrc): + rootsThreshold = high(int) + +proc GC_prepareOrc*(): int {.inline.} = roots.len + +proc GC_partialCollect*(limit: int) = + partialCollect(limit) + +proc GC_fullCollect* = + ## Forces a full garbage collection pass. With `--mm:orc` triggers the cycle + ## collector. This is an alias for `GC_runOrc`. + collectCycles() + +proc GC_enableMarkAndSweep*() = + ## For `--mm:orc` an alias for `GC_enableOrc`. + GC_enableOrc() + +proc GC_disableMarkAndSweep*() = + ## For `--mm:orc` an alias for `GC_disableOrc`. + GC_disableOrc() + +const + acyclicFlag = 1 # see also cggtypes.nim, proc genTypeInfoV2Impl + +when optimizedOrc: + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 and (s.rc and maybeCycle) != 0 +else: + template markedAsCyclic(s: Cell; desc: PNimTypeV2): bool = + (desc.flags and acyclicFlag) == 0 + +proc rememberCycle(isDestroyAction: bool; s: Cell; desc: PNimTypeV2) {.noinline.} = + if isDestroyAction: + if s.rootIdx > 0: + unregisterCycle(s) + else: + # do not call 'rememberCycle' again unless this cell + # got an 'incRef' event: + if s.rootIdx == 0 and markedAsCyclic(s, desc): + s.setColor colBlack + registerCycle(s, desc) + +proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + rememberCycle(result, cell, cast[ptr PNimTypeV2](p)[]) + +proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + if result: + if cell.rootIdx > 0: + unregisterCycle(cell) + +proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p %s\n", p, desc.name) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + rememberCycle(result, cell, desc) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index aa86ee379..5509d0070 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -28,7 +28,30 @@ const doNotUnmap = not (defined(amd64) or defined(i386)) or defined(windows) or defined(nimAllocNoUnmap) -when defined(emscripten) and not defined(StandaloneHeapSize): +when defined(nimAllocPagesViaMalloc): + when not defined(gcArc) and not defined(gcOrc) and not defined(gcAtomicArc): + {.error: "-d:nimAllocPagesViaMalloc is only supported with --mm:arc or --mm:atomicArc or --mm:orc".} + + proc osTryAllocPages(size: int): pointer {.inline.} = + let base = c_malloc(csize_t size + PageSize - 1 + sizeof(uint32)) + if base == nil: raiseOutOfMem() + # memory layout: padding + offset (4 bytes) + user_data + # in order to deallocate: read offset at user_data - 4 bytes, + # then deallocate user_data - offset + let offset = PageSize - (cast[int](base) and (PageSize - 1)) + cast[ptr uint32](base +! (offset - sizeof(uint32)))[] = uint32(offset) + result = base +! offset + + proc osAllocPages(size: int): pointer {.inline.} = + result = osTryAllocPages(size) + if result == nil: raiseOutOfMem() + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # read offset at p - 4 bytes, then deallocate (p - offset) pointer + let offset = cast[ptr uint32](p -! sizeof(uint32))[] + c_free(p -! offset) + +elif defined(emscripten) and not defined(StandaloneHeapSize): const PROT_READ = 1 # page can be read PROT_WRITE = 2 # page can be written @@ -57,12 +80,12 @@ when defined(emscripten) and not defined(StandaloneHeapSize): let pos = cast[int](result) # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + var new_pos = cast[int](pos) +% (PageSize - (pos %% PageSize)) if (new_pos-pos) < sizeof(EmscriptenMMapBlock): new_pos = new_pos +% PageSize result = cast[pointer](new_pos) - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](result) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) mmapDescr.realSize = realSize @@ -73,130 +96,25 @@ when defined(emscripten) and not defined(StandaloneHeapSize): proc osTryAllocPages(size: int): pointer = osAllocPages(size) proc osDeallocPages(p: pointer, size: int) {.inline.} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](p) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) munmap(mmapDescr.realPointer, mmapDescr.realSize) elif defined(genode) and not defined(StandaloneHeapSize): include genode/alloc # osAllocPages, osTryAllocPages, osDeallocPages -elif defined(nintendoswitch) and not defined(StandaloneHeapSize): - - import nintendoswitch/switch_memory - - type - PSwitchBlock = ptr NSwitchBlock - ## This will hold the heap pointer data in a separate - ## block of memory that is PageSize bytes above - ## the requested memory. It's the only good way - ## to pass around data with heap allocations - NSwitchBlock {.pure, inheritable.} = object - realSize: int - heap: pointer # pointer to main heap alloc - heapMirror: pointer # pointer to virtmem mapped heap - - proc alignSize(size: int): int {.inline.} = - ## Align a size integer to be in multiples of PageSize - ## The nintendo switch will not allocate memory that is not - ## aligned to 0x1000 bytes and will just crash. - (size + (PageSize - 1)) and not (PageSize - 1) - - proc deallocate(heapMirror: pointer, heap: pointer, size: int) = - # Unmap the allocated memory - discard svcUnmapMemory(heapMirror, heap, size.uint64) - # These should be called (theoretically), but referencing them crashes the switch. - # The above call seems to free all heap memory, so these are not needed. - # virtmemFreeMap(nswitchBlock.heapMirror, nswitchBlock.realSize.csize) - # free(nswitchBlock.heap) - - proc freeMem(p: pointer) = - # Retrieve the switch block data from the pointer we set before - # The data is located just sizeof(NSwitchBlock) bytes below - # the top of the pointer to the heap - let - nswitchDescrPos = cast[ByteAddress](p) -% sizeof(NSwitchBlock) - nswitchBlock = cast[PSwitchBlock](nswitchDescrPos) - - deallocate( - nswitchBlock.heapMirror, nswitchBlock.heap, nswitchBlock.realSize - ) - - proc storeHeapData(address, heapMirror, heap: pointer, size: int) {.inline.} = - ## Store data in the heap for deallocation purposes later - - # the position of our heap pointer data. Since we allocated PageSize extra - # bytes, we should have a buffer on top of the requested size of at least - # PageSize bytes, which is much larger than sizeof(NSwitchBlock). So we - # decrement the address by sizeof(NSwitchBlock) and use that address - # to store our pointer data - let nswitchBlockPos = cast[ByteAddress](address) -% sizeof(NSwitchBlock) - - # We need to store this in a pointer obj (PSwitchBlock) so that the data sticks - # at the address we've chosen. If NSwitchBlock is used here, the data will - # be all 0 when we try to retrieve it later. - var nswitchBlock = cast[PSwitchBlock](nswitchBlockPos) - nswitchBlock.realSize = size - nswitchBlock.heap = heap - nswitchBlock.heapMirror = heapMirror - - proc getOriginalHeapPosition(address: pointer, difference: int): pointer {.inline.} = - ## This function sets the heap back to the originally requested - ## size - let - pos = cast[int](address) - newPos = cast[ByteAddress](pos) +% difference - - return cast[pointer](newPos) - - template allocPages(size: int, outOfMemoryStmt: untyped): untyped = - # This is to ensure we get a block of memory the requested - # size, as well as space to store our structure - let realSize = alignSize(size + sizeof(NSwitchBlock)) - - let heap = memalign(PageSize, realSize) - - if heap.isNil: - outOfMemoryStmt - - let heapMirror = virtmemReserveMap(realSize.csize) - result = heapMirror - - let rc = svcMapMemory(heapMirror, heap, realSize.uint64) - # Any return code not equal 0 means an error in libnx - if rc.uint32 != 0: - deallocate(heapMirror, heap, realSize) - outOfMemoryStmt - - # set result to be the original size requirement - result = getOriginalHeapPosition(result, realSize - size) - - storeHeapData(result, heapMirror, heap, realSize) - - proc osAllocPages(size: int): pointer {.inline.} = - allocPages(size): - raiseOutOfMem() - - proc osTryAllocPages(size: int): pointer = - allocPages(size): - return nil - - proc osDeallocPages(p: pointer, size: int) = - # Note that in order for the Switch not to crash, a call to - # deallocHeap(runFinalizers = true, allowGcAfterwards = false) - # must be run before gfxExit(). The Switch requires all memory - # to be deallocated before the graphics application has exited. - # - # gfxExit() can be found in <switch/gfx/gfx.h> in the github - # repo https://github.com/switchbrew/libnx - when reallyOsDealloc: - freeMem(p) - elif defined(posix) and not defined(StandaloneHeapSize): const PROT_READ = 1 # page can be read PROT_WRITE = 2 # page can be written - when defined(macosx) or defined(bsd): + when defined(netbsd) or defined(openbsd): + # OpenBSD security for setjmp/longjmp coroutines + var MAP_STACK {.importc: "MAP_STACK", header: "<sys/mman.h>".}: cint + else: + const MAP_STACK = 0 # avoid sideeffects + + when defined(macosx) or defined(freebsd): const MAP_ANONYMOUS = 0x1000 const MAP_PRIVATE = 0x02 # Changes are private elif defined(solaris): @@ -210,7 +128,7 @@ elif defined(posix) and not defined(StandaloneHeapSize): elif defined(haiku): const MAP_ANONYMOUS = 0x08 const MAP_PRIVATE = 0x02 - else: + else: # posix including netbsd or openbsd var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint MAP_PRIVATE {.importc: "MAP_PRIVATE", header: "<sys/mman.h>".}: cint @@ -222,13 +140,13 @@ elif defined(posix) and not defined(StandaloneHeapSize): proc osAllocPages(size: int): pointer {.inline.} = result = mmap(nil, cast[csize_t](size), PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + MAP_ANONYMOUS or MAP_PRIVATE or MAP_STACK, -1, 0) if result == nil or result == cast[pointer](-1): raiseOutOfMem() proc osTryAllocPages(size: int): pointer {.inline.} = result = mmap(nil, cast[csize_t](size), PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + MAP_ANONYMOUS or MAP_PRIVATE or MAP_STACK, -1, 0) if result == cast[pointer](-1): result = nil proc osDeallocPages(p: pointer, size: int) {.inline.} = @@ -271,7 +189,7 @@ elif defined(windows) and not defined(StandaloneHeapSize): when reallyOsDealloc: if virtualFree(p, 0, MEM_RELEASE) == 0: cprintf "virtualFree failing!" - quit 1 + rawQuit 1 #VirtualFree(p, size, MEM_DECOMMIT) elif hostOS == "standalone" or defined(StandaloneHeapSize): diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index 6e39dc7f2..0619f3fca 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -10,6 +10,8 @@ ## Platform detection for NimScript. This module is included by the system module! ## Do not import it directly! +# CPU architectures have alias names mapped in tools/niminst/makefile.nimf + type CpuPlatform* {.pure.} = enum ## the CPU this program will run on. none, ## unknown CPU @@ -33,14 +35,17 @@ type vm, ## Some Virtual machine: Nim's VM or JavaScript avr, ## AVR based processor msp430, ## TI MSP430 microcontroller - riscv64 ## RISC-V 64-bit processor - wasm32 ## WASM, 32-bit + riscv32, ## RISC-V 32-bit processor + riscv64, ## RISC-V 64-bit processor + wasm32, ## WASM, 32-bit + e2k, ## MCST Elbrus 2000 + loongarch64, ## LoongArch 64-bit processor + s390x ## IBM Z OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, irix, netbsd, freebsd, openbsd, aix, palmos, qnx, amiga, - atari, netware, macos, macosx, haiku, android, js, nimVM, - standalone, nintendoswitch + atari, netware, macos, macosx, haiku, android, js, standalone, nintendoswitch const targetOS* = when defined(windows): OsPlatform.windows @@ -65,7 +70,6 @@ const elif defined(haiku): OsPlatform.haiku elif defined(android): OsPlatform.android elif defined(js): OsPlatform.js - elif defined(nimVM): OsPlatform.nimVM elif defined(standalone): OsPlatform.standalone elif defined(nintendoswitch): OsPlatform.nintendoswitch else: OsPlatform.none @@ -91,7 +95,11 @@ const elif defined(vm): CpuPlatform.vm elif defined(avr): CpuPlatform.avr elif defined(msp430): CpuPlatform.msp430 + elif defined(riscv32): CpuPlatform.riscv32 elif defined(riscv64): CpuPlatform.riscv64 elif defined(wasm32): CpuPlatform.wasm32 + elif defined(e2k): CpuPlatform.e2k + elif defined(loongarch64): CpuPlatform.loongarch64 + elif defined(s390x): CpuPlatform.s390x else: CpuPlatform.none ## the CPU this program will run on. diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index 0649f1176..e7eb6ac82 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -60,13 +60,13 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = b = b.prev var - profilingRequestedHook*: proc (): bool {.nimcall, locks: 0, gcsafe.} + profilingRequestedHook*: proc (): bool {.nimcall, gcsafe.} ## set this variable to provide a procedure that implements a profiler in ## user space. See the `nimprof` module for a reference implementation. when defined(memProfiler): type - MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, locks: 0, gcsafe.} + MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, gcsafe.} var profilerHook*: MemProfilerHook diff --git a/lib/system/rawquits.nim b/lib/system/rawquits.nim new file mode 100644 index 000000000..f0ead10c6 --- /dev/null +++ b/lib/system/rawquits.nim @@ -0,0 +1,27 @@ +import system/ctypes + +when defined(nimNoQuit): + proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(genode): + import genode/env + + var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr + + type GenodeEnv = GenodeEnvPtr + ## Opaque type representing Genode environment. + + proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, + importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "<base/sleep.h>".} + + proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = + systemEnv.rawQuit(errorcode) + + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc rawQuit(errorcode: cint) {. + magic: "Exit", importc: "exit", header: "<stdlib.h>", noreturn.} \ No newline at end of file diff --git a/lib/system/refs_v2.nim b/lib/system/refs_v2.nim deleted file mode 100644 index f6f3ba01c..000000000 --- a/lib/system/refs_v2.nim +++ /dev/null @@ -1,176 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2019 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -#[ -In this new runtime we simplify the object layouts a bit: The runtime type -information is only accessed for the objects that have it and it's always -at offset 0 then. The ``ref`` object header is independent from the -runtime type and only contains a reference count. - -Object subtyping is checked via the generated 'name'. This should have -comparable overhead to the old pointer chasing approach but has the benefit -that it works across DLL boundaries. - -The generated name is a concatenation of the object names in the hierarchy -so that a subtype check becomes a substring check. For example:: - - type - ObjectA = object of RootObj - ObjectB = object of ObjectA - -ObjectA's ``name`` is "|ObjectA|RootObj|". -ObjectB's ``name`` is "|ObjectB|ObjectA|RootObj|". - -Now to check for ``x of ObjectB`` we need to check -for ``x.typ.name.hasSubstring("|ObjectB|")``. In the actual implementation, -however, we could also use a -hash of ``package & "." & module & "." & name`` to save space. - -]# - -const - rcIncrement = 0b1000 # so that lowest 3 bits are not touched - rcMask = 0b111 - -type - RefHeader = object - rc: int # the object header is now a single RC field. - # we could remove it in non-debug builds for the 'owned ref' - # design but this seems unwise. - Cell = ptr RefHeader - -template `+!`(p: pointer, s: int): pointer = - cast[pointer](cast[int](p) +% s) - -template `-!`(p: pointer, s: int): pointer = - cast[pointer](cast[int](p) -% s) - -template head(p: pointer): Cell = - cast[Cell](cast[int](p) -% sizeof(RefHeader)) - -const - traceCollector = defined(traceArc) - -proc nimNewObj(size: int): pointer {.compilerRtl.} = - let s = size + sizeof(RefHeader) - when defined(nimscript): - discard - elif defined(useMalloc): - var orig = c_malloc(cuint s) - nimZeroMem(orig, s) - result = orig +! sizeof(RefHeader) - elif compileOption("threads"): - result = allocShared0(s) +! sizeof(RefHeader) - else: - result = alloc0(s) +! sizeof(RefHeader) - when traceCollector: - cprintf("[Allocated] %p\n", result -! sizeof(RefHeader)) - -proc nimNewObjUninit(size: int): pointer {.compilerRtl.} = - # Same as 'newNewObj' but do not initialize the memory to zero. - # The codegen proved for us that this is not necessary. - let s = size + sizeof(RefHeader) - when defined(nimscript): - discard - elif defined(useMalloc): - var orig = cast[ptr RefHeader](c_malloc(cuint s)) - elif compileOption("threads"): - var orig = cast[ptr RefHeader](allocShared(s)) - else: - var orig = cast[ptr RefHeader](alloc(s)) - orig.rc = 0 - result = orig +! sizeof(RefHeader) - when traceCollector: - cprintf("[Allocated] %p\n", result -! sizeof(RefHeader)) - -proc nimDecWeakRef(p: pointer) {.compilerRtl, inl.} = - dec head(p).rc, rcIncrement - -proc nimIncRef(p: pointer) {.compilerRtl, inl.} = - inc head(p).rc, rcIncrement - when traceCollector: - cprintf("[INCREF] %p\n", head(p)) - -proc nimRawDispose(p: pointer) {.compilerRtl.} = - when not defined(nimscript): - when traceCollector: - cprintf("[Freed] %p\n", p -! sizeof(RefHeader)) - when defined(nimOwnedEnabled): - if head(p).rc >= rcIncrement: - cstderr.rawWrite "[FATAL] dangling references exist\n" - quit 1 - when defined(useMalloc): - c_free(p -! sizeof(RefHeader)) - elif compileOption("threads"): - deallocShared(p -! sizeof(RefHeader)) - else: - dealloc(p -! sizeof(RefHeader)) - -template dispose*[T](x: owned(ref T)) = nimRawDispose(cast[pointer](x)) -#proc dispose*(x: pointer) = nimRawDispose(x) - -proc nimDestroyAndDispose(p: pointer) {.compilerRtl, raises: [].} = - let d = cast[ptr PNimType](p)[].destructor - if d != nil: cast[DestructorProc](d)(p) - when false: - cstderr.rawWrite cast[ptr PNimType](p)[].name - cstderr.rawWrite "\n" - if d == nil: - cstderr.rawWrite "bah, nil\n" - else: - cstderr.rawWrite "has destructor!\n" - nimRawDispose(p) - -when defined(gcOrc): - include cyclicrefs_v2 - -proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} = - if p != nil: - var cell = head(p) - if (cell.rc and not rcMask) == 0: - result = true - when traceCollector: - cprintf("[ABOUT TO DESTROY] %p\n", cell) - else: - dec cell.rc, rcIncrement - # According to Lins it's correct to do nothing else here. - when traceCollector: - cprintf("[DeCREF] %p\n", cell) - -proc GC_unref*[T](x: ref T) = - ## New runtime only supports this operation for 'ref T'. - if nimDecRefIsLast(cast[pointer](x)): - # XXX this does NOT work for virtual destructors! - `=destroy`(x[]) - nimRawDispose(cast[pointer](x)) - -proc GC_ref*[T](x: ref T) = - ## New runtime only supports this operation for 'ref T'. - if x != nil: nimIncRef(cast[pointer](x)) - -template GC_fullCollect* = - ## Forces a full garbage collection pass. With ``--gc:arc`` a nop. - discard - -template setupForeignThreadGc* = - ## With ``--gc:arc`` a nop. - discard - -template tearDownForeignThreadGc* = - ## With ``--gc:arc`` a nop. - discard - -proc isObj(obj: PNimType, subclass: cstring): bool {.compilerRtl, inl.} = - proc strstr(s, sub: cstring): cstring {.header: "<string.h>", importc.} - - result = strstr(obj.name, subclass) != nil - -proc chckObj(obj: PNimType, subclass: cstring) {.compilerRtl.} = - # checks if obj is of type subclass: - if not isObj(obj, subclass): sysFatal(ObjectConversionError, "invalid object conversion") diff --git a/lib/system/repr.nim b/lib/system/repr.nim index cc5ba2fee..13118e40b 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -16,14 +16,9 @@ proc reprInt(x: int64): string {.compilerproc.} = return $x proc reprFloat(x: float): string {.compilerproc.} = return $x proc reprPointer(x: pointer): string {.compilerproc.} = - when defined(nimNoArrayToCstringConversion): - result = newString(60) - let n = c_sprintf(addr result[0], "%p", x) - setLen(result, n) - else: - var buf: array[0..59, char] - discard c_sprintf(buf, "%p", x) - return $buf + result = newString(60) + let n = c_snprintf(cast[cstring](addr result[0]), csize_t(60), "%p", x) + setLen(result, n) proc reprStrAux(result: var string, s: cstring; len: int) = if cast[pointer](s) == nil: @@ -77,6 +72,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = result = $e & " (invalid data!)" +include system/repr_impl + type PByteArray = ptr UncheckedArray[byte] # array[0xffff, byte] @@ -102,6 +99,7 @@ proc reprSetAux(result: var string, p: pointer, typ: PNimType) = of 4: u = cast[ptr uint32](p)[] of 8: u = cast[ptr uint64](p)[] else: + u = uint64(0) var a = cast[PByteArray](p) for i in 0 .. typ.size*8-1: if (uint(a[i shr 3]) and (1'u shl (i and 7))) != 0: @@ -157,7 +155,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..typ.size div bs - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), typ.base, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), typ.base, cl) add result, "]" when defined(nimSeqsV2): @@ -172,7 +170,7 @@ when not defined(useNimRtl): template payloadPtr(x: untyped): untyped = cast[PGenericSeq](x).p else: - const payloadOffset = GenericSeqSize + const payloadOffset = GenericSeqSize ## the payload offset always depends on the alignment of the member type. template payloadPtr(x: untyped): untyped = x proc reprSequence(result: var string, p: pointer, typ: PNimType, @@ -185,7 +183,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..cast[PGenericSeq](p).len-1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](payloadPtr(p)) + payloadOffset + i*bs), + reprAux(result, cast[pointer](cast[int](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), typ.base, cl) add result, "]" @@ -196,14 +194,14 @@ when not defined(useNimRtl): of nkSlot: add result, $n.name add result, " = " - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) of nkList: for i in 0..n.len-1: if i > 0: add result, ",\n" reprRecordAux(result, p, n.sons[i], cl) of nkCase: var m = selectBranch(p, n) - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) if m != nil: reprRecordAux(result, p, m, cl) proc reprRecord(result: var string, p: pointer, typ: PNimType, @@ -285,7 +283,7 @@ when not defined(useNimRtl): of tyString: let sp = cast[ptr string](p) reprStrAux(result, sp[].cstring, sp[].len) - of tyCString: + of tyCstring: let cs = cast[ptr cstring](p)[] if cs.isNil: add result, "nil" else: reprStrAux(result, cs, cs.len) @@ -309,7 +307,7 @@ when not defined(useNimRtl): var bs = elemtyp.size for i in 0..length - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), elemtyp, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), elemtyp, cl) add result, "]" deinitReprClosure(cl) @@ -324,5 +322,6 @@ when not defined(useNimRtl): else: var p = p reprAux(result, addr(p), typ, cl) - add result, "\n" + when defined(nimLegacyReprWithNewline): # see PR #16034 + add result, "\n" deinitReprClosure(cl) diff --git a/lib/system/repr_impl.nim b/lib/system/repr_impl.nim new file mode 100644 index 000000000..b9ec1890f --- /dev/null +++ b/lib/system/repr_impl.nim @@ -0,0 +1,15 @@ +#[ +other APIs common to system/repr and system/reprjs could be refactored here, eg: +* reprChar +* reprBool +* reprStr + +Another possibility in future work would be to have a single include file instead +of system/repr and system/reprjs, and use `when defined(js)` inside it. +]# + +proc reprDiscriminant*(e: int, typ: PNimType): string {.compilerRtl.} = + case typ.kind + of tyEnum: reprEnum(e, typ) + of tyBool: $(e != 0) + else: $e diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index beca1d842..d2aef536c 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -1,53 +1,87 @@ +include system/inclrtl + +when defined(nimPreviewSlimSystem): + import std/formatfloat + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## imported from typetraits -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} ## imported from typetraits +proc rangeBase(T: typedesc): typedesc {.magic: "TypeTrait".} + # skip one level of range; return the base type of a range type + proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} -proc repr*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int): string = + ## Same as $x + $x -proc repr*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int64): string = + ## Same as $x + $x -proc repr*(x: float): string {.magic: "FloatToStr", noSideEffect.} - ## repr for a float argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: uint64): string {.noSideEffect.} = + ## Same as $x + $x + +proc repr*(x: float): string = + ## Same as $x + $x proc repr*(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## repr for a boolean argument. Returns `x` ## converted to the string "false" or "true". -proc repr*(x: char): string {.magic: "CharToStr", noSideEffect.} +proc repr*(x: char): string {.noSideEffect, raises: [].} = ## repr for a character argument. Returns `x` - ## converted to a string. + ## converted to an escaped string. ## - ## .. code-block:: Nim - ## assert $'c' == "c" - -proc repr*(x: cstring): string {.magic: "CStrToStr", noSideEffect.} - ## repr for a CString argument. Returns `x` - ## converted to a string. + ## ```Nim + ## assert repr('c') == "'c'" + ## ``` + result = "'" + # Elides string creations if not needed + if x in {'\\', '\0'..'\31', '\127'..'\255'}: + result.add '\\' + if x in {'\0'..'\31', '\127'..'\255'}: + result.add $x.uint8 + else: + result.add x + result.add '\'' -proc repr*(x: string): string {.magic: "StrToStr", noSideEffect.} +proc repr*(x: string | cstring): string {.noSideEffect, raises: [].} = ## repr for a string argument. Returns `x` - ## as it is. This operator is useful for generic code, so - ## that ``$expr`` also works if ``expr`` is already a string. + ## converted to a quoted and escaped string. + result = "\"" + for i in 0..<x.len: + if x[i] in {'"', '\\', '\0'..'\31', '\127'..'\255'}: + result.add '\\' + case x[i]: + of '\n': + result.add "n\n" + of '\0'..'\9', '\11'..'\31', '\127'..'\255': + result.add $x[i].uint8 + else: + result.add x[i] + result.add '\"' -proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} +proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect, raises: [].} ## repr for an enumeration argument. This works for ## any enumeration type thanks to compiler magic. ## ## If a `repr` operator for a concrete enumeration is provided, this is ## used instead. (In other words: *Overwriting* is possible.) +proc reprDiscriminant*(e: int): string {.compilerproc.} = + # repr and reprjs can use `PNimType` to symbolize `e`; making this work here + # would require a way to pass the set of enum stringified values to cgen. + $e + proc repr*(p: pointer): string = ## repr of pointer as its hexadecimal value - if p == nil: + if p == nil: result = "nil" else: when nimvm: @@ -61,12 +95,21 @@ proc repr*(p: pointer): string = result[j] = HexChars[n and 0xF] n = n shr 4 -template repr*(x: distinct): string = - repr(distinctBase(typeof(x))(x)) +proc repr*(p: proc | iterator {.closure.}): string = + ## repr of a proc as its address + repr(cast[ptr pointer](unsafeAddr p)[]) + +template repr*[T: distinct|(range and not enum)](x: T): string = + when T is range: # add a branch to handle range + repr(rangeBase(typeof(x))(x)) + elif T is distinct: + repr(distinctBase(typeof(x))(x)) + else: + {.error: "cannot happen".} template repr*(t: typedesc): string = $t -proc reprObject[T: tuple|object](res: var string, x: T) = +proc reprObject[T: tuple|object](res: var string, x: T) {.noSideEffect, raises: [].} = res.add '(' var firstElement = true const isNamed = T is object or isNamedTuple(T) @@ -87,28 +130,28 @@ proc reprObject[T: tuple|object](res: var string, x: T) = res.add(')') -proc repr*[T: tuple|object](x: T): string = +proc repr*[T: tuple|object](x: T): string {.noSideEffect, raises: [].} = ## Generic `repr` operator for tuples that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(23, 45) == "(23, 45)" ## $(a: 23, b: 45) == "(a: 23, b: 45)" ## $() == "()" + ## ``` when T is object: result = $typeof(x) reprObject(result, x) - -proc repr*[T](x: ptr T): string = - result.add repr(pointer(x)) & " " - result.add repr(x[]) -proc repr*[T](x: ref T | ptr T): string = +proc repr*[T](x: ref T | ptr T): string {.noSideEffect, raises: [].} = if isNil(x): return "nil" - result = $typeof(x) - reprObject(result, x[]) + when T is object: + result = $typeof(x) + reprObject(result, x[]) + else: + result = when typeof(x) is ref: "ref " else: "ptr " + result.add repr(x[]) -proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string = +proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string {.noSideEffect, raises: [].} = result = prefix var firstElement = true for value in items(x): @@ -122,29 +165,19 @@ proc collectionToRepr[T](x: T, prefix, separator, suffix: string): string = proc repr*[T](x: set[T]): string = ## Generic `repr` operator for sets that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## ${23, 45} == "{23, 45}" + ## ``` collectionToRepr(x, "{", ", ", "}") proc repr*[T](x: seq[T]): string = ## Generic `repr` operator for seqs that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45]) == "@[23, 45]" + ## ``` collectionToRepr(x, "@[", ", ", "]") -proc repr*[T, U](x: HSlice[T, U]): string = - ## Generic `repr` operator for slices that is lifted from the components - ## of `x`. Example: - ## - ## .. code-block:: Nim - ## $(1 .. 5) == "1 .. 5" - result = repr(x.a) - result.add(" .. ") - result.add(repr(x.b)) - proc repr*[T, IDX](x: array[IDX, T]): string = ## Generic `repr` operator for arrays that is lifted from the components. collectionToRepr(x, "[", ", ", "]") @@ -152,7 +185,10 @@ proc repr*[T, IDX](x: array[IDX, T]): string = proc repr*[T](x: openArray[T]): string = ## Generic `repr` operator for openarrays that is lifted from the components ## of `x`. Example: - ## - ## .. code-block:: Nim + ## ```Nim ## $(@[23, 45].toOpenArray(0, 1)) == "[23, 45]" + ## ``` collectionToRepr(x, "[", ", ", "]") + +proc repr*[T](x: UncheckedArray[T]): string = + "[...]" diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index c45053537..761d66aec 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -8,37 +8,35 @@ # # The generic ``repr`` procedure for the javascript backend. -proc reprInt(x: int64): string {.compilerproc.} = return $x -proc reprFloat(x: float): string {.compilerproc.} = - # Js toString doesn't differentiate between 1.0 and 1, - # but we do. - if $x == $(x.int): $x & ".0" - else: $x +when defined(nimPreviewSlimSystem): + import std/formatfloat + +proc reprInt(x: int64): string {.compilerproc.} = $x +proc reprInt(x: uint64): string {.compilerproc.} = $x +proc reprInt(x: int): string {.compilerproc.} = $x +proc reprFloat(x: float): string {.compilerproc.} = $x proc reprPointer(p: pointer): string {.compilerproc.} = # Do we need to generate the full 8bytes ? In js a pointer is an int anyway var tmp: int - {. emit: """ - if (`p`_Idx == null) { - `tmp` = 0; - } else { - `tmp` = `p`_Idx; - } - """ .} + {.emit: "`tmp` = `p`_Idx || 0;".} result = $tmp proc reprBool(x: bool): string {.compilerRtl.} = if x: result = "true" else: result = "false" -proc isUndefined[T](x: T): bool {.inline.} = {.emit: "`result` = `x` === undefined;"} - proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = - if not typ.node.sons[e].isUndefined: - result = makeNimstrLit(typ.node.sons[e].name) + var tmp: bool + let item = typ.node.sons[e] + {.emit: "`tmp` = `item` !== undefined;".} + if tmp: + result = makeNimstrLit(item.name) else: result = $e & " (invalid data!)" +include system/repr_impl + proc reprChar(x: char): string {.compilerRtl.} = result = "\'" case x @@ -48,7 +46,7 @@ proc reprChar(x: char): string {.compilerRtl.} = else: add(result, x) add(result, "\'") -proc reprStrAux(result: var string, s: cstring, len: int) = +proc reprStrAux(result: var string, s: cstring | string, len: int) = add(result, "\"") for i in 0 .. len-1: let c = s[i] @@ -63,17 +61,7 @@ proc reprStrAux(result: var string, s: cstring, len: int) = add(result, "\"") proc reprStr(s: string): string {.compilerRtl.} = - result = "" - var sIsNil = false - asm """`sIsNil` = `s` === null""" - if sIsNil: # cast[pointer](s).isNil: - # Handle nil strings here because they don't have a length field in js - # TODO: check for null/undefined before generating call to length in js? - # Also: c backend repr of a nil string is <pointer>"", but repr of an - # array of string that is not initialized is [nil, nil, ...] ?? - add(result, "nil") - else: - reprStrAux(result, s, s.len) + reprStrAux(result, s, s.len) proc addSetElem(result: var string, elem: int, typ: PNimType) = # Dispatch each set element to the correct repr<Type> proc @@ -114,7 +102,6 @@ proc reprSetAux(result: var string, s: int, typ: PNimType) = add(result, "}") proc reprSet(e: int, typ: PNimType): string {.compilerRtl.} = - result = "" reprSetAux(result, e, typ) type @@ -130,24 +117,9 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, cl: var ReprClosure) proc reprArray(a: pointer, typ: PNimType, cl: var ReprClosure): string {.compilerRtl.} = - var isNilArrayOrSeq: bool - # isnil is not enough here as it would try to deref `a` without knowing what's inside - {. emit: """ - if (`a` == null) { - `isNilArrayOrSeq` = true; - } else if (`a`[0] == null) { - `isNilArrayOrSeq` = true; - } else { - `isNilArrayOrSeq` = false; - }; - """ .} - if typ.kind == tySequence and isNilArrayOrSeq: - return "nil" - # We prepend @ to seq, the C backend prepends the pointer to the seq. result = if typ.kind == tySequence: "@[" else: "[" var len: int = 0 - var i: int = 0 {. emit: "`len` = `a`.length;\n" .} var dereffed: pointer = a @@ -163,8 +135,8 @@ proc reprArray(a: pointer, typ: PNimType, add(result, "]") -proc isPointedToNil(p: pointer): bool {.inline.}= - {. emit: "if (`p` === null) {`result` = true};\n" .} +proc isPointedToNil(p: pointer): bool = + {. emit: "if (`p` === null) {`result` = true;}\n" .} proc reprRef(result: var string, p: pointer, typ: PNimType, cl: var ReprClosure) = @@ -205,7 +177,6 @@ proc reprRecordAux(result: var string, o: pointer, typ: PNimType, cl: var ReprCl add(result, "]") proc reprRecord(o: pointer, typ: PNimType, cl: var ReprClosure): string {.compilerRtl.} = - result = "" reprRecordAux(result, o, typ, cl) @@ -223,8 +194,12 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, return dec(cl.recDepth) case typ.kind - of tyInt..tyInt64, tyUInt..tyUInt64: + of tyInt..tyInt32, tyUInt..tyUInt32: add(result, reprInt(cast[int](p))) + of tyInt64: + add(result, reprInt(cast[int64](p))) + of tyUInt64: + add(result, reprInt(cast[uint64](p))) of tyChar: add(result, reprChar(cast[char](p))) of tyBool: @@ -235,7 +210,7 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, var fp: int {. emit: "`fp` = `p`;\n" .} add(result, reprStr(cast[string](p))) - of tyCString: + of tyCstring: var fp: cstring {. emit: "`fp` = `p`;\n" .} if fp.isNil: @@ -271,6 +246,6 @@ proc reprAux(result: var string, p: pointer, typ: PNimType, proc reprAny(p: pointer, typ: PNimType): string {.compilerRtl.} = var cl: ReprClosure initReprClosure(cl) - result = "" reprAux(result, p, typ, cl) - add(result, "\n") \ No newline at end of file + when defined(nimLegacyReprWithNewline): # see PR #16034 + add result, "\n" diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 3bd5e291b..572e77408 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -8,14 +8,16 @@ # -# import typetraits +# import std/typetraits # strs already imported allocateds for us. -proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".} + +# Some optimizations here may be not to empty-seq-initialize some symbols, then StrictNotNil complains. +{.push warning[StrictNotNil]: off.} # See https://github.com/nim-lang/Nim/issues/21401 + ## Default seq implementation used by Nim's core. type - NimSeqPayloadBase = object cap: int @@ -23,35 +25,85 @@ type cap: int data: UncheckedArray[T] - NimSeqV2*[T] = object + NimSeqV2*[T] = object # \ + # if you change this implementation, also change seqs_v2_reimpl.nim! len: int p: ptr NimSeqPayload[T] + NimRawSeq = object + len: int + p: pointer + const nimSeqVersion {.core.} = 2 # XXX make code memory safe for overflows in '*' -proc newSeqPayload(cap, elemSize: int): pointer {.compilerRtl, raises: [].} = +proc newSeqPayload(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises: [].} = # we have to use type erasure here as Nim does not support generic # compilerProcs. Oh well, this will all be inlined anyway. if cap > 0: - var p = cast[ptr NimSeqPayloadBase](allocShared0(cap * elemSize + sizeof(NimSeqPayloadBase))) + var p = cast[ptr NimSeqPayloadBase](alignedAlloc0(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize, elemAlign)) p.cap = cap result = p else: result = nil -proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize: int): pointer {. - noSideEffect, raises: [].} = +proc newSeqPayloadUninit(cap, elemSize, elemAlign: int): pointer {.compilerRtl, raises: [].} = + # Used in `newSeqOfCap()`. + if cap > 0: + var p = cast[ptr NimSeqPayloadBase](alignedAlloc(align(sizeof(NimSeqPayloadBase), elemAlign) + cap * elemSize, elemAlign)) + p.cap = cap + result = p + else: + result = nil + +template `+!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +template `-!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) -% s) + +proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + noSideEffect, tags: [], raises: [], compilerRtl.} = {.noSideEffect.}: - template `+!`(p: pointer, s: int): pointer = - cast[pointer](cast[int](p) +% s) + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + if addlen <= 0: + result = p + elif p == nil: + result = newSeqPayload(len+addlen, elemSize, elemAlign) + else: + # Note: this means we cannot support things that have internal pointers as + # they get reallocated here. This needs to be documented clearly. + var p = cast[ptr NimSeqPayloadBase](p) + let oldCap = p.cap and not strlitFlag + let newCap = max(resize(oldCap), len+addlen) + var q: ptr NimSeqPayloadBase + if (p.cap and strlitFlag) == strlitFlag: + q = cast[ptr NimSeqPayloadBase](alignedAlloc(headerSize + elemSize * newCap, elemAlign)) + copyMem(q +! headerSize, p +! headerSize, len * elemSize) + else: + let oldSize = headerSize + elemSize * oldCap + let newSize = headerSize + elemSize * newCap + q = cast[ptr NimSeqPayloadBase](alignedRealloc(p, oldSize, newSize, elemAlign)) + + zeroMem(q +! headerSize +! len * elemSize, addlen * elemSize) + q.cap = newCap + result = q - const headerSize = sizeof(NimSeqPayloadBase) +proc zeroNewElements(len: int; q: pointer; addlen, elemSize, elemAlign: int) {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) + zeroMem(q +! headerSize +! len * elemSize, addlen * elemSize) + +proc prepareSeqAddUninit(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. + noSideEffect, tags: [], raises: [], compilerRtl.} = + {.noSideEffect.}: + let headerSize = align(sizeof(NimSeqPayloadBase), elemAlign) if addlen <= 0: result = p elif p == nil: - result = newSeqPayload(len+addlen, elemSize) + result = newSeqPayloadUninit(len+addlen, elemSize, elemAlign) else: # Note: this means we cannot support things that have internal pointers as # they get reallocated here. This needs to be documented clearly. @@ -59,58 +111,108 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize: int): pointer {. let oldCap = p.cap and not strlitFlag let newCap = max(resize(oldCap), len+addlen) if (p.cap and strlitFlag) == strlitFlag: - var q = cast[ptr NimSeqPayloadBase](allocShared0(headerSize + elemSize * newCap)) + var q = cast[ptr NimSeqPayloadBase](alignedAlloc(headerSize + elemSize * newCap, elemAlign)) copyMem(q +! headerSize, p +! headerSize, len * elemSize) q.cap = newCap result = q else: let oldSize = headerSize + elemSize * oldCap let newSize = headerSize + elemSize * newCap - var q = cast[ptr NimSeqPayloadBase](reallocShared0(p, oldSize, newSize)) + var q = cast[ptr NimSeqPayloadBase](alignedRealloc(p, oldSize, newSize, elemAlign)) q.cap = newCap result = q -proc shrink*[T](x: var seq[T]; newLen: Natural) = +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = when nimvm: - setLen(x, newLen) + {.cast(tags: []).}: + setLen(x, newLen) else: - mixin `=destroy` - sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'" + #sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'" when not supportsCopyMem(T): for i in countdown(x.len - 1, newLen): - `=destroy`(x[i]) + reset x[i] # XXX This is wrong for const seqs that were moved into 'x'! - cast[ptr NimSeqV2[T]](addr x).len = newLen + {.noSideEffect.}: + cast[ptr NimSeqV2[T]](addr x).len = newLen -proc grow*[T](x: var seq[T]; newLen: Natural; value: T) = +proc grow*[T](x: var seq[T]; newLen: Natural; value: T) {.nodestroy.} = let oldLen = x.len + #sysAssert newLen >= x.len, "invalid newLen parameter for 'grow'" if newLen <= oldLen: return var xu = cast[ptr NimSeqV2[T]](addr x) - if xu.p == nil or xu.p.cap < newLen: - xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, newLen - oldLen, sizeof(T))) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newLen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newLen - oldLen, sizeof(T), alignof(T))) xu.len = newLen for i in oldLen .. newLen-1: - xu.p.data[i] = value + wasMoved(xu.p.data[i]) + `=copy`(xu.p.data[i], value) -proc add*[T](x: var seq[T]; value: sink T) {.magic: "AppendSeqElem", noSideEffect, nodestroy.} = +proc add*[T](x: var seq[T]; y: sink T) {.magic: "AppendSeqElem", noSideEffect, nodestroy.} = ## Generic proc for adding a data item `y` to a container `x`. ## ## For containers that have an order, `add` means *append*. New generic ## containers should also call their adding proc `add` for consistency. ## Generic code becomes much easier to write if the Nim naming scheme is ## respected. - let oldLen = x.len - var xu = cast[ptr NimSeqV2[T]](addr x) - if xu.p == nil or xu.p.cap < oldLen+1: - xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, 1, sizeof(T))) - xu.len = oldLen+1 - # .nodestroy means `xu.p.data[oldLen] = value` is compiled into a - # copyMem(). This is fine as know by construction that - # in `xu.p.data[oldLen]` there is nothing to destroy. - # We also save the `wasMoved + destroy` pair for the sink parameter. - xu.p.data[oldLen] = value - -proc setLen[T](s: var seq[T], newlen: Natural) = + {.cast(noSideEffect).}: + let oldLen = x.len + var xu = cast[ptr NimSeqV2[T]](addr x) + if xu.p == nil or (xu.p.cap and not strlitFlag) < oldLen+1: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, 1, sizeof(T), alignof(T))) + xu.len = oldLen+1 + # .nodestroy means `xu.p.data[oldLen] = value` is compiled into a + # copyMem(). This is fine as know by construction that + # in `xu.p.data[oldLen]` there is nothing to destroy. + # We also save the `wasMoved + destroy` pair for the sink parameter. + xu.p.data[oldLen] = y + +proc setLen[T](s: var seq[T], newlen: Natural) {.nodestroy.} = + {.noSideEffect.}: + if newlen < s.len: + shrink(s, newlen) + else: + let oldLen = s.len + if newlen <= oldLen: return + var xu = cast[ptr NimSeqV2[T]](addr s) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newlen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) + xu.len = newlen + for i in oldLen..<newlen: + xu.p.data[i] = default(T) + +proc newSeq[T](s: var seq[T], len: Natural) = + shrink(s, 0) + setLen(s, len) + +proc sameSeqPayload(x: pointer, y: pointer): bool {.compilerRtl, inl.} = + result = cast[ptr NimRawSeq](x)[].p == cast[ptr NimRawSeq](y)[].p + + +func capacity*[T](self: seq[T]): int {.inline.} = + ## Returns the current capacity of the seq. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var lst = newSeqOfCap[string](cap = 42) + lst.add "Nim" + assert lst.capacity == 42 + + let sek = cast[ptr NimSeqV2[T]](unsafeAddr self) + result = if sek.p != nil: sek.p.cap and not strlitFlag else: 0 + +func setLenUninit*[T](s: var seq[T], newlen: Natural) {.nodestroy.} = + ## Sets the length of seq `s` to `newlen`. `T` may be any sequence type. + ## New slots will not be initialized. + ## + ## If the current length is greater than the new length, + ## `s` will be truncated. + ## ```nim + ## var x = @[10, 20] + ## x.setLenUninit(5) + ## x[4] = 50 + ## assert x[4] == 50 + ## x.setLenUninit(1) + ## assert x == @[10] + ## ``` {.noSideEffect.}: if newlen < s.len: shrink(s, newlen) @@ -118,6 +220,8 @@ proc setLen[T](s: var seq[T], newlen: Natural) = let oldLen = s.len if newlen <= oldLen: return var xu = cast[ptr NimSeqV2[T]](addr s) - if xu.p == nil or xu.p.cap < newlen: - xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, newlen - oldLen, sizeof(T))) + if xu.p == nil or (xu.p.cap and not strlitFlag) < newlen: + xu.p = cast[typeof(xu.p)](prepareSeqAddUninit(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) xu.len = newlen + +{.pop.} # See https://github.com/nim-lang/Nim/issues/21401 diff --git a/lib/system/seqs_v2_reimpl.nim b/lib/system/seqs_v2_reimpl.nim new file mode 100644 index 000000000..09b7e7ac4 --- /dev/null +++ b/lib/system/seqs_v2_reimpl.nim @@ -0,0 +1,24 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +type + NimSeqPayloadReimpl = object + cap: int + data: pointer + + NimSeqV2Reimpl = object + len: int + p: ptr NimSeqPayloadReimpl + +template frees(s: NimSeqV2Reimpl) = + if s.p != nil and (s.p.cap and strlitFlag) != strlitFlag: + when compileOption("threads"): + deallocShared(s.p) + else: + dealloc(s.p) \ No newline at end of file diff --git a/lib/system/setops.nim b/lib/system/setops.nim index 97f042d93..67aa3097a 100644 --- a/lib/system/setops.nim +++ b/lib/system/setops.nim @@ -1,94 +1,89 @@ -proc incl*[T](x: var set[T], y: T) {.magic: "Incl", noSideEffect.} - ## Includes element ``y`` in the set ``x``. +func incl*[T](x: var set[T], y: T) {.magic: "Incl".} = + ## Includes element `y` in the set `x`. ## - ## This is the same as ``x = x + {y}``, but it might be more efficient. - ## - ## .. code-block:: Nim - ## var a = {1, 3, 5} - ## a.incl(2) # a <- {1, 2, 3, 5} - ## a.incl(4) # a <- {1, 2, 3, 4, 5} + ## This is the same as `x = x + {y}`, but it might be more efficient. + runnableExamples: + var a = {1, 3, 5} + a.incl(2) + assert a == {1, 2, 3, 5} + a.incl(4) + assert a == {1, 2, 3, 4, 5} -template incl*[T](x: var set[T], y: set[T]) = - ## Includes the set ``y`` in the set ``x``. - ## - ## .. code-block:: Nim - ## var a = {1, 3, 5, 7} - ## var b = {4, 5, 6} - ## a.incl(b) # a <- {1, 3, 4, 5, 6, 7} +when not defined(nimHasCallsitePragma): + {.pragma: callsite.} + +template incl*[T](x: var set[T], y: set[T]) {.callsite.} = + ## Includes the set `y` in the set `x`. + runnableExamples: + var a = {1, 3, 5, 7} + var b = {4, 5, 6} + a.incl(b) + assert a == {1, 3, 4, 5, 6, 7} x = x + y -proc excl*[T](x: var set[T], y: T) {.magic: "Excl", noSideEffect.} - ## Excludes element ``y`` from the set ``x``. - ## - ## This is the same as ``x = x - {y}``, but it might be more efficient. +func excl*[T](x: var set[T], y: T) {.magic: "Excl".} = + ## Excludes element `y` from the set `x`. ## - ## .. code-block:: Nim - ## var b = {2, 3, 5, 6, 12, 545} - ## b.excl(5) # b <- {2, 3, 6, 12, 545} + ## This is the same as `x = x - {y}`, but it might be more efficient. + runnableExamples: + var b = {2, 3, 5, 6, 12, 54} + b.excl(5) + assert b == {2, 3, 6, 12, 54} -template excl*[T](x: var set[T], y: set[T]) = - ## Excludes the set ``y`` from the set ``x``. - ## - ## .. code-block:: Nim - ## var a = {1, 3, 5, 7} - ## var b = {3, 4, 5} - ## a.excl(b) # a <- {1, 7} +template excl*[T](x: var set[T], y: set[T]) {.callsite.} = + ## Excludes the set `y` from the set `x`. + runnableExamples: + var a = {1, 3, 5, 7} + var b = {3, 4, 5} + a.excl(b) + assert a == {1, 7} x = x - y -proc card*[T](x: set[T]): int {.magic: "Card", noSideEffect.} - ## Returns the cardinality of the set ``x``, i.e. the number of elements +func card*[T](x: set[T]): int {.magic: "Card".} = + ## Returns the cardinality of the set `x`, i.e. the number of elements ## in the set. - ## - ## .. code-block:: Nim - ## var a = {1, 3, 5, 7} - ## echo card(a) # => 4 + runnableExamples: + var a = {1, 3, 5, 7} + assert card(a) == 4 + var b = {1, 3, 5, 7, 5} + assert card(b) == 4 # repeated 5 doesn't count -proc len*[T](x: set[T]): int {.magic: "Card", noSideEffect.} +func len*[T](x: set[T]): int {.magic: "Card".} ## An alias for `card(x)`. -proc `*`*[T](x, y: set[T]): set[T] {.magic: "MulSet", noSideEffect.} +func `*`*[T](x, y: set[T]): set[T] {.magic: "MulSet".} = ## This operator computes the intersection of two sets. - ## - ## .. code-block:: Nim - ## let - ## a = {1, 2, 3} - ## b = {2, 3, 4} - ## echo a * b # => {2, 3} -proc `+`*[T](x, y: set[T]): set[T] {.magic: "PlusSet", noSideEffect.} + runnableExamples: + assert {1, 2, 3} * {2, 3, 4} == {2, 3} + +func `+`*[T](x, y: set[T]): set[T] {.magic: "PlusSet".} = ## This operator computes the union of two sets. - ## - ## .. code-block:: Nim - ## let - ## a = {1, 2, 3} - ## b = {2, 3, 4} - ## echo a + b # => {1, 2, 3, 4} -proc `-`*[T](x, y: set[T]): set[T] {.magic: "MinusSet", noSideEffect.} + runnableExamples: + assert {1, 2, 3} + {2, 3, 4} == {1, 2, 3, 4} + +func `-`*[T](x, y: set[T]): set[T] {.magic: "MinusSet".} = ## This operator computes the difference of two sets. - ## - ## .. code-block:: Nim - ## let - ## a = {1, 2, 3} - ## b = {2, 3, 4} - ## echo a - b # => {1} + runnableExamples: + assert {1, 2, 3} - {2, 3, 4} == {1} -proc contains*[T](x: set[T], y: T): bool {.magic: "InSet", noSideEffect.} - ## One should overload this proc if one wants to overload the ``in`` operator. +func contains*[T](x: set[T], y: T): bool {.magic: "InSet".} = + ## One should overload this proc if one wants to overload the `in` operator. ## - ## The parameters are in reverse order! ``a in b`` is a template for - ## ``contains(b, a)``. + ## The parameters are in reverse order! `a in b` is a template for + ## `contains(b, a)`. ## This is because the unification algorithm that Nim uses for overload ## resolution works from left to right. - ## But for the ``in`` operator that would be the wrong direction for this + ## But for the `in` operator that would be the wrong direction for this ## piece of code: - ## - ## .. code-block:: Nim - ## var s: set[range['a'..'z']] = {'a'..'c'} - ## assert s.contains('c') - ## assert 'b' in s - ## - ## If ``in`` had been declared as ``[T](elem: T, s: set[T])`` then ``T`` would - ## have been bound to ``char``. But ``s`` is not compatible to type - ## ``set[char]``! The solution is to bind ``T`` to ``range['a'..'z']``. This - ## is achieved by reversing the parameters for ``contains``; ``in`` then + runnableExamples: + var s: set[range['a'..'z']] = {'a'..'c'} + assert s.contains('c') + assert 'b' in s + assert 'd' notin s + assert set['a'..'z'] is set[range['a'..'z']] + ## If `in` had been declared as `[T](elem: T, s: set[T])` then `T` would + ## have been bound to `char`. But `s` is not compatible to type + ## `set[char]`! The solution is to bind `T` to `range['a'..'z']`. This + ## is achieved by reversing the parameters for `contains`; `in` then ## passes its arguments in reverse order. diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 0df50957d..97431c296 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -9,33 +9,20 @@ # set handling -type - NimSet = array[0..4*2048-1, uint8] -# bitops can't be imported here, therefore the code duplication. - -proc countBits32(n: uint32): int {.compilerproc.} = - # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - var v = uint32(n) - v = v - ((v shr 1'u32) and 0x55555555'u32) - v = (v and 0x33333333'u32) + ((v shr 2'u32) and 0x33333333'u32) - result = (((v + (v shr 4'u32) and 0xF0F0F0F'u32) * 0x1010101'u32) shr 24'u32).int - -proc countBits64(n: uint64): int {.compilerproc, inline.} = - # generic formula is from: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - var v = uint64(n) - v = v - ((v shr 1'u64) and 0x5555555555555555'u64) - v = (v and 0x3333333333333333'u64) + ((v shr 2'u64) and 0x3333333333333333'u64) - v = (v + (v shr 4'u64) and 0x0F0F0F0F0F0F0F0F'u64) - result = ((v * 0x0101010101010101'u64) shr 56'u64).int - -proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = +proc cardSetImpl(s: ptr UncheckedArray[uint8], len: int): int {.inline.} = var i = 0 + result = 0 + var num = 0'u64 when defined(x86) or defined(amd64): while i < len - 8: - inc(result, countBits64((cast[ptr uint64](s[i].unsafeAddr))[])) + copyMem(addr num, addr s[i], 8) + inc(result, countBits64(num)) inc(i, 8) while i < len: inc(result, countBits32(uint32(s[i]))) inc(i, 1) + +proc cardSet(s: ptr UncheckedArray[uint8], len: int): int {.compilerproc, inline.} = + result = cardSetImpl(s, len) diff --git a/lib/system/stacktraces.nim b/lib/system/stacktraces.nim new file mode 100644 index 000000000..42be9d94f --- /dev/null +++ b/lib/system/stacktraces.nim @@ -0,0 +1,83 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Additional code for customizable stack traces. Unstable API, for internal +# usage only. + +const + reraisedFromBegin* = -10 + reraisedFromEnd* = -100 + maxStackTraceLines* = 128 + +when defined(nimStackTraceOverride): + ## Procedure types for overriding the default stack trace. + type + cuintptr_t* {.importc: "uintptr_t", nodecl.} = uint + ## This is the same as the type `uintptr_t` in C. + + StackTraceOverrideGetTracebackProc* = proc (): string {. + nimcall, gcsafe, raises: [], tags: [], noinline.} + StackTraceOverrideGetProgramCountersProc* = proc (maxLength: cint): seq[cuintptr_t] {. + nimcall, gcsafe, raises: [], tags: [], noinline.} + StackTraceOverrideGetDebuggingInfoProc* = + proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {. + nimcall, gcsafe, raises: [], tags: [], noinline.} + + # Default procedures (not normally used, because people opting in on this + # override are supposed to register their own versions). + var + stackTraceOverrideGetTraceback: StackTraceOverrideGetTracebackProc = + proc (): string {.nimcall, gcsafe, raises: [], tags: [], noinline.} = + discard + #result = "Stack trace override procedure not registered.\n" + stackTraceOverrideGetProgramCounters: StackTraceOverrideGetProgramCountersProc = + proc (maxLength: cint): seq[cuintptr_t] {.nimcall, gcsafe, raises: [], tags: [], noinline.} = + discard + stackTraceOverrideGetDebuggingInfo: StackTraceOverrideGetDebuggingInfoProc = + proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {. + nimcall, gcsafe, raises: [], tags: [], noinline.} = + discard + + # Custom procedure registration. + proc registerStackTraceOverride*(overrideProc: StackTraceOverrideGetTracebackProc) = + ## Override the default stack trace inside rawWriteStackTrace() with your + ## own procedure. + stackTraceOverrideGetTraceback = overrideProc + proc registerStackTraceOverrideGetProgramCounters*(overrideProc: StackTraceOverrideGetProgramCountersProc) = + stackTraceOverrideGetProgramCounters = overrideProc + proc registerStackTraceOverrideGetDebuggingInfo*(overrideProc: StackTraceOverrideGetDebuggingInfoProc) = + stackTraceOverrideGetDebuggingInfo = overrideProc + + # Custom stack trace manipulation. + proc auxWriteStackTraceWithOverride*(s: var string) = + add(s, stackTraceOverrideGetTraceback()) + + proc auxWriteStackTraceWithOverride*(s: var seq[StackTraceEntry]) = + let programCounters = stackTraceOverrideGetProgramCounters(maxStackTraceLines) + if s.len == 0: + s = newSeqOfCap[StackTraceEntry](programCounters.len) + for programCounter in programCounters: + s.add(StackTraceEntry(programCounter: cast[uint](programCounter))) + + # We may have more stack trace lines in the output, due to inlined procedures. + proc addDebuggingInfo*(s: seq[StackTraceEntry]): seq[StackTraceEntry] = + var programCounters: seq[cuintptr_t] + # We process program counters in groups from complete stack traces, because + # we have logic that keeps track of certain functions being inlined or not. + for entry in s: + if entry.procname.isNil and entry.programCounter != 0: + programCounters.add(cast[cuintptr_t](entry.programCounter)) + elif entry.procname.isNil and (entry.line == reraisedFromBegin or entry.line == reraisedFromEnd): + result.add(stackTraceOverrideGetDebuggingInfo(programCounters, maxStackTraceLines)) + programCounters = @[] + result.add(entry) + else: + result.add(entry) + if programCounters.len > 0: + result.add(stackTraceOverrideGetDebuggingInfo(programCounters, maxStackTraceLines)) diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index a111ec563..89046253b 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -9,17 +9,28 @@ # Compilerprocs for strings that do not depend on the string implementation. +import std/private/digitsutils + + proc cmpStrings(a, b: string): int {.inline, compilerproc.} = let alen = a.len let blen = b.len let minlen = min(alen, blen) if minlen > 0: - result = c_memcmp(unsafeAddr a[0], unsafeAddr b[0], cast[csize_t](minlen)) + result = c_memcmp(unsafeAddr a[0], unsafeAddr b[0], cast[csize_t](minlen)).int if result == 0: result = alen - blen else: result = alen - blen +proc leStrings(a, b: string): bool {.inline, compilerproc.} = + # required by upcoming backends (NIR). + cmpStrings(a, b) <= 0 + +proc ltStrings(a, b: string): bool {.inline, compilerproc.} = + # required by upcoming backends (NIR). + cmpStrings(a, b) < 0 + proc eqStrings(a, b: string): bool {.inline, compilerproc.} = let alen = a.len let blen = b.len @@ -30,7 +41,7 @@ proc eqStrings(a, b: string): bool {.inline, compilerproc.} = proc hashString(s: string): int {.compilerproc.} = # the compiler needs exactly the same hash function! # this used to be used for efficient generation of string case statements - var h : uint = 0 + var h = 0'u for i in 0..len(s)-1: h = h + uint(s[i]) h = h + h shl 10 @@ -40,71 +51,28 @@ proc hashString(s: string): int {.compilerproc.} = h = h + h shl 15 result = cast[int](h) -proc addInt*(result: var string; x: int64) = - ## Converts integer to its string representation and appends it to `result`. - ## - ## .. code-block:: Nim - ## var - ## a = "123" - ## b = 45 - ## a.addInt(b) # a <- "12345" - let base = result.len - setLen(result, base + sizeof(x)*4) +proc eqCstrings(a, b: cstring): bool {.inline, compilerproc.} = + if pointer(a) == pointer(b): result = true + elif a.isNil or b.isNil: result = false + else: result = c_strcmp(a, b) == 0 + +proc hashCstring(s: cstring): int {.compilerproc.} = + # the compiler needs exactly the same hash function! + # this used to be used for efficient generation of cstring case statements + if s.isNil: return 0 + var h : uint = 0 var i = 0 - var y = x while true: - var d = y div 10 - result[base+i] = chr(abs(int(y - d*10)) + ord('0')) - inc(i) - y = d - if y == 0: break - if x < 0: - result[base+i] = '-' - inc(i) - setLen(result, base+i) - # mirror the string: - for j in 0..i div 2 - 1: - swap(result[base+j], result[base+i-j-1]) - -proc add*(result: var string; x: int64) {.deprecated: - "Deprecated since v0.20, use 'addInt'".} = - addInt(result, x) - -proc nimIntToStr(x: int): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x - -proc addCstringN(result: var string, buf: cstring; buflen: int) = - # no nimvm support needed, so it doesn't need to be fast here either - let oldLen = result.len - let newLen = oldLen + buflen - result.setLen newLen - copyMem(result[oldLen].addr, buf, buflen) - -import formatfloat - -proc addFloat*(result: var string; x: float) = - ## Converts float to its string representation and appends it to `result`. - ## - ## .. code-block:: Nim - ## var - ## a = "123" - ## b = 45.67 - ## a.addFloat(b) # a <- "12345.67" - when nimvm: - result.add $x - else: - var buffer: array[65, char] - let n = writeFloatToBuffer(buffer, x) - result.addCstringN(cstring(buffer[0].addr), n) - -proc add*(result: var string; x: float) {.deprecated: - "Deprecated since v0.20, use 'addFloat'".} = - addFloat(result, x) - -proc nimFloatToStr(f: float): string {.compilerproc.} = - result = newStringOfCap(8) - result.addFloat f + let c = s[i] + if c == '\0': break + h = h + uint(c) + h = h + h shl 10 + h = h xor (h shr 6) + inc i + h = h + h shl 3 + h = h xor (h shr 11) + h = h + h shl 15 + result = cast[int](h) proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "<stdlib.h>", noSideEffect.} @@ -115,21 +83,24 @@ const 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22] -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, - start = 0): int {.compilerproc.} = + +{.push staticBoundChecks: off.} + +proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, + ): int {.compilerproc.} = # This routine attempt to parse float that can parsed quickly. - # ie whose integer part can fit inside a 53bits integer. + # i.e. whose integer part can fit inside a 53bits integer. # their real exponent must also be <= 22. If the float doesn't follow # these restrictions, transform the float into this form: # INTEGER * 10 ^ exponent and leave the work to standard `strtod()`. # This avoid the problems of decimal character portability. # see: http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ var - i = start + i = 0 sign = 1.0 kdigits, fdigits = 0 - exponent: int - integer: uint64 + exponent = 0 + integer = uint64(0) fracExponent = 0 expSign = 1 firstDigit = -1 @@ -148,7 +119,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, if s[i+2] == 'N' or s[i+2] == 'n': if i+3 >= s.len or s[i+3] notin IdentChars: number = NaN - return i+3 - start + return i+3 return 0 # Inf? @@ -157,7 +128,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, if s[i+2] == 'F' or s[i+2] == 'f': if i+3 >= s.len or s[i+3] notin IdentChars: number = Inf*sign - return i+3 - start + return i+3 return 0 if i < s.len and s[i] in {'0'..'9'}: @@ -191,8 +162,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # if has no digits: return error if kdigits + fdigits <= 0 and - (i == start or # no char consumed (empty string). - (i == start + 1 and hasSign)): # or only '+' or '- + (i == 0 or # no char consumed (empty string). + (i == 1 and hasSign)): # or only '+' or '- return 0 if i+1 < s.len and s[i] in {'e', 'E'}: @@ -215,11 +186,13 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, # if exponent greater than can be represented: +/- zero or infinity if absExponent > 999: - if expNegative: + if integer == 0: + number = 0.0 + elif expNegative: number = 0.0*sign else: number = Inf*sign - return i - start + return i # if integer is representable in 53 bits: fast path # max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits) @@ -231,29 +204,30 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, number = sign * integer.float / powtens[absExponent] else: number = sign * integer.float * powtens[absExponent] - return i - start + return i # if exponent is greater try to fit extra exponent above 22 by multiplying # integer part is there is space left. let slop = 15 - kdigits - fdigits - if absExponent <= 22 + slop and not expNegative: + if absExponent <= 22 + slop and not expNegative: number = sign * integer.float * powtens[slop] * powtens[absExponent-slop] - return i - start + return i # if failed: slow path with strtod. var t: array[500, char] # flaviu says: 325 is the longest reasonable literal var ti = 0 let maxlen = t.high - "e+000".len # reserve enough space for exponent - result = i - start - i = start + let endPos = i + result = endPos + i = 0 # re-parse without error checking, any error should be handled by the code above. - if i < s.len and s[i] == '.': i.inc - while i < s.len and s[i] in {'0'..'9','+','-'}: + if i < endPos and s[i] == '.': i.inc + while i < endPos and s[i] in {'0'..'9','+','-'}: if ti < maxlen: t[ti] = s[i]; inc(ti) inc(i) - while i < s.len and s[i] in {'.', '_'}: # skip underscore and decimal point + while i < endPos and s[i] in {'.', '_'}: # skip underscore and decimal point inc(i) # insert exponent @@ -268,15 +242,9 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, t[ti-2] = ('0'.ord + absExponent mod 10).char absExponent = absExponent div 10 t[ti-3] = ('0'.ord + absExponent mod 10).char + number = c_strtod(cast[cstring](addr t), nil) - when defined(nimNoArrayToCstringConversion): - number = c_strtod(addr t, nil) - else: - number = c_strtod(t, nil) - -proc nimInt64ToStr(x: int64): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x +{.pop.} # staticBoundChecks proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" @@ -285,26 +253,6 @@ proc nimCharToStr(x: char): string {.compilerRtl.} = result = newString(1) result[0] = x -proc `$`*(x: uint64): string {.noSideEffect, raises: [].} = - ## The stringify operator for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - if x == 0: - result = "0" - else: - result = newString(60) - var i = 0 - var n = x - while n != 0: - let nn = n div 10'u64 - result[i] = char(n - 10'u64 * nn + ord('0')) - inc i - n = nn - result.setLen i - - let half = i div 2 - # Reverse - for t in 0 .. half-1: swap(result[t], result[i-t-1]) - when defined(gcDestructors): proc GC_getStatistics*(): string = result = "[GC] total memory: " diff --git a/lib/system/strs_v2.nim b/lib/system/strs_v2.nim index 6f28ad237..404b4f78d 100644 --- a/lib/system/strs_v2.nim +++ b/lib/system/strs_v2.nim @@ -29,45 +29,77 @@ template contentSize(cap): int = cap + 1 + sizeof(NimStrPayloadBase) template frees(s) = if not isLiteral(s): - deallocShared(s.p) + when compileOption("threads"): + deallocShared(s.p) + else: + dealloc(s.p) + +template allocPayload(newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](allocShared(contentSize(newLen))) + else: + cast[ptr NimStrPayload](alloc(contentSize(newLen))) + +template allocPayload0(newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](allocShared0(contentSize(newLen))) + else: + cast[ptr NimStrPayload](alloc0(contentSize(newLen))) + +template reallocPayload(p: pointer, newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](reallocShared(p, contentSize(newLen))) + else: + cast[ptr NimStrPayload](realloc(p, contentSize(newLen))) + +template reallocPayload0(p: pointer; oldLen, newLen: int): ptr NimStrPayload = + when compileOption("threads"): + cast[ptr NimStrPayload](reallocShared0(p, contentSize(oldLen), contentSize(newLen))) + else: + cast[ptr NimStrPayload](realloc0(p, contentSize(oldLen), contentSize(newLen))) proc resize(old: int): int {.inline.} = if old <= 0: result = 4 - elif old < 65536: result = old * 2 + elif old <= high(int16): result = old * 2 else: result = old * 3 div 2 # for large arrays * 3/2 is better -proc prepareAdd(s: var NimStringV2; addlen: int) {.compilerRtl.} = +proc prepareAdd(s: var NimStringV2; addLen: int) {.compilerRtl.} = + let newLen = s.len + addLen if isLiteral(s): - if addlen > 0: - let oldP = s.p - # can't mutate a literal, so we need a fresh copy here: - s.p = cast[ptr NimStrPayload](allocShared0(contentSize(s.len + addlen))) - s.p.cap = s.len + addlen - if s.len > 0: - # we are about to append, so there is no need to copy the \0 terminator: - copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len) + let oldP = s.p + # can't mutate a literal, so we need a fresh copy here: + s.p = allocPayload(newLen) + s.p.cap = newLen + if s.len > 0: + # we are about to append, so there is no need to copy the \0 terminator: + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], min(s.len, newLen)) + elif oldP == nil: + # In the case of `newString(0) & ""`, since `src.len == 0`, `appendString` + # will not set the `\0` terminator, so we set it here. + s.p.data[0] = '\0' else: let oldCap = s.p.cap and not strlitFlag - if s.len + addlen > oldCap: - let newCap = max(s.len + addlen, resize(oldCap)) - s.p = cast[ptr NimStrPayload](reallocShared0(s.p, contentSize(oldCap), contentSize(newCap))) + if newLen > oldCap: + let newCap = max(newLen, resize(oldCap)) + s.p = reallocPayload(s.p, newCap) s.p.cap = newCap + if newLen < newCap: + zeroMem(cast[pointer](addr s.p.data[newLen+1]), newCap - newLen) -proc nimAddCharV1(s: var NimStringV2; c: char) {.compilerRtl.} = +proc nimAddCharV1(s: var NimStringV2; c: char) {.compilerRtl, inl.} = + #if (s.p == nil) or (s.len+1 > s.p.cap and not strlitFlag): prepareAdd(s, 1) s.p.data[s.len] = c - s.p.data[s.len+1] = '\0' inc s.len + s.p.data[s.len] = '\0' proc toNimStr(str: cstring, len: int): NimStringV2 {.compilerproc.} = if len <= 0: result = NimStringV2(len: 0, p: nil) else: - var p = cast[ptr NimStrPayload](allocShared0(contentSize(len))) + var p = allocPayload(len) p.cap = len - if len > 0: - # we are about to append, so there is no need to copy the \0 terminator: - copyMem(unsafeAddr p.data[0], str, len) + copyMem(unsafeAddr p.data[0], str, len+1) result = NimStringV2(len: len, p: p) proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = @@ -76,7 +108,7 @@ proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = proc nimToCStringConv(s: NimStringV2): cstring {.compilerproc, nonReloadable, inline.} = if s.len == 0: result = cstring"" - else: result = cstring(unsafeAddr s.p.data) + else: result = cast[cstring](unsafeAddr s.p.data) proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inline.} = if src.len > 0: @@ -86,36 +118,55 @@ proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inlin proc appendChar(dest: var NimStringV2; c: char) {.compilerproc, inline.} = dest.p.data[dest.len] = c - dest.p.data[dest.len+1] = '\0' inc dest.len + dest.p.data[dest.len] = '\0' proc rawNewString(space: int): NimStringV2 {.compilerproc.} = # this is also 'system.newStringOfCap'. if space <= 0: result = NimStringV2(len: 0, p: nil) else: - var p = cast[ptr NimStrPayload](allocShared0(contentSize(space))) + var p = allocPayload(space) p.cap = space + p.data[0] = '\0' result = NimStringV2(len: 0, p: p) proc mnewString(len: int): NimStringV2 {.compilerproc.} = if len <= 0: result = NimStringV2(len: 0, p: nil) else: - var p = cast[ptr NimStrPayload](allocShared0(contentSize(len))) + var p = allocPayload0(len) p.cap = len result = NimStringV2(len: len, p: p) proc setLengthStrV2(s: var NimStringV2, newLen: int) {.compilerRtl.} = if newLen == 0: - frees(s) - s.p = nil - elif newLen > s.len or isLiteral(s): - prepareAdd(s, newLen - s.len) + discard "do not free the buffer here, pattern 's.setLen 0' is common for avoiding allocations" + else: + if isLiteral(s): + let oldP = s.p + s.p = allocPayload(newLen) + s.p.cap = newLen + if s.len > 0: + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], min(s.len, newLen)) + if newLen > s.len: + zeroMem(cast[pointer](addr s.p.data[s.len]), newLen - s.len + 1) + else: + s.p.data[newLen] = '\0' + else: + zeroMem(cast[pointer](addr s.p.data[0]), newLen + 1) + elif newLen > s.len: + let oldCap = s.p.cap and not strlitFlag + if newLen > oldCap: + let newCap = max(newLen, resize(oldCap)) + s.p = reallocPayload0(s.p, oldCap, newCap) + s.p.cap = newCap + + s.p.data[newLen] = '\0' s.len = newLen proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} = - if a.p == b.p: return + if a.p == b.p and a.len == b.len: return if isLiteral(b): # we can shallow copy literals: frees(a) @@ -127,15 +178,47 @@ proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} = # 'let y = newStringOfCap(); var x = y' # on the other hand... These get turned into moves now. frees(a) - a.p = cast[ptr NimStrPayload](allocShared0(contentSize(b.len))) + a.p = allocPayload(b.len) a.p.cap = b.len a.len = b.len copyMem(unsafeAddr a.p.data[0], unsafeAddr b.p.data[0], b.len+1) -proc nimPrepareStrMutationV2(s: var NimStringV2) {.compilerRtl.} = +proc nimPrepareStrMutationImpl(s: var NimStringV2) = + let oldP = s.p + # can't mutate a literal, so we need a fresh copy here: + s.p = allocPayload(s.len) + s.p.cap = s.len + copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len+1) + +proc nimPrepareStrMutationV2(s: var NimStringV2) {.compilerRtl, inl.} = if s.p != nil and (s.p.cap and strlitFlag) == strlitFlag: - let oldP = s.p - # can't mutate a literal, so we need a fresh copy here: - s.p = cast[ptr NimStrPayload](allocShared0(contentSize(s.len))) - s.p.cap = s.len - copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len+1) + nimPrepareStrMutationImpl(s) + +proc prepareMutation*(s: var string) {.inline.} = + # string literals are "copy on write", so you need to call + # `prepareMutation` before modifying the strings via `addr`. + {.cast(noSideEffect).}: + let s = unsafeAddr s + nimPrepareStrMutationV2(cast[ptr NimStringV2](s)[]) + +proc nimAddStrV1(s: var NimStringV2; src: NimStringV2) {.compilerRtl, inl.} = + #if (s.p == nil) or (s.len+1 > s.p.cap and not strlitFlag): + prepareAdd(s, src.len) + appendString s, src + +proc nimDestroyStrV1(s: NimStringV2) {.compilerRtl, inl.} = + frees(s) + +proc nimStrAtLe(s: string; idx: int; ch: char): bool {.compilerRtl, inl.} = + result = idx < s.len and s[idx] <= ch + +func capacity*(self: string): int {.inline.} = + ## Returns the current capacity of the string. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var str = newStringOfCap(cap = 42) + str.add "Nim" + assert str.capacity == 42 + + let str = cast[ptr NimStringV2](unsafeAddr self) + result = if str.p != nil: str.p.cap and not strlitFlag else: 0 diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim deleted file mode 100644 index fa4164b95..000000000 --- a/lib/system/syslocks.nim +++ /dev/null @@ -1,226 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Low level system locks and condition vars. - -{.push stackTrace: off.} - -when defined(Windows): - type - Handle = int - - SysLock {.importc: "CRITICAL_SECTION", - header: "<windows.h>", final, pure.} = object # CRITICAL_SECTION in WinApi - DebugInfo: pointer - LockCount: int32 - RecursionCount: int32 - OwningThread: int - LockSemaphore: int - SpinCount: int - - SysCond = Handle - - proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", - header: "<windows.h>".} - ## Initializes the lock `L`. - - proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", - header: "<windows.h>".} - ## Tries to acquire the lock `L`. - - proc tryAcquireSys(L: var SysLock): bool {.inline.} = - result = tryAcquireSysAux(L) != 0'i32 - - proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", - header: "<windows.h>".} - ## Acquires the lock `L`. - - proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", - header: "<windows.h>".} - ## Releases the lock `L`. - - proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", - header: "<windows.h>".} - - proc createEvent(lpEventAttributes: pointer, - bManualReset, bInitialState: int32, - lpName: cstring): SysCond {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CreateEventA".} - - proc closeHandle(hObject: Handle) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "CloseHandle".} - proc waitForSingleObject(hHandle: Handle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", noSideEffect.} - - proc signalSysCond(hEvent: SysCond) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "SetEvent".} - - proc initSysCond(cond: var SysCond) {.inline.} = - cond = createEvent(nil, 0'i32, 0'i32, nil) - proc deinitSysCond(cond: var SysCond) {.inline.} = - closeHandle(cond) - proc waitSysCond(cond: var SysCond, lock: var SysLock) = - releaseSys(lock) - discard waitForSingleObject(cond, -1'i32) - acquireSys(lock) - - proc waitSysCondWindows(cond: var SysCond) = - discard waitForSingleObject(cond, -1'i32) - -elif defined(genode): - const - Header = "genode_cpp/syslocks.h" - type - SysLock {.importcpp: "Nim::SysLock", pure, final, - header: Header.} = object - SysCond {.importcpp: "Nim::SysCond", pure, final, - header: Header.} = object - - proc initSysLock(L: var SysLock) = discard - proc deinitSys(L: var SysLock) = discard - proc acquireSys(L: var SysLock) {.noSideEffect, importcpp.} - proc tryAcquireSys(L: var SysLock): bool {.noSideEffect, importcpp.} - proc releaseSys(L: var SysLock) {.noSideEffect, importcpp.} - - proc initSysCond(L: var SysCond) = discard - proc deinitSysCond(L: var SysCond) = discard - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - noSideEffect, importcpp.} - proc signalSysCond(cond: var SysCond) {. - noSideEffect, importcpp.} - -else: - type - SysLockObj {.importc: "pthread_mutex_t", pure, final, - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[40 div sizeof(clong), clong] - - SysLockAttr {.importc: "pthread_mutexattr_t", pure, final - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[4 div sizeof(cint), cint] # actually a cint - - SysCondObj {.importc: "pthread_cond_t", pure, final, - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[48 div sizeof(clonglong), clonglong] - - SysCondAttr {.importc: "pthread_condattr_t", pure, final - header: """#include <sys/types.h> - #include <pthread.h>""".} = object - when defined(linux) and defined(amd64): - abi: array[4 div sizeof(cint), cint] # actually a cint - - SysLockType = distinct cint - - proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. - importc: "pthread_mutex_init", header: "<pthread.h>", noSideEffect.} - proc deinitSysAux(L: var SysLockObj) {.noSideEffect, - importc: "pthread_mutex_destroy", header: "<pthread.h>".} - - proc acquireSysAux(L: var SysLockObj) {.noSideEffect, - importc: "pthread_mutex_lock", header: "<pthread.h>".} - proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, - importc: "pthread_mutex_trylock", header: "<pthread.h>".} - - proc releaseSysAux(L: var SysLockObj) {.noSideEffect, - importc: "pthread_mutex_unlock", header: "<pthread.h>".} - - when defined(ios): - # iOS will behave badly if sync primitives are moved in memory. In order - # to prevent this once and for all, we're doing an extra malloc when - # initializing the primitive. - type - SysLock = ptr SysLockObj - SysCond = ptr SysCondObj - - when not declared(c_malloc): - proc c_malloc(size: csize): pointer {. - importc: "malloc", header: "<stdlib.h>".} - proc c_free(p: pointer) {. - importc: "free", header: "<stdlib.h>".} - - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) = - L = cast[SysLock](c_malloc(sizeof(SysLockObj))) - initSysLockAux(L[], attr) - - proc deinitSys(L: var SysLock) = - deinitSysAux(L[]) - c_free(L) - - template acquireSys(L: var SysLock) = - acquireSysAux(L[]) - template tryAcquireSys(L: var SysLock): bool = - tryAcquireSysAux(L[]) == 0'i32 - template releaseSys(L: var SysLock) = - releaseSysAux(L[]) - else: - type - SysLock = SysLockObj - SysCond = SysCondObj - - template initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) = - initSysLockAux(L, attr) - template deinitSys(L: var SysLock) = - deinitSysAux(L) - template acquireSys(L: var SysLock) = - acquireSysAux(L) - template tryAcquireSys(L: var SysLock): bool = - tryAcquireSysAux(L) == 0'i32 - template releaseSys(L: var SysLock) = - releaseSysAux(L) - - when insideRLocksModule: - proc SysLockType_Reentrant: SysLockType = - {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "<pthread.h>", noSideEffect.} - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.} - - else: - proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {. - importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.} - proc deinitSysCondAux(cond: var SysCondObj) {.noSideEffect, - importc: "pthread_cond_destroy", header: "<pthread.h>".} - - proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj) {. - importc: "pthread_cond_wait", header: "<pthread.h>", noSideEffect.} - proc signalSysCondAux(cond: var SysCondObj) {. - importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.} - - when defined(ios): - proc initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = - cond = cast[SysCond](c_malloc(sizeof(SysCondObj))) - initSysCondAux(cond[], cond_attr) - - proc deinitSysCond(cond: var SysCond) = - deinitSysCondAux(cond[]) - c_free(cond) - - template waitSysCond(cond: var SysCond, lock: var SysLock) = - waitSysCondAux(cond[], lock[]) - template signalSysCond(cond: var SysCond) = - signalSysCondAux(cond[]) - else: - template initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) = - initSysCondAux(cond, cond_attr) - template deinitSysCond(cond: var SysCond) = - deinitSysCondAux(cond) - - template waitSysCond(cond: var SysCond, lock: var SysLock) = - waitSysCondAux(cond, lock) - template signalSysCond(cond: var SysCond) = - signalSysCondAux(cond) - -{.pop.} diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim deleted file mode 100644 index dc2d13578..000000000 --- a/lib/system/sysspawn.nim +++ /dev/null @@ -1,194 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements Nim's 'spawn'. - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -{.push stackTrace:off.} - -# We declare our own condition variables here to get rid of the dummy lock -# on Windows: - -type - CondVar = object - c: SysCond - when defined(posix): - stupidLock: SysLock - counter: int - -proc createCondVar(): CondVar = - initSysCond(result.c) - when defined(posix): - initSysLock(result.stupidLock) - #acquireSys(result.stupidLock) - -proc destroyCondVar(c: var CondVar) {.inline.} = - deinitSysCond(c.c) - -proc await(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - while cv.counter <= 0: - waitSysCond(cv.c, cv.stupidLock) - dec cv.counter - releaseSys(cv.stupidLock) - else: - waitSysCondWindows(cv.c) - -proc signal(cv: var CondVar) = - when defined(posix): - acquireSys(cv.stupidLock) - inc cv.counter - releaseSys(cv.stupidLock) - signalSysCond(cv.c) - -type - FastCondVar = object - event, slowPath: bool - slow: CondVar - -proc createFastCondVar(): FastCondVar = - initSysCond(result.slow.c) - when defined(posix): - initSysLock(result.slow.stupidLock) - #acquireSys(result.slow.stupidLock) - result.event = false - result.slowPath = false - -proc await(cv: var FastCondVar) = - #for i in 0 .. 50: - # if cas(addr cv.event, true, false): - # # this is a HIT: Triggers > 95% in my tests. - # return - # cpuRelax() - #cv.slowPath = true - # XXX For some reason this crashes some test programs - await(cv.slow) - cv.event = false - -proc signal(cv: var FastCondVar) = - cv.event = true - #if cas(addr cv.slowPath, true, false): - signal(cv.slow) - -type - Barrier* {.compilerProc.} = object - counter: int - cv: CondVar - -proc barrierEnter*(b: ptr Barrier) {.compilerProc.} = - atomicInc b.counter - -proc barrierLeave*(b: ptr Barrier) {.compilerProc.} = - atomicDec b.counter - if b.counter <= 0: signal(b.cv) - -proc openBarrier*(b: ptr Barrier) {.compilerProc.} = - b.counter = 0 - b.cv = createCondVar() - -proc closeBarrier*(b: ptr Barrier) {.compilerProc.} = - await(b.cv) - destroyCondVar(b.cv) - -{.pop.} - -# ---------------------------------------------------------------------------- - -type - WorkerProc = proc (thread, args: pointer) {.nimcall, gcsafe.} - Worker = object - taskArrived: CondVar - taskStarted: FastCondVar #\ - # task data: - f: WorkerProc - data: pointer - ready: bool # put it here for correct alignment! - -proc nimArgsPassingDone(p: pointer) {.compilerProc.} = - let w = cast[ptr Worker](p) - signal(w.taskStarted) - -var gSomeReady = createFastCondVar() - -proc slave(w: ptr Worker) {.thread.} = - while true: - w.ready = true # If we instead signal "workerReady" we need the scheduler - # to notice this. The scheduler could then optimize the - # layout of the worker threads (e.g. keep the list sorted) - # so that no search for a "ready" thread is necessary. - # This might be implemented later, but is more tricky than - # it looks because 'spawn' itself can run concurrently. - signal(gSomeReady) - await(w.taskArrived) - assert(not w.ready) - # shield against spurious wakeups: - if w.data != nil: - w.f(w, w.data) - w.data = nil - -const NumThreads = 4 - -var - workers: array[NumThreads, Thread[ptr Worker]] - workersData: array[NumThreads, Worker] - -proc setup() = - for i in 0 ..< NumThreads: - workersData[i].taskArrived = createCondVar() - workersData[i].taskStarted = createFastCondVar() - createThread(workers[i], slave, addr(workersData[i])) - -proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a 'spawn' or a direct call is - ## preferable. If it returns 'true' a 'spawn' may make sense. In general - ## it is not necessary to call this directly; use 'spawnX' instead. - result = gSomeReady.event - -proc spawn*(call: typed) {.magic: "Spawn".} - ## always spawns a new task, so that the 'call' is never executed on - ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - -template spawnX*(call: typed) = - ## spawns a new task if a CPU core is ready, otherwise executes the - ## call in the calling thread. Usually it is advised to - ## use 'spawn' in order to not block the producer for an unknown - ## amount of time. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - if preferSpawn(): spawn call - else: call - -proc nimSpawn(fn: WorkerProc; data: pointer) {.compilerProc.} = - # implementation of 'spawn' that is used by the code generator. - while true: - for i in 0.. high(workers): - let w = addr(workersData[i]) - if cas(addr w.ready, true, false): - w.data = data - w.f = fn - signal(w.taskArrived) - await(w.taskStarted) - return - await(gSomeReady) - -proc sync*() = - ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate - ## waiting, you have to use an explicit barrier. - while true: - var allReady = true - for i in 0 .. high(workers): - if not allReady: break - allReady = allReady and workersData[i].ready - if allReady: break - await(gSomeReady) - -setup() diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 853397e0a..3621c4960 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -15,6 +15,13 @@ # we don't use refcounts because that's a behaviour # the programmer may not want + +proc dataPointer(a: PGenericSeq, elemAlign: int): pointer = + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign)) + +proc dataPointer(a: PGenericSeq, elemAlign, elemSize, index: int): pointer = + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) + proc resize(old: int): int {.inline.} = if old <= 0: result = 4 elif old < 65536: result = old * 2 @@ -40,27 +47,22 @@ else: template allocStrNoInit(size: untyped): untyped = cast[NimString](newObjNoInit(addr(strDesc), size)) -proc rawNewStringNoInit(space: int): NimString {.compilerproc.} = - var s = space - if s < 7: s = 7 +proc rawNewStringNoInit(space: int): NimString = + let s = max(space, 7) result = allocStrNoInit(sizeof(TGenericSeq) + s + 1) result.reserved = s - result.len = 0 when defined(gogc): result.elemSize = 1 proc rawNewString(space: int): NimString {.compilerproc.} = - var s = space - if s < 7: s = 7 - result = allocStr(sizeof(TGenericSeq) + s + 1) - result.reserved = s + result = rawNewStringNoInit(space) result.len = 0 - when defined(gogc): - result.elemSize = 1 + result.data[0] = '\0' proc mnewString(len: int): NimString {.compilerproc.} = - result = rawNewString(len) + result = rawNewStringNoInit(len) result.len = len + zeroMem(addr result.data[0], len + 1) proc copyStrLast(s: NimString, start, last: int): NimString {.compilerproc.} = # This is not used by most recent versions of the compiler anymore, but @@ -68,13 +70,10 @@ proc copyStrLast(s: NimString, start, last: int): NimString {.compilerproc.} = let start = max(start, 0) if s == nil: return nil let len = min(last, s.len-1) - start + 1 - if len > 0: - result = rawNewStringNoInit(len) - result.len = len - copyMem(addr(result.data), addr(s.data[start]), len) - result.data[len] = '\0' - else: - result = rawNewString(len) + result = rawNewStringNoInit(len) + result.len = len + copyMem(addr(result.data), addr(s.data[start]), len) + result.data[len] = '\0' proc copyStr(s: NimString, start: int): NimString {.compilerproc.} = # This is not used by most recent versions of the compiler anymore, but @@ -84,12 +83,13 @@ proc copyStr(s: NimString, start: int): NimString {.compilerproc.} = proc nimToCStringConv(s: NimString): cstring {.compilerproc, nonReloadable, inline.} = if s == nil or s.len == 0: result = cstring"" - else: result = cstring(addr s.data) + else: result = cast[cstring](addr s.data) proc toNimStr(str: cstring, len: int): NimString {.compilerproc.} = result = rawNewStringNoInit(len) result.len = len - copyMem(addr(result.data), str, len + 1) + copyMem(addr(result.data), str, len) + result.data[len] = '\0' proc cstrToNimstr(str: cstring): NimString {.compilerRtl.} = if str == nil: NimString(nil) @@ -153,13 +153,9 @@ proc addChar(s: NimString, c: char): NimString = result = s if result.len >= result.space: let r = resize(result.space) - when defined(nimIncrSeqV3): - result = rawNewStringNoInit(r) - result.len = s.len - copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) - else: - result = cast[NimString](growObj(result, - sizeof(TGenericSeq) + r + 1)) + result = rawNewStringNoInit(r) + result.len = s.len + copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) result.reserved = r result.data[result.len] = c result.data[result.len+1] = '\0' @@ -198,17 +194,14 @@ proc addChar(s: NimString, c: char): NimString = proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.} = if dest == nil: - result = rawNewStringNoInit(addlen) + result = rawNewString(addlen) elif dest.len + addlen <= dest.space: result = dest else: # slow path: let sp = max(resize(dest.space), dest.len + addlen) - when defined(nimIncrSeqV3): - result = rawNewStringNoInit(sp) - result.len = dest.len - copyMem(addr result.data[0], unsafeAddr(dest.data[0]), dest.len+1) - else: - result = cast[NimString](growObj(dest, sizeof(TGenericSeq) + sp + 1)) + result = rawNewStringNoInit(sp) + result.len = dest.len + copyMem(addr result.data[0], unsafeAddr(dest.data[0]), dest.len+1) result.reserved = sp #result = rawNewString(sp) #copyMem(result, dest, dest.len + sizeof(TGenericSeq)) @@ -227,25 +220,25 @@ proc appendChar(dest: NimString, c: char) {.compilerproc, inline.} = proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.} = let n = max(newLen, 0) if s == nil: - result = mnewString(newLen) + if n == 0: + return s + else: + result = mnewString(n) elif n <= s.space: result = s else: - let sp = max(resize(s.space), newLen) - when defined(nimIncrSeqV3): - result = rawNewStringNoInit(sp) - result.len = s.len - copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len+1) - zeroMem(addr result.data[s.len], newLen - s.len) - result.reserved = sp - else: - result = resizeString(s, n) + let sp = max(resize(s.space), n) + result = rawNewStringNoInit(sp) + result.len = s.len + copyMem(addr result.data[0], unsafeAddr(s.data[0]), s.len) + zeroMem(addr result.data[s.len], n - s.len) + result.reserved = sp result.len = n result.data[n] = '\0' # ----------------- sequences ---------------------------------------------- -proc incrSeq(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerproc.} = +proc incrSeq(seq: PGenericSeq, elemSize, elemAlign: int): PGenericSeq {.compilerproc.} = # increments the length by one: # this is needed for supporting ``add``; # @@ -255,23 +248,18 @@ proc incrSeq(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerproc.} = result = seq if result.len >= result.space: let r = resize(result.space) - result = cast[PGenericSeq](growObj(result, elemSize * r + - GenericSeqSize)) + result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, elemAlign) + elemSize * r)) result.reserved = r inc(result.len) -proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerproc.} = +proc incrSeqV2(seq: PGenericSeq, elemSize, elemAlign: int): PGenericSeq {.compilerproc.} = # incrSeq version 2 result = seq if result.len >= result.space: let r = resize(result.space) - result = cast[PGenericSeq](growObj(result, elemSize * r + - GenericSeqSize)) + result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, elemAlign) + elemSize * r)) result.reserved = r -template `+!`(p: pointer, s: int): pointer = - cast[pointer](cast[int](p) +% s) - proc incrSeqV3(s: PGenericSeq, typ: PNimType): PGenericSeq {.compilerproc.} = if s == nil: result = cast[PGenericSeq](newSeq(typ, 1)) @@ -280,46 +268,28 @@ proc incrSeqV3(s: PGenericSeq, typ: PNimType): PGenericSeq {.compilerproc.} = result = s if result.len >= result.space: let r = resize(result.space) - when defined(nimIncrSeqV3): - result = cast[PGenericSeq](newSeq(typ, r)) - result.len = s.len - copyMem(result +! GenericSeqSize, s +! GenericSeqSize, s.len * typ.base.size) - # since we steal the content from 's', it's crucial to set s's len to 0. - s.len = 0 - else: - result = cast[PGenericSeq](growObj(result, typ.base.size * r + - GenericSeqSize)) - result.reserved = r + result = cast[PGenericSeq](newSeq(typ, r)) + result.len = s.len + copyMem(dataPointer(result, typ.base.align), dataPointer(s, typ.base.align), s.len * typ.base.size) + # since we steal the content from 's', it's crucial to set s's len to 0. + s.len = 0 -proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. +proc setLengthSeq(seq: PGenericSeq, elemSize, elemAlign, newLen: int): PGenericSeq {. compilerRtl, inl.} = result = seq if result.space < newLen: let r = max(resize(result.space), newLen) - result = cast[PGenericSeq](growObj(result, elemSize * r + - GenericSeqSize)) + result = cast[PGenericSeq](growObj(result, align(GenericSeqSize, elemAlign) + elemSize * r)) result.reserved = r elif newLen < result.len: # we need to decref here, otherwise the GC leaks! when not defined(boehmGC) and not defined(nogc) and not defined(gcMarkAndSweep) and not defined(gogc) and not defined(gcRegions): - when false: # compileOption("gc", "v2"): + if ntfNoRefs notin extGetCellType(result).base.flags: for i in newLen..result.len-1: - let len0 = gch.tempStack.len - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), - extGetCellType(result).base, waPush) - let len1 = gch.tempStack.len - for i in len0 ..< len1: - doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) - gch.tempStack.len = len0 - else: - if ntfNoRefs notin extGetCellType(result).base.flags: - for i in newLen..result.len-1: - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), - extGetCellType(result).base, waZctDecRef) + forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), + extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). @@ -327,46 +297,67 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. # presence of user defined destructors, the user will expect the cell to be # "destroyed" thus creating the same problem. We can destroy the cell in the # finalizer of the sequence, but this makes destruction non-deterministic. - zeroMem(cast[pointer](cast[ByteAddress](result) +% GenericSeqSize +% - (newLen*%elemSize)), (result.len-%newLen) *% elemSize) + zeroMem(dataPointer(result, elemAlign, elemSize, newLen), (result.len-%newLen) *% elemSize) result.len = newLen proc setLengthSeqV2(s: PGenericSeq, typ: PNimType, newLen: int): PGenericSeq {. compilerRtl.} = sysAssert typ.kind == tySequence, "setLengthSeqV2: type is not a seq" if s == nil: - result = cast[PGenericSeq](newSeq(typ, newLen)) + if newLen == 0: + result = s + else: + result = cast[PGenericSeq](newSeq(typ, newLen)) else: - when defined(nimIncrSeqV3): - let elemSize = typ.base.size - if s.space < newLen: - let r = max(resize(s.space), newLen) - result = cast[PGenericSeq](newSeq(typ, r)) - copyMem(result +! GenericSeqSize, s +! GenericSeqSize, s.len * elemSize) - # since we steal the content from 's', it's crucial to set s's len to 0. - s.len = 0 - elif newLen < s.len: - result = s - # we need to decref here, otherwise the GC leaks! - when not defined(boehmGC) and not defined(nogc) and - not defined(gcMarkAndSweep) and not defined(gogc) and - not defined(gcRegions): - if ntfNoRefs notin typ.base.flags: - for i in newLen..result.len-1: - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), - extGetCellType(result).base, waZctDecRef) - - # XXX: zeroing out the memory can still result in crashes if a wiped-out - # cell is aliased by another pointer (ie proc parameter or a let variable). - # This is a tough problem, because even if we don't zeroMem here, in the - # presence of user defined destructors, the user will expect the cell to be - # "destroyed" thus creating the same problem. We can destroy the cell in the - # finalizer of the sequence, but this makes destruction non-deterministic. - zeroMem(cast[pointer](cast[ByteAddress](result) +% GenericSeqSize +% - (newLen*%elemSize)), (result.len-%newLen) *% elemSize) - else: - result = s - result.len = newLen + let elemSize = typ.base.size + let elemAlign = typ.base.align + if s.space < newLen: + let r = max(resize(s.space), newLen) + result = cast[PGenericSeq](newSeq(typ, r)) + copyMem(dataPointer(result, elemAlign), dataPointer(s, elemAlign), s.len * elemSize) + # since we steal the content from 's', it's crucial to set s's len to 0. + s.len = 0 + elif newLen < s.len: + result = s + # we need to decref here, otherwise the GC leaks! + when not defined(boehmGC) and not defined(nogc) and + not defined(gcMarkAndSweep) and not defined(gogc) and + not defined(gcRegions): + if ntfNoRefs notin typ.base.flags: + for i in newLen..result.len-1: + forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), + extGetCellType(result).base, waZctDecRef) + + # XXX: zeroing out the memory can still result in crashes if a wiped-out + # cell is aliased by another pointer (ie proc parameter or a let variable). + # This is a tough problem, because even if we don't zeroMem here, in the + # presence of user defined destructors, the user will expect the cell to be + # "destroyed" thus creating the same problem. We can destroy the cell in the + # finalizer of the sequence, but this makes destruction non-deterministic. + zeroMem(dataPointer(result, elemAlign, elemSize, newLen), (result.len-%newLen) *% elemSize) else: - result = setLengthSeq(s, typ.base.size, newLen) + result = s + zeroMem(dataPointer(result, elemAlign, elemSize, result.len), (newLen-%result.len) *% elemSize) + result.len = newLen + +func capacity*(self: string): int {.inline.} = + ## Returns the current capacity of the string. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var str = newStringOfCap(cap = 42) + str.add "Nim" + assert str.capacity == 42 + + let str = cast[NimString](self) + result = if str != nil: str.space else: 0 + +func capacity*[T](self: seq[T]): int {.inline.} = + ## Returns the current capacity of the seq. + # See https://github.com/nim-lang/RFCs/issues/460 + runnableExamples: + var lst = newSeqOfCap[string](cap = 42) + lst.add "Nim" + assert lst.capacity == 42 + + let sek = cast[PGenericSeq](self) + result = if sek != nil: sek.space else: 0 diff --git a/lib/system/threadids.nim b/lib/system/threadids.nim new file mode 100644 index 000000000..3a6eadcbb --- /dev/null +++ b/lib/system/threadids.nim @@ -0,0 +1,103 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# we need to cache current threadId to not perform syscall all the time +var threadId {.threadvar.}: int + +when defined(windows): + proc getCurrentThreadId(): int32 {. + stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getCurrentThreadId()) + result = threadId + +elif defined(linux): + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} + when defined(amd64): + const NR_gettid = clong(186) + else: + var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: clong + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(NR_gettid)) + result = threadId + +elif defined(dragonfly): + proc lwp_gettid(): int32 {.importc, header: "unistd.h".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_gettid()) + result = threadId + +elif defined(openbsd): + proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getthrid()) + result = threadId + +elif defined(netbsd): + proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_self()) + result = threadId + +elif defined(freebsd): + proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} + var SYS_thr_self {.importc:"SYS_thr_self", header:"<sys/syscall.h>".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + var tid = 0.cint + if threadId == 0: + discard syscall(SYS_thr_self, addr tid) + threadId = tid + result = threadId + +elif defined(macosx): + proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} + var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"<sys/syscall.h>".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(SYS_thread_selfid)) + result = threadId + +elif defined(solaris): + type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int + proc thr_self(): thread_t {.importc, header: "<thread.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(thr_self()) + result = threadId + +elif defined(haiku): + type thr_id {.importc: "thread_id", header: "<OS.h>".} = distinct int32 + proc find_thread(name: cstring): thr_id {.importc, header: "<OS.h>".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(find_thread(nil)) + result = threadId diff --git a/lib/system/threadimpl.nim b/lib/system/threadimpl.nim new file mode 100644 index 000000000..285b8f5e7 --- /dev/null +++ b/lib/system/threadimpl.nim @@ -0,0 +1,111 @@ +var + nimThreadDestructionHandlers* {.rtlThreadVar.}: seq[proc () {.closure, gcsafe, raises: [].}] +when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcRegions): + proc deallocOsPages() {.rtl, raises: [].} +proc threadTrouble() {.raises: [], gcsafe.} +# create for the main thread. Note: do not insert this data into the list +# of all threads; it's not to be stopped etc. +when not defined(useNimRtl): + #when not defined(createNimRtl): initStackBottom() + when declared(initGC): + initGC() + when not emulatedThreadVars: + type ThreadType {.pure.} = enum + None = 0, + NimThread = 1, + ForeignThread = 2 + var + threadType {.rtlThreadVar.}: ThreadType + + threadType = ThreadType.NimThread + +when defined(gcDestructors): + proc deallocThreadStorage(p: pointer) = c_free(p) +else: + template deallocThreadStorage(p: pointer) = deallocShared(p) + +template afterThreadRuns() = + for i in countdown(nimThreadDestructionHandlers.len-1, 0): + nimThreadDestructionHandlers[i]() + +proc onThreadDestruction*(handler: proc () {.closure, gcsafe, raises: [].}) = + ## Registers a *thread local* handler that is called at the thread's + ## destruction. + ## + ## A thread is destructed when the `.thread` proc returns + ## normally or when it raises an exception. Note that unhandled exceptions + ## in a thread nevertheless cause the whole process to die. + nimThreadDestructionHandlers.add handler + +when defined(boehmgc): + type GCStackBaseProc = proc(sb: pointer, t: pointer) {.noconv.} + proc boehmGC_call_with_stack_base(sbp: GCStackBaseProc, p: pointer) + {.importc: "GC_call_with_stack_base", boehmGC.} + proc boehmGC_register_my_thread(sb: pointer) + {.importc: "GC_register_my_thread", boehmGC.} + proc boehmGC_unregister_my_thread() + {.importc: "GC_unregister_my_thread", boehmGC.} + + proc threadProcWrapDispatch[TArg](sb: pointer, thrd: pointer) {.noconv, raises: [].} = + boehmGC_register_my_thread(sb) + try: + let thrd = cast[ptr Thread[TArg]](thrd) + when TArg is void: + thrd.dataFn() + else: + thrd.dataFn(thrd.data) + except: + threadTrouble() + finally: + afterThreadRuns() + boehmGC_unregister_my_thread() +else: + proc threadProcWrapDispatch[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + try: + when TArg is void: + thrd.dataFn() + else: + when defined(nimV2): + thrd.dataFn(thrd.data) + else: + var x: TArg + deepCopy(x, thrd.data) + thrd.dataFn(x) + except: + threadTrouble() + finally: + afterThreadRuns() + when hasAllocStack: + deallocThreadStorage(thrd.rawStack) + +proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + when defined(boehmgc): + boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) + elif not defined(nogc) and not defined(gogc) and not defined(gcRegions) and not usesDestructors: + var p {.volatile.}: pointer + # init the GC for refc/markandsweep + nimGC_setStackBottom(addr(p)) + when declared(initGC): + initGC() + when declared(threadType): + threadType = ThreadType.NimThread + threadProcWrapDispatch[TArg](thrd) + when declared(deallocOsPages): deallocOsPages() + else: + threadProcWrapDispatch(thrd) + +template nimThreadProcWrapperBody*(closure: untyped): untyped = + var thrd = cast[ptr Thread[TArg]](closure) + var core = thrd.core + when declared(globalsSlot): threadVarSetValue(globalsSlot, thrd.core) + threadProcWrapStackFrame(thrd) + # Since an unhandled exception terminates the whole process (!), there is + # no need for a ``try finally`` here, nor would it be correct: The current + # exception is tried to be re-raised by the code-gen after the ``finally``! + # However this is doomed to fail, because we already unmapped every heap + # page! + + # mark as not running anymore: + thrd.core = nil + thrd.dataFn = nil + deallocThreadStorage(cast[pointer](core)) diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index 117af4c25..e6ad9dca5 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -1,83 +1,33 @@ +import std/private/threadtypes when defined(windows): type - SysThread* = Handle - WinThreadProc = proc (x: pointer): int32 {.stdcall.} - - proc createThread(lpThreadAttributes: pointer, dwStackSize: int32, - lpStartAddress: WinThreadProc, - lpParameter: pointer, - dwCreationFlags: int32, - lpThreadId: var int32): SysThread {. - stdcall, dynlib: "kernel32", importc: "CreateThread".} - - proc winSuspendThread(hThread: SysThread): int32 {. - stdcall, dynlib: "kernel32", importc: "SuspendThread".} - - proc winResumeThread(hThread: SysThread): int32 {. - stdcall, dynlib: "kernel32", importc: "ResumeThread".} - - proc waitForMultipleObjects(nCount: int32, - lpHandles: ptr SysThread, - bWaitAll: int32, - dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".} - - proc terminateThread(hThread: SysThread, dwExitCode: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "TerminateThread".} - - proc getCurrentThreadId(): int32 {. - stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} - - type ThreadVarSlot = distinct int32 - when true: - proc threadVarAlloc(): ThreadVarSlot {. - importc: "TlsAlloc", stdcall, header: "<windows.h>".} - proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. - importc: "TlsSetValue", stdcall, header: "<windows.h>".} - proc tlsGetValue(dwTlsIndex: ThreadVarSlot): pointer {. - importc: "TlsGetValue", stdcall, header: "<windows.h>".} - - proc getLastError(): uint32 {. - importc: "GetLastError", stdcall, header: "<windows.h>".} - proc setLastError(x: uint32) {. - importc: "SetLastError", stdcall, header: "<windows.h>".} - - proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer = - let realLastError = getLastError() - result = tlsGetValue(dwTlsIndex) - setLastError(realLastError) - else: - proc threadVarAlloc(): ThreadVarSlot {. - importc: "TlsAlloc", stdcall, dynlib: "kernel32".} - proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. - importc: "TlsSetValue", stdcall, dynlib: "kernel32".} - proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer {. - importc: "TlsGetValue", stdcall, dynlib: "kernel32".} + proc threadVarAlloc(): ThreadVarSlot {. + importc: "TlsAlloc", stdcall, header: "<windows.h>".} + proc threadVarSetValue(dwTlsIndex: ThreadVarSlot, lpTlsValue: pointer) {. + importc: "TlsSetValue", stdcall, header: "<windows.h>".} + proc tlsGetValue(dwTlsIndex: ThreadVarSlot): pointer {. + importc: "TlsGetValue", stdcall, header: "<windows.h>".} - proc setThreadAffinityMask(hThread: SysThread, dwThreadAffinityMask: uint) {. - importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".} + proc getLastError(): uint32 {. + importc: "GetLastError", stdcall, header: "<windows.h>".} + proc setLastError(x: uint32) {. + importc: "SetLastError", stdcall, header: "<windows.h>".} + + proc threadVarGetValue(dwTlsIndex: ThreadVarSlot): pointer = + let realLastError = getLastError() + result = tlsGetValue(dwTlsIndex) + setLastError(realLastError) elif defined(genode): - import genode/env const GenodeHeader = "genode_cpp/threads.h" + type - SysThread* {.importcpp: "Nim::SysThread", - header: GenodeHeader, final, pure.} = object - GenodeThreadProc = proc (x: pointer) {.noconv.} ThreadVarSlot = int - proc initThread(s: var SysThread, - env: GenodeEnv, - stackSize: culonglong, - entry: GenodeThreadProc, - arg: pointer, - affinity: cuint) {. - importcpp: "#.initThread(@)".} - proc threadVarAlloc(): ThreadVarSlot = 0 proc offMainThread(): bool {. @@ -113,60 +63,18 @@ else: when not defined(haiku): {.passc: "-pthread".} - const - schedh = "#define _GNU_SOURCE\n#include <sched.h>" - pthreadh = "#define _GNU_SOURCE\n#include <pthread.h>" - - when not declared(Time): - when defined(linux): - type Time = clong - else: - type Time = int - when (defined(linux) or defined(nintendoswitch)) and defined(amd64): type - SysThread* {.importc: "pthread_t", - header: "<sys/types.h>" .} = distinct culong - Pthread_attr {.importc: "pthread_attr_t", - header: "<sys/types.h>".} = object - abi: array[56 div sizeof(clong), clong] ThreadVarSlot {.importc: "pthread_key_t", header: "<sys/types.h>".} = distinct cuint elif defined(openbsd) and defined(amd64): type - SysThread* {.importc: "pthread_t", header: "<pthread.h>".} = object - Pthread_attr {.importc: "pthread_attr_t", - header: "<pthread.h>".} = object ThreadVarSlot {.importc: "pthread_key_t", header: "<pthread.h>".} = cint else: type - SysThread* {.importc: "pthread_t", header: "<sys/types.h>".} = object - Pthread_attr {.importc: "pthread_attr_t", - header: "<sys/types.h>".} = object ThreadVarSlot {.importc: "pthread_key_t", header: "<sys/types.h>".} = object - type - Timespec {.importc: "struct timespec", header: "<time.h>".} = object - tv_sec: Time - tv_nsec: clong - - proc pthread_attr_init(a1: var Pthread_attr): cint {. - importc, header: pthreadh.} - proc pthread_attr_setstacksize(a1: var Pthread_attr, a2: int): cint {. - importc, header: pthreadh.} - proc pthread_attr_destroy(a1: var Pthread_attr): cint {. - importc, header: pthreadh.} - - proc pthread_create(a1: var SysThread, a2: var Pthread_attr, - a3: proc (x: pointer): pointer {.noconv.}, - a4: pointer): cint {.importc: "pthread_create", - header: pthreadh.} - proc pthread_join(a1: SysThread, a2: ptr pointer): cint {. - importc, header: pthreadh.} - - proc pthread_cancel(a1: SysThread): cint {. - importc: "pthread_cancel", header: pthreadh.} proc pthread_getspecific(a1: ThreadVarSlot): pointer {. importc: "pthread_getspecific", header: pthreadh.} @@ -186,44 +94,13 @@ else: proc threadVarGetValue(s: ThreadVarSlot): pointer {.inline.} = result = pthread_getspecific(s) - type CpuSet {.importc: "cpu_set_t", header: schedh.} = object - when defined(linux) and defined(amd64): - abi: array[1024 div (8 * sizeof(culong)), culong] - - proc cpusetZero(s: var CpuSet) {.importc: "CPU_ZERO", header: schedh.} - proc cpusetIncl(cpu: cint; s: var CpuSet) {. - importc: "CPU_SET", header: schedh.} - - proc setAffinity(thread: SysThread; setsize: csize_t; s: var CpuSet) {. - importc: "pthread_setaffinity_np", header: pthreadh.} - - -const - emulatedThreadVars = compileOption("tlsEmulation") when emulatedThreadVars: # the compiler generates this proc for us, so that we can get the size of # the thread local var block; we use this only for sanity checking though proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".} -# we preallocate a fixed size for thread local storage, so that no heap -# allocations are needed. Currently less than 16K are used on a 64bit machine. -# We use ``float`` for proper alignment: -const nimTlsSize {.intdefine.} = 16000 -type - ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float] - PGcThread = ptr GcThread - GcThread {.pure, inheritable.} = object - when emulatedThreadVars: - tls: ThreadLocalStorage - else: - nil - when hasSharedHeap: - next, prev: PGcThread - stackBottom, stackTop: pointer - stackSize: int - else: - nil + when emulatedThreadVars: var globalsSlot: ThreadVarSlot @@ -245,4 +122,4 @@ when not defined(useNimRtl): if nimThreadVarsSize() > sizeof(ThreadLocalStorage): c_fprintf(cstderr, """too large thread local storage size requested, use -d:\"nimTlsSize=X\" to setup even more or stop using unittest.nim""") - quit 1 + rawQuit 1 diff --git a/lib/system/threads.nim b/lib/system/threads.nim deleted file mode 100644 index 54b07e467..000000000 --- a/lib/system/threads.nim +++ /dev/null @@ -1,426 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Thread support for Nim. -## -## **Note**: This is part of the system module. Do not import it directly. -## To activate thread support you need to compile -## with the ``--threads:on`` command line switch. -## -## Nim's memory model for threads is quite different from other common -## programming languages (C, Pascal): Each thread has its own -## (garbage collected) heap and sharing of memory is restricted. This helps -## to prevent race conditions and improves efficiency. See `the manual for -## details of this memory model <manual.html#threads>`_. -## -## Examples -## ======== -## -## .. code-block:: Nim -## -## import locks -## -## var -## thr: array[0..4, Thread[tuple[a,b: int]]] -## L: Lock -## -## proc threadFunc(interval: tuple[a,b: int]) {.thread.} = -## for i in interval.a..interval.b: -## acquire(L) # lock stdout -## echo i -## release(L) -## -## initLock(L) -## -## for i in 0..high(thr): -## createThread(thr[i], threadFunc, (i*10, i*10+5)) -## joinThreads(thr) - -when not declared(ThisIsSystem): - {.error: "You must not import this module explicitly".} - -const - StackGuardSize = 4096 - ThreadStackMask = - when defined(genode): - 1024*64*sizeof(int)-1 - else: - 1024*256*sizeof(int)-1 - ThreadStackSize = ThreadStackMask+1 - StackGuardSize - -#const globalsSlot = ThreadVarSlot(0) -#sysAssert checkSlot.int == globalsSlot.int - -# create for the main thread. Note: do not insert this data into the list -# of all threads; it's not to be stopped etc. -when not defined(useNimRtl): - #when not defined(createNimRtl): initStackBottom() - when declared(initGC): - initGC() - when not emulatedThreadVars: - type ThreadType {.pure.} = enum - None = 0, - NimThread = 1, - ForeignThread = 2 - var - threadType {.rtlThreadVar.}: ThreadType - - threadType = ThreadType.NimThread - -# We jump through some hops here to ensure that Nim thread procs can have -# the Nim calling convention. This is needed because thread procs are -# ``stdcall`` on Windows and ``noconv`` on UNIX. Alternative would be to just -# use ``stdcall`` since it is mapped to ``noconv`` on UNIX anyway. - -type - Thread* {.pure, final.}[TArg] = object - core: PGcThread - sys: SysThread - when TArg is void: - dataFn: proc () {.nimcall, gcsafe.} - else: - dataFn: proc (m: TArg) {.nimcall, gcsafe.} - data: TArg - -var - threadDestructionHandlers {.rtlThreadVar.}: seq[proc () {.closure, gcsafe.}] - -proc onThreadDestruction*(handler: proc () {.closure, gcsafe.}) = - ## Registers a *thread local* handler that is called at the thread's - ## destruction. - ## - ## A thread is destructed when the ``.thread`` proc returns - ## normally or when it raises an exception. Note that unhandled exceptions - ## in a thread nevertheless cause the whole process to die. - when not defined(nimNoNilSeqs): - if threadDestructionHandlers.isNil: - threadDestructionHandlers = @[] - threadDestructionHandlers.add handler - -template afterThreadRuns() = - for i in countdown(threadDestructionHandlers.len-1, 0): - threadDestructionHandlers[i]() - -when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcRegions): - proc deallocOsPages() {.rtl.} - -when defined(boehmgc): - type GCStackBaseProc = proc(sb: pointer, t: pointer) {.noconv.} - proc boehmGC_call_with_stack_base(sbp: GCStackBaseProc, p: pointer) - {.importc: "GC_call_with_stack_base", boehmGC.} - proc boehmGC_register_my_thread(sb: pointer) - {.importc: "GC_register_my_thread", boehmGC.} - proc boehmGC_unregister_my_thread() - {.importc: "GC_unregister_my_thread", boehmGC.} - - proc threadProcWrapDispatch[TArg](sb: pointer, thrd: pointer) {.noconv.} = - boehmGC_register_my_thread(sb) - try: - let thrd = cast[ptr Thread[TArg]](thrd) - when TArg is void: - thrd.dataFn() - else: - thrd.dataFn(thrd.data) - finally: - afterThreadRuns() - boehmGC_unregister_my_thread() -else: - proc threadProcWrapDispatch[TArg](thrd: ptr Thread[TArg]) = - try: - when TArg is void: - thrd.dataFn() - else: - when defined(nimV2): - thrd.dataFn(thrd.data) - else: - var x: TArg - deepCopy(x, thrd.data) - thrd.dataFn(x) - finally: - afterThreadRuns() - -proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = - when defined(boehmgc): - boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) - elif not defined(nogc) and not defined(gogc) and not defined(gcRegions) and not usesDestructors: - var p {.volatile.}: proc(a: ptr Thread[TArg]) {.nimcall, gcsafe.} = - threadProcWrapDispatch[TArg] - # init the GC for refc/markandsweep - nimGC_setStackBottom(addr(p)) - initGC() - when declared(threadType): - threadType = ThreadType.NimThread - p(thrd) - when declared(deallocOsPages): deallocOsPages() - else: - threadProcWrapDispatch(thrd) - -template threadProcWrapperBody(closure: untyped): untyped = - var thrd = cast[ptr Thread[TArg]](closure) - var core = thrd.core - when declared(globalsSlot): threadVarSetValue(globalsSlot, thrd.core) - threadProcWrapStackFrame(thrd) - # Since an unhandled exception terminates the whole process (!), there is - # no need for a ``try finally`` here, nor would it be correct: The current - # exception is tried to be re-raised by the code-gen after the ``finally``! - # However this is doomed to fail, because we already unmapped every heap - # page! - - # mark as not running anymore: - thrd.core = nil - thrd.dataFn = nil - deallocShared(cast[pointer](core)) - -{.push stack_trace:off.} -when defined(windows): - proc threadProcWrapper[TArg](closure: pointer): int32 {.stdcall.} = - threadProcWrapperBody(closure) - # implicitly return 0 -elif defined(genode): - proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = - threadProcWrapperBody(closure) -else: - proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = - threadProcWrapperBody(closure) -{.pop.} - -proc running*[TArg](t: Thread[TArg]): bool {.inline.} = - ## Returns true if `t` is running. - result = t.dataFn != nil - -proc handle*[TArg](t: Thread[TArg]): SysThread {.inline.} = - ## Returns the thread handle of `t`. - result = t.sys - -when hostOS == "windows": - const MAXIMUM_WAIT_OBJECTS = 64 - - proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = - ## Waits for the thread `t` to finish. - discard waitForSingleObject(t.sys, -1'i32) - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## Waits for every thread in `t` to finish. - var a: array[MAXIMUM_WAIT_OBJECTS, SysThread] - var k = 0 - while k < len(t): - var count = min(len(t) - k, MAXIMUM_WAIT_OBJECTS) - for i in 0..(count - 1): a[i] = t[i + k].sys - discard waitForMultipleObjects(int32(count), - cast[ptr SysThread](addr(a)), 1, -1) - inc(k, MAXIMUM_WAIT_OBJECTS) - -elif defined(genode): - proc joinThread*[TArg](t: Thread[TArg]) {.importcpp.} - ## Waits for the thread `t` to finish. - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## Waits for every thread in `t` to finish. - for i in 0..t.high: joinThread(t[i]) - -else: - proc joinThread*[TArg](t: Thread[TArg]) {.inline.} = - ## Waits for the thread `t` to finish. - discard pthread_join(t.sys, nil) - - proc joinThreads*[TArg](t: varargs[Thread[TArg]]) = - ## Waits for every thread in `t` to finish. - for i in 0..t.high: joinThread(t[i]) - -when false: - # XXX a thread should really release its heap here somehow: - proc destroyThread*[TArg](t: var Thread[TArg]) = - ## Forces the thread `t` to terminate. This is potentially dangerous if - ## you don't have full control over `t` and its acquired resources. - when hostOS == "windows": - discard TerminateThread(t.sys, 1'i32) - else: - discard pthread_cancel(t.sys) - when declared(registerThread): unregisterThread(addr(t)) - t.dataFn = nil - ## if thread `t` already exited, `t.core` will be `null`. - if not isNil(t.core): - deallocShared(t.core) - t.core = nil - -when hostOS == "windows": - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread, nimcall.}, - param: TArg) = - ## Creates a new thread `t` and starts its execution. - ## - ## Entry point is the proc `tp`. - ## `param` is passed to `tp`. `TArg` can be ``void`` if you - ## don't need to pass any data to the thread. - t.core = cast[PGcThread](allocShared0(sizeof(GcThread))) - - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.core.stackSize = ThreadStackSize - var dummyThreadId: int32 - t.sys = createThread(nil, ThreadStackSize, threadProcWrapper[TArg], - addr(t), 0'i32, dummyThreadId) - if t.sys <= 0: - raise newException(ResourceExhaustedError, "cannot create thread") - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - ## Pins a thread to a `CPU`:idx:. - ## - ## In other words sets a thread's `affinity`:idx:. - ## If you don't know what this means, you shouldn't use this proc. - setThreadAffinityMask(t.sys, uint(1 shl cpu)) - -elif defined(genode): - var affinityOffset: cuint = 1 - ## CPU affinity offset for next thread, safe to roll-over. - - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread, nimcall.}, - param: TArg) = - t.core = cast[PGcThread](allocShared0(sizeof(GcThread))) - - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.stackSize = ThreadStackSize - t.sys.initThread( - runtimeEnv, - ThreadStackSize.culonglong, - threadProcWrapper[TArg], addr(t), affinityOffset) - inc affinityOffset - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - {.hint: "cannot change Genode thread CPU affinity after initialization".} - discard - -else: - proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread, nimcall.}, - param: TArg) = - ## Creates a new thread `t` and starts its execution. - ## - ## Entry point is the proc `tp`. `param` is passed to `tp`. - ## `TArg` can be ``void`` if you - ## don't need to pass any data to the thread. - t.core = cast[PGcThread](allocShared0(sizeof(GcThread))) - - when TArg isnot void: t.data = param - t.dataFn = tp - when hasSharedHeap: t.core.stackSize = ThreadStackSize - var a {.noinit.}: Pthread_attr - doAssert pthread_attr_init(a) == 0 - doAssert pthread_attr_setstacksize(a, ThreadStackSize) == 0 - if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0: - raise newException(ResourceExhaustedError, "cannot create thread") - doAssert pthread_attr_destroy(a) == 0 - - proc pinToCpu*[Arg](t: var Thread[Arg]; cpu: Natural) = - ## Pins a thread to a `CPU`:idx:. - ## - ## In other words sets a thread's `affinity`:idx:. - ## If you don't know what this means, you shouldn't use this proc. - when not defined(macosx): - var s {.noinit.}: CpuSet - cpusetZero(s) - cpusetIncl(cpu.cint, s) - setAffinity(t.sys, csize_t(sizeof(s)), s) - -proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = - createThread[void](t, tp) - -# we need to cache current threadId to not perform syscall all the time -var threadId {.threadvar.}: int - -when defined(windows): - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(getCurrentThreadId()) - result = threadId - -elif defined(linux): - proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".} - when defined(amd64): - const NR_gettid = clong(186) - else: - var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: clong - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(NR_gettid)) - result = threadId - -elif defined(dragonfly): - proc lwp_gettid(): int32 {.importc, header: "unistd.h".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_gettid()) - result = threadId - -elif defined(openbsd): - proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".} - - proc getThreadId*(): int = - ## get the ID of the currently running thread. - if threadId == 0: - threadId = int(getthrid()) - result = threadId - -elif defined(netbsd): - proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_self()) - result = threadId - -elif defined(freebsd): - proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} - var SYS_thr_self {.importc:"SYS_thr_self", header:"<sys/syscall.h>"}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - var tid = 0.cint - if threadId == 0: - discard syscall(SYS_thr_self, addr tid) - threadId = tid - result = threadId - -elif defined(macosx): - proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".} - var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"<sys/syscall.h>".}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(SYS_thread_selfid)) - result = threadId - -elif defined(solaris): - type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int - proc thr_self(): thread_t {.importc, header: "<thread.h>".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(thr_self()) - result = threadId - -elif defined(haiku): - type thr_id {.importc: "thread_id", header: "<OS.h>".} = distinct int32 - proc find_thread(name: cstring): thr_id {.importc, header: "<OS.h>".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(find_thread(nil)) - result = threadId diff --git a/lib/system/timers.nim b/lib/system/timers.nim index 56575abb1..ffb0f7716 100644 --- a/lib/system/timers.nim +++ b/lib/system/timers.nim @@ -31,7 +31,7 @@ when defined(windows): result = Nanos(float64(a.int64 - b.int64) * performanceCounterRate) -elif defined(macosx): +elif defined(macosx) and not defined(emscripten): type MachTimebaseInfoData {.pure, final, importc: "mach_timebase_info_data_t", diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim deleted file mode 100644 index 3048de82a..000000000 --- a/lib/system/widestrs.nim +++ /dev/null @@ -1,211 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# Nim support for C/C++'s `wide strings`:idx:. This is part of the system -# module! Do not import it directly! - -#when not declared(ThisIsSystem): -# {.error: "You must not import this module explicitly".} - -type - Utf16Char* = distinct int16 - -when defined(nimv2): - - type - WideCString* = ptr UncheckedArray[Utf16Char] - - WideCStringObj* = object - bytes: int - data: WideCString - - proc `=destroy`(a: var WideCStringObj) = - if a.data != nil: - deallocShared(a.data) - a.data = nil - - proc `=`(a: var WideCStringObj; b: WideCStringObj) {.error.} - - proc `=sink`(a: var WideCStringObj; b: WideCStringObj) = - a.bytes = b.bytes - a.data = b.data - - proc createWide(a: var WideCStringObj; bytes: int) = - a.bytes = bytes - a.data = cast[typeof(a.data)](allocShared0(bytes)) - - template `[]`*(a: WideCStringObj; idx: int): Utf16Char = a.data[idx] - template `[]=`*(a: WideCStringObj; idx: int; val: Utf16Char) = a.data[idx] = val - - template nullWide(): untyped = WideCStringObj(bytes: 0, data: nil) - - converter toWideCString*(x: WideCStringObj): WideCString {.inline.} = - result = x.data - -else: - template nullWide(): untyped = nil - - type - WideCString* = ref UncheckedArray[Utf16Char] - WideCStringObj* = WideCString - - template createWide(a; L) = - unsafeNew(a, L * 4 + 2) - -proc ord(arg: Utf16Char): int = int(cast[uint16](arg)) - -proc len*(w: WideCString): int = - ## returns the length of a widestring. This traverses the whole string to - ## find the binary zero end marker! - while int16(w[result]) != 0'i16: inc result - -const - UNI_REPLACEMENT_CHAR = Utf16Char(0xFFFD'i16) - UNI_MAX_BMP = 0x0000FFFF - UNI_MAX_UTF16 = 0x0010FFFF - UNI_MAX_UTF32 = 0x7FFFFFFF - UNI_MAX_LEGAL_UTF32 = 0x0010FFFF - - halfShift = 10 - halfBase = 0x0010000 - halfMask = 0x3FF - - UNI_SUR_HIGH_START = 0xD800 - UNI_SUR_HIGH_END = 0xDBFF - UNI_SUR_LOW_START = 0xDC00 - UNI_SUR_LOW_END = 0xDFFF - UNI_REPL = 0xFFFD - -template ones(n: untyped): untyped = ((1 shl n)-1) - -template fastRuneAt(s: cstring, i, L: int, result: untyped, doInc = true) = - ## Returns the unicode character ``s[i]`` in `result`. If ``doInc == true`` - ## `i` is incremented by the number of bytes that have been processed. - bind ones - - if ord(s[i]) <= 127: - result = ord(s[i]) - when doInc: inc(i) - elif ord(s[i]) shr 5 == 0b110: - #assert(ord(s[i+1]) shr 6 == 0b10) - if i <= L - 2: - result = (ord(s[i]) and (ones(5))) shl 6 or (ord(s[i+1]) and ones(6)) - when doInc: inc(i, 2) - else: - result = UNI_REPL - when doInc: inc(i) - elif ord(s[i]) shr 4 == 0b1110: - if i <= L - 3: - #assert(ord(s[i+1]) shr 6 == 0b10) - #assert(ord(s[i+2]) shr 6 == 0b10) - result = (ord(s[i]) and ones(4)) shl 12 or - (ord(s[i+1]) and ones(6)) shl 6 or - (ord(s[i+2]) and ones(6)) - when doInc: inc(i, 3) - else: - result = UNI_REPL - when doInc: inc(i) - elif ord(s[i]) shr 3 == 0b11110: - if i <= L - 4: - #assert(ord(s[i+1]) shr 6 == 0b10) - #assert(ord(s[i+2]) shr 6 == 0b10) - #assert(ord(s[i+3]) shr 6 == 0b10) - result = (ord(s[i]) and ones(3)) shl 18 or - (ord(s[i+1]) and ones(6)) shl 12 or - (ord(s[i+2]) and ones(6)) shl 6 or - (ord(s[i+3]) and ones(6)) - when doInc: inc(i, 4) - else: - result = UNI_REPL - when doInc: inc(i) - else: - result = 0xFFFD - when doInc: inc(i) - -iterator runes(s: cstring, L: int): int = - var - i = 0 - result: int - while i < L: - fastRuneAt(s, i, L, result, true) - yield result - -proc newWideCString*(source: cstring, L: int): WideCStringObj = - createWide(result, L * 4 + 2) - #result = cast[wideCString](alloc(L * 4 + 2)) - var d = 0 - for ch in runes(source, L): - - if ch <= UNI_MAX_BMP: - if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_LOW_END: - result[d] = UNI_REPLACEMENT_CHAR - else: - result[d] = cast[Utf16Char](uint16(ch)) - elif ch > UNI_MAX_UTF16: - result[d] = UNI_REPLACEMENT_CHAR - else: - let ch = ch - halfBase - result[d] = cast[Utf16Char](uint16((ch shr halfShift) + UNI_SUR_HIGH_START)) - inc d - result[d] = cast[Utf16Char](uint16((ch and halfMask) + UNI_SUR_LOW_START)) - inc d - result[d] = Utf16Char(0) - -proc newWideCString*(s: cstring): WideCStringObj = - if s.isNil: return nullWide - - result = newWideCString(s, s.len) - -proc newWideCString*(s: string): WideCStringObj = - result = newWideCString(s, s.len) - -proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string = - result = newStringOfCap(estimate + estimate shr 2) - - var i = 0 - while w[i].int16 != 0'i16: - var ch = ord(w[i]) - inc i - if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END: - # If the 16 bits following the high surrogate are in the source buffer... - let ch2 = ord(w[i]) - - # If it's a low surrogate, convert to UTF32: - if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END: - ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase - inc i - else: - #invalid UTF-16 - ch = replacement - elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END: - #invalid UTF-16 - ch = replacement - - if ch < 0x80: - result.add chr(ch) - elif ch < 0x800: - result.add chr((ch shr 6) or 0xc0) - result.add chr((ch and 0x3f) or 0x80) - elif ch < 0x10000: - result.add chr((ch shr 12) or 0xe0) - result.add chr(((ch shr 6) and 0x3f) or 0x80) - result.add chr((ch and 0x3f) or 0x80) - elif ch <= 0x10FFFF: - result.add chr((ch shr 18) or 0xf0) - result.add chr(((ch shr 12) and 0x3f) or 0x80) - result.add chr(((ch shr 6) and 0x3f) or 0x80) - result.add chr((ch and 0x3f) or 0x80) - else: - # replacement char(in case user give very large number): - result.add chr(0xFFFD shr 12 or 0b1110_0000) - result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) - result.add chr(0xFFFD and ones(6) or 0b10_0000_00) - -proc `$`*(s: WideCString): string = - result = s $ 80 diff --git a/lib/system_overview.rst b/lib/system_overview.rst index 23284dabf..cc0643bf1 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -1,19 +1,23 @@ +.. default-role:: code +.. include:: ../doc/rstcommon.rst + The System module imports several separate modules, and their documentation is in separate files: * `iterators <iterators.html>`_ +* `exceptions <exceptions.html>`_ * `assertions <assertions.html>`_ * `dollars <dollars.html>`_ -* `io <io.html>`_ -* `widestrs <widestrs.html>`_ +* `ctypes <ctypes.html>`_ Here is a short overview of the most commonly used functions from the `system` module. Function names in the tables below are clickable and will take you to the full documentation of the function. -The amount of available functions is much larger. Use the table of contents -on the left-hand side and/or `Ctrl+F` to navigate through this module. +There are many more functions available than the ones listed in this overview. +Use the table of contents on the left-hand side and/or `Ctrl+F` to navigate +through this module. Strings and characters @@ -35,7 +39,7 @@ Proc Usage * `strutils module <strutils.html>`_ for common string functions * `strformat module <strformat.html>`_ for string interpolation and formatting * `unicode module <unicode.html>`_ for Unicode UTF-8 handling -* `strscans <strscans.html>`_ for ``scanf`` and ``scanp`` macros, which offer +* `strscans <strscans.html>`_ for `scanf` and `scanp` macros, which offer easier substring extraction than regular expressions * `strtabs module <strtabs.html>`_ for efficient hash tables (dictionaries, in some programming languages) mapping from strings to strings @@ -45,24 +49,27 @@ Proc Usage Seqs ---- -======================================== ========================================== -Proc Usage -======================================== ========================================== -`newSeq<#newSeq>`_ Create a new sequence of a given length -`newSeqOfCap<#newSeqOfCap,Natural>`_ Create a new sequence with zero length - and a given capacity -`setLen<#setLen,seq[T][T],Natural>`_ Set the length of a sequence -`len<#len,seq[T][T]>`_ Return the length of a sequence -`@<#@,array[IDX,T]>`_ Turn an array into a sequence -`add<#add,seq[T][T],T>`_ Add an item to the sequence -`insert<#insert,seq[T][T],T>`_ Insert an item at a specific position -`delete<#delete,seq[T][T],Natural>`_ Delete an item while preserving the - order of elements (`O(n)` operation) -`del<#del,seq[T][T],Natural>`_ `O(1)` removal, doesn't preserve the order -`pop<#pop,seq[T][T]>`_ Remove and return last item of a sequence -`x & y<#&,seq[T][T],seq[T][T]>`_ Concatenate two sequences -`x[a..b]<#[],openArray[T],HSlice[U,V]>`_ Slice of a sequence (both ends included) -======================================== ========================================== +============================================================= ========================================== +Proc Usage +============================================================= ========================================== +`newSeq<#newSeq>`_ Create a new sequence of a given length +`newSeqOfCap<#newSeqOfCap,Natural>`_ Create a new sequence with zero length + and a given capacity +`setLen<#setLen,seq[T],Natural>`_ Set the length of a sequence +`len<#len,seq[T]>`_ Return the length of a sequence +`@<#@,openArray[T]>`_ Turn an array into a sequence +`add<#add,seq[T],sinkT>`_ Add an item to the sequence +`insert<#insert,seq[T],sinkT>`_ Insert an item at a specific position +`delete<#delete,seq[T],Natural>`_ Delete an item while preserving the + order of elements (`O(n)` operation) +`del<#del,seq[T],Natural>`_ `O(1)` removal, doesn't preserve the order +`pop<#pop,seq[T]>`_ Remove and return last item of a sequence +`x & y<#&,seq[T],seq[T]>`_ Concatenate two sequences +`x[a .. b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence (both ends included) +`x[a .. ^b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence but `b` is a + reversed index (both ends included) +`x[a ..< b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_ Slice of a sequence (excluded upper bound) +============================================================= ========================================== **See also:** * `sequtils module <sequtils.html>`_ for operations on container @@ -92,6 +99,7 @@ Proc Usage =============================== ====================================== **See also:** +* `setutils module <setutils.html>`_ for bit set convenience functions * `sets module <sets.html>`_ for hash sets * `intsets module <intsets.html>`_ for efficient int sets @@ -149,19 +157,21 @@ Proc Usage Misc ---- -============================================= ============================================ -Proc Usage -============================================= ============================================ -`is<#is,T,S>`_ Check if two arguments are of the same type -`isnot<#isnot.t,untyped,untyped>`_ Negated version of `is` -`!=<#!%3D.t,untyped,untyped>`_ Not equals -`addr<#addr,T>`_ Take the address of a memory location -`T and F<#and,bool,bool>`_ Boolean `and` -`T or F<#or,bool,bool>`_ Boolean `or` -`T xor F<#xor,bool,bool>`_ Boolean `xor` (exclusive or) -`not T<#not,bool>`_ Boolean `not` -`a .. b<#..,T,U>`_ Binary slice that constructs an interval - `[a, b]` -[a ..< b](#..<.t,untyped,untyped) Interval `[a, b)` (excluded upper bound) -[runnableExamples](#runnableExamples,untyped) Create testable documentation -============================================= ============================================ +==================================================== ============================================ +Proc Usage +==================================================== ============================================ +`is<#is,T,S>`_ Check if two arguments are of the same type +`isnot<#isnot.t,untyped,untyped>`_ Negated version of `is` +`!=<#!%3D.t,untyped,untyped>`_ Not equals +`addr<#addr,T>`_ Take the address of a memory location +`T and F<#and,bool,bool>`_ Boolean `and` +`T or F<#or,bool,bool>`_ Boolean `or` +`T xor F<#xor,bool,bool>`_ Boolean `xor` (exclusive or) +`not T<#not,bool>`_ Boolean `not` +`a[^x]<#^.t,int>`_ Take the element at the reversed index `x` +`a .. b<#..,sinkT,sinkU>`_ Binary slice that constructs an interval + `[a, b]` +`a ..^ b<#..^.t,untyped,untyped>`_ Interval `[a, b]` but `b` as reversed index +[a ..< b](#..<.t,untyped,untyped) Interval `[a, b)` (excluded upper bound) +[runnableExamples](#runnableExamples,string,untyped) Create testable documentation +==================================================== ============================================ diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim index c17f2f455..207172f8c 100644 --- a/lib/windows/registry.nim +++ b/lib/windows/registry.nim @@ -9,7 +9,10 @@ ## This module is experimental and its interface may change. -import winlean, os +import std/oserrors + +when defined(nimPreviewSlimSystem): + import std/widestrs type HKEY* = uint @@ -46,24 +49,26 @@ template call(f) = proc getUnicodeValue*(path, key: string; handle: HKEY): string = let hh = newWideCString path let kk = newWideCString key - var bufsize: int32 + var bufSize: int32 # try a couple of different flag settings: var flags: int32 = RRF_RT_ANY - let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufsize) + let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufSize) if err != 0: var newHandle: HKEY call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) - call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufsize) - var res = newWideCString("", bufsize) - call regGetValue(newHandle, nil, kk, flags, nil, cast[pointer](res), - addr bufsize) - result = res $ bufsize + call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufSize) + if bufSize > 0: + var res = newWideCString(bufSize) + call regGetValue(newHandle, nil, kk, flags, nil, addr res[0], + addr bufSize) + result = res $ bufSize call regCloseKey(newHandle) else: - var res = newWideCString("", bufsize) - call regGetValue(handle, hh, kk, flags, nil, cast[pointer](res), - addr bufsize) - result = res $ bufsize + if bufSize > 0: + var res = newWideCString(bufSize) + call regGetValue(handle, hh, kk, flags, nil, addr res[0], + addr bufSize) + result = res $ bufSize proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, dwType: int32; lpData: WideCString; cbData: int32): int32 {. diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index b5eabb6aa..79681376b 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -10,29 +10,29 @@ ## This module implements a small wrapper for some needed Win API procedures, ## so that the Nim compiler does not depend on the huge Windows module. -{.deadCodeElim: on.} # dce option deprecated - -import dynlib +import std/dynlib when defined(nimHasStyleChecks): {.push styleChecks: off.} {.passc: "-DWIN32_LEAN_AND_MEAN".} -const - useWinUnicode* = not defined(useWinAnsi) +when defined(nimPreviewSlimSystem): + from std/syncio import FileHandle + import std/widestrs -when useWinUnicode: - type WinChar* = Utf16Char -else: - type WinChar* = char +type WinChar* = Utf16Char +# See https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types type Handle* = int LONG* = int32 ULONG* = int32 PULONG* = ptr int WINBOOL* = int32 + ## `WINBOOL` uses opposite convention as posix, !=0 meaning success. + # xxx this should be distinct int32, distinct would make code less error prone + PBOOL* = ptr WINBOOL DWORD* = int32 PDWORD* = ptr DWORD LPINT* = ptr int32 @@ -40,13 +40,14 @@ type PULONG_PTR* = ptr uint HDC* = Handle HGLRC* = Handle + BYTE* = uint8 - SECURITY_ATTRIBUTES* {.final, pure.} = object + SECURITY_ATTRIBUTES* = object nLength*: int32 lpSecurityDescriptor*: pointer bInheritHandle*: WINBOOL - STARTUPINFO* {.final, pure.} = object + STARTUPINFO* = object cb*: int32 lpReserved*: cstring lpDesktop*: cstring @@ -66,17 +67,17 @@ type hStdOutput*: Handle hStdError*: Handle - PROCESS_INFORMATION* {.final, pure.} = object + PROCESS_INFORMATION* = object hProcess*: Handle hThread*: Handle dwProcessId*: int32 dwThreadId*: int32 - FILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT + FILETIME* = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT dwLowDateTime*: DWORD dwHighDateTime*: DWORD - BY_HANDLE_FILE_INFORMATION* {.final, pure.} = object + BY_HANDLE_FILE_INFORMATION* = object dwFileAttributes*: DWORD ftCreationTime*: FILETIME ftLastAccessTime*: FILETIME @@ -88,7 +89,7 @@ type nFileIndexHigh*: DWORD nFileIndexLow*: DWORD - OSVERSIONINFO* {.final, pure.} = object + OSVERSIONINFO* = object dwOSVersionInfoSize*: DWORD dwMajorVersion*: DWORD dwMinorVersion*: DWORD @@ -96,6 +97,12 @@ type dwPlatformId*: DWORD szCSDVersion*: array[0..127, WinChar] + Protoent* = object + p_name*: cstring + p_aliases*: cstringArray + p_proto*: cshort + + const STARTF_USESHOWWINDOW* = 1'i32 STARTF_USESTDHANDLES* = 256'i32 @@ -128,35 +135,41 @@ const CREATE_NO_WINDOW* = 0x08000000'i32 -when useWinUnicode: - proc getVersionExW*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetVersionExW".} -else: - proc getVersionExA*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetVersionExA".} + HANDLE_FLAG_INHERIT* = 0x00000001'i32 + +proc isSuccess*(a: WINBOOL): bool {.inline.} = + ## Returns true if `a != 0`. Windows uses a different convention than POSIX, + ## where `a == 0` is commonly used on success. + a != 0 +proc getVersionExW*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetVersionExW", sideEffect.} +proc getVersionExA*(lpVersionInfo: ptr OSVERSIONINFO): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetVersionExA", sideEffect.} -proc getVersion*(): DWORD {.stdcall, dynlib: "kernel32", importc: "GetVersion".} +proc getVersion*(): DWORD {.stdcall, dynlib: "kernel32", importc: "GetVersion", sideEffect.} proc closeHandle*(hObject: Handle): WINBOOL {.stdcall, dynlib: "kernel32", importc: "CloseHandle".} proc readFile*(hFile: Handle, buffer: pointer, nNumberOfBytesToRead: int32, lpNumberOfBytesRead: ptr int32, lpOverlapped: pointer): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "ReadFile".} + stdcall, dynlib: "kernel32", importc: "ReadFile", sideEffect.} proc writeFile*(hFile: Handle, buffer: pointer, nNumberOfBytesToWrite: int32, lpNumberOfBytesWritten: ptr int32, lpOverlapped: pointer): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "WriteFile".} + stdcall, dynlib: "kernel32", importc: "WriteFile", sideEffect.} proc createPipe*(hReadPipe, hWritePipe: var Handle, lpPipeAttributes: var SECURITY_ATTRIBUTES, nSize: int32): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreatePipe".} + stdcall, dynlib: "kernel32", importc: "CreatePipe", sideEffect.} proc createNamedPipe*(lpName: WideCString, dwOpenMode, dwPipeMode, nMaxInstances, nOutBufferSize, nInBufferSize, nDefaultTimeOut: int32, lpSecurityAttributes: ptr SECURITY_ATTRIBUTES): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateNamedPipeW".} + stdcall, dynlib: "kernel32", importc: "CreateNamedPipeW", sideEffect.} proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, nBufferSize: int32 = 0, @@ -165,37 +178,25 @@ proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, lpBytesLeftThisMessage: ptr int32 = nil): bool {. stdcall, dynlib: "kernel32", importc: "PeekNamedPipe".} -when useWinUnicode: - proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment, lpCurrentDirectory: WideCString, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessW".} - -else: - proc createProcessA*(lpApplicationName, lpCommandLine: cstring, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment: pointer, lpCurrentDirectory: cstring, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessA".} - +proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, + lpProcessAttributes: ptr SECURITY_ATTRIBUTES, + lpThreadAttributes: ptr SECURITY_ATTRIBUTES, + bInheritHandles: WINBOOL, dwCreationFlags: int32, + lpEnvironment, lpCurrentDirectory: WideCString, + lpStartupInfo: var STARTUPINFO, + lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} proc suspendThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", - importc: "SuspendThread".} + importc: "SuspendThread", sideEffect.} proc resumeThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", - importc: "ResumeThread".} + importc: "ResumeThread", sideEffect.} proc waitForSingleObject*(hHandle: Handle, dwMilliseconds: int32): int32 {. - stdcall, dynlib: "kernel32", importc: "WaitForSingleObject".} + stdcall, dynlib: "kernel32", importc: "WaitForSingleObject", sideEffect.} proc terminateProcess*(hProcess: Handle, uExitCode: int): WINBOOL {.stdcall, - dynlib: "kernel32", importc: "TerminateProcess".} + dynlib: "kernel32", importc: "TerminateProcess", sideEffect.} proc getExitCodeProcess*(hProcess: Handle, lpExitCode: var int32): WINBOOL {. stdcall, dynlib: "kernel32", importc: "GetExitCodeProcess".} @@ -203,77 +204,47 @@ proc getExitCodeProcess*(hProcess: Handle, lpExitCode: var int32): WINBOOL {. proc getStdHandle*(nStdHandle: int32): Handle {.stdcall, dynlib: "kernel32", importc: "GetStdHandle".} proc setStdHandle*(nStdHandle: int32, hHandle: Handle): WINBOOL {.stdcall, - dynlib: "kernel32", importc: "SetStdHandle".} + dynlib: "kernel32", importc: "SetStdHandle", sideEffect.} proc flushFileBuffers*(hFile: Handle): WINBOOL {.stdcall, dynlib: "kernel32", - importc: "FlushFileBuffers".} + importc: "FlushFileBuffers", sideEffect.} proc getLastError*(): int32 {.importc: "GetLastError", - stdcall, dynlib: "kernel32".} + stdcall, dynlib: "kernel32", sideEffect.} proc setLastError*(error: int32) {.importc: "SetLastError", - stdcall, dynlib: "kernel32".} - -when useWinUnicode: - proc formatMessageW*(dwFlags: int32, lpSource: pointer, - dwMessageId, dwLanguageId: int32, - lpBuffer: pointer, nSize: int32, - arguments: pointer): int32 {. - importc: "FormatMessageW", stdcall, dynlib: "kernel32".} -else: - proc formatMessageA*(dwFlags: int32, lpSource: pointer, + stdcall, dynlib: "kernel32", sideEffect.} + +proc formatMessageW*(dwFlags: int32, lpSource: pointer, dwMessageId, dwLanguageId: int32, lpBuffer: pointer, nSize: int32, arguments: pointer): int32 {. - importc: "FormatMessageA", stdcall, dynlib: "kernel32".} + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} proc localFree*(p: pointer) {. importc: "LocalFree", stdcall, dynlib: "kernel32".} -when useWinUnicode: - proc getCurrentDirectoryW*(nBufferLength: int32, - lpBuffer: WideCString): int32 {. - importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall.} - proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. - importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall.} - proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. - importc: "CreateDirectoryW", dynlib: "kernel32", stdcall.} - proc removeDirectoryW*(lpPathName: WideCString): int32 {. - importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall.} - proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW".} - - proc getModuleFileNameW*(handle: Handle, buf: WideCString, - size: int32): int32 {.importc: "GetModuleFileNameW", - dynlib: "kernel32", stdcall.} -else: - proc getCurrentDirectoryA*(nBufferLength: int32, lpBuffer: cstring): int32 {. - importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall.} - proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. - importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall.} - proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. - importc: "CreateDirectoryA", dynlib: "kernel32", stdcall.} - proc removeDirectoryA*(lpPathName: cstring): int32 {. - importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall.} - proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA".} - - proc getModuleFileNameA*(handle: Handle, buf: cstring, size: int32): int32 {. - importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} - -when useWinUnicode: - proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall.} - proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, - security: pointer=nil): int32 {. - importc:"CreateHardLinkW", dynlib: "kernel32", stdcall.} -else: - proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall.} - proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, - security: pointer=nil): int32 {. - importc:"CreateHardLinkA", dynlib: "kernel32", stdcall.} +proc getCurrentDirectoryW*(nBufferLength: int32, + lpBuffer: WideCString): int32 {. + importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. + importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. + importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc removeDirectoryW*(lpPathName: WideCString): int32 {. + importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} + +proc getModuleFileNameW*(handle: Handle, buf: WideCString, + size: int32): int32 {.importc: "GetModuleFileNameW", + dynlib: "kernel32", stdcall.} + +proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} +proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, + security: pointer=nil): int32 {. + importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} const FILE_ATTRIBUTE_READONLY* = 0x00000001'i32 @@ -324,112 +295,66 @@ type cFileName*: array[0..(MAX_PATH) - 1, WinChar] cAlternateFileName*: array[0..13, WinChar] -when useWinUnicode: - proc findFirstFileW*(lpFileName: WideCString, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileW".} - proc findNextFileW*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileW".} -else: - proc findFirstFileA*(lpFileName: cstring, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileA".} - proc findNextFileA*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileA".} +proc findFirstFileW*(lpFileName: WideCString, + lpFindFileData: var WIN32_FIND_DATA): Handle {. + stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} +proc findNextFileW*(hFindFile: Handle, + lpFindFileData: var WIN32_FIND_DATA): int32 {. + stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} proc findClose*(hFindFile: Handle) {.stdcall, dynlib: "kernel32", importc: "FindClose".} -when useWinUnicode: - proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, - lpBuffer: WideCString, - lpFilePart: var WideCString): int32 {. +proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, + lpBuffer: WideCString, + lpFilePart: var WideCString): int32 {. + stdcall, dynlib: "kernel32", + importc: "GetFullPathNameW", sideEffect.} +proc getFileAttributesW*(lpFileName: WideCString): int32 {. stdcall, dynlib: "kernel32", - importc: "GetFullPathNameW".} - proc getFileAttributesW*(lpFileName: WideCString): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesW".} - proc setFileAttributesW*(lpFileName: WideCString, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesW".} - - proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: WINBOOL): WINBOOL {. - importc: "CopyFileW", stdcall, dynlib: "kernel32".} - - proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. - importc: "MoveFileW", stdcall, dynlib: "kernel32".} - proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, - flags: DWORD): WINBOOL {. - importc: "MoveFileExW", stdcall, dynlib: "kernel32".} - - proc getEnvironmentStringsW*(): WideCString {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW".} - proc freeEnvironmentStringsW*(para1: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW".} - - proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", - stdcall, dynlib: "kernel32".} + importc: "GetFileAttributesW", sideEffect.} +proc setFileAttributesW*(lpFileName: WideCString, + dwFileAttributes: int32): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} -else: - proc getFullPathNameA*(lpFileName: cstring, nBufferLength: int32, - lpBuffer: cstring, lpFilePart: var cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFullPathNameA".} - proc getFileAttributesA*(lpFileName: cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesA".} - proc setFileAttributesA*(lpFileName: cstring, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesA".} - - proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. - importc: "CopyFileA", stdcall, dynlib: "kernel32".} - - proc moveFileA*(lpExistingFileName, lpNewFileName: cstring): WINBOOL {. - importc: "MoveFileA", stdcall, dynlib: "kernel32".} - proc moveFileExA*(lpExistingFileName, lpNewFileName: cstring, - flags: DWORD): WINBOOL {. - importc: "MoveFileExA", stdcall, dynlib: "kernel32".} - - proc getEnvironmentStringsA*(): cstring {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsA".} - proc freeEnvironmentStringsA*(para1: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsA".} - - proc getCommandLineA*(): cstring {. - importc: "GetCommandLineA", stdcall, dynlib: "kernel32".} +proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, + bFailIfExists: WINBOOL): WINBOOL {. + importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} + +proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. + importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, + flags: DWORD): WINBOOL {. + importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} + +proc getEnvironmentStringsW*(): WideCString {. + stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} +proc freeEnvironmentStringsW*(para1: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} + +proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", + stdcall, dynlib: "kernel32", sideEffect.} proc rdFileTime*(f: FILETIME): int64 = - result = ze64(f.dwLowDateTime) or (ze64(f.dwHighDateTime) shl 32) + result = int64(cast[uint32](f.dwLowDateTime)) or (int64(cast[uint32](f.dwHighDateTime)) shl 32) proc rdFileSize*(f: WIN32_FIND_DATA): int64 = - result = ze64(f.nFileSizeLow) or (ze64(f.nFileSizeHigh) shl 32) + result = int64(cast[uint32](f.nFileSizeLow)) or (int64(cast[uint32](f.nFileSizeHigh)) shl 32) proc getSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. - importc: "GetSystemTimeAsFileTime", dynlib: "kernel32", stdcall.} + importc: "GetSystemTimeAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} proc sleep*(dwMilliseconds: int32){.stdcall, dynlib: "kernel32", - importc: "Sleep".} + importc: "Sleep", sideEffect.} -when useWinUnicode: - proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: WideCString, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW".} - -else: - proc shellExecuteA*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: cstring, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteA".} +proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, + lpParameters, lpDirectory: WideCString, + nShowCmd: int32): Handle{. + stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} proc getFileInformationByHandle*(hFile: Handle, lpFileInformation: ptr BY_HANDLE_FILE_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "GetFileInformationByHandle".} + stdcall, dynlib: "kernel32", importc: "GetFileInformationByHandle", sideEffect.} const WSADESCRIPTION_LEN* = 256 @@ -444,7 +369,7 @@ const ws2dll = "Ws2_32.dll" -proc wsaGetLastError*(): cint {.importc: "WSAGetLastError", dynlib: ws2dll.} +proc wsaGetLastError*(): cint {.importc: "WSAGetLastError", dynlib: ws2dll, sideEffect.} type SocketHandle* = distinct int @@ -463,7 +388,7 @@ type PSockAddr = ptr SockAddr - InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object + InAddr* {.importc: "IN_ADDR", header: "winsock2.h", union.} = object s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", @@ -487,9 +412,9 @@ type Sockaddr_storage* {.importc: "SOCKADDR_STORAGE", header: "winsock2.h".} = object ss_family*: uint16 - ss_pad1: array[6, byte] - ss_align: int64 - ss_pad2: array[112, byte] + ss_pad1 {.importc: "__ss_pad1".}: array[6, byte] + ss_align {.importc: "__ss_align".}: int64 + ss_pad2 {.importc: "__ss_pad2".}: array[112, byte] Servent* = object s_name*: cstring @@ -557,19 +482,27 @@ var proc `==`*(x, y: SocketHandle): bool {.borrow.} proc getservbyname*(name, proto: cstring): ptr Servent {. - stdcall, importc: "getservbyname", dynlib: ws2dll.} + stdcall, importc: "getservbyname", dynlib: ws2dll, sideEffect.} proc getservbyport*(port: cint, proto: cstring): ptr Servent {. - stdcall, importc: "getservbyport", dynlib: ws2dll.} + stdcall, importc: "getservbyport", dynlib: ws2dll, sideEffect.} proc gethostbyaddr*(ip: ptr InAddr, len: cuint, theType: cint): ptr Hostent {. - stdcall, importc: "gethostbyaddr", dynlib: ws2dll.} + stdcall, importc: "gethostbyaddr", dynlib: ws2dll, sideEffect.} proc gethostbyname*(name: cstring): ptr Hostent {. - stdcall, importc: "gethostbyname", dynlib: ws2dll.} + stdcall, importc: "gethostbyname", dynlib: ws2dll, sideEffect.} proc gethostname*(hostname: cstring, len: cint): cint {. - stdcall, importc: "gethostname", dynlib: ws2dll.} + stdcall, importc: "gethostname", dynlib: ws2dll, sideEffect.} + +proc getprotobyname*( + name: cstring +): ptr Protoent {.stdcall, importc: "getprotobyname", dynlib: ws2dll, sideEffect.} + +proc getprotobynumber*( + proto: cint +): ptr Protoent {.stdcall, importc: "getprotobynumber", dynlib: ws2dll, sideEffect.} proc socket*(af, typ, protocol: cint): SocketHandle {. stdcall, importc: "socket", dynlib: ws2dll.} @@ -644,7 +577,7 @@ proc getaddrinfo*(nodename, servname: cstring, hints: ptr AddrInfo, res: var ptr AddrInfo): cint {. stdcall, importc: "getaddrinfo", dynlib: ws2dll.} -proc freeaddrinfo*(ai: ptr AddrInfo) {. +proc freeAddrInfo*(ai: ptr AddrInfo) {. stdcall, importc: "freeaddrinfo", dynlib: ws2dll.} proc inet_ntoa*(i: InAddr): cstring {. @@ -695,12 +628,13 @@ const # Error Constants const - ERROR_FILE_NOT_FOUND* = 2 + ERROR_FILE_NOT_FOUND* = 2 ## https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499- ERROR_PATH_NOT_FOUND* = 3 ERROR_ACCESS_DENIED* = 5 ERROR_NO_MORE_FILES* = 18 ERROR_LOCK_VIOLATION* = 33 ERROR_HANDLE_EOF* = 38 + ERROR_FILE_EXISTS* = 80 ERROR_BAD_ARGUMENTS* = 165 proc duplicateHandle*(hSourceProcessHandle: Handle, hSourceHandle: Handle, @@ -710,6 +644,9 @@ proc duplicateHandle*(hSourceProcessHandle: Handle, hSourceHandle: Handle, dwOptions: DWORD): WINBOOL{.stdcall, dynlib: "kernel32", importc: "DuplicateHandle".} +proc getHandleInformation*(hObject: Handle, lpdwFlags: ptr DWORD): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "GetHandleInformation".} + proc setHandleInformation*(hObject: Handle, dwMask: DWORD, dwFlags: DWORD): WINBOOL {.stdcall, dynlib: "kernel32", importc: "SetHandleInformation".} @@ -717,22 +654,20 @@ proc setHandleInformation*(hObject: Handle, dwMask: DWORD, proc getCurrentProcess*(): Handle{.stdcall, dynlib: "kernel32", importc: "GetCurrentProcess".} -when useWinUnicode: - proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD, - lpSecurityAttributes: pointer, - dwCreationDisposition, dwFlagsAndAttributes: DWORD, - hTemplateFile: Handle): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileW".} - proc deleteFileW*(pathName: WideCString): int32 {. - importc: "DeleteFileW", dynlib: "kernel32", stdcall.} -else: - proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD, - lpSecurityAttributes: pointer, - dwCreationDisposition, dwFlagsAndAttributes: DWORD, - hTemplateFile: Handle): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileA".} - proc deleteFileA*(pathName: cstring): int32 {. - importc: "DeleteFileA", dynlib: "kernel32", stdcall.} +proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD, + lpSecurityAttributes: pointer, + dwCreationDisposition, dwFlagsAndAttributes: DWORD, + hTemplateFile: Handle): Handle {. + stdcall, dynlib: "kernel32", importc: "CreateFileW".} +proc deleteFileW*(pathName: WideCString): int32 {. + importc: "DeleteFileW", dynlib: "kernel32", stdcall.} +proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD, + lpSecurityAttributes: pointer, + dwCreationDisposition, dwFlagsAndAttributes: DWORD, + hTemplateFile: Handle): Handle {. + stdcall, dynlib: "kernel32", importc: "CreateFileA".} +proc deleteFileA*(pathName: cstring): int32 {. + importc: "DeleteFileA", dynlib: "kernel32", stdcall.} proc setEndOfFile*(hFile: Handle): WINBOOL {.stdcall, dynlib: "kernel32", importc: "SetEndOfFile".} @@ -765,13 +700,6 @@ proc createFileMappingW*(hFile: Handle, lpName: pointer): Handle {. stdcall, dynlib: "kernel32", importc: "CreateFileMappingW".} -when not useWinUnicode: - proc createFileMappingA*(hFile: Handle, - lpFileMappingAttributes: pointer, - flProtect, dwMaximumSizeHigh: DWORD, - dwMaximumSizeLow: DWORD, lpName: cstring): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".} - proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, dynlib: "kernel32", importc: "UnmapViewOfFile".} @@ -791,7 +719,7 @@ type POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD, para3: POVERLAPPED){.stdcall.} - GUID* {.final, pure.} = object + GUID* = object D1*: int32 D2*: int16 D3*: int16 @@ -810,6 +738,7 @@ const WSAEINPROGRESS* = 10036 WSAEINTR* = 10004 WSAEWOULDBLOCK* = 10035 + WSAESHUTDOWN* = 10058 ERROR_NETNAME_DELETED* = 64 STATUS_PENDING* = 0x103 @@ -902,6 +831,9 @@ proc getProcessTimes*(hProcess: Handle; lpCreationTime, lpExitTime, lpKernelTime, lpUserTime: var FILETIME): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetProcessTimes".} +proc getSystemTimePreciseAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. + importc: "GetSystemTimePreciseAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} + type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.gcsafe, stdcall, tags: [].} @@ -943,7 +875,7 @@ proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = var ver: OSVERSIONINFO ver.dwOSVersionInfoSize = sizeof(ver).DWORD - let res = when useWinUnicode: getVersionExW(ver.addr) else: getVersionExA(ver.addr) + let res = getVersionExW(ver.addr) if res == 0: result = nil elif ver.dwMajorVersion >= 6: @@ -961,13 +893,13 @@ type dwRemoteAddressLength: DWORD, lpdwBytesReceived: ptr DWORD, lpOverlapped: POVERLAPPED): bool {. - stdcall,gcsafe.} + stdcall, gcsafe, raises: [].} WSAPROC_CONNECTEX* = proc (s: SocketHandle, name: ptr SockAddr, namelen: cint, lpSendBuffer: pointer, dwSendDataLength: DWORD, lpdwBytesSent: ptr DWORD, lpOverlapped: POVERLAPPED): bool {. - stdcall,gcsafe.} + stdcall, gcsafe, raises: [].} WSAPROC_GETACCEPTEXSOCKADDRS* = proc(lpOutputBuffer: pointer, dwReceiveDataLength: DWORD, @@ -977,7 +909,7 @@ type LocalSockaddrLength: ptr cint, RemoteSockaddr: ptr PSockAddr, RemoteSockaddrLength: ptr cint) {. - stdcall,gcsafe.} + stdcall, gcsafe, raises: [].} const WT_EXECUTEDEFAULT* = 0x00000000'i32 @@ -1005,7 +937,7 @@ const PROCESS_QUERY_LIMITED_INFORMATION* = 0x00001000'i32 PROCESS_SET_LIMITED_INFORMATION* = 0x00002000'i32 type - WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32): void {.stdcall.} + WAITORTIMERCALLBACK* = proc(para1: pointer, para2: int32) {.stdcall.} proc postQueuedCompletionStatus*(CompletionPort: Handle, dwNumberOfBytesTransferred: DWORD, @@ -1027,16 +959,10 @@ proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwProcessId: DWORD): Handle {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} -when defined(useWinAnsi): - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: cstring): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventA".} -else: - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: ptr Utf16Char): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} +proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} proc setEvent*(hEvent: Handle): cint {.stdcall, dynlib: "kernel32", importc: "SetEvent".} @@ -1068,7 +994,7 @@ proc wsaResetEvent*(hEvent: Handle): bool {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} type - KEY_EVENT_RECORD* {.final, pure.} = object + KEY_EVENT_RECORD* = object eventType*: int16 bKeyDown*: WINBOOL wRepeatCount*: int16 @@ -1077,17 +1003,12 @@ type uChar*: int16 dwControlKeyState*: DWORD -when defined(useWinAnsi): - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} -else: - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} +proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} type - LPFIBER_START_ROUTINE* = proc (param: pointer): void {.stdcall.} + LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} const FIBER_FLAG_FLOAT_SWITCH* = 0x01 @@ -1096,12 +1017,12 @@ proc CreateFiber*(stackSize: int, fn: LPFIBER_START_ROUTINE, param: pointer): po proc CreateFiberEx*(stkCommit: int, stkReserve: int, flags: int32, fn: LPFIBER_START_ROUTINE, param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} proc ConvertThreadToFiber*(param: pointer): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} proc ConvertThreadToFiberEx*(param: pointer, flags: int32): pointer {.stdcall, discardable, dynlib: "kernel32", importc.} -proc DeleteFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} -proc SwitchToFiber*(fiber: pointer): void {.stdcall, discardable, dynlib: "kernel32", importc.} +proc DeleteFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} +proc SwitchToFiber*(fiber: pointer) {.stdcall, discardable, dynlib: "kernel32", importc.} proc GetCurrentFiber*(): pointer {.stdcall, importc, header: "windows.h".} proc toFILETIME*(t: int64): FILETIME = - ## Convert the Windows file time timestamp ``t`` to ``FILETIME``. + ## Convert the Windows file time timestamp `t` to `FILETIME`. result = FILETIME(dwLowDateTime: cast[DWORD](t), dwHighDateTime: DWORD(t shr 32)) type @@ -1111,5 +1032,42 @@ proc setFileTime*(hFile: Handle, lpCreationTime: LPFILETIME, lpLastAccessTime: LPFILETIME, lpLastWriteTime: LPFILETIME): WINBOOL {.stdcall, dynlib: "kernel32", importc: "SetFileTime".} +type + # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority + SID_IDENTIFIER_AUTHORITY* {.importc, header: "<windows.h>".} = object + value* {.importc: "Value".}: array[6, BYTE] + # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid + SID* {.importc, header: "<windows.h>".} = object + Revision: BYTE + SubAuthorityCount: BYTE + IdentifierAuthority: SID_IDENTIFIER_AUTHORITY + SubAuthority: ptr ptr DWORD + PSID* = ptr SID + +const + # https://docs.microsoft.com/en-us/windows/win32/secauthz/sid-components + # https://github.com/mirror/mingw-w64/blob/84c950bdab7c999ace49fe8383856be77f88c4a8/mingw-w64-headers/include/winnt.h#L2994 + SECURITY_NT_AUTHORITY* = [BYTE(0), BYTE(0), BYTE(0), BYTE(0), BYTE(0), BYTE(5)] + SECURITY_BUILTIN_DOMAIN_RID* = 32 + DOMAIN_ALIAS_RID_ADMINS* = 544 + +proc allocateAndInitializeSid*(pIdentifierAuthority: ptr SID_IDENTIFIER_AUTHORITY, + nSubAuthorityCount: BYTE, + nSubAuthority0: DWORD, + nSubAuthority1: DWORD, + nSubAuthority2: DWORD, + nSubAuthority3: DWORD, + nSubAuthority4: DWORD, + nSubAuthority5: DWORD, + nSubAuthority6: DWORD, + nSubAuthority7: DWORD, + pSid: ptr PSID): WINBOOL + {.stdcall, dynlib: "Advapi32", importc: "AllocateAndInitializeSid".} +proc checkTokenMembership*(tokenHandle: Handle, sidToCheck: PSID, + isMember: PBOOL): WINBOOL + {.stdcall, dynlib: "Advapi32", importc: "CheckTokenMembership".} +proc freeSid*(pSid: PSID): PSID + {.stdcall, dynlib: "Advapi32", importc: "FreeSid".} + when defined(nimHasStyleChecks): {.pop.} # {.push styleChecks: off.} diff --git a/lib/wrappers/iup.nim b/lib/wrappers/iup.nim deleted file mode 100644 index 1a9e6ab3e..000000000 --- a/lib/wrappers/iup.nim +++ /dev/null @@ -1,957 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# C header files translated by hand -# Licence of IUP follows: - - -# **************************************************************************** -# Copyright (C) 1994-2009 Tecgraf, PUC-Rio. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# **************************************************************************** - -{.deadCodeElim: on.} # dce option deprecated - -when defined(windows): - const dllname = "iup(|30|27|26|25|24).dll" -elif defined(macosx): - const dllname = "libiup(|3.0|2.7|2.6|2.5|2.4).dylib" -else: - const dllname = "libiup(|3.0|2.7|2.6|2.5|2.4).so(|.1)" - -const - IUP_NAME* = "IUP - Portable User Interface" - IUP_COPYRIGHT* = "Copyright (C) 1994-2009 Tecgraf, PUC-Rio." - IUP_DESCRIPTION* = "Portable toolkit for building graphical user interfaces." - constIUP_VERSION* = "3.0" - constIUP_VERSION_NUMBER* = 300000 - constIUP_VERSION_DATE* = "2009/07/18" - -type - Ihandle = object - PIhandle* = ptr Ihandle - - Icallback* = proc (arg: PIhandle): cint {.cdecl.} - -# pre-defined dialogs -proc fileDlg*: PIhandle {.importc: "IupFileDlg", dynlib: dllname, cdecl.} -proc messageDlg*: PIhandle {.importc: "IupMessageDlg", dynlib: dllname, cdecl.} -proc colorDlg*: PIhandle {.importc: "IupColorDlg", dynlib: dllname, cdecl.} -proc fontDlg*: PIhandle {.importc: "IupFontDlg", dynlib: dllname, cdecl.} - -proc getFile*(arq: cstring): cint {. - importc: "IupGetFile", dynlib: dllname, cdecl.} -proc message*(title, msg: cstring) {. - importc: "IupMessage", dynlib: dllname, cdecl.} -proc messagef*(title, format: cstring) {. - importc: "IupMessagef", dynlib: dllname, cdecl, varargs.} -proc alarm*(title, msg, b1, b2, b3: cstring): cint {. - importc: "IupAlarm", dynlib: dllname, cdecl.} -proc scanf*(format: cstring): cint {. - importc: "IupScanf", dynlib: dllname, cdecl, varargs.} -proc listDialog*(theType: cint, title: cstring, size: cint, - list: cstringArray, op, maxCol, maxLin: cint, - marks: ptr cint): cint {. - importc: "IupListDialog", dynlib: dllname, cdecl.} -proc getText*(title, text: cstring): cint {. - importc: "IupGetText", dynlib: dllname, cdecl.} -proc getColor*(x, y: cint, r, g, b: var byte): cint {. - importc: "IupGetColor", dynlib: dllname, cdecl.} - -type - Iparamcb* = proc (dialog: PIhandle, paramIndex: cint, - userData: pointer): cint {.cdecl.} - -proc getParam*(title: cstring, action: Iparamcb, userData: pointer, - format: cstring): cint {. - importc: "IupGetParam", cdecl, varargs, dynlib: dllname.} -proc getParamv*(title: cstring, action: Iparamcb, userData: pointer, - format: cstring, paramCount, paramExtra: cint, - paramData: pointer): cint {. - importc: "IupGetParamv", cdecl, dynlib: dllname.} - - -# Functions - -proc open*(argc: ptr cint, argv: ptr cstringArray): cint {. - importc: "IupOpen", cdecl, dynlib: dllname.} -proc close*() {.importc: "IupClose", cdecl, dynlib: dllname.} -proc imageLibOpen*() {.importc: "IupImageLibOpen", cdecl, dynlib: dllname.} - -proc mainLoop*(): cint {.importc: "IupMainLoop", cdecl, dynlib: dllname, - discardable.} -proc loopStep*(): cint {.importc: "IupLoopStep", cdecl, dynlib: dllname, - discardable.} -proc mainLoopLevel*(): cint {.importc: "IupMainLoopLevel", cdecl, - dynlib: dllname, discardable.} -proc flush*() {.importc: "IupFlush", cdecl, dynlib: dllname.} -proc exitLoop*() {.importc: "IupExitLoop", cdecl, dynlib: dllname.} - -proc update*(ih: PIhandle) {.importc: "IupUpdate", cdecl, dynlib: dllname.} -proc updateChildren*(ih: PIhandle) {.importc: "IupUpdateChildren", cdecl, dynlib: dllname.} -proc redraw*(ih: PIhandle, children: cint) {.importc: "IupRedraw", cdecl, dynlib: dllname.} -proc refresh*(ih: PIhandle) {.importc: "IupRefresh", cdecl, dynlib: dllname.} - -proc mapFont*(iupfont: cstring): cstring {.importc: "IupMapFont", cdecl, dynlib: dllname.} -proc unMapFont*(driverfont: cstring): cstring {.importc: "IupUnMapFont", cdecl, dynlib: dllname.} -proc help*(url: cstring): cint {.importc: "IupHelp", cdecl, dynlib: dllname.} -proc load*(filename: cstring): cstring {.importc: "IupLoad", cdecl, dynlib: dllname.} - -proc iupVersion*(): cstring {.importc: "IupVersion", cdecl, dynlib: dllname.} -proc iupVersionDate*(): cstring {.importc: "IupVersionDate", cdecl, dynlib: dllname.} -proc iupVersionNumber*(): cint {.importc: "IupVersionNumber", cdecl, dynlib: dllname.} -proc setLanguage*(lng: cstring) {.importc: "IupSetLanguage", cdecl, dynlib: dllname.} -proc getLanguage*(): cstring {.importc: "IupGetLanguage", cdecl, dynlib: dllname.} - -proc destroy*(ih: PIhandle) {.importc: "IupDestroy", cdecl, dynlib: dllname.} -proc detach*(child: PIhandle) {.importc: "IupDetach", cdecl, dynlib: dllname.} -proc append*(ih, child: PIhandle): PIhandle {. - importc: "IupAppend", cdecl, dynlib: dllname, discardable.} -proc insert*(ih, refChild, child: PIhandle): PIhandle {. - importc: "IupInsert", cdecl, dynlib: dllname, discardable.} -proc getChild*(ih: PIhandle, pos: cint): PIhandle {. - importc: "IupGetChild", cdecl, dynlib: dllname.} -proc getChildPos*(ih, child: PIhandle): cint {. - importc: "IupGetChildPos", cdecl, dynlib: dllname.} -proc getChildCount*(ih: PIhandle): cint {. - importc: "IupGetChildCount", cdecl, dynlib: dllname.} -proc getNextChild*(ih, child: PIhandle): PIhandle {. - importc: "IupGetNextChild", cdecl, dynlib: dllname.} -proc getBrother*(ih: PIhandle): PIhandle {. - importc: "IupGetBrother", cdecl, dynlib: dllname.} -proc getParent*(ih: PIhandle): PIhandle {. - importc: "IupGetParent", cdecl, dynlib: dllname.} -proc getDialog*(ih: PIhandle): PIhandle {. - importc: "IupGetDialog", cdecl, dynlib: dllname.} -proc getDialogChild*(ih: PIhandle, name: cstring): PIhandle {. - importc: "IupGetDialogChild", cdecl, dynlib: dllname.} -proc reparent*(ih, newParent: PIhandle): cint {. - importc: "IupReparent", cdecl, dynlib: dllname.} - -proc popup*(ih: PIhandle, x, y: cint): cint {. - importc: "IupPopup", cdecl, dynlib: dllname, discardable.} -proc show*(ih: PIhandle): cint {. - importc: "IupShow", cdecl, dynlib: dllname, discardable.} -proc showXY*(ih: PIhandle, x, y: cint): cint {. - importc: "IupShowXY", cdecl, dynlib: dllname, discardable.} -proc hide*(ih: PIhandle): cint {. - importc: "IupHide", cdecl, dynlib: dllname, discardable.} -proc map*(ih: PIhandle): cint {. - importc: "IupMap", cdecl, dynlib: dllname, discardable.} -proc unmap*(ih: PIhandle) {. - importc: "IupUnmap", cdecl, dynlib: dllname, discardable.} - -proc setAttribute*(ih: PIhandle, name, value: cstring) {. - importc: "IupSetAttribute", cdecl, dynlib: dllname.} -proc storeAttribute*(ih: PIhandle, name, value: cstring) {. - importc: "IupStoreAttribute", cdecl, dynlib: dllname.} -proc setAttributes*(ih: PIhandle, str: cstring): PIhandle {. - importc: "IupSetAttributes", cdecl, dynlib: dllname.} -proc getAttribute*(ih: PIhandle, name: cstring): cstring {. - importc: "IupGetAttribute", cdecl, dynlib: dllname.} -proc getAttributes*(ih: PIhandle): cstring {. - importc: "IupGetAttributes", cdecl, dynlib: dllname.} -proc getInt*(ih: PIhandle, name: cstring): cint {. - importc: "IupGetInt", cdecl, dynlib: dllname.} -proc getInt2*(ih: PIhandle, name: cstring): cint {. - importc: "IupGetInt2", cdecl, dynlib: dllname.} -proc getIntInt*(ih: PIhandle, name: cstring, i1, i2: var cint): cint {. - importc: "IupGetIntInt", cdecl, dynlib: dllname.} -proc getFloat*(ih: PIhandle, name: cstring): cfloat {. - importc: "IupGetFloat", cdecl, dynlib: dllname.} -proc setfAttribute*(ih: PIhandle, name, format: cstring) {. - importc: "IupSetfAttribute", cdecl, dynlib: dllname, varargs.} -proc getAllAttributes*(ih: PIhandle, names: cstringArray, n: cint): cint {. - importc: "IupGetAllAttributes", cdecl, dynlib: dllname.} -proc setAtt*(handleName: cstring, ih: PIhandle, name: cstring): PIhandle {. - importc: "IupSetAtt", cdecl, dynlib: dllname, varargs, discardable.} - -proc setGlobal*(name, value: cstring) {. - importc: "IupSetGlobal", cdecl, dynlib: dllname.} -proc storeGlobal*(name, value: cstring) {. - importc: "IupStoreGlobal", cdecl, dynlib: dllname.} -proc getGlobal*(name: cstring): cstring {. - importc: "IupGetGlobal", cdecl, dynlib: dllname.} - -proc setFocus*(ih: PIhandle): PIhandle {. - importc: "IupSetFocus", cdecl, dynlib: dllname.} -proc getFocus*(): PIhandle {. - importc: "IupGetFocus", cdecl, dynlib: dllname.} -proc previousField*(ih: PIhandle): PIhandle {. - importc: "IupPreviousField", cdecl, dynlib: dllname.} -proc nextField*(ih: PIhandle): PIhandle {. - importc: "IupNextField", cdecl, dynlib: dllname.} - -proc getCallback*(ih: PIhandle, name: cstring): Icallback {. - importc: "IupGetCallback", cdecl, dynlib: dllname.} -proc setCallback*(ih: PIhandle, name: cstring, fn: Icallback): Icallback {. - importc: "IupSetCallback", cdecl, dynlib: dllname, discardable.} - -proc setCallbacks*(ih: PIhandle, name: cstring, fn: Icallback): PIhandle {. - importc: "IupSetCallbacks", cdecl, dynlib: dllname, varargs, discardable.} - -proc getFunction*(name: cstring): Icallback {. - importc: "IupGetFunction", cdecl, dynlib: dllname.} -proc setFunction*(name: cstring, fn: Icallback): Icallback {. - importc: "IupSetFunction", cdecl, dynlib: dllname, discardable.} -proc getActionName*(): cstring {. - importc: "IupGetActionName", cdecl, dynlib: dllname.} - -proc getHandle*(name: cstring): PIhandle {. - importc: "IupGetHandle", cdecl, dynlib: dllname.} -proc setHandle*(name: cstring, ih: PIhandle): PIhandle {. - importc: "IupSetHandle", cdecl, dynlib: dllname.} -proc getAllNames*(names: cstringArray, n: cint): cint {. - importc: "IupGetAllNames", cdecl, dynlib: dllname.} -proc getAllDialogs*(names: cstringArray, n: cint): cint {. - importc: "IupGetAllDialogs", cdecl, dynlib: dllname.} -proc getName*(ih: PIhandle): cstring {. - importc: "IupGetName", cdecl, dynlib: dllname.} - -proc setAttributeHandle*(ih: PIhandle, name: cstring, ihNamed: PIhandle) {. - importc: "IupSetAttributeHandle", cdecl, dynlib: dllname.} -proc getAttributeHandle*(ih: PIhandle, name: cstring): PIhandle {. - importc: "IupGetAttributeHandle", cdecl, dynlib: dllname.} - -proc getClassName*(ih: PIhandle): cstring {. - importc: "IupGetClassName", cdecl, dynlib: dllname.} -proc getClassType*(ih: PIhandle): cstring {. - importc: "IupGetClassType", cdecl, dynlib: dllname.} -proc getClassAttributes*(classname: cstring, names: cstringArray, - n: cint): cint {. - importc: "IupGetClassAttributes", cdecl, dynlib: dllname.} -proc saveClassAttributes*(ih: PIhandle) {. - importc: "IupSaveClassAttributes", cdecl, dynlib: dllname.} -proc setClassDefaultAttribute*(classname, name, value: cstring) {. - importc: "IupSetClassDefaultAttribute", cdecl, dynlib: dllname.} - -proc create*(classname: cstring): PIhandle {. - importc: "IupCreate", cdecl, dynlib: dllname.} -proc createv*(classname: cstring, params: pointer): PIhandle {. - importc: "IupCreatev", cdecl, dynlib: dllname.} -proc createp*(classname: cstring, first: pointer): PIhandle {. - importc: "IupCreatep", cdecl, dynlib: dllname, varargs.} - -proc fill*(): PIhandle {.importc: "IupFill", cdecl, dynlib: dllname.} -proc radio*(child: PIhandle): PIhandle {. - importc: "IupRadio", cdecl, dynlib: dllname.} -proc vbox*(child: PIhandle): PIhandle {. - importc: "IupVbox", cdecl, dynlib: dllname, varargs.} -proc vboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupVboxv", cdecl, dynlib: dllname.} -proc zbox*(child: PIhandle): PIhandle {. - importc: "IupZbox", cdecl, dynlib: dllname, varargs.} -proc zboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupZboxv", cdecl, dynlib: dllname.} -proc hbox*(child: PIhandle): PIhandle {. - importc: "IupHbox", cdecl, dynlib: dllname, varargs.} -proc hboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupHboxv", cdecl, dynlib: dllname.} - -proc normalizer*(ihFirst: PIhandle): PIhandle {. - importc: "IupNormalizer", cdecl, dynlib: dllname, varargs.} -proc normalizerv*(ihList: ptr PIhandle): PIhandle {. - importc: "IupNormalizerv", cdecl, dynlib: dllname.} - -proc cbox*(child: PIhandle): PIhandle {. - importc: "IupCbox", cdecl, dynlib: dllname, varargs.} -proc cboxv*(children: ptr PIhandle): PIhandle {. - importc: "IupCboxv", cdecl, dynlib: dllname.} -proc sbox*(child: PIhandle): PIhandle {. - importc: "IupSbox", cdecl, dynlib: dllname.} - -proc frame*(child: PIhandle): PIhandle {. - importc: "IupFrame", cdecl, dynlib: dllname.} - -proc image*(width, height: cint, pixmap: pointer): PIhandle {. - importc: "IupImage", cdecl, dynlib: dllname.} -proc imageRGB*(width, height: cint, pixmap: pointer): PIhandle {. - importc: "IupImageRGB", cdecl, dynlib: dllname.} -proc imageRGBA*(width, height: cint, pixmap: pointer): PIhandle {. - importc: "IupImageRGBA", cdecl, dynlib: dllname.} - -proc item*(title, action: cstring): PIhandle {. - importc: "IupItem", cdecl, dynlib: dllname.} -proc submenu*(title: cstring, child: PIhandle): PIhandle {. - importc: "IupSubmenu", cdecl, dynlib: dllname.} -proc separator*(): PIhandle {. - importc: "IupSeparator", cdecl, dynlib: dllname.} -proc menu*(child: PIhandle): PIhandle {. - importc: "IupMenu", cdecl, dynlib: dllname, varargs.} -proc menuv*(children: ptr PIhandle): PIhandle {. - importc: "IupMenuv", cdecl, dynlib: dllname.} - -proc button*(title, action: cstring): PIhandle {. - importc: "IupButton", cdecl, dynlib: dllname.} -proc link*(url, title: cstring): PIhandle {. - importc: "IupLink", cdecl, dynlib: dllname.} -proc canvas*(action: cstring): PIhandle {. - importc: "IupCanvas", cdecl, dynlib: dllname.} -proc dialog*(child: PIhandle): PIhandle {. - importc: "IupDialog", cdecl, dynlib: dllname.} -proc user*(): PIhandle {. - importc: "IupUser", cdecl, dynlib: dllname.} -proc label*(title: cstring): PIhandle {. - importc: "IupLabel", cdecl, dynlib: dllname.} -proc list*(action: cstring): PIhandle {. - importc: "IupList", cdecl, dynlib: dllname.} -proc text*(action: cstring): PIhandle {. - importc: "IupText", cdecl, dynlib: dllname.} -proc multiLine*(action: cstring): PIhandle {. - importc: "IupMultiLine", cdecl, dynlib: dllname.} -proc toggle*(title, action: cstring): PIhandle {. - importc: "IupToggle", cdecl, dynlib: dllname.} -proc timer*(): PIhandle {. - importc: "IupTimer", cdecl, dynlib: dllname.} -proc progressBar*(): PIhandle {. - importc: "IupProgressBar", cdecl, dynlib: dllname.} -proc val*(theType: cstring): PIhandle {. - importc: "IupVal", cdecl, dynlib: dllname.} -proc tabs*(child: PIhandle): PIhandle {. - importc: "IupTabs", cdecl, dynlib: dllname, varargs.} -proc tabsv*(children: ptr PIhandle): PIhandle {. - importc: "IupTabsv", cdecl, dynlib: dllname.} -proc tree*(): PIhandle {.importc: "IupTree", cdecl, dynlib: dllname.} - -proc spin*(): PIhandle {.importc: "IupSpin", cdecl, dynlib: dllname.} -proc spinbox*(child: PIhandle): PIhandle {. - importc: "IupSpinbox", cdecl, dynlib: dllname.} - -# IupText utilities -proc textConvertLinColToPos*(ih: PIhandle, lin, col: cint, pos: var cint) {. - importc: "IupTextConvertLinColToPos", cdecl, dynlib: dllname.} -proc textConvertPosToLinCol*(ih: PIhandle, pos: cint, lin, col: var cint) {. - importc: "IupTextConvertPosToLinCol", cdecl, dynlib: dllname.} - -proc convertXYToPos*(ih: PIhandle, x, y: cint): cint {. - importc: "IupConvertXYToPos", cdecl, dynlib: dllname.} - -# IupTree utilities -proc treeSetUserId*(ih: PIhandle, id: cint, userid: pointer): cint {. - importc: "IupTreeSetUserId", cdecl, dynlib: dllname, discardable.} -proc treeGetUserId*(ih: PIhandle, id: cint): pointer {. - importc: "IupTreeGetUserId", cdecl, dynlib: dllname.} -proc treeGetId*(ih: PIhandle, userid: pointer): cint {. - importc: "IupTreeGetId", cdecl, dynlib: dllname.} - -proc treeSetAttribute*(ih: PIhandle, name: cstring, id: cint, value: cstring) {. - importc: "IupTreeSetAttribute", cdecl, dynlib: dllname.} -proc treeStoreAttribute*(ih: PIhandle, name: cstring, id: cint, value: cstring) {. - importc: "IupTreeStoreAttribute", cdecl, dynlib: dllname.} -proc treeGetAttribute*(ih: PIhandle, name: cstring, id: cint): cstring {. - importc: "IupTreeGetAttribute", cdecl, dynlib: dllname.} -proc treeGetInt*(ih: PIhandle, name: cstring, id: cint): cint {. - importc: "IupTreeGetInt", cdecl, dynlib: dllname.} -proc treeGetFloat*(ih: PIhandle, name: cstring, id: cint): cfloat {. - importc: "IupTreeGetFloat", cdecl, dynlib: dllname.} -proc treeSetfAttribute*(ih: PIhandle, name: cstring, id: cint, format: cstring) {. - importc: "IupTreeSetfAttribute", cdecl, dynlib: dllname, varargs.} - - -# Common Return Values -const - IUP_ERROR* = cint(1) - IUP_NOERROR* = cint(0) - IUP_OPENED* = cint(-1) - IUP_INVALID* = cint(-1) - - # Callback Return Values - IUP_IGNORE* = cint(-1) - IUP_DEFAULT* = cint(-2) - IUP_CLOSE* = cint(-3) - IUP_CONTINUE* = cint(-4) - - # IupPopup and IupShowXY Parameter Values - IUP_CENTER* = cint(0xFFFF) - IUP_LEFT* = cint(0xFFFE) - IUP_RIGHT* = cint(0xFFFD) - IUP_MOUSEPOS* = cint(0xFFFC) - IUP_CURRENT* = cint(0xFFFB) - IUP_CENTERPARENT* = cint(0xFFFA) - IUP_TOP* = IUP_LEFT - IUP_BOTTOM* = IUP_RIGHT - - # SHOW_CB Callback Values - IUP_SHOW* = cint(0) - IUP_RESTORE* = cint(1) - IUP_MINIMIZE* = cint(2) - IUP_MAXIMIZE* = cint(3) - IUP_HIDE* = cint(4) - - # SCROLL_CB Callback Values - IUP_SBUP* = cint(0) - IUP_SBDN* = cint(1) - IUP_SBPGUP* = cint(2) - IUP_SBPGDN* = cint(3) - IUP_SBPOSV* = cint(4) - IUP_SBDRAGV* = cint(5) - IUP_SBLEFT* = cint(6) - IUP_SBRIGHT* = cint(7) - IUP_SBPGLEFT* = cint(8) - IUP_SBPGRIGHT* = cint(9) - IUP_SBPOSH* = cint(10) - IUP_SBDRAGH* = cint(11) - - # Mouse Button Values and Macros - IUP_BUTTON1* = cint(ord('1')) - IUP_BUTTON2* = cint(ord('2')) - IUP_BUTTON3* = cint(ord('3')) - IUP_BUTTON4* = cint(ord('4')) - IUP_BUTTON5* = cint(ord('5')) - -proc isShift*(s: cstring): bool = return s[0] == 'S' -proc isControl*(s: cstring): bool = return s[1] == 'C' -proc isButton1*(s: cstring): bool = return s[2] == '1' -proc isButton2*(s: cstring): bool = return s[3] == '2' -proc isbutton3*(s: cstring): bool = return s[4] == '3' -proc isDouble*(s: cstring): bool = return s[5] == 'D' -proc isAlt*(s: cstring): bool = return s[6] == 'A' -proc isSys*(s: cstring): bool = return s[7] == 'Y' -proc isButton4*(s: cstring): bool = return s[8] == '4' -proc isButton5*(s: cstring): bool = return s[9] == '5' - -# Pre-Defined Masks -const - IUP_MASK_FLOAT* = "[+/-]?(/d+/.?/d*|/./d+)" - IUP_MASK_UFLOAT* = "(/d+/.?/d*|/./d+)" - IUP_MASK_EFLOAT* = "[+/-]?(/d+/.?/d*|/./d+)([eE][+/-]?/d+)?" - IUP_MASK_INT* = "[+/-]?/d+" - IUP_MASK_UINT* = "/d+" - -# from 32 to 126, all character sets are equal, -# the key code i the same as the character code. -const - K_SP* = cint(ord(' ')) - K_exclam* = cint(ord('!')) - K_quotedbl* = cint(ord('\"')) - K_numbersign* = cint(ord('#')) - K_dollar* = cint(ord('$')) - K_percent* = cint(ord('%')) - K_ampersand* = cint(ord('&')) - K_apostrophe* = cint(ord('\'')) - K_parentleft* = cint(ord('(')) - K_parentright* = cint(ord(')')) - K_asterisk* = cint(ord('*')) - K_plus* = cint(ord('+')) - K_comma* = cint(ord(',')) - K_minus* = cint(ord('-')) - K_period* = cint(ord('.')) - K_slash* = cint(ord('/')) - K_0* = cint(ord('0')) - K_1* = cint(ord('1')) - K_2* = cint(ord('2')) - K_3* = cint(ord('3')) - K_4* = cint(ord('4')) - K_5* = cint(ord('5')) - K_6* = cint(ord('6')) - K_7* = cint(ord('7')) - K_8* = cint(ord('8')) - K_9* = cint(ord('9')) - K_colon* = cint(ord(':')) - K_semicolon* = cint(ord(';')) - K_less* = cint(ord('<')) - K_equal* = cint(ord('=')) - K_greater* = cint(ord('>')) - K_question* = cint(ord('?')) - K_at* = cint(ord('@')) - K_upperA* = cint(ord('A')) - K_upperB* = cint(ord('B')) - K_upperC* = cint(ord('C')) - K_upperD* = cint(ord('D')) - K_upperE* = cint(ord('E')) - K_upperF* = cint(ord('F')) - K_upperG* = cint(ord('G')) - K_upperH* = cint(ord('H')) - K_upperI* = cint(ord('I')) - K_upperJ* = cint(ord('J')) - K_upperK* = cint(ord('K')) - K_upperL* = cint(ord('L')) - K_upperM* = cint(ord('M')) - K_upperN* = cint(ord('N')) - K_upperO* = cint(ord('O')) - K_upperP* = cint(ord('P')) - K_upperQ* = cint(ord('Q')) - K_upperR* = cint(ord('R')) - K_upperS* = cint(ord('S')) - K_upperT* = cint(ord('T')) - K_upperU* = cint(ord('U')) - K_upperV* = cint(ord('V')) - K_upperW* = cint(ord('W')) - K_upperX* = cint(ord('X')) - K_upperY* = cint(ord('Y')) - K_upperZ* = cint(ord('Z')) - K_bracketleft* = cint(ord('[')) - K_backslash* = cint(ord('\\')) - K_bracketright* = cint(ord(']')) - K_circum* = cint(ord('^')) - K_underscore* = cint(ord('_')) - K_grave* = cint(ord('`')) - K_lowera* = cint(ord('a')) - K_lowerb* = cint(ord('b')) - K_lowerc* = cint(ord('c')) - K_lowerd* = cint(ord('d')) - K_lowere* = cint(ord('e')) - K_lowerf* = cint(ord('f')) - K_lowerg* = cint(ord('g')) - K_lowerh* = cint(ord('h')) - K_loweri* = cint(ord('i')) - K_lowerj* = cint(ord('j')) - K_lowerk* = cint(ord('k')) - K_lowerl* = cint(ord('l')) - K_lowerm* = cint(ord('m')) - K_lowern* = cint(ord('n')) - K_lowero* = cint(ord('o')) - K_lowerp* = cint(ord('p')) - K_lowerq* = cint(ord('q')) - K_lowerr* = cint(ord('r')) - K_lowers* = cint(ord('s')) - K_lowert* = cint(ord('t')) - K_loweru* = cint(ord('u')) - K_lowerv* = cint(ord('v')) - K_lowerw* = cint(ord('w')) - K_lowerx* = cint(ord('x')) - K_lowery* = cint(ord('y')) - K_lowerz* = cint(ord('z')) - K_braceleft* = cint(ord('{')) - K_bar* = cint(ord('|')) - K_braceright* = cint(ord('}')) - K_tilde* = cint(ord('~')) - -proc isPrint*(c: cint): bool = return c > 31 and c < 127 - -# also define the escape sequences that have keys associated -const - K_BS* = cint(ord('\b')) - K_TAB* = cint(ord('\t')) - K_LF* = cint(10) - K_CR* = cint(13) - -# IUP Extended Key Codes, range start at 128 -# Modifiers use 256 interval -# These key code definitions are specific to IUP - -proc isXkey*(c: cint): bool = return c > 128 -proc isShiftXkey*(c: cint): bool = return c > 256 and c < 512 -proc isCtrlXkey*(c: cint): bool = return c > 512 and c < 768 -proc isAltXkey*(c: cint): bool = return c > 768 and c < 1024 -proc isSysXkey*(c: cint): bool = return c > 1024 and c < 1280 - -proc iUPxCODE*(c: cint): cint = return c + cint(128) # Normal (must be above 128) -proc iUPsxCODE*(c: cint): cint = - return c + cint(256) - # Shift (must have range to include the standard keys and the normal - # extended keys, so must be above 256 - -proc iUPcxCODE*(c: cint): cint = return c + cint(512) # Ctrl -proc iUPmxCODE*(c: cint): cint = return c + cint(768) # Alt -proc iUPyxCODE*(c: cint): cint = return c + cint(1024) # Sys (Win or Apple) - -const - IUP_NUMMAXCODES* = 1280 ## 5*256=1280 Normal+Shift+Ctrl+Alt+Sys - - K_HOME* = iUPxCODE(1) - K_UP* = iUPxCODE(2) - K_PGUP* = iUPxCODE(3) - K_LEFT* = iUPxCODE(4) - K_MIDDLE* = iUPxCODE(5) - K_RIGHT* = iUPxCODE(6) - K_END* = iUPxCODE(7) - K_DOWN* = iUPxCODE(8) - K_PGDN* = iUPxCODE(9) - K_INS* = iUPxCODE(10) - K_DEL* = iUPxCODE(11) - K_PAUSE* = iUPxCODE(12) - K_ESC* = iUPxCODE(13) - K_ccedilla* = iUPxCODE(14) - K_F1* = iUPxCODE(15) - K_F2* = iUPxCODE(16) - K_F3* = iUPxCODE(17) - K_F4* = iUPxCODE(18) - K_F5* = iUPxCODE(19) - K_F6* = iUPxCODE(20) - K_F7* = iUPxCODE(21) - K_F8* = iUPxCODE(22) - K_F9* = iUPxCODE(23) - K_F10* = iUPxCODE(24) - K_F11* = iUPxCODE(25) - K_F12* = iUPxCODE(26) - K_Print* = iUPxCODE(27) - K_Menu* = iUPxCODE(28) - - K_acute* = iUPxCODE(29) # no Shift/Ctrl/Alt - - K_sHOME* = iUPsxCODE(K_HOME) - K_sUP* = iUPsxCODE(K_UP) - K_sPGUP* = iUPsxCODE(K_PGUP) - K_sLEFT* = iUPsxCODE(K_LEFT) - K_sMIDDLE* = iUPsxCODE(K_MIDDLE) - K_sRIGHT* = iUPsxCODE(K_RIGHT) - K_sEND* = iUPsxCODE(K_END) - K_sDOWN* = iUPsxCODE(K_DOWN) - K_sPGDN* = iUPsxCODE(K_PGDN) - K_sINS* = iUPsxCODE(K_INS) - K_sDEL* = iUPsxCODE(K_DEL) - K_sSP* = iUPsxCODE(K_SP) - K_sTAB* = iUPsxCODE(K_TAB) - K_sCR* = iUPsxCODE(K_CR) - K_sBS* = iUPsxCODE(K_BS) - K_sPAUSE* = iUPsxCODE(K_PAUSE) - K_sESC* = iUPsxCODE(K_ESC) - K_sCcedilla* = iUPsxCODE(K_ccedilla) - K_sF1* = iUPsxCODE(K_F1) - K_sF2* = iUPsxCODE(K_F2) - K_sF3* = iUPsxCODE(K_F3) - K_sF4* = iUPsxCODE(K_F4) - K_sF5* = iUPsxCODE(K_F5) - K_sF6* = iUPsxCODE(K_F6) - K_sF7* = iUPsxCODE(K_F7) - K_sF8* = iUPsxCODE(K_F8) - K_sF9* = iUPsxCODE(K_F9) - K_sF10* = iUPsxCODE(K_F10) - K_sF11* = iUPsxCODE(K_F11) - K_sF12* = iUPsxCODE(K_F12) - K_sPrint* = iUPsxCODE(K_Print) - K_sMenu* = iUPsxCODE(K_Menu) - - K_cHOME* = iUPcxCODE(K_HOME) - K_cUP* = iUPcxCODE(K_UP) - K_cPGUP* = iUPcxCODE(K_PGUP) - K_cLEFT* = iUPcxCODE(K_LEFT) - K_cMIDDLE* = iUPcxCODE(K_MIDDLE) - K_cRIGHT* = iUPcxCODE(K_RIGHT) - K_cEND* = iUPcxCODE(K_END) - K_cDOWN* = iUPcxCODE(K_DOWN) - K_cPGDN* = iUPcxCODE(K_PGDN) - K_cINS* = iUPcxCODE(K_INS) - K_cDEL* = iUPcxCODE(K_DEL) - K_cSP* = iUPcxCODE(K_SP) - K_cTAB* = iUPcxCODE(K_TAB) - K_cCR* = iUPcxCODE(K_CR) - K_cBS* = iUPcxCODE(K_BS) - K_cPAUSE* = iUPcxCODE(K_PAUSE) - K_cESC* = iUPcxCODE(K_ESC) - K_cCcedilla* = iUPcxCODE(K_ccedilla) - K_cF1* = iUPcxCODE(K_F1) - K_cF2* = iUPcxCODE(K_F2) - K_cF3* = iUPcxCODE(K_F3) - K_cF4* = iUPcxCODE(K_F4) - K_cF5* = iUPcxCODE(K_F5) - K_cF6* = iUPcxCODE(K_F6) - K_cF7* = iUPcxCODE(K_F7) - K_cF8* = iUPcxCODE(K_F8) - K_cF9* = iUPcxCODE(K_F9) - K_cF10* = iUPcxCODE(K_F10) - K_cF11* = iUPcxCODE(K_F11) - K_cF12* = iUPcxCODE(K_F12) - K_cPrint* = iUPcxCODE(K_Print) - K_cMenu* = iUPcxCODE(K_Menu) - - K_mHOME* = iUPmxCODE(K_HOME) - K_mUP* = iUPmxCODE(K_UP) - K_mPGUP* = iUPmxCODE(K_PGUP) - K_mLEFT* = iUPmxCODE(K_LEFT) - K_mMIDDLE* = iUPmxCODE(K_MIDDLE) - K_mRIGHT* = iUPmxCODE(K_RIGHT) - K_mEND* = iUPmxCODE(K_END) - K_mDOWN* = iUPmxCODE(K_DOWN) - K_mPGDN* = iUPmxCODE(K_PGDN) - K_mINS* = iUPmxCODE(K_INS) - K_mDEL* = iUPmxCODE(K_DEL) - K_mSP* = iUPmxCODE(K_SP) - K_mTAB* = iUPmxCODE(K_TAB) - K_mCR* = iUPmxCODE(K_CR) - K_mBS* = iUPmxCODE(K_BS) - K_mPAUSE* = iUPmxCODE(K_PAUSE) - K_mESC* = iUPmxCODE(K_ESC) - K_mCcedilla* = iUPmxCODE(K_ccedilla) - K_mF1* = iUPmxCODE(K_F1) - K_mF2* = iUPmxCODE(K_F2) - K_mF3* = iUPmxCODE(K_F3) - K_mF4* = iUPmxCODE(K_F4) - K_mF5* = iUPmxCODE(K_F5) - K_mF6* = iUPmxCODE(K_F6) - K_mF7* = iUPmxCODE(K_F7) - K_mF8* = iUPmxCODE(K_F8) - K_mF9* = iUPmxCODE(K_F9) - K_mF10* = iUPmxCODE(K_F10) - K_mF11* = iUPmxCODE(K_F11) - K_mF12* = iUPmxCODE(K_F12) - K_mPrint* = iUPmxCODE(K_Print) - K_mMenu* = iUPmxCODE(K_Menu) - - K_yHOME* = iUPyxCODE(K_HOME) - K_yUP* = iUPyxCODE(K_UP) - K_yPGUP* = iUPyxCODE(K_PGUP) - K_yLEFT* = iUPyxCODE(K_LEFT) - K_yMIDDLE* = iUPyxCODE(K_MIDDLE) - K_yRIGHT* = iUPyxCODE(K_RIGHT) - K_yEND* = iUPyxCODE(K_END) - K_yDOWN* = iUPyxCODE(K_DOWN) - K_yPGDN* = iUPyxCODE(K_PGDN) - K_yINS* = iUPyxCODE(K_INS) - K_yDEL* = iUPyxCODE(K_DEL) - K_ySP* = iUPyxCODE(K_SP) - K_yTAB* = iUPyxCODE(K_TAB) - K_yCR* = iUPyxCODE(K_CR) - K_yBS* = iUPyxCODE(K_BS) - K_yPAUSE* = iUPyxCODE(K_PAUSE) - K_yESC* = iUPyxCODE(K_ESC) - K_yCcedilla* = iUPyxCODE(K_ccedilla) - K_yF1* = iUPyxCODE(K_F1) - K_yF2* = iUPyxCODE(K_F2) - K_yF3* = iUPyxCODE(K_F3) - K_yF4* = iUPyxCODE(K_F4) - K_yF5* = iUPyxCODE(K_F5) - K_yF6* = iUPyxCODE(K_F6) - K_yF7* = iUPyxCODE(K_F7) - K_yF8* = iUPyxCODE(K_F8) - K_yF9* = iUPyxCODE(K_F9) - K_yF10* = iUPyxCODE(K_F10) - K_yF11* = iUPyxCODE(K_F11) - K_yF12* = iUPyxCODE(K_F12) - K_yPrint* = iUPyxCODE(K_Print) - K_yMenu* = iUPyxCODE(K_Menu) - - K_sPlus* = iUPsxCODE(K_plus) - K_sComma* = iUPsxCODE(K_comma) - K_sMinus* = iUPsxCODE(K_minus) - K_sPeriod* = iUPsxCODE(K_period) - K_sSlash* = iUPsxCODE(K_slash) - K_sAsterisk* = iUPsxCODE(K_asterisk) - - K_cupperA* = iUPcxCODE(K_upperA) - K_cupperB* = iUPcxCODE(K_upperB) - K_cupperC* = iUPcxCODE(K_upperC) - K_cupperD* = iUPcxCODE(K_upperD) - K_cupperE* = iUPcxCODE(K_upperE) - K_cupperF* = iUPcxCODE(K_upperF) - K_cupperG* = iUPcxCODE(K_upperG) - K_cupperH* = iUPcxCODE(K_upperH) - K_cupperI* = iUPcxCODE(K_upperI) - K_cupperJ* = iUPcxCODE(K_upperJ) - K_cupperK* = iUPcxCODE(K_upperK) - K_cupperL* = iUPcxCODE(K_upperL) - K_cupperM* = iUPcxCODE(K_upperM) - K_cupperN* = iUPcxCODE(K_upperN) - K_cupperO* = iUPcxCODE(K_upperO) - K_cupperP* = iUPcxCODE(K_upperP) - K_cupperQ* = iUPcxCODE(K_upperQ) - K_cupperR* = iUPcxCODE(K_upperR) - K_cupperS* = iUPcxCODE(K_upperS) - K_cupperT* = iUPcxCODE(K_upperT) - K_cupperU* = iUPcxCODE(K_upperU) - K_cupperV* = iUPcxCODE(K_upperV) - K_cupperW* = iUPcxCODE(K_upperW) - K_cupperX* = iUPcxCODE(K_upperX) - K_cupperY* = iUPcxCODE(K_upperY) - K_cupperZ* = iUPcxCODE(K_upperZ) - K_c1* = iUPcxCODE(K_1) - K_c2* = iUPcxCODE(K_2) - K_c3* = iUPcxCODE(K_3) - K_c4* = iUPcxCODE(K_4) - K_c5* = iUPcxCODE(K_5) - K_c6* = iUPcxCODE(K_6) - K_c7* = iUPcxCODE(K_7) - K_c8* = iUPcxCODE(K_8) - K_c9* = iUPcxCODE(K_9) - K_c0* = iUPcxCODE(K_0) - K_cPlus* = iUPcxCODE(K_plus) - K_cComma* = iUPcxCODE(K_comma) - K_cMinus* = iUPcxCODE(K_minus) - K_cPeriod* = iUPcxCODE(K_period) - K_cSlash* = iUPcxCODE(K_slash) - K_cSemicolon* = iUPcxCODE(K_semicolon) - K_cEqual* = iUPcxCODE(K_equal) - K_cBracketleft* = iUPcxCODE(K_bracketleft) - K_cBracketright* = iUPcxCODE(K_bracketright) - K_cBackslash* = iUPcxCODE(K_backslash) - K_cAsterisk* = iUPcxCODE(K_asterisk) - - K_mupperA* = iUPmxCODE(K_upperA) - K_mupperB* = iUPmxCODE(K_upperB) - K_mupperC* = iUPmxCODE(K_upperC) - K_mupperD* = iUPmxCODE(K_upperD) - K_mupperE* = iUPmxCODE(K_upperE) - K_mupperF* = iUPmxCODE(K_upperF) - K_mupperG* = iUPmxCODE(K_upperG) - K_mupperH* = iUPmxCODE(K_upperH) - K_mupperI* = iUPmxCODE(K_upperI) - K_mupperJ* = iUPmxCODE(K_upperJ) - K_mupperK* = iUPmxCODE(K_upperK) - K_mupperL* = iUPmxCODE(K_upperL) - K_mupperM* = iUPmxCODE(K_upperM) - K_mupperN* = iUPmxCODE(K_upperN) - K_mupperO* = iUPmxCODE(K_upperO) - K_mupperP* = iUPmxCODE(K_upperP) - K_mupperQ* = iUPmxCODE(K_upperQ) - K_mupperR* = iUPmxCODE(K_upperR) - K_mupperS* = iUPmxCODE(K_upperS) - K_mupperT* = iUPmxCODE(K_upperT) - K_mupperU* = iUPmxCODE(K_upperU) - K_mupperV* = iUPmxCODE(K_upperV) - K_mupperW* = iUPmxCODE(K_upperW) - K_mupperX* = iUPmxCODE(K_upperX) - K_mupperY* = iUPmxCODE(K_upperY) - K_mupperZ* = iUPmxCODE(K_upperZ) - K_m1* = iUPmxCODE(K_1) - K_m2* = iUPmxCODE(K_2) - K_m3* = iUPmxCODE(K_3) - K_m4* = iUPmxCODE(K_4) - K_m5* = iUPmxCODE(K_5) - K_m6* = iUPmxCODE(K_6) - K_m7* = iUPmxCODE(K_7) - K_m8* = iUPmxCODE(K_8) - K_m9* = iUPmxCODE(K_9) - K_m0* = iUPmxCODE(K_0) - K_mPlus* = iUPmxCODE(K_plus) - K_mComma* = iUPmxCODE(K_comma) - K_mMinus* = iUPmxCODE(K_minus) - K_mPeriod* = iUPmxCODE(K_period) - K_mSlash* = iUPmxCODE(K_slash) - K_mSemicolon* = iUPmxCODE(K_semicolon) - K_mEqual* = iUPmxCODE(K_equal) - K_mBracketleft* = iUPmxCODE(K_bracketleft) - K_mBracketright* = iUPmxCODE(K_bracketright) - K_mBackslash* = iUPmxCODE(K_backslash) - K_mAsterisk* = iUPmxCODE(K_asterisk) - - K_yA* = iUPyxCODE(K_upperA) - K_yB* = iUPyxCODE(K_upperB) - K_yC* = iUPyxCODE(K_upperC) - K_yD* = iUPyxCODE(K_upperD) - K_yE* = iUPyxCODE(K_upperE) - K_yF* = iUPyxCODE(K_upperF) - K_yG* = iUPyxCODE(K_upperG) - K_yH* = iUPyxCODE(K_upperH) - K_yI* = iUPyxCODE(K_upperI) - K_yJ* = iUPyxCODE(K_upperJ) - K_yK* = iUPyxCODE(K_upperK) - K_yL* = iUPyxCODE(K_upperL) - K_yM* = iUPyxCODE(K_upperM) - K_yN* = iUPyxCODE(K_upperN) - K_yO* = iUPyxCODE(K_upperO) - K_yP* = iUPyxCODE(K_upperP) - K_yQ* = iUPyxCODE(K_upperQ) - K_yR* = iUPyxCODE(K_upperR) - K_yS* = iUPyxCODE(K_upperS) - K_yT* = iUPyxCODE(K_upperT) - K_yU* = iUPyxCODE(K_upperU) - K_yV* = iUPyxCODE(K_upperV) - K_yW* = iUPyxCODE(K_upperW) - K_yX* = iUPyxCODE(K_upperX) - K_yY* = iUPyxCODE(K_upperY) - K_yZ* = iUPyxCODE(K_upperZ) - K_y1* = iUPyxCODE(K_1) - K_y2* = iUPyxCODE(K_2) - K_y3* = iUPyxCODE(K_3) - K_y4* = iUPyxCODE(K_4) - K_y5* = iUPyxCODE(K_5) - K_y6* = iUPyxCODE(K_6) - K_y7* = iUPyxCODE(K_7) - K_y8* = iUPyxCODE(K_8) - K_y9* = iUPyxCODE(K_9) - K_y0* = iUPyxCODE(K_0) - K_yPlus* = iUPyxCODE(K_plus) - K_yComma* = iUPyxCODE(K_comma) - K_yMinus* = iUPyxCODE(K_minus) - K_yPeriod* = iUPyxCODE(K_period) - K_ySlash* = iUPyxCODE(K_slash) - K_ySemicolon* = iUPyxCODE(K_semicolon) - K_yEqual* = iUPyxCODE(K_equal) - K_yBracketleft* = iUPyxCODE(K_bracketleft) - K_yBracketright* = iUPyxCODE(K_bracketright) - K_yBackslash* = iUPyxCODE(K_backslash) - K_yAsterisk* = iUPyxCODE(K_asterisk) - -proc controlsOpen*(): cint {.cdecl, importc: "IupControlsOpen", dynlib: dllname.} -proc controlsClose*() {.cdecl, importc: "IupControlsClose", dynlib: dllname.} - -proc oldValOpen*() {.cdecl, importc: "IupOldValOpen", dynlib: dllname.} -proc oldTabsOpen*() {.cdecl, importc: "IupOldTabsOpen", dynlib: dllname.} - -proc colorbar*(): PIhandle {.cdecl, importc: "IupColorbar", dynlib: dllname.} -proc cells*(): PIhandle {.cdecl, importc: "IupCells", dynlib: dllname.} -proc colorBrowser*(): PIhandle {.cdecl, importc: "IupColorBrowser", dynlib: dllname.} -proc gauge*(): PIhandle {.cdecl, importc: "IupGauge", dynlib: dllname.} -proc dial*(theType: cstring): PIhandle {.cdecl, importc: "IupDial", dynlib: dllname.} -proc matrix*(action: cstring): PIhandle {.cdecl, importc: "IupMatrix", dynlib: dllname.} - -# IupMatrix utilities -proc matSetAttribute*(ih: PIhandle, name: cstring, lin, col: cint, - value: cstring) {. - cdecl, importc: "IupMatSetAttribute", dynlib: dllname.} -proc matStoreAttribute*(ih: PIhandle, name: cstring, lin, col: cint, - value: cstring) {.cdecl, - importc: "IupMatStoreAttribute", dynlib: dllname.} -proc matGetAttribute*(ih: PIhandle, name: cstring, lin, col: cint): cstring {. - cdecl, importc: "IupMatGetAttribute", dynlib: dllname.} -proc matGetInt*(ih: PIhandle, name: cstring, lin, col: cint): cint {. - cdecl, importc: "IupMatGetInt", dynlib: dllname.} -proc matGetFloat*(ih: PIhandle, name: cstring, lin, col: cint): cfloat {. - cdecl, importc: "IupMatGetFloat", dynlib: dllname.} -proc matSetfAttribute*(ih: PIhandle, name: cstring, lin, col: cint, - format: cstring) {.cdecl, - importc: "IupMatSetfAttribute", - dynlib: dllname, varargs.} - -# Used by IupColorbar -const - IUP_PRIMARY* = -1 - IUP_SECONDARY* = -2 - -# Initialize PPlot widget class -proc pPlotOpen*() {.cdecl, importc: "IupPPlotOpen", dynlib: dllname.} - -# Create an PPlot widget instance -proc pPlot*: PIhandle {.cdecl, importc: "IupPPlot", dynlib: dllname.} - -# Add dataset to plot -proc pPlotBegin*(ih: PIhandle, strXdata: cint) {. - cdecl, importc: "IupPPlotBegin", dynlib: dllname.} -proc pPlotAdd*(ih: PIhandle, x, y: cfloat) {. - cdecl, importc: "IupPPlotAdd", dynlib: dllname.} -proc pPlotAddStr*(ih: PIhandle, x: cstring, y: cfloat) {. - cdecl, importc: "IupPPlotAddStr", dynlib: dllname.} -proc pPlotEnd*(ih: PIhandle): cint {. - cdecl, importc: "IupPPlotEnd", dynlib: dllname.} - -proc pPlotInsertStr*(ih: PIhandle, index, sampleIndex: cint, x: cstring, - y: cfloat) {.cdecl, importc: "IupPPlotInsertStr", - dynlib: dllname.} -proc pPlotInsert*(ih: PIhandle, index, sampleIndex: cint, - x, y: cfloat) {. - cdecl, importc: "IupPPlotInsert", dynlib: dllname.} - -# convert from plot coordinates to pixels -proc pPlotTransform*(ih: PIhandle, x, y: cfloat, ix, iy: var cint) {. - cdecl, importc: "IupPPlotTransform", dynlib: dllname.} - -# Plot on the given device. Uses a "cdCanvas*". -proc pPlotPaintTo*(ih: PIhandle, cnv: pointer) {. - cdecl, importc: "IupPPlotPaintTo", dynlib: dllname.} - - diff --git a/lib/wrappers/linenoise/linenoise.c b/lib/wrappers/linenoise/linenoise.c index ae185fece..be792b96b 100644 --- a/lib/wrappers/linenoise/linenoise.c +++ b/lib/wrappers/linenoise/linenoise.c @@ -765,7 +765,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) { * when ctrl+d is typed. * * The function returns the length of the current buffer. */ -static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt, linenoiseData* data) { struct linenoiseState l; @@ -827,6 +827,7 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, return (int)l.len; case CTRL_C: /* ctrl-c */ errno = EAGAIN; + data->status = linenoiseStatus_ctrl_C; return -1; case BACKSPACE: /* backspace */ case 8: /* ctrl-h */ @@ -839,6 +840,7 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, } else { history_len--; free(history[history_len]); + data->status = linenoiseStatus_ctrl_D; return -1; } break; @@ -979,7 +981,7 @@ void linenoisePrintKeyCodes(void) { /* This function calls the line editing function linenoiseEdit() using * the STDIN file descriptor set in raw mode. */ -static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { +static int linenoiseRaw(char *buf, size_t buflen, const char *prompt, linenoiseData* data) { int count; if (buflen == 0) { @@ -988,7 +990,7 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { } if (enableRawMode(STDIN_FILENO) == -1) return -1; - count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); + count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt, data); disableRawMode(STDIN_FILENO); printf("\n"); return count; @@ -1035,7 +1037,7 @@ static char *linenoiseNoTTY(void) { * for a blacklist of stupid terminals, and later either calls the line * editing function or uses dummy fgets() so that you will be able to type * something even in the most desperate of the conditions. */ -char *linenoise(const char *prompt) { +char *linenoiseExtra(const char *prompt, linenoiseData* data) { char buf[LINENOISE_MAX_LINE]; int count; @@ -1056,12 +1058,18 @@ char *linenoise(const char *prompt) { } return strdup(buf); } else { - count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt); + count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt, data); if (count == -1) return NULL; return strdup(buf); } } +char *linenoise(const char *prompt) { + linenoiseData data; + data.status = linenoiseStatus_ctrl_unknown; + return linenoiseExtra(prompt, &data); +} + /* This is just a wrapper the user may want to call in order to make sure * the linenoise returned buffer is freed with the same allocator it was * created with. Useful when the main program is using an alternative diff --git a/lib/wrappers/linenoise/linenoise.h b/lib/wrappers/linenoise/linenoise.h index ed20232c5..aa86ccb78 100644 --- a/lib/wrappers/linenoise/linenoise.h +++ b/lib/wrappers/linenoise/linenoise.h @@ -48,6 +48,16 @@ typedef struct linenoiseCompletions { char **cvec; } linenoiseCompletions; +typedef enum linenoiseStatus { + linenoiseStatus_ctrl_unknown, + linenoiseStatus_ctrl_C, + linenoiseStatus_ctrl_D +} linenoiseStatus; + +typedef struct linenoiseData { + linenoiseStatus status; +} linenoiseData; + typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); typedef void(linenoiseFreeHintsCallback)(void *); @@ -57,6 +67,7 @@ void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); void linenoiseAddCompletion(linenoiseCompletions *, const char *); char *linenoise(const char *prompt); +char *linenoiseExtra(const char *prompt, linenoiseData* data); void linenoiseFree(void *ptr); int linenoiseHistoryAdd(const char *line); int linenoiseHistorySetMaxLen(int len); diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index a6260eb12..186b3b252 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -9,14 +9,14 @@ type Completions* = object - len*: csize + len*: csize_t cvec*: cstringArray CompletionCallback* = proc (a2: cstring; a3: ptr Completions) {.cdecl.} {.compile: "linenoise.c".} -proc setCompletionCallback*(a2: ptr CompletionCallback) {. +proc setCompletionCallback*(a2: CompletionCallback) {. importc: "linenoiseSetCompletionCallback".} proc addCompletion*(a2: ptr Completions; a3: cstring) {. importc: "linenoiseAddCompletion".} @@ -32,3 +32,41 @@ proc printKeyCodes*() {.importc: "linenoisePrintKeyCodes".} proc free*(s: cstring) {.importc: "free", header: "<stdlib.h>".} +when defined(nimExperimentalLinenoiseExtra) and not defined(windows): + # C interface + type LinenoiseStatus = enum + linenoiseStatus_ctrl_unknown + linenoiseStatus_ctrl_C + linenoiseStatus_ctrl_D + + type LinenoiseData* = object + status: LinenoiseStatus + + proc linenoiseExtra(prompt: cstring, data: ptr LinenoiseData): cstring {.importc.} + + # stable nim interface + type Status* = enum + lnCtrlUnkown + lnCtrlC + lnCtrlD + + type ReadLineResult* = object + line*: string + status*: Status + + proc readLineStatus*(prompt: string, result: var ReadLineResult) = + ## line editing API that allows returning the line entered and an indicator + ## of which control key was entered, allowing user to distinguish between + ## for example ctrl-C vs ctrl-D. + runnableExamples("-d:nimExperimentalLinenoiseExtra -r:off"): + var ret: ReadLineResult + while true: + readLineStatus("name: ", ret) # ctrl-D will exit, ctrl-C will go to next prompt + if ret.line.len > 0: echo ret.line + if ret.status == lnCtrlD: break + echo "exiting" + var data: LinenoiseData + let buf = linenoiseExtra(prompt, data.addr) + result.line = $buf + free(buf) + result.status = data.status.ord.Status diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim deleted file mode 100644 index f8d4739ae..000000000 --- a/lib/wrappers/mysql.nim +++ /dev/null @@ -1,1116 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} # dce option deprecated -{.push, callconv: cdecl.} -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -when defined(Unix): - when defined(macosx): - const - lib = "(libmysqlclient|libmariadbclient)(|.20|.19|.18|.17|.16|.15).dylib" - else: - const - lib = "(libmysqlclient|libmariadbclient).so(|.20|.19|.18|.17|.16|.15)" -when defined(Windows): - const - lib = "(libmysql.dll|libmariadb.dll)" -type - my_bool* = bool - Pmy_bool* = ptr my_bool - PVIO* = pointer - Pgptr* = ptr gptr - gptr* = cstring - Pmy_socket* = ptr my_socket - my_socket* = cint - PPByte* = pointer - cuint* = cint - -# ------------ Start of declaration in "mysql_com.h" --------------------- -# -# ** Common definition between mysql server & client -# -# Field/table name length - -const - NAME_LEN* = 64 - HOSTNAME_LENGTH* = 60 - USERNAME_LENGTH* = 16 - SERVER_VERSION_LENGTH* = 60 - SQLSTATE_LENGTH* = 5 - LOCAL_HOST* = "localhost" - LOCAL_HOST_NAMEDPIPE* = '.' - -const - NAMEDPIPE* = "MySQL" - SERVICENAME* = "MySQL" - -type - Enum_server_command* = enum - COM_SLEEP, COM_QUIT, COM_INIT_DB, COM_QUERY, COM_FIELD_LIST, COM_CREATE_DB, - COM_DROP_DB, COM_REFRESH, COM_SHUTDOWN, COM_STATISTICS, COM_PROCESS_INFO, - COM_CONNECT, COM_PROCESS_KILL, COM_DEBUG, COM_PING, COM_TIME, - COM_DELAYED_INSERT, COM_CHANGE_USER, COM_BINLOG_DUMP, COM_TABLE_DUMP, - COM_CONNECT_OUT, COM_REGISTER_SLAVE, COM_STMT_PREPARE, COM_STMT_EXECUTE, - COM_STMT_SEND_LONG_DATA, COM_STMT_CLOSE, COM_STMT_RESET, COM_SET_OPTION, - COM_STMT_FETCH, COM_END -{.deprecated: [Tenum_server_command: Enum_server_command].} - -const - SCRAMBLE_LENGTH* = 20 # Length of random string sent by server on handshake; - # this is also length of obfuscated password, - # received from client - SCRAMBLE_LENGTH_323* = 8 # length of password stored in the db: - # new passwords are preceded with '*' - SCRAMBLED_PASSWORD_CHAR_LENGTH* = SCRAMBLE_LENGTH * 2 + 1 - SCRAMBLED_PASSWORD_CHAR_LENGTH_323* = SCRAMBLE_LENGTH_323 * 2 - NOT_NULL_FLAG* = 1 # Field can't be NULL - PRI_KEY_FLAG* = 2 # Field is part of a primary key - UNIQUE_KEY_FLAG* = 4 # Field is part of a unique key - MULTIPLE_KEY_FLAG* = 8 # Field is part of a key - BLOB_FLAG* = 16 # Field is a blob - UNSIGNED_FLAG* = 32 # Field is unsigned - ZEROFILL_FLAG* = 64 # Field is zerofill - BINARY_FLAG* = 128 # Field is binary - # The following are only sent to new clients - ENUM_FLAG* = 256 # field is an enum - AUTO_INCREMENT_FLAG* = 512 # field is a autoincrement field - TIMESTAMP_FLAG* = 1024 # Field is a timestamp - SET_FLAG* = 2048 # field is a set - NO_DEFAULT_VALUE_FLAG* = 4096 # Field doesn't have default value - NUM_FLAG* = 32768 # Field is num (for clients) - PART_KEY_FLAG* = 16384 # Intern; Part of some key - GROUP_FLAG* = 32768 # Intern: Group field - UNIQUE_FLAG* = 65536 # Intern: Used by sql_yacc - BINCMP_FLAG* = 131072 # Intern: Used by sql_yacc - REFRESH_GRANT* = 1 # Refresh grant tables - REFRESH_LOG* = 2 # Start on new log file - REFRESH_TABLES* = 4 # close all tables - REFRESH_HOSTS* = 8 # Flush host cache - REFRESH_STATUS* = 16 # Flush status variables - REFRESH_THREADS* = 32 # Flush thread cache - REFRESH_SLAVE* = 64 # Reset master info and restart slave thread - REFRESH_MASTER* = 128 # Remove all bin logs in the index and truncate the index - # The following can't be set with mysql_refresh() - REFRESH_READ_LOCK* = 16384 # Lock tables for read - REFRESH_FAST* = 32768 # Intern flag - REFRESH_QUERY_CACHE* = 65536 # RESET (remove all queries) from query cache - REFRESH_QUERY_CACHE_FREE* = 0x00020000 # pack query cache - REFRESH_DES_KEY_FILE* = 0x00040000 - REFRESH_USER_RESOURCES* = 0x00080000 - CLIENT_LONG_PASSWORD* = 1 # new more secure passwords - CLIENT_FOUND_ROWS* = 2 # Found instead of affected rows - CLIENT_LONG_FLAG* = 4 # Get all column flags - CLIENT_CONNECT_WITH_DB* = 8 # One can specify db on connect - CLIENT_NO_SCHEMA* = 16 # Don't allow database.table.column - CLIENT_COMPRESS* = 32 # Can use compression protocol - CLIENT_ODBC* = 64 # Odbc client - CLIENT_LOCAL_FILES* = 128 # Can use LOAD DATA LOCAL - CLIENT_IGNORE_SPACE* = 256 # Ignore spaces before '(' - CLIENT_PROTOCOL_41* = 512 # New 4.1 protocol - CLIENT_INTERACTIVE* = 1024 # This is an interactive client - CLIENT_SSL* = 2048 # Switch to SSL after handshake - CLIENT_IGNORE_SIGPIPE* = 4096 # IGNORE sigpipes - CLIENT_TRANSACTIONS* = 8192 # Client knows about transactions - CLIENT_RESERVED* = 16384 # Old flag for 4.1 protocol - CLIENT_SECURE_CONNECTION* = 32768 # New 4.1 authentication - CLIENT_MULTI_STATEMENTS* = 65536 # Enable/disable multi-stmt support - CLIENT_MULTI_RESULTS* = 131072 # Enable/disable multi-results - CLIENT_REMEMBER_OPTIONS*: int = 1 shl 31 - SERVER_STATUS_IN_TRANS* = 1 # Transaction has started - SERVER_STATUS_AUTOCOMMIT* = 2 # Server in auto_commit mode - SERVER_STATUS_MORE_RESULTS* = 4 # More results on server - SERVER_MORE_RESULTS_EXISTS* = 8 # Multi query - next query exists - SERVER_QUERY_NO_GOOD_INDEX_USED* = 16 - SERVER_QUERY_NO_INDEX_USED* = 32 # The server was able to fulfill the clients request and opened a - # read-only non-scrollable cursor for a query. This flag comes - # in reply to COM_STMT_EXECUTE and COM_STMT_FETCH commands. - SERVER_STATUS_CURSOR_EXISTS* = 64 # This flag is sent when a read-only cursor is exhausted, in reply to - # COM_STMT_FETCH command. - SERVER_STATUS_LAST_ROW_SENT* = 128 - SERVER_STATUS_DB_DROPPED* = 256 # A database was dropped - SERVER_STATUS_NO_BACKSLASH_ESCAPES* = 512 - ERRMSG_SIZE* = 200 - NET_READ_TIMEOUT* = 30 # Timeout on read - NET_WRITE_TIMEOUT* = 60 # Timeout on write - NET_WAIT_TIMEOUT* = 8 * 60 * 60 # Wait for new query - ONLY_KILL_QUERY* = 1 - -const - MAX_TINYINT_WIDTH* = 3 # Max width for a TINY w.o. sign - MAX_SMALLINT_WIDTH* = 5 # Max width for a SHORT w.o. sign - MAX_MEDIUMINT_WIDTH* = 8 # Max width for a INT24 w.o. sign - MAX_INT_WIDTH* = 10 # Max width for a LONG w.o. sign - MAX_BIGINT_WIDTH* = 20 # Max width for a LONGLONG - MAX_CHAR_WIDTH* = 255 # Max length for a CHAR column - MAX_BLOB_WIDTH* = 8192 # Default width for blob - -type - Pst_net* = ptr St_net - St_net*{.final.} = object - vio*: PVIO - buff*: cstring - buff_end*: cstring - write_pos*: cstring - read_pos*: cstring - fd*: my_socket # For Perl DBI/dbd - max_packet*: int - max_packet_size*: int - pkt_nr*: cuint - compress_pkt_nr*: cuint - write_timeout*: cuint - read_timeout*: cuint - retry_count*: cuint - fcntl*: cint - compress*: my_bool # The following variable is set if we are doing several queries in one - # command ( as in LOAD TABLE ... FROM MASTER ), - # and do not want to confuse the client with OK at the wrong time - remain_in_buf*: int - len*: int - buf_length*: int - where_b*: int - return_status*: ptr cint - reading_or_writing*: char - save_char*: cchar - no_send_ok*: my_bool # For SPs and other things that do multiple stmts - no_send_eof*: my_bool # For SPs' first version read-only cursors - no_send_error*: my_bool # Set if OK packet is already sent, and - # we do not need to send error messages - # Pointer to query object in query cache, do not equal NULL (0) for - # queries in cache that have not stored its results yet - # $endif - last_error*: array[0..(ERRMSG_SIZE) - 1, char] - sqlstate*: array[0..(SQLSTATE_LENGTH + 1) - 1, char] - last_errno*: cuint - error*: char - query_cache_query*: gptr - report_error*: my_bool # We should report error (we have unreported error) - return_errno*: my_bool - - NET* = St_net - PNET* = ptr NET -{.deprecated: [Tst_net: St_net, TNET: NET].} - -const - packet_error* = - 1 - -type - Enum_field_types* = enum # For backward compatibility - TYPE_DECIMAL, TYPE_TINY, TYPE_SHORT, TYPE_LONG, TYPE_FLOAT, TYPE_DOUBLE, - TYPE_NULL, TYPE_TIMESTAMP, TYPE_LONGLONG, TYPE_INT24, TYPE_DATE, TYPE_TIME, - TYPE_DATETIME, TYPE_YEAR, TYPE_NEWDATE, TYPE_VARCHAR, TYPE_BIT, - TYPE_NEWDECIMAL = 246, TYPE_ENUM = 247, TYPE_SET = 248, - TYPE_TINY_BLOB = 249, TYPE_MEDIUM_BLOB = 250, TYPE_LONG_BLOB = 251, - TYPE_BLOB = 252, TYPE_VAR_STRING = 253, TYPE_STRING = 254, - TYPE_GEOMETRY = 255 -{.deprecated: [Tenum_field_types: Enum_field_types].} - -const - CLIENT_MULTI_QUERIES* = CLIENT_MULTI_STATEMENTS - FIELD_TYPE_DECIMAL* = TYPE_DECIMAL - FIELD_TYPE_NEWDECIMAL* = TYPE_NEWDECIMAL - FIELD_TYPE_TINY* = TYPE_TINY - FIELD_TYPE_SHORT* = TYPE_SHORT - FIELD_TYPE_LONG* = TYPE_LONG - FIELD_TYPE_FLOAT* = TYPE_FLOAT - FIELD_TYPE_DOUBLE* = TYPE_DOUBLE - FIELD_TYPE_NULL* = TYPE_NULL - FIELD_TYPE_TIMESTAMP* = TYPE_TIMESTAMP - FIELD_TYPE_LONGLONG* = TYPE_LONGLONG - FIELD_TYPE_INT24* = TYPE_INT24 - FIELD_TYPE_DATE* = TYPE_DATE - FIELD_TYPE_TIME* = TYPE_TIME - FIELD_TYPE_DATETIME* = TYPE_DATETIME - FIELD_TYPE_YEAR* = TYPE_YEAR - FIELD_TYPE_NEWDATE* = TYPE_NEWDATE - FIELD_TYPE_ENUM* = TYPE_ENUM - FIELD_TYPE_SET* = TYPE_SET - FIELD_TYPE_TINY_BLOB* = TYPE_TINY_BLOB - FIELD_TYPE_MEDIUM_BLOB* = TYPE_MEDIUM_BLOB - FIELD_TYPE_LONG_BLOB* = TYPE_LONG_BLOB - FIELD_TYPE_BLOB* = TYPE_BLOB - FIELD_TYPE_VAR_STRING* = TYPE_VAR_STRING - FIELD_TYPE_STRING* = TYPE_STRING - FIELD_TYPE_CHAR* = TYPE_TINY - FIELD_TYPE_INTERVAL* = TYPE_ENUM - FIELD_TYPE_GEOMETRY* = TYPE_GEOMETRY - FIELD_TYPE_BIT* = TYPE_BIT # Shutdown/kill enums and constants - # Bits for THD::killable. - SHUTDOWN_KILLABLE_CONNECT* = chr(1 shl 0) - SHUTDOWN_KILLABLE_TRANS* = chr(1 shl 1) - SHUTDOWN_KILLABLE_LOCK_TABLE* = chr(1 shl 2) - SHUTDOWN_KILLABLE_UPDATE* = chr(1 shl 3) - -type - Enum_shutdown_level* = enum - SHUTDOWN_DEFAULT = 0, SHUTDOWN_WAIT_CONNECTIONS = 1, - SHUTDOWN_WAIT_TRANSACTIONS = 2, SHUTDOWN_WAIT_UPDATES = 8, - SHUTDOWN_WAIT_ALL_BUFFERS = 16, SHUTDOWN_WAIT_CRITICAL_BUFFERS = 17, - KILL_QUERY = 254, KILL_CONNECTION = 255 - Enum_cursor_type* = enum # options for mysql_set_option - CURSOR_TYPE_NO_CURSOR = 0, CURSOR_TYPE_READ_ONLY = 1, - CURSOR_TYPE_FOR_UPDATE = 2, CURSOR_TYPE_SCROLLABLE = 4 - Enum_mysql_set_option* = enum - OPTION_MULTI_STATEMENTS_ON, OPTION_MULTI_STATEMENTS_OFF -{.deprecated: [Tenum_shutdown_level: Enum_shutdown_level, - Tenum_cursor_type: Enum_cursor_type, - Tenum_mysql_set_option: Enum_mysql_set_option].} - -proc my_net_init*(net: PNET, vio: PVIO): my_bool{.cdecl, dynlib: lib, - importc: "my_net_init".} -proc my_net_local_init*(net: PNET){.cdecl, dynlib: lib, - importc: "my_net_local_init".} -proc net_end*(net: PNET){.cdecl, dynlib: lib, importc: "net_end".} -proc net_clear*(net: PNET){.cdecl, dynlib: lib, importc: "net_clear".} -proc net_realloc*(net: PNET, len: int): my_bool{.cdecl, dynlib: lib, - importc: "net_realloc".} -proc net_flush*(net: PNET): my_bool{.cdecl, dynlib: lib, importc: "net_flush".} -proc my_net_write*(net: PNET, packet: cstring, length: int): my_bool{.cdecl, - dynlib: lib, importc: "my_net_write".} -proc net_write_command*(net: PNET, command: char, header: cstring, - head_len: int, packet: cstring, length: int): my_bool{. - cdecl, dynlib: lib, importc: "net_write_command".} -proc net_real_write*(net: PNET, packet: cstring, length: int): cint{.cdecl, - dynlib: lib, importc: "net_real_write".} -proc my_net_read*(net: PNET): int{.cdecl, dynlib: lib, importc: "my_net_read".} - # The following function is not meant for normal usage - # Currently it's used internally by manager.c -type - Psockaddr* = ptr Sockaddr - Sockaddr*{.final.} = object # undefined structure -{.deprecated: [Tsockaddr: Sockaddr].} - -proc my_connect*(s: my_socket, name: Psockaddr, namelen: cuint, timeout: cuint): cint{. - cdecl, dynlib: lib, importc: "my_connect".} -type - Prand_struct* = ptr Rand_struct - Rand_struct*{.final.} = object # The following is for user defined functions - seed1*: int - seed2*: int - max_value*: int - max_value_dbl*: cdouble - - Item_result* = enum - STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT, DECIMAL_RESULT - PItem_result* = ptr Item_result - Pst_udf_args* = ptr St_udf_args - St_udf_args*{.final.} = object - arg_count*: cuint # Number of arguments - arg_type*: PItem_result # Pointer to item_results - args*: cstringArray # Pointer to item_results - lengths*: ptr int # Length of string arguments - maybe_null*: cstring # Length of string arguments - attributes*: cstringArray # Pointer to attribute name - attribute_lengths*: ptr int # Length of attribute arguments - - UDF_ARGS* = St_udf_args - PUDF_ARGS* = ptr UDF_ARGS # This holds information about the result - Pst_udf_init* = ptr St_udf_init - St_udf_init*{.final.} = object - maybe_null*: my_bool # 1 if function can return NULL - decimals*: cuint # for real functions - max_length*: int # For string functions - theptr*: cstring # free pointer for function data - const_item*: my_bool # free pointer for function data - - UDF_INIT* = St_udf_init - PUDF_INIT* = ptr UDF_INIT # Constants when using compression -{.deprecated: [Trand_stuct: Rand_struct, TItem_result: Item_result, - Tst_udf_args: St_udf_args, TUDF_ARGS: UDF_ARGS, - Tst_udf_init: St_udf_init, TUDF_INIT: UDF_INIT].} - -const - NET_HEADER_SIZE* = 4 # standard header size - COMP_HEADER_SIZE* = 3 # compression header extra size - # Prototypes to password functions - # These functions are used for authentication by client and server and - # implemented in sql/password.c - -proc randominit*(para1: Prand_struct, seed1: int, seed2: int){.cdecl, - dynlib: lib, importc: "randominit".} -proc my_rnd*(para1: Prand_struct): cdouble{.cdecl, dynlib: lib, - importc: "my_rnd".} -proc create_random_string*(fto: cstring, len: cuint, rand_st: Prand_struct){. - cdecl, dynlib: lib, importc: "create_random_string".} -proc hash_password*(fto: int, password: cstring, password_len: cuint){.cdecl, - dynlib: lib, importc: "hash_password".} -proc make_scrambled_password_323*(fto: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "make_scrambled_password_323".} -proc scramble_323*(fto: cstring, message: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "scramble_323".} -proc check_scramble_323*(para1: cstring, message: cstring, salt: int): my_bool{. - cdecl, dynlib: lib, importc: "check_scramble_323".} -proc get_salt_from_password_323*(res: ptr int, password: cstring){.cdecl, - dynlib: lib, importc: "get_salt_from_password_323".} -proc make_password_from_salt_323*(fto: cstring, salt: ptr int){.cdecl, - dynlib: lib, importc: "make_password_from_salt_323".} -proc octet2hex*(fto: cstring, str: cstring, length: cuint): cstring{.cdecl, - dynlib: lib, importc: "octet2hex".} -proc make_scrambled_password*(fto: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "make_scrambled_password".} -proc scramble*(fto: cstring, message: cstring, password: cstring){.cdecl, - dynlib: lib, importc: "scramble".} -proc check_scramble*(reply: cstring, message: cstring, hash_stage2: pointer): my_bool{. - cdecl, dynlib: lib, importc: "check_scramble".} -proc get_salt_from_password*(res: pointer, password: cstring){.cdecl, - dynlib: lib, importc: "get_salt_from_password".} -proc make_password_from_salt*(fto: cstring, hash_stage2: pointer){.cdecl, - dynlib: lib, importc: "make_password_from_salt".} - # end of password.c -proc get_tty_password*(opt_message: cstring): cstring{.cdecl, dynlib: lib, - importc: "get_tty_password".} -proc errno_to_sqlstate*(errno: cuint): cstring{.cdecl, dynlib: lib, - importc: "mysql_errno_to_sqlstate".} - # Some other useful functions -proc modify_defaults_file*(file_location: cstring, option: cstring, - option_value: cstring, section_name: cstring, - remove_option: cint): cint{.cdecl, dynlib: lib, - importc: "load_defaults".} -proc load_defaults*(conf_file: cstring, groups: cstringArray, argc: ptr cint, - argv: ptr cstringArray): cint{.cdecl, dynlib: lib, - importc: "load_defaults".} -proc my_init*(): my_bool{.cdecl, dynlib: lib, importc: "my_init".} -proc my_thread_init*(): my_bool{.cdecl, dynlib: lib, importc: "my_thread_init".} -proc my_thread_end*(){.cdecl, dynlib: lib, importc: "my_thread_end".} -const - NULL_LENGTH*: int = int(not (0)) # For net_store_length - -const - STMT_HEADER* = 4 - LONG_DATA_HEADER* = 6 # ------------ Stop of declaration in "mysql_com.h" ----------------------- - # $include "mysql_time.h" - # $include "mysql_version.h" - # $include "typelib.h" - # $include "my_list.h" /* for LISTs used in 'MYSQL' and 'MYSQL_STMT' */ - # var - # mysql_port : cuint;cvar;external; - # mysql_unix_port : Pchar;cvar;external; - -const - CLIENT_NET_READ_TIMEOUT* = 365 * 24 * 3600 # Timeout on read - CLIENT_NET_WRITE_TIMEOUT* = 365 * 24 * 3600 # Timeout on write - -type - Pst_mysql_field* = ptr St_mysql_field - St_mysql_field*{.final.} = object - name*: cstring # Name of column - org_name*: cstring # Original column name, if an alias - table*: cstring # Table of column if column was a field - org_table*: cstring # Org table name, if table was an alias - db*: cstring # Database for table - catalog*: cstring # Catalog for table - def*: cstring # Default value (set by mysql_list_fields) - len*: int # Width of column (create length) - max_length*: int # Max width for selected set - name_length*: cuint - org_name_length*: cuint - table_length*: cuint - org_table_length*: cuint - db_length*: cuint - catalog_length*: cuint - def_length*: cuint - flags*: cuint # Div flags - decimals*: cuint # Number of decimals in field - charsetnr*: cuint # Character set - ftype*: Enum_field_types # Type of field. See mysql_com.h for types - extension*: pointer - - FIELD* = St_mysql_field - PFIELD* = ptr FIELD - PROW* = ptr ROW # return data as array of strings - ROW* = cstringArray - PFIELD_OFFSET* = ptr FIELD_OFFSET # offset to current field - FIELD_OFFSET* = cuint -{.deprecated: [Tst_mysql_field: St_mysql_field, TFIELD: FIELD, TROW: ROW, - TFIELD_OFFSET: FIELD_OFFSET].} - -proc IS_PRI_KEY*(n: int32): bool -proc IS_NOT_NULL*(n: int32): bool -proc IS_BLOB*(n: int32): bool -proc IS_NUM*(t: Enum_field_types): bool -proc INTERNAL_NUM_FIELD*(f: Pst_mysql_field): bool -proc IS_NUM_FIELD*(f: Pst_mysql_field): bool -type - my_ulonglong* = int64 - Pmy_ulonglong* = ptr my_ulonglong - -const - COUNT_ERROR* = not (my_ulonglong(0)) - -type - Pst_mysql_rows* = ptr St_mysql_rows - St_mysql_rows*{.final.} = object - next*: Pst_mysql_rows # list of rows - data*: ROW - len*: int - - ROWS* = St_mysql_rows - PROWS* = ptr ROWS - PROW_OFFSET* = ptr ROW_OFFSET # offset to current row - ROW_OFFSET* = ROWS -{.deprecated: [Tst_mysql_rows: St_mysql_rows, TROWS: ROWS, - TROW_OFFSET: ROW_OFFSET].} - -const - ALLOC_MAX_BLOCK_TO_DROP* = 4096 - ALLOC_MAX_BLOCK_USAGE_BEFORE_DROP* = 10 # struct for once_alloc (block) - -type - Pst_used_mem* = ptr St_used_mem - St_used_mem*{.final.} = object - next*: Pst_used_mem # Next block in use - left*: cuint # memory left in block - size*: cuint # size of block - - USED_MEM* = St_used_mem - PUSED_MEM* = ptr USED_MEM - Pst_mem_root* = ptr St_mem_root - St_mem_root*{.final.} = object - free*: PUSED_MEM # blocks with free memory in it - used*: PUSED_MEM # blocks almost without free memory - pre_alloc*: PUSED_MEM # preallocated block - min_malloc*: cuint # if block have less memory it will be put in 'used' list - block_size*: cuint # initial block size - block_num*: cuint # allocated blocks counter - # first free block in queue test counter (if it exceed - # MAX_BLOCK_USAGE_BEFORE_DROP block will be dropped in 'used' list) - first_block_usage*: cuint - error_handler*: proc (){.cdecl.} - - MEM_ROOT* = St_mem_root - PMEM_ROOT* = ptr MEM_ROOT # ------------ Stop of declaration in "my_alloc.h" ---------------------- -{.deprecated: [Tst_used_mem: St_used_mem, TUSED_MEM: USED_MEM, - Tst_mem_root: St_mem_root, TMEM_ROOT: MEM_ROOT].} - -type - Pst_mysql_data* = ptr St_mysql_data - St_mysql_data*{.final.} = object - rows*: my_ulonglong - fields*: cuint - data*: PROWS - alloc*: MEM_ROOT - prev_ptr*: ptr PROWS - - DATA* = St_mysql_data - PDATA* = ptr DATA - Option* = enum - OPT_CONNECT_TIMEOUT, OPT_COMPRESS, OPT_NAMED_PIPE, INIT_COMMAND, - READ_DEFAULT_FILE, READ_DEFAULT_GROUP, SET_CHARSET_DIR, SET_CHARSET_NAME, - OPT_LOCAL_INFILE, OPT_PROTOCOL, SHARED_MEMORY_BASE_NAME, OPT_READ_TIMEOUT, - OPT_WRITE_TIMEOUT, OPT_USE_RESULT, OPT_USE_REMOTE_CONNECTION, - OPT_USE_EMBEDDED_CONNECTION, OPT_GUESS_CONNECTION, SET_CLIENT_IP, - SECURE_AUTH, REPORT_DATA_TRUNCATION, OPT_RECONNECT -{.deprecated: [Tst_mysql_data: St_mysql_data, TDATA: DATA, Toption: Option].} - -const - MAX_MYSQL_MANAGER_ERR* = 256 - MAX_MYSQL_MANAGER_MSG* = 256 - MANAGER_OK* = 200 - MANAGER_INFO* = 250 - MANAGER_ACCESS* = 401 - MANAGER_CLIENT_ERR* = 450 - MANAGER_INTERNAL_ERR* = 500 - -type - St_dynamic_array*{.final.} = object - buffer*: cstring - elements*: cuint - max_element*: cuint - alloc_increment*: cuint - size_of_element*: cuint - - DYNAMIC_ARRAY* = St_dynamic_array - Pst_dynamic_array* = ptr St_dynamic_array - Pst_mysql_options* = ptr St_mysql_options - St_mysql_options*{.final.} = object - connect_timeout*: cuint - read_timeout*: cuint - write_timeout*: cuint - port*: cuint - protocol*: cuint - client_flag*: int - host*: cstring - user*: cstring - password*: cstring - unix_socket*: cstring - db*: cstring - init_commands*: Pst_dynamic_array - my_cnf_file*: cstring - my_cnf_group*: cstring - charset_dir*: cstring - charset_name*: cstring - ssl_key*: cstring # PEM key file - ssl_cert*: cstring # PEM cert file - ssl_ca*: cstring # PEM CA file - ssl_capath*: cstring # PEM directory of CA-s? - ssl_cipher*: cstring # cipher to use - shared_memory_base_name*: cstring - max_allowed_packet*: int - use_ssl*: my_bool # if to use SSL or not - compress*: my_bool - named_pipe*: my_bool # On connect, find out the replication role of the server, and - # establish connections to all the peers - rpl_probe*: my_bool # Each call to mysql_real_query() will parse it to tell if it is a read - # or a write, and direct it to the slave or the master - rpl_parse*: my_bool # If set, never read from a master, only from slave, when doing - # a read that is replication-aware - no_master_reads*: my_bool - separate_thread*: my_bool - methods_to_use*: Option - client_ip*: cstring - secure_auth*: my_bool # Refuse client connecting to server if it uses old (pre-4.1.1) protocol - report_data_truncation*: my_bool # 0 - never report, 1 - always report (default) - # function pointers for local infile support - local_infile_init*: proc (para1: var pointer, para2: cstring, para3: pointer): cint{. - cdecl.} - local_infile_read*: proc (para1: pointer, para2: cstring, para3: cuint): cint - local_infile_end*: proc (para1: pointer) - local_infile_error*: proc (para1: pointer, para2: cstring, para3: cuint): cint - local_infile_userdata*: pointer - - Status* = enum - STATUS_READY, STATUS_GET_RESULT, STATUS_USE_RESULT - Protocol_type* = enum # There are three types of queries - the ones that have to go to - # the master, the ones that go to a slave, and the administrative - # type which must happen on the pivot connection - PROTOCOL_DEFAULT, PROTOCOL_TCP, PROTOCOL_SOCKET, PROTOCOL_PIPE, - PROTOCOL_MEMORY - Rpl_type* = enum - RPL_MASTER, RPL_SLAVE, RPL_ADMIN - Charset_info_st*{.final.} = object - number*: cuint - primary_number*: cuint - binary_number*: cuint - state*: cuint - csname*: cstring - name*: cstring - comment*: cstring - tailoring*: cstring - ftype*: cstring - to_lower*: cstring - to_upper*: cstring - sort_order*: cstring - contractions*: ptr int16 - sort_order_big*: ptr ptr int16 - tab_to_uni*: ptr int16 - tab_from_uni*: pointer # was ^MY_UNI_IDX - state_map*: cstring - ident_map*: cstring - strxfrm_multiply*: cuint - mbminlen*: cuint - mbmaxlen*: cuint - min_sort_char*: int16 - max_sort_char*: int16 - escape_with_backslash_is_dangerous*: my_bool - cset*: pointer # was ^MY_CHARSET_HANDLER - coll*: pointer # was ^MY_COLLATION_HANDLER; - - CHARSET_INFO* = Charset_info_st - Pcharset_info_st* = ptr Charset_info_st - Pcharacter_set* = ptr Character_set - Character_set*{.final.} = object - number*: cuint - state*: cuint - csname*: cstring - name*: cstring - comment*: cstring - dir*: cstring - mbminlen*: cuint - mbmaxlen*: cuint - - MY_CHARSET_INFO* = Character_set - PMY_CHARSET_INFO* = ptr MY_CHARSET_INFO - Pst_mysql_methods* = ptr St_mysql_methods - Pst_mysql* = ptr St_mysql - St_mysql*{.final.} = object - net*: NET # Communication parameters - connector_fd*: gptr # ConnectorFd for SSL - host*: cstring - user*: cstring - passwd*: cstring - unix_socket*: cstring - server_version*: cstring - host_info*: cstring - info*: cstring - db*: cstring - charset*: Pcharset_info_st - fields*: PFIELD - field_alloc*: MEM_ROOT - affected_rows*: my_ulonglong - insert_id*: my_ulonglong # id if insert on table with NEXTNR - extra_info*: my_ulonglong # Used by mysqlshow, not used by mysql 5.0 and up - thread_id*: int # Id for connection in server - packet_length*: int - port*: cuint - client_flag*: int - server_capabilities*: int - protocol_version*: cuint - field_count*: cuint - server_status*: cuint - server_language*: cuint - warning_count*: cuint - options*: St_mysql_options - status*: Status - free_me*: my_bool # If free in mysql_close - reconnect*: my_bool # set to 1 if automatic reconnect - scramble*: array[0..(SCRAMBLE_LENGTH + 1) - 1, char] # session-wide random string - # Set if this is the original connection, not a master or a slave we have - # added though mysql_rpl_probe() or mysql_set_master()/ mysql_add_slave() - rpl_pivot*: my_bool # Pointers to the master, and the next slave connections, points to - # itself if lone connection. - master*: Pst_mysql - next_slave*: Pst_mysql - last_used_slave*: Pst_mysql # needed for round-robin slave pick - last_used_con*: Pst_mysql # needed for send/read/store/use result to work correctly with replication - stmts*: pointer # was PList, list of all statements - methods*: Pst_mysql_methods - thd*: pointer # Points to boolean flag in MYSQL_RES or MYSQL_STMT. We set this flag - # from mysql_stmt_close if close had to cancel result set of this object. - unbuffered_fetch_owner*: Pmy_bool - - MySQL* = St_mysql - PMySQL* = ptr MySQL - Pst_mysql_res* = ptr St_mysql_res - St_mysql_res*{.final.} = object - row_count*: my_ulonglong - fields*: PFIELD - data*: PDATA - data_cursor*: PROWS - lengths*: ptr int # column lengths of current row - handle*: PMySQL # for unbuffered reads - field_alloc*: MEM_ROOT - field_count*: cuint - current_field*: cuint - row*: ROW # If unbuffered read - current_row*: ROW # buffer to current row - eof*: my_bool # Used by mysql_fetch_row - unbuffered_fetch_cancelled*: my_bool # mysql_stmt_close() had to cancel this result - methods*: Pst_mysql_methods - - RES* = St_mysql_res - PRES* = ptr RES - Pst_mysql_stmt* = ptr St_mysql_stmt - PSTMT* = ptr STMT - St_mysql_methods*{.final.} = object - read_query_result*: proc (MySQL: PMySQL): my_bool{.cdecl.} - advanced_command*: proc (MySQL: PMySQL, command: Enum_server_command, header: cstring, - header_length: int, arg: cstring, arg_length: int, - skip_check: my_bool): my_bool - read_rows*: proc (MySQL: PMySQL, fields: PFIELD, fields_count: cuint): PDATA - use_result*: proc (MySQL: PMySQL): PRES - fetch_lengths*: proc (fto: ptr int, column: ROW, field_count: cuint) - flush_use_result*: proc (MySQL: PMySQL) - list_fields*: proc (MySQL: PMySQL): PFIELD - read_prepare_result*: proc (MySQL: PMySQL, stmt: PSTMT): my_bool - stmt_execute*: proc (stmt: PSTMT): cint - read_binary_rows*: proc (stmt: PSTMT): cint - unbuffered_fetch*: proc (MySQL: PMySQL, row: cstringArray): cint - free_embedded_thd*: proc (MySQL: PMySQL) - read_statistics*: proc (MySQL: PMySQL): cstring - next_result*: proc (MySQL: PMySQL): my_bool - read_change_user_result*: proc (MySQL: PMySQL, buff: cstring, passwd: cstring): cint - read_rowsfrom_cursor*: proc (stmt: PSTMT): cint - - METHODS* = St_mysql_methods - PMETHODS* = ptr METHODS - Pst_mysql_manager* = ptr St_mysql_manager - St_mysql_manager*{.final.} = object - net*: NET - host*: cstring - user*: cstring - passwd*: cstring - port*: cuint - free_me*: my_bool - eof*: my_bool - cmd_status*: cint - last_errno*: cint - net_buf*: cstring - net_buf_pos*: cstring - net_data_end*: cstring - net_buf_size*: cint - last_error*: array[0..(MAX_MYSQL_MANAGER_ERR) - 1, char] - - MANAGER* = St_mysql_manager - PMANAGER* = ptr MANAGER - Pst_mysql_parameters* = ptr St_mysql_parameters - St_mysql_parameters*{.final.} = object - p_max_allowed_packet*: ptr int - p_net_buffer_length*: ptr int - - PARAMETERS* = St_mysql_parameters - PPARAMETERS* = ptr PARAMETERS - Enum_mysql_stmt_state* = enum - STMT_INIT_DONE = 1, STMT_PREPARE_DONE, STMT_EXECUTE_DONE, STMT_FETCH_DONE - Pst_mysql_bind* = ptr St_mysql_bind - St_mysql_bind*{.final.} = object - len*: int # output length pointer - is_null*: Pmy_bool # Pointer to null indicator - buffer*: pointer # buffer to get/put data - error*: Pmy_bool # set this if you want to track data truncations happened during fetch - buffer_type*: Enum_field_types # buffer type - buffer_length*: int # buffer length, must be set for str/binary - # Following are for internal use. Set by mysql_stmt_bind_param - row_ptr*: ptr byte # for the current data position - offset*: int # offset position for char/binary fetch - length_value*: int # Used if length is 0 - param_number*: cuint # For null count and error messages - pack_length*: cuint # Internal length for packed data - error_value*: my_bool # used if error is 0 - is_unsigned*: my_bool # set if integer type is unsigned - long_data_used*: my_bool # If used with mysql_send_long_data - is_null_value*: my_bool # Used if is_null is 0 - store_param_func*: proc (net: PNET, param: Pst_mysql_bind){.cdecl.} - fetch_result*: proc (para1: Pst_mysql_bind, para2: PFIELD, row: PPByte) - skip_result*: proc (para1: Pst_mysql_bind, para2: PFIELD, row: PPByte) - - BIND* = St_mysql_bind - PBIND* = ptr BIND # statement handler - St_mysql_stmt*{.final.} = object - mem_root*: MEM_ROOT # root allocations - mysql*: PMySQL # connection handle - params*: PBIND # input parameters - `bind`*: PBIND # input parameters - fields*: PFIELD # result set metadata - result*: DATA # cached result set - data_cursor*: PROWS # current row in cached result - affected_rows*: my_ulonglong # copy of mysql->affected_rows after statement execution - insert_id*: my_ulonglong - read_row_func*: proc (stmt: Pst_mysql_stmt, row: PPByte): cint{.cdecl.} - stmt_id*: int # Id for prepared statement - flags*: int # i.e. type of cursor to open - prefetch_rows*: int # number of rows per one COM_FETCH - server_status*: cuint # Copied from mysql->server_status after execute/fetch to know - # server-side cursor status for this statement. - last_errno*: cuint # error code - param_count*: cuint # input parameter count - field_count*: cuint # number of columns in result set - state*: Enum_mysql_stmt_state # statement state - last_error*: array[0..(ERRMSG_SIZE) - 1, char] # error message - sqlstate*: array[0..(SQLSTATE_LENGTH + 1) - 1, char] - send_types_to_server*: my_bool # Types of input parameters should be sent to server - bind_param_done*: my_bool # input buffers were supplied - bind_result_done*: char # output buffers were supplied - unbuffered_fetch_cancelled*: my_bool - update_max_length*: my_bool - - STMT* = St_mysql_stmt - - Enum_stmt_attr_type* = enum - STMT_ATTR_UPDATE_MAX_LENGTH, STMT_ATTR_CURSOR_TYPE, STMT_ATTR_PREFETCH_ROWS -{.deprecated: [Tst_dynamic_array: St_dynamic_array, Tst_mysql_options: St_mysql_options, - TDYNAMIC_ARRAY: DYNAMIC_ARRAY, Tprotocol_type: Protocol_type, - Trpl_type: Rpl_type, Tcharset_info_st: Charset_info_st, - TCHARSET_INFO: CHARSET_INFO, Tcharacter_set: Character_set, - TMY_CHARSET_INFO: MY_CHARSET_INFO, Tst_mysql: St_mysql, - Tst_mysql_methods: St_mysql_methods, TMySql: MySql, - Tst_mysql_res: St_mysql_res, TMETHODS: METHODS, TRES: RES, - Tst_mysql_manager: St_mysql_manager, TMANAGER: MANAGER, - Tst_mysql_parameters: St_mysql_parameters, TPARAMETERS: PARAMETERS, - Tenum_mysql_stmt_state: Enum_mysql_stmt_state, - Tst_mysql_bind: St_mysql_bind, TBIND: BIND, Tst_mysql_stmt: St_mysql_stmt, - TSTMT: STMT, Tenum_stmt_attr_type: Enum_stmt_attr_type, - Tstatus: Status].} - -proc server_init*(argc: cint, argv: cstringArray, groups: cstringArray): cint{. - cdecl, dynlib: lib, importc: "mysql_server_init".} -proc server_end*(){.cdecl, dynlib: lib, importc: "mysql_server_end".} - # mysql_server_init/end need to be called when using libmysqld or - # libmysqlclient (exactly, mysql_server_init() is called by mysql_init() so - # you don't need to call it explicitly; but you need to call - # mysql_server_end() to free memory). The names are a bit misleading - # (mysql_SERVER* to be used when using libmysqlCLIENT). So we add more general - # names which suit well whether you're using libmysqld or libmysqlclient. We - # intend to promote these aliases over the mysql_server* ones. -proc library_init*(argc: cint, argv: cstringArray, groups: cstringArray): cint{. - cdecl, dynlib: lib, importc: "mysql_server_init".} -proc library_end*(){.cdecl, dynlib: lib, importc: "mysql_server_end".} -proc get_parameters*(): PPARAMETERS{.stdcall, dynlib: lib, - importc: "mysql_get_parameters".} - # Set up and bring down a thread; these function should be called - # for each thread in an application which opens at least one MySQL - # connection. All uses of the connection(s) should be between these - # function calls. -proc thread_init*(): my_bool{.stdcall, dynlib: lib, importc: "mysql_thread_init".} -proc thread_end*(){.stdcall, dynlib: lib, importc: "mysql_thread_end".} - # Functions to get information from the MYSQL and MYSQL_RES structures - # Should definitely be used if one uses shared libraries. -proc num_rows*(res: PRES): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_num_rows".} -proc num_fields*(res: PRES): cuint{.stdcall, dynlib: lib, - importc: "mysql_num_fields".} -proc eof*(res: PRES): my_bool{.stdcall, dynlib: lib, importc: "mysql_eof".} -proc fetch_field_direct*(res: PRES, fieldnr: cuint): PFIELD{.stdcall, - dynlib: lib, importc: "mysql_fetch_field_direct".} -proc fetch_fields*(res: PRES): PFIELD{.stdcall, dynlib: lib, - importc: "mysql_fetch_fields".} -proc row_tell*(res: PRES): ROW_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_row_tell".} -proc field_tell*(res: PRES): FIELD_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_field_tell".} -proc field_count*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_field_count".} -proc affected_rows*(MySQL: PMySQL): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_affected_rows".} -proc insert_id*(MySQL: PMySQL): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_insert_id".} -proc errno*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, importc: "mysql_errno".} -proc error*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_error".} -proc sqlstate*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_sqlstate".} -proc warning_count*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_warning_count".} -proc info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_info".} -proc thread_id*(MySQL: PMySQL): int{.stdcall, dynlib: lib, importc: "mysql_thread_id".} -proc character_set_name*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_character_set_name".} -proc set_character_set*(MySQL: PMySQL, csname: cstring): int32{.stdcall, dynlib: lib, - importc: "mysql_set_character_set".} -proc init*(MySQL: PMySQL): PMySQL{.stdcall, dynlib: lib, importc: "mysql_init".} -proc ssl_set*(MySQL: PMySQL, key: cstring, cert: cstring, ca: cstring, capath: cstring, - cipher: cstring): my_bool{.stdcall, dynlib: lib, - importc: "mysql_ssl_set".} -proc change_user*(MySQL: PMySQL, user: cstring, passwd: cstring, db: cstring): my_bool{. - stdcall, dynlib: lib, importc: "mysql_change_user".} -proc real_connect*(MySQL: PMySQL, host: cstring, user: cstring, passwd: cstring, - db: cstring, port: cuint, unix_socket: cstring, - clientflag: int): PMySQL{.stdcall, dynlib: lib, - importc: "mysql_real_connect".} -proc select_db*(MySQL: PMySQL, db: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_select_db".} -proc query*(MySQL: PMySQL, q: cstring): cint{.stdcall, dynlib: lib, importc: "mysql_query".} -proc send_query*(MySQL: PMySQL, q: cstring, len: int): cint{.stdcall, dynlib: lib, - importc: "mysql_send_query".} -proc real_query*(MySQL: PMySQL, q: cstring, len: int): cint{.stdcall, dynlib: lib, - importc: "mysql_real_query".} -proc store_result*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, - importc: "mysql_store_result".} -proc use_result*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, importc: "mysql_use_result".} - # perform query on master -proc master_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_master_query".} -proc master_send_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, - dynlib: lib, importc: "mysql_master_send_query".} - # perform query on slave -proc slave_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_slave_query".} -proc slave_send_query*(MySQL: PMySQL, q: cstring, len: int): my_bool{.stdcall, - dynlib: lib, importc: "mysql_slave_send_query".} -proc get_character_set_info*(MySQL: PMySQL, charset: PMY_CHARSET_INFO){.stdcall, - dynlib: lib, importc: "mysql_get_character_set_info".} - # local infile support -const - LOCAL_INFILE_ERROR_LEN* = 512 - -# procedure mysql_set_local_infile_handler(mysql:PMYSQL; local_infile_init:function (para1:Ppointer; para2:Pchar; para3:pointer):longint; local_infile_read:function (para1:pointer; para2:Pchar; para3:dword):longint; local_infile_end:procedure (_pa -# para6:pointer);cdecl;external mysqllib name 'mysql_set_local_infile_handler'; - -proc set_local_infile_default*(MySQL: PMySQL){.cdecl, dynlib: lib, - importc: "mysql_set_local_infile_default".} - # enable/disable parsing of all queries to decide if they go on master or - # slave -proc enable_rpl_parse*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_enable_rpl_parse".} -proc disable_rpl_parse*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_disable_rpl_parse".} - # get the value of the parse flag -proc rpl_parse_enabled*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, - importc: "mysql_rpl_parse_enabled".} - # enable/disable reads from master -proc enable_reads_from_master*(MySQL: PMySQL){.stdcall, dynlib: lib, - importc: "mysql_enable_reads_from_master".} -proc disable_reads_from_master*(MySQL: PMySQL){.stdcall, dynlib: lib, importc: "mysql_disable_reads_from_master".} - # get the value of the master read flag -proc reads_from_master_enabled*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_reads_from_master_enabled".} -proc rpl_query_type*(q: cstring, length: cint): Rpl_type{.stdcall, dynlib: lib, - importc: "mysql_rpl_query_type".} - # discover the master and its slaves -proc rpl_probe*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_rpl_probe".} - # set the master, close/free the old one, if it is not a pivot -proc set_master*(MySQL: PMySQL, host: cstring, port: cuint, user: cstring, passwd: cstring): cint{. - stdcall, dynlib: lib, importc: "mysql_set_master".} -proc add_slave*(MySQL: PMySQL, host: cstring, port: cuint, user: cstring, passwd: cstring): cint{. - stdcall, dynlib: lib, importc: "mysql_add_slave".} -proc shutdown*(MySQL: PMySQL, shutdown_level: Enum_shutdown_level): cint{.stdcall, - dynlib: lib, importc: "mysql_shutdown".} -proc dump_debug_info*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, - importc: "mysql_dump_debug_info".} -proc refresh*(sql: PMySQL, refresh_options: cuint): cint{.stdcall, dynlib: lib, - importc: "mysql_refresh".} -proc kill*(MySQL: PMySQL, pid: int): cint{.stdcall, dynlib: lib, importc: "mysql_kill".} -proc set_server_option*(MySQL: PMySQL, option: Enum_mysql_set_option): cint{.stdcall, - dynlib: lib, importc: "mysql_set_server_option".} -proc ping*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, importc: "mysql_ping".} -proc stat*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, importc: "mysql_stat".} -proc get_server_info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_server_info".} -proc get_client_info*(): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_client_info".} -proc get_client_version*(): int{.stdcall, dynlib: lib, - importc: "mysql_get_client_version".} -proc get_host_info*(MySQL: PMySQL): cstring{.stdcall, dynlib: lib, - importc: "mysql_get_host_info".} -proc get_server_version*(MySQL: PMySQL): int{.stdcall, dynlib: lib, - importc: "mysql_get_server_version".} -proc get_proto_info*(MySQL: PMySQL): cuint{.stdcall, dynlib: lib, - importc: "mysql_get_proto_info".} -proc list_dbs*(MySQL: PMySQL, wild: cstring): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_dbs".} -proc list_tables*(MySQL: PMySQL, wild: cstring): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_tables".} -proc list_processes*(MySQL: PMySQL): PRES{.stdcall, dynlib: lib, - importc: "mysql_list_processes".} -proc options*(MySQL: PMySQL, option: Option, arg: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_options".} -proc free_result*(result: PRES){.stdcall, dynlib: lib, - importc: "mysql_free_result".} -proc data_seek*(result: PRES, offset: my_ulonglong){.stdcall, dynlib: lib, - importc: "mysql_data_seek".} -proc row_seek*(result: PRES, offset: ROW_OFFSET): ROW_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_row_seek".} -proc field_seek*(result: PRES, offset: FIELD_OFFSET): FIELD_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_field_seek".} -proc fetch_row*(result: PRES): ROW{.stdcall, dynlib: lib, - importc: "mysql_fetch_row".} -proc fetch_lengths*(result: PRES): ptr int{.stdcall, dynlib: lib, - importc: "mysql_fetch_lengths".} -proc fetch_field*(result: PRES): PFIELD{.stdcall, dynlib: lib, - importc: "mysql_fetch_field".} -proc list_fields*(MySQL: PMySQL, table: cstring, wild: cstring): PRES{.stdcall, - dynlib: lib, importc: "mysql_list_fields".} -proc escape_string*(fto: cstring, `from`: cstring, from_length: int): int{. - stdcall, dynlib: lib, importc: "mysql_escape_string".} -proc hex_string*(fto: cstring, `from`: cstring, from_length: int): int{.stdcall, - dynlib: lib, importc: "mysql_hex_string".} -proc real_escape_string*(MySQL: PMySQL, fto: cstring, `from`: cstring, len: int): int{. - stdcall, dynlib: lib, importc: "mysql_real_escape_string".} -proc debug*(debug: cstring){.stdcall, dynlib: lib, importc: "mysql_debug".} - # function mysql_odbc_escape_string(mysql:PMYSQL; fto:Pchar; to_length:dword; from:Pchar; from_length:dword; - # param:pointer; extend_buffer:function (para1:pointer; to:Pchar; length:Pdword):Pchar):Pchar;stdcall;external mysqllib name 'mysql_odbc_escape_string'; -proc myodbc_remove_escape*(MySQL: PMySQL, name: cstring){.stdcall, dynlib: lib, - importc: "myodbc_remove_escape".} -proc thread_safe*(): cuint{.stdcall, dynlib: lib, importc: "mysql_thread_safe".} -proc embedded*(): my_bool{.stdcall, dynlib: lib, importc: "mysql_embedded".} -proc manager_init*(con: PMANAGER): PMANAGER{.stdcall, dynlib: lib, - importc: "mysql_manager_init".} -proc manager_connect*(con: PMANAGER, host: cstring, user: cstring, - passwd: cstring, port: cuint): PMANAGER{.stdcall, - dynlib: lib, importc: "mysql_manager_connect".} -proc manager_close*(con: PMANAGER){.stdcall, dynlib: lib, - importc: "mysql_manager_close".} -proc manager_command*(con: PMANAGER, cmd: cstring, cmd_len: cint): cint{. - stdcall, dynlib: lib, importc: "mysql_manager_command".} -proc manager_fetch_line*(con: PMANAGER, res_buf: cstring, res_buf_size: cint): cint{. - stdcall, dynlib: lib, importc: "mysql_manager_fetch_line".} -proc read_query_result*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_read_query_result".} -proc stmt_init*(MySQL: PMySQL): PSTMT{.stdcall, dynlib: lib, importc: "mysql_stmt_init".} -proc stmt_prepare*(stmt: PSTMT, query: cstring, len: int): cint{.stdcall, - dynlib: lib, importc: "mysql_stmt_prepare".} -proc stmt_execute*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_execute".} -proc stmt_fetch*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_fetch".} -proc stmt_fetch_column*(stmt: PSTMT, `bind`: PBIND, column: cuint, offset: int): cint{. - stdcall, dynlib: lib, importc: "mysql_stmt_fetch_column".} -proc stmt_store_result*(stmt: PSTMT): cint{.stdcall, dynlib: lib, - importc: "mysql_stmt_store_result".} -proc stmt_param_count*(stmt: PSTMT): int{.stdcall, dynlib: lib, - importc: "mysql_stmt_param_count".} -proc stmt_attr_set*(stmt: PSTMT, attr_type: Enum_stmt_attr_type, attr: pointer): my_bool{. - stdcall, dynlib: lib, importc: "mysql_stmt_attr_set".} -proc stmt_attr_get*(stmt: PSTMT, attr_type: Enum_stmt_attr_type, attr: pointer): my_bool{. - stdcall, dynlib: lib, importc: "mysql_stmt_attr_get".} -proc stmt_bind_param*(stmt: PSTMT, bnd: PBIND): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_bind_param".} -proc stmt_bind_result*(stmt: PSTMT, bnd: PBIND): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_bind_result".} -proc stmt_close*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_close".} -proc stmt_reset*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_reset".} -proc stmt_free_result*(stmt: PSTMT): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_free_result".} -proc stmt_send_long_data*(stmt: PSTMT, param_number: cuint, data: cstring, - len: int): my_bool{.stdcall, dynlib: lib, - importc: "mysql_stmt_send_long_data".} -proc stmt_result_metadata*(stmt: PSTMT): PRES{.stdcall, dynlib: lib, - importc: "mysql_stmt_result_metadata".} -proc stmt_param_metadata*(stmt: PSTMT): PRES{.stdcall, dynlib: lib, - importc: "mysql_stmt_param_metadata".} -proc stmt_errno*(stmt: PSTMT): cuint{.stdcall, dynlib: lib, - importc: "mysql_stmt_errno".} -proc stmt_error*(stmt: PSTMT): cstring{.stdcall, dynlib: lib, - importc: "mysql_stmt_error".} -proc stmt_sqlstate*(stmt: PSTMT): cstring{.stdcall, dynlib: lib, - importc: "mysql_stmt_sqlstate".} -proc stmt_row_seek*(stmt: PSTMT, offset: ROW_OFFSET): ROW_OFFSET{.stdcall, - dynlib: lib, importc: "mysql_stmt_row_seek".} -proc stmt_row_tell*(stmt: PSTMT): ROW_OFFSET{.stdcall, dynlib: lib, - importc: "mysql_stmt_row_tell".} -proc stmt_data_seek*(stmt: PSTMT, offset: my_ulonglong){.stdcall, dynlib: lib, - importc: "mysql_stmt_data_seek".} -proc stmt_num_rows*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_num_rows".} -proc stmt_affected_rows*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_affected_rows".} -proc stmt_insert_id*(stmt: PSTMT): my_ulonglong{.stdcall, dynlib: lib, - importc: "mysql_stmt_insert_id".} -proc stmt_field_count*(stmt: PSTMT): cuint{.stdcall, dynlib: lib, - importc: "mysql_stmt_field_count".} -proc commit*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_commit".} -proc rollback*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, importc: "mysql_rollback".} -proc autocommit*(MySQL: PMySQL, auto_mode: my_bool): my_bool{.stdcall, dynlib: lib, - importc: "mysql_autocommit".} -proc more_results*(MySQL: PMySQL): my_bool{.stdcall, dynlib: lib, - importc: "mysql_more_results".} -proc next_result*(MySQL: PMySQL): cint{.stdcall, dynlib: lib, importc: "mysql_next_result".} -proc close*(sock: PMySQL){.stdcall, dynlib: lib, importc: "mysql_close".} - # status return codes -const - NO_DATA* = 100 - DATA_TRUNCATED* = 101 - -proc reload*(x: PMySQL): cint -when defined(USE_OLD_FUNCTIONS): - proc connect*(MySQL: PMySQL, host: cstring, user: cstring, passwd: cstring): PMySQL{.stdcall, - dynlib: lib, importc: "mysql_connect".} - proc create_db*(MySQL: PMySQL, DB: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_create_db".} - proc drop_db*(MySQL: PMySQL, DB: cstring): cint{.stdcall, dynlib: lib, - importc: "mysql_drop_db".} -proc net_safe_read*(MySQL: PMySQL): cuint{.cdecl, dynlib: lib, importc: "net_safe_read".} - -proc IS_PRI_KEY(n: int32): bool = - result = (n and PRI_KEY_FLAG) != 0 - -proc IS_NOT_NULL(n: int32): bool = - result = (n and NOT_NULL_FLAG) != 0 - -proc IS_BLOB(n: int32): bool = - result = (n and BLOB_FLAG) != 0 - -proc IS_NUM_FIELD(f: Pst_mysql_field): bool = - result = (f.flags and NUM_FLAG) != 0 - -proc IS_NUM(t: Enum_field_types): bool = - result = (t <= FIELD_TYPE_INT24) or (t == FIELD_TYPE_YEAR) or - (t == FIELD_TYPE_NEWDECIMAL) - -proc INTERNAL_NUM_FIELD(f: Pst_mysql_field): bool = - result = (f.ftype <= FIELD_TYPE_INT24) and - ((f.ftype != FIELD_TYPE_TIMESTAMP) or (f.len == 14) or (f.len == 8)) or - (f.ftype == FIELD_TYPE_YEAR) - -proc reload(x: PMySQL): cint = - result = refresh(x, REFRESH_GRANT) - -{.pop.} -when defined(nimHasStyleChecks): - {.pop.} diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim deleted file mode 100644 index 14dd10ae3..000000000 --- a/lib/wrappers/odbcsql.nim +++ /dev/null @@ -1,841 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} # dce option deprecated - -when not defined(ODBCVER): - const - ODBCVER = 0x0351 ## define ODBC version 3.51 by default - -when defined(windows): - {.push callconv: stdcall.} - const odbclib = "odbc32.dll" -else: - {.push callconv: cdecl.} - const odbclib = "libodbc.so" - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -# DATA TYPES CORRESPONDENCE -# BDE fields ODBC types -# ---------- ------------------ -# ftBlob SQL_BINARY -# ftBoolean SQL_BIT -# ftDate SQL_TYPE_DATE -# ftTime SQL_TYPE_TIME -# ftDateTime SQL_TYPE_TIMESTAMP -# ftInteger SQL_INTEGER -# ftSmallint SQL_SMALLINT -# ftFloat SQL_DOUBLE -# ftString SQL_CHAR -# ftMemo SQL_BINARY // SQL_VARCHAR -# - -type - TSqlChar* = char - TSqlSmallInt* = int16 - SqlUSmallInt* = int16 - SqlHandle* = pointer - SqlHEnv* = SqlHandle - SqlHDBC* = SqlHandle - SqlHStmt* = SqlHandle - SqlHDesc* = SqlHandle - TSqlInteger* = int - SqlUInteger* = int - SqlPointer* = pointer - TSqlReal* = cfloat - TSqlDouble* = cdouble - TSqlFloat* = cdouble - SqlHWND* = pointer - PSQLCHAR* = cstring - PSQLINTEGER* = ptr TSqlInteger - PSQLUINTEGER* = ptr SqlUInteger - PSQLSMALLINT* = ptr TSqlSmallInt - PSQLUSMALLINT* = ptr SqlUSmallInt - PSQLREAL* = ptr TSqlReal - PSQLDOUBLE* = ptr TSqlDouble - PSQLFLOAT* = ptr TSqlFloat - PSQLHANDLE* = ptr SqlHandle - -const # SQL data type codes - SQL_UNKNOWN_TYPE* = 0 - SQL_LONGVARCHAR* = (- 1) - SQL_BINARY* = (- 2) - SQL_VARBINARY* = (- 3) - SQL_LONGVARBINARY* = (- 4) - SQL_BIGINT* = (- 5) - SQL_TINYINT* = (- 6) - SQL_BIT* = (- 7) - SQL_WCHAR* = (- 8) - SQL_WVARCHAR* = (- 9) - SQL_WLONGVARCHAR* = (- 10) - SQL_CHAR* = 1 - SQL_NUMERIC* = 2 - SQL_DECIMAL* = 3 - SQL_INTEGER* = 4 - SQL_SMALLINT* = 5 - SQL_FLOAT* = 6 - SQL_REAL* = 7 - SQL_DOUBLE* = 8 - SQL_DATETIME* = 9 - SQL_VARCHAR* = 12 - SQL_TYPE_DATE* = 91 - SQL_TYPE_TIME* = 92 - SQL_TYPE_TIMESTAMP* = 93 - SQL_DATE* = 9 - SQL_TIME* = 10 - SQL_TIMESTAMP* = 11 - SQL_INTERVAL* = 10 - SQL_GUID* = - 11 # interval codes - -when ODBCVER >= 0x0300: - const - SQL_CODE_YEAR* = 1 - SQL_CODE_MONTH* = 2 - SQL_CODE_DAY* = 3 - SQL_CODE_HOUR* = 4 - SQL_CODE_MINUTE* = 5 - SQL_CODE_SECOND* = 6 - SQL_CODE_YEAR_TO_MONTH* = 7 - SQL_CODE_DAY_TO_HOUR* = 8 - SQL_CODE_DAY_TO_MINUTE* = 9 - SQL_CODE_DAY_TO_SECOND* = 10 - SQL_CODE_HOUR_TO_MINUTE* = 11 - SQL_CODE_HOUR_TO_SECOND* = 12 - SQL_CODE_MINUTE_TO_SECOND* = 13 - SQL_INTERVAL_YEAR* = 100 + SQL_CODE_YEAR - SQL_INTERVAL_MONTH* = 100 + SQL_CODE_MONTH - SQL_INTERVAL_DAY* = 100 + SQL_CODE_DAY - SQL_INTERVAL_HOUR* = 100 + SQL_CODE_HOUR - SQL_INTERVAL_MINUTE* = 100 + SQL_CODE_MINUTE - SQL_INTERVAL_SECOND* = 100 + SQL_CODE_SECOND - SQL_INTERVAL_YEAR_TO_MONTH* = 100 + SQL_CODE_YEAR_TO_MONTH - SQL_INTERVAL_DAY_TO_HOUR* = 100 + SQL_CODE_DAY_TO_HOUR - SQL_INTERVAL_DAY_TO_MINUTE* = 100 + SQL_CODE_DAY_TO_MINUTE - SQL_INTERVAL_DAY_TO_SECOND* = 100 + SQL_CODE_DAY_TO_SECOND - SQL_INTERVAL_HOUR_TO_MINUTE* = 100 + SQL_CODE_HOUR_TO_MINUTE - SQL_INTERVAL_HOUR_TO_SECOND* = 100 + SQL_CODE_HOUR_TO_SECOND - SQL_INTERVAL_MINUTE_TO_SECOND* = 100 + SQL_CODE_MINUTE_TO_SECOND -else: - const - SQL_INTERVAL_YEAR* = - 80 - SQL_INTERVAL_MONTH* = - 81 - SQL_INTERVAL_YEAR_TO_MONTH* = - 82 - SQL_INTERVAL_DAY* = - 83 - SQL_INTERVAL_HOUR* = - 84 - SQL_INTERVAL_MINUTE* = - 85 - SQL_INTERVAL_SECOND* = - 86 - SQL_INTERVAL_DAY_TO_HOUR* = - 87 - SQL_INTERVAL_DAY_TO_MINUTE* = - 88 - SQL_INTERVAL_DAY_TO_SECOND* = - 89 - SQL_INTERVAL_HOUR_TO_MINUTE* = - 90 - SQL_INTERVAL_HOUR_TO_SECOND* = - 91 - SQL_INTERVAL_MINUTE_TO_SECOND* = - 92 - - -when ODBCVER < 0x0300: - const - SQL_UNICODE* = - 95 - SQL_UNICODE_VARCHAR* = - 96 - SQL_UNICODE_LONGVARCHAR* = - 97 - SQL_UNICODE_CHAR* = SQL_UNICODE -else: - # The previous definitions for SQL_UNICODE_ are historical and obsolete - const - SQL_UNICODE* = SQL_WCHAR - SQL_UNICODE_VARCHAR* = SQL_WVARCHAR - SQL_UNICODE_LONGVARCHAR* = SQL_WLONGVARCHAR - SQL_UNICODE_CHAR* = SQL_WCHAR -const # C datatype to SQL datatype mapping - SQL_C_CHAR* = SQL_CHAR - SQL_C_LONG* = SQL_INTEGER - SQL_C_SHORT* = SQL_SMALLINT - SQL_C_FLOAT* = SQL_REAL - SQL_C_DOUBLE* = SQL_DOUBLE - SQL_C_NUMERIC* = SQL_NUMERIC - SQL_C_DEFAULT* = 99 - SQL_SIGNED_OFFSET* = - 20 - SQL_UNSIGNED_OFFSET* = - 22 - SQL_C_DATE* = SQL_DATE - SQL_C_TIME* = SQL_TIME - SQL_C_TIMESTAMP* = SQL_TIMESTAMP - SQL_C_TYPE_DATE* = SQL_TYPE_DATE - SQL_C_TYPE_TIME* = SQL_TYPE_TIME - SQL_C_TYPE_TIMESTAMP* = SQL_TYPE_TIMESTAMP - SQL_C_INTERVAL_YEAR* = SQL_INTERVAL_YEAR - SQL_C_INTERVAL_MONTH* = SQL_INTERVAL_MONTH - SQL_C_INTERVAL_DAY* = SQL_INTERVAL_DAY - SQL_C_INTERVAL_HOUR* = SQL_INTERVAL_HOUR - SQL_C_INTERVAL_MINUTE* = SQL_INTERVAL_MINUTE - SQL_C_INTERVAL_SECOND* = SQL_INTERVAL_SECOND - SQL_C_INTERVAL_YEAR_TO_MONTH* = SQL_INTERVAL_YEAR_TO_MONTH - SQL_C_INTERVAL_DAY_TO_HOUR* = SQL_INTERVAL_DAY_TO_HOUR - SQL_C_INTERVAL_DAY_TO_MINUTE* = SQL_INTERVAL_DAY_TO_MINUTE - SQL_C_INTERVAL_DAY_TO_SECOND* = SQL_INTERVAL_DAY_TO_SECOND - SQL_C_INTERVAL_HOUR_TO_MINUTE* = SQL_INTERVAL_HOUR_TO_MINUTE - SQL_C_INTERVAL_HOUR_TO_SECOND* = SQL_INTERVAL_HOUR_TO_SECOND - SQL_C_INTERVAL_MINUTE_TO_SECOND* = SQL_INTERVAL_MINUTE_TO_SECOND - SQL_C_BINARY* = SQL_BINARY - SQL_C_BIT* = SQL_BIT - SQL_C_SBIGINT* = SQL_BIGINT + SQL_SIGNED_OFFSET # SIGNED BIGINT - SQL_C_UBIGINT* = SQL_BIGINT + SQL_UNSIGNED_OFFSET # UNSIGNED BIGINT - SQL_C_TINYINT* = SQL_TINYINT - SQL_C_SLONG* = SQL_C_LONG + SQL_SIGNED_OFFSET # SIGNED INTEGER - SQL_C_SSHORT* = SQL_C_SHORT + SQL_SIGNED_OFFSET # SIGNED SMALLINT - SQL_C_STINYINT* = SQL_TINYINT + SQL_SIGNED_OFFSET # SIGNED TINYINT - SQL_C_ULONG* = SQL_C_LONG + SQL_UNSIGNED_OFFSET # UNSIGNED INTEGER - SQL_C_USHORT* = SQL_C_SHORT + SQL_UNSIGNED_OFFSET # UNSIGNED SMALLINT - SQL_C_UTINYINT* = SQL_TINYINT + SQL_UNSIGNED_OFFSET # UNSIGNED TINYINT - SQL_C_BOOKMARK* = SQL_C_ULONG # BOOKMARK - SQL_C_GUID* = SQL_GUID - SQL_TYPE_NULL* = 0 - -when ODBCVER < 0x0300: - const - SQL_TYPE_MIN* = SQL_BIT - SQL_TYPE_MAX* = SQL_VARCHAR - -const - SQL_C_VARBOOKMARK* = SQL_C_BINARY - SQL_API_SQLDESCRIBEPARAM* = 58 - SQL_NO_TOTAL* = - 4 - -type - SQL_DATE_STRUCT* {.final, pure.} = object - Year*: TSqlSmallInt - Month*: SqlUSmallInt - Day*: SqlUSmallInt - - PSQL_DATE_STRUCT* = ptr SQL_DATE_STRUCT - SQL_TIME_STRUCT* {.final, pure.} = object - Hour*: SqlUSmallInt - Minute*: SqlUSmallInt - Second*: SqlUSmallInt - - PSQL_TIME_STRUCT* = ptr SQL_TIME_STRUCT - SQL_TIMESTAMP_STRUCT* {.final, pure.} = object - Year*: SqlUSmallInt - Month*: SqlUSmallInt - Day*: SqlUSmallInt - Hour*: SqlUSmallInt - Minute*: SqlUSmallInt - Second*: SqlUSmallInt - Fraction*: SqlUInteger - - PSQL_TIMESTAMP_STRUCT* = ptr SQL_TIMESTAMP_STRUCT - -const - SQL_NAME_LEN* = 128 - SQL_OV_ODBC3* = 3 - SQL_OV_ODBC2* = 2 - SQL_ATTR_ODBC_VERSION* = 200 # Options for SQLDriverConnect - SQL_DRIVER_NOPROMPT* = 0 - SQL_DRIVER_COMPLETE* = 1 - SQL_DRIVER_PROMPT* = 2 - SQL_DRIVER_COMPLETE_REQUIRED* = 3 - SQL_IS_POINTER* = (- 4) # whether an attribute is a pointer or not - SQL_IS_UINTEGER* = (- 5) - SQL_IS_INTEGER* = (- 6) - SQL_IS_USMALLINT* = (- 7) - SQL_IS_SMALLINT* = (- 8) # SQLExtendedFetch "fFetchType" values - SQL_FETCH_BOOKMARK* = 8 - SQL_SCROLL_OPTIONS* = 44 # SQL_USE_BOOKMARKS options - SQL_UB_OFF* = 0 - SQL_UB_ON* = 1 - SQL_UB_DEFAULT* = SQL_UB_OFF - SQL_UB_FIXED* = SQL_UB_ON - SQL_UB_VARIABLE* = 2 # SQL_SCROLL_OPTIONS masks - SQL_SO_FORWARD_ONLY* = 0x00000001 - SQL_SO_KEYSET_DRIVEN* = 0x00000002 - SQL_SO_DYNAMIC* = 0x00000004 - SQL_SO_MIXED* = 0x00000008 - SQL_SO_STATIC* = 0x00000010 - SQL_BOOKMARK_PERSISTENCE* = 82 - SQL_STATIC_SENSITIVITY* = 83 # SQL_BOOKMARK_PERSISTENCE values - SQL_BP_CLOSE* = 0x00000001 - SQL_BP_DELETE* = 0x00000002 - SQL_BP_DROP* = 0x00000004 - SQL_BP_TRANSACTION* = 0x00000008 - SQL_BP_UPDATE* = 0x00000010 - SQL_BP_OTHER_HSTMT* = 0x00000020 - SQL_BP_SCROLL* = 0x00000040 - SQL_DYNAMIC_CURSOR_ATTRIBUTES1* = 144 - SQL_DYNAMIC_CURSOR_ATTRIBUTES2* = 145 - SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES1* = 146 - SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES2* = 147 - SQL_INDEX_KEYWORDS* = 148 - SQL_INFO_SCHEMA_VIEWS* = 149 - SQL_KEYSET_CURSOR_ATTRIBUTES1* = 150 - SQL_KEYSET_CURSOR_ATTRIBUTES2* = 151 - SQL_STATIC_CURSOR_ATTRIBUTES1* = 167 - SQL_STATIC_CURSOR_ATTRIBUTES2* = 168 # supported SQLFetchScroll FetchOrientation's - SQL_CA1_NEXT* = 1 - SQL_CA1_ABSOLUTE* = 2 - SQL_CA1_RELATIVE* = 4 - SQL_CA1_BOOKMARK* = 8 # supported SQLSetPos LockType's - SQL_CA1_LOCK_NO_CHANGE* = 0x00000040 - SQL_CA1_LOCK_EXCLUSIVE* = 0x00000080 - SQL_CA1_LOCK_UNLOCK* = 0x00000100 # supported SQLSetPos Operations - SQL_CA1_POS_POSITION* = 0x00000200 - SQL_CA1_POS_UPDATE* = 0x00000400 - SQL_CA1_POS_DELETE* = 0x00000800 - SQL_CA1_POS_REFRESH* = 0x00001000 # positioned updates and deletes - SQL_CA1_POSITIONED_UPDATE* = 0x00002000 - SQL_CA1_POSITIONED_DELETE* = 0x00004000 - SQL_CA1_SELECT_FOR_UPDATE* = 0x00008000 # supported SQLBulkOperations operations - SQL_CA1_BULK_ADD* = 0x00010000 - SQL_CA1_BULK_UPDATE_BY_BOOKMARK* = 0x00020000 - SQL_CA1_BULK_DELETE_BY_BOOKMARK* = 0x00040000 - SQL_CA1_BULK_FETCH_BY_BOOKMARK* = 0x00080000 # supported values for SQL_ATTR_SCROLL_CONCURRENCY - SQL_CA2_READ_ONLY_CONCURRENCY* = 1 - SQL_CA2_LOCK_CONCURRENCY* = 2 - SQL_CA2_OPT_ROWVER_CONCURRENCY* = 4 - SQL_CA2_OPT_VALUES_CONCURRENCY* = 8 # sensitivity of the cursor to its own inserts, deletes, and updates - SQL_CA2_SENSITIVITY_ADDITIONS* = 0x00000010 - SQL_CA2_SENSITIVITY_DELETIONS* = 0x00000020 - SQL_CA2_SENSITIVITY_UPDATES* = 0x00000040 # semantics of SQL_ATTR_MAX_ROWS - SQL_CA2_MAX_ROWS_SELECT* = 0x00000080 - SQL_CA2_MAX_ROWS_INSERT* = 0x00000100 - SQL_CA2_MAX_ROWS_DELETE* = 0x00000200 - SQL_CA2_MAX_ROWS_UPDATE* = 0x00000400 - SQL_CA2_MAX_ROWS_CATALOG* = 0x00000800 - SQL_CA2_MAX_ROWS_AFFECTS_ALL* = (SQL_CA2_MAX_ROWS_SELECT or - SQL_CA2_MAX_ROWS_INSERT or SQL_CA2_MAX_ROWS_DELETE or - SQL_CA2_MAX_ROWS_UPDATE or SQL_CA2_MAX_ROWS_CATALOG) # semantics of - # SQL_DIAG_CURSOR_ROW_COUNT - SQL_CA2_CRC_EXACT* = 0x00001000 - SQL_CA2_CRC_APPROXIMATE* = 0x00002000 # the kinds of positioned statements that can be simulated - SQL_CA2_SIMULATE_NON_UNIQUE* = 0x00004000 - SQL_CA2_SIMULATE_TRY_UNIQUE* = 0x00008000 - SQL_CA2_SIMULATE_UNIQUE* = 0x00010000 # Operations in SQLBulkOperations - SQL_ADD* = 4 - SQL_SETPOS_MAX_OPTION_VALUE* = SQL_ADD - SQL_UPDATE_BY_BOOKMARK* = 5 - SQL_DELETE_BY_BOOKMARK* = 6 - SQL_FETCH_BY_BOOKMARK* = 7 # Operations in SQLSetPos - SQL_POSITION* = 0 - SQL_REFRESH* = 1 - SQL_UPDATE* = 2 - SQL_DELETE* = 3 # Lock options in SQLSetPos - SQL_LOCK_NO_CHANGE* = 0 - SQL_LOCK_EXCLUSIVE* = 1 - SQL_LOCK_UNLOCK* = 2 # SQLExtendedFetch "rgfRowStatus" element values - SQL_ROW_SUCCESS* = 0 - SQL_ROW_DELETED* = 1 - SQL_ROW_UPDATED* = 2 - SQL_ROW_NOROW* = 3 - SQL_ROW_ADDED* = 4 - SQL_ROW_ERROR* = 5 - SQL_ROW_SUCCESS_WITH_INFO* = 6 - SQL_ROW_PROCEED* = 0 - SQL_ROW_IGNORE* = 1 - SQL_MAX_DSN_LENGTH* = 32 # maximum data source name size - SQL_MAX_OPTION_STRING_LENGTH* = 256 - SQL_ODBC_CURSORS* = 110 - SQL_ATTR_ODBC_CURSORS* = SQL_ODBC_CURSORS # SQL_ODBC_CURSORS options - SQL_CUR_USE_IF_NEEDED* = 0 - SQL_CUR_USE_ODBC* = 1 - SQL_CUR_USE_DRIVER* = 2 - SQL_CUR_DEFAULT* = SQL_CUR_USE_DRIVER - SQL_PARAM_TYPE_UNKNOWN* = 0 - SQL_PARAM_INPUT* = 1 - SQL_PARAM_INPUT_OUTPUT* = 2 - SQL_RESULT_COL* = 3 - SQL_PARAM_OUTPUT* = 4 - SQL_RETURN_VALUE* = 5 # special length/indicator values - SQL_NULL_DATA* = (- 1) - SQL_DATA_AT_EXEC* = (- 2) - SQL_SUCCESS* = 0 - SQL_SUCCESS_WITH_INFO* = 1 - SQL_NO_DATA* = 100 - SQL_ERROR* = (- 1) - SQL_INVALID_HANDLE* = (- 2) - SQL_STILL_EXECUTING* = 2 - SQL_NEED_DATA* = 99 # flags for null-terminated string - SQL_NTS* = (- 3) # maximum message length - SQL_MAX_MESSAGE_LENGTH* = 512 # date/time length constants - SQL_DATE_LEN* = 10 - SQL_TIME_LEN* = 8 # add P+1 if precision is nonzero - SQL_TIMESTAMP_LEN* = 19 # add P+1 if precision is nonzero - # handle type identifiers - SQL_HANDLE_ENV* = 1 - SQL_HANDLE_DBC* = 2 - SQL_HANDLE_STMT* = 3 - SQL_HANDLE_DESC* = 4 # environment attribute - SQL_ATTR_OUTPUT_NTS* = 10001 # connection attributes - SQL_ATTR_AUTO_IPD* = 10001 - SQL_ATTR_METADATA_ID* = 10014 # statement attributes - SQL_ATTR_APP_ROW_DESC* = 10010 - SQL_ATTR_APP_PARAM_DESC* = 10011 - SQL_ATTR_IMP_ROW_DESC* = 10012 - SQL_ATTR_IMP_PARAM_DESC* = 10013 - SQL_ATTR_CURSOR_SCROLLABLE* = (- 1) - SQL_ATTR_CURSOR_SENSITIVITY* = (- 2) - SQL_QUERY_TIMEOUT* = 0 - SQL_MAX_ROWS* = 1 - SQL_NOSCAN* = 2 - SQL_MAX_LENGTH* = 3 - SQL_ASYNC_ENABLE* = 4 # same as SQL_ATTR_ASYNC_ENABLE */ - SQL_BIND_TYPE* = 5 - SQL_CURSOR_TYPE* = 6 - SQL_CONCURRENCY* = 7 - SQL_KEYSET_SIZE* = 8 - SQL_ROWSET_SIZE* = 9 - SQL_SIMULATE_CURSOR* = 10 - SQL_RETRIEVE_DATA* = 11 - SQL_USE_BOOKMARKS* = 12 - SQL_GET_BOOKMARK* = 13 # GetStmtOption Only */ - SQL_ROW_NUMBER* = 14 # GetStmtOption Only */ - SQL_ATTR_CURSOR_TYPE* = SQL_CURSOR_TYPE - SQL_ATTR_CONCURRENCY* = SQL_CONCURRENCY - SQL_ATTR_FETCH_BOOKMARK_PTR* = 16 - SQL_ATTR_ROW_STATUS_PTR* = 25 - SQL_ATTR_ROWS_FETCHED_PTR* = 26 - SQL_AUTOCOMMIT* = 102 - SQL_ATTR_AUTOCOMMIT* = SQL_AUTOCOMMIT - SQL_ATTR_ROW_NUMBER* = SQL_ROW_NUMBER - SQL_TXN_ISOLATION* = 108 - SQL_ATTR_TXN_ISOLATION* = SQL_TXN_ISOLATION - SQL_ATTR_MAX_ROWS* = SQL_MAX_ROWS - SQL_ATTR_USE_BOOKMARKS* = SQL_USE_BOOKMARKS #* connection attributes */ - SQL_ACCESS_MODE* = 101 # SQL_AUTOCOMMIT =102; - SQL_LOGIN_TIMEOUT* = 103 - SQL_OPT_TRACE* = 104 - SQL_OPT_TRACEFILE* = 105 - SQL_TRANSLATE_DLL* = 106 - SQL_TRANSLATE_OPTION* = 107 # SQL_TXN_ISOLATION =108; - SQL_CURRENT_QUALIFIER* = 109 # SQL_ODBC_CURSORS =110; - SQL_QUIET_MODE* = 111 - SQL_PACKET_SIZE* = 112 #* connection attributes with new names */ - SQL_ATTR_ACCESS_MODE* = SQL_ACCESS_MODE # SQL_ATTR_AUTOCOMMIT =SQL_AUTOCOMMIT; - SQL_ATTR_CONNECTION_DEAD* = 1209 #* GetConnectAttr only */ - SQL_ATTR_CONNECTION_TIMEOUT* = 113 - SQL_ATTR_CURRENT_CATALOG* = SQL_CURRENT_QUALIFIER - SQL_ATTR_DISCONNECT_BEHAVIOR* = 114 - SQL_ATTR_ENLIST_IN_DTC* = 1207 - SQL_ATTR_ENLIST_IN_XA* = 1208 - SQL_ATTR_LOGIN_TIMEOUT* = SQL_LOGIN_TIMEOUT # SQL_ATTR_ODBC_CURSORS =SQL_ODBC_CURSORS; - SQL_ATTR_PACKET_SIZE* = SQL_PACKET_SIZE - SQL_ATTR_QUIET_MODE* = SQL_QUIET_MODE - SQL_ATTR_TRACE* = SQL_OPT_TRACE - SQL_ATTR_TRACEFILE* = SQL_OPT_TRACEFILE - SQL_ATTR_TRANSLATE_LIB* = SQL_TRANSLATE_DLL - SQL_ATTR_TRANSLATE_OPTION* = SQL_TRANSLATE_OPTION # SQL_ATTR_TXN_ISOLATION =SQL_TXN_ISOLATION; - #* SQL_ACCESS_MODE options */ - SQL_MODE_READ_WRITE* = 0 - SQL_MODE_READ_ONLY* = 1 - SQL_MODE_DEFAULT* = SQL_MODE_READ_WRITE #* SQL_AUTOCOMMIT options */ - SQL_AUTOCOMMIT_OFF* = 0 - SQL_AUTOCOMMIT_ON* = 1 - SQL_AUTOCOMMIT_DEFAULT* = SQL_AUTOCOMMIT_ON # SQL_ATTR_CURSOR_SCROLLABLE values - SQL_NONSCROLLABLE* = 0 - SQL_SCROLLABLE* = 1 # SQL_CURSOR_TYPE options - SQL_CURSOR_FORWARD_ONLY* = 0 - SQL_CURSOR_KEYSET_DRIVEN* = 1 - SQL_CURSOR_DYNAMIC* = 2 - SQL_CURSOR_STATIC* = 3 - SQL_CURSOR_TYPE_DEFAULT* = SQL_CURSOR_FORWARD_ONLY # Default value - # SQL_CONCURRENCY options - SQL_CONCUR_READ_ONLY* = 1 - SQL_CONCUR_LOCK* = 2 - SQL_CONCUR_ROWVER* = 3 - SQL_CONCUR_VALUES* = 4 - SQL_CONCUR_DEFAULT* = SQL_CONCUR_READ_ONLY # Default value - # identifiers of fields in the SQL descriptor - SQL_DESC_COUNT* = 1001 - SQL_DESC_TYPE* = 1002 - SQL_DESC_LENGTH* = 1003 - SQL_DESC_OCTET_LENGTH_PTR* = 1004 - SQL_DESC_PRECISION* = 1005 - SQL_DESC_SCALE* = 1006 - SQL_DESC_DATETIME_INTERVAL_CODE* = 1007 - SQL_DESC_NULLABLE* = 1008 - SQL_DESC_INDICATOR_PTR* = 1009 - SQL_DESC_DATA_PTR* = 1010 - SQL_DESC_NAME* = 1011 - SQL_DESC_UNNAMED* = 1012 - SQL_DESC_OCTET_LENGTH* = 1013 - SQL_DESC_ALLOC_TYPE* = 1099 # identifiers of fields in the diagnostics area - SQL_DIAG_RETURNCODE* = 1 - SQL_DIAG_NUMBER* = 2 - SQL_DIAG_ROW_COUNT* = 3 - SQL_DIAG_SQLSTATE* = 4 - SQL_DIAG_NATIVE* = 5 - SQL_DIAG_MESSAGE_TEXT* = 6 - SQL_DIAG_DYNAMIC_FUNCTION* = 7 - SQL_DIAG_CLASS_ORIGIN* = 8 - SQL_DIAG_SUBCLASS_ORIGIN* = 9 - SQL_DIAG_CONNECTION_NAME* = 10 - SQL_DIAG_SERVER_NAME* = 11 - SQL_DIAG_DYNAMIC_FUNCTION_CODE* = 12 # dynamic function codes - SQL_DIAG_ALTER_TABLE* = 4 - SQL_DIAG_CREATE_INDEX* = (- 1) - SQL_DIAG_CREATE_TABLE* = 77 - SQL_DIAG_CREATE_VIEW* = 84 - SQL_DIAG_DELETE_WHERE* = 19 - SQL_DIAG_DROP_INDEX* = (- 2) - SQL_DIAG_DROP_TABLE* = 32 - SQL_DIAG_DROP_VIEW* = 36 - SQL_DIAG_DYNAMIC_DELETE_CURSOR* = 38 - SQL_DIAG_DYNAMIC_UPDATE_CURSOR* = 81 - SQL_DIAG_GRANT* = 48 - SQL_DIAG_INSERT* = 50 - SQL_DIAG_REVOKE* = 59 - SQL_DIAG_SELECT_CURSOR* = 85 - SQL_DIAG_UNKNOWN_STATEMENT* = 0 - SQL_DIAG_UPDATE_WHERE* = 82 # Statement attribute values for cursor sensitivity - SQL_UNSPECIFIED* = 0 - SQL_INSENSITIVE* = 1 - SQL_SENSITIVE* = 2 # GetTypeInfo() request for all data types - SQL_ALL_TYPES* = 0 # Default conversion code for SQLBindCol(), SQLBindParam() and SQLGetData() - SQL_DEFAULT* = 99 # SQLGetData() code indicating that the application row descriptor - # specifies the data type - SQL_ARD_TYPE* = (- 99) # SQL date/time type subcodes - SQL_CODE_DATE* = 1 - SQL_CODE_TIME* = 2 - SQL_CODE_TIMESTAMP* = 3 # CLI option values - SQL_FALSE* = 0 - SQL_TRUE* = 1 # values of NULLABLE field in descriptor - SQL_NO_NULLS* = 0 - SQL_NULLABLE* = 1 # Value returned by SQLGetTypeInfo() to denote that it is - # not known whether or not a data type supports null values. - SQL_NULLABLE_UNKNOWN* = 2 - SQL_CLOSE* = 0 - SQL_DROP* = 1 - SQL_UNBIND* = 2 - SQL_RESET_PARAMS* = 3 # Codes used for FetchOrientation in SQLFetchScroll(), - # and in SQLDataSources() - SQL_FETCH_NEXT* = 1 - SQL_FETCH_FIRST* = 2 - SQL_FETCH_FIRST_USER* = 31 - SQL_FETCH_FIRST_SYSTEM* = 32 # Other codes used for FetchOrientation in SQLFetchScroll() - SQL_FETCH_LAST* = 3 - SQL_FETCH_PRIOR* = 4 - SQL_FETCH_ABSOLUTE* = 5 - SQL_FETCH_RELATIVE* = 6 - SQL_NULL_HENV* = SqlHEnv(nil) - SQL_NULL_HDBC* = SqlHDBC(nil) - SQL_NULL_HSTMT* = SqlHStmt(nil) - SQL_NULL_HDESC* = SqlHDesc(nil) #* null handle used in place of parent handle when allocating HENV */ - SQL_NULL_HANDLE* = SqlHandle(nil) #* Values that may appear in the result set of SQLSpecialColumns() */ - SQL_SCOPE_CURROW* = 0 - SQL_SCOPE_TRANSACTION* = 1 - SQL_SCOPE_SESSION* = 2 #* Column types and scopes in SQLSpecialColumns. */ - SQL_BEST_ROWID* = 1 - SQL_ROWVER* = 2 - SQL_ROW_IDENTIFIER* = 1 #* Reserved values for UNIQUE argument of SQLStatistics() */ - SQL_INDEX_UNIQUE* = 0 - SQL_INDEX_ALL* = 1 #* Reserved values for RESERVED argument of SQLStatistics() */ - SQL_QUICK* = 0 - SQL_ENSURE* = 1 #* Values that may appear in the result set of SQLStatistics() */ - SQL_TABLE_STAT* = 0 - SQL_INDEX_CLUSTERED* = 1 - SQL_INDEX_HASHED* = 2 - SQL_INDEX_OTHER* = 3 - SQL_SCROLL_CONCURRENCY* = 43 - SQL_TXN_CAPABLE* = 46 - SQL_TRANSACTION_CAPABLE* = SQL_TXN_CAPABLE - SQL_USER_NAME* = 47 - SQL_TXN_ISOLATION_OPTION* = 72 - SQL_TRANSACTION_ISOLATION_OPTION* = SQL_TXN_ISOLATION_OPTION - SQL_OJ_CAPABILITIES* = 115 - SQL_OUTER_JOIN_CAPABILITIES* = SQL_OJ_CAPABILITIES - SQL_XOPEN_CLI_YEAR* = 10000 - SQL_CURSOR_SENSITIVITY* = 10001 - SQL_DESCRIBE_PARAMETER* = 10002 - SQL_CATALOG_NAME* = 10003 - SQL_COLLATION_SEQ* = 10004 - SQL_MAX_IDENTIFIER_LEN* = 10005 - SQL_MAXIMUM_IDENTIFIER_LENGTH* = SQL_MAX_IDENTIFIER_LEN - SQL_SCCO_READ_ONLY* = 1 - SQL_SCCO_LOCK* = 2 - SQL_SCCO_OPT_ROWVER* = 4 - SQL_SCCO_OPT_VALUES* = 8 #* SQL_TXN_CAPABLE values */ - SQL_TC_NONE* = 0 - SQL_TC_DML* = 1 - SQL_TC_ALL* = 2 - SQL_TC_DDL_COMMIT* = 3 - SQL_TC_DDL_IGNORE* = 4 #* SQL_TXN_ISOLATION_OPTION bitmasks */ - SQL_TXN_READ_UNCOMMITTED* = 1 - SQL_TRANSACTION_READ_UNCOMMITTED* = SQL_TXN_READ_UNCOMMITTED - SQL_TXN_READ_COMMITTED* = 2 - SQL_TRANSACTION_READ_COMMITTED* = SQL_TXN_READ_COMMITTED - SQL_TXN_REPEATABLE_READ* = 4 - SQL_TRANSACTION_REPEATABLE_READ* = SQL_TXN_REPEATABLE_READ - SQL_TXN_SERIALIZABLE* = 8 - SQL_TRANSACTION_SERIALIZABLE* = SQL_TXN_SERIALIZABLE - SQL_SS_ADDITIONS* = 1 - SQL_SS_DELETIONS* = 2 - SQL_SS_UPDATES* = 4 # SQLColAttributes defines - SQL_COLUMN_COUNT* = 0 - SQL_COLUMN_NAME* = 1 - SQL_COLUMN_TYPE* = 2 - SQL_COLUMN_LENGTH* = 3 - SQL_COLUMN_PRECISION* = 4 - SQL_COLUMN_SCALE* = 5 - SQL_COLUMN_DISPLAY_SIZE* = 6 - SQL_COLUMN_NULLABLE* = 7 - SQL_COLUMN_UNSIGNED* = 8 - SQL_COLUMN_MONEY* = 9 - SQL_COLUMN_UPDATABLE* = 10 - SQL_COLUMN_AUTO_INCREMENT* = 11 - SQL_COLUMN_CASE_SENSITIVE* = 12 - SQL_COLUMN_SEARCHABLE* = 13 - SQL_COLUMN_TYPE_NAME* = 14 - SQL_COLUMN_TABLE_NAME* = 15 - SQL_COLUMN_OWNER_NAME* = 16 - SQL_COLUMN_QUALIFIER_NAME* = 17 - SQL_COLUMN_LABEL* = 18 - SQL_COLATT_OPT_MAX* = SQL_COLUMN_LABEL - SQL_COLUMN_DRIVER_START* = 1000 - SQL_DESC_ARRAY_SIZE* = 20 - SQL_DESC_ARRAY_STATUS_PTR* = 21 - SQL_DESC_AUTO_UNIQUE_VALUE* = SQL_COLUMN_AUTO_INCREMENT - SQL_DESC_BASE_COLUMN_NAME* = 22 - SQL_DESC_BASE_TABLE_NAME* = 23 - SQL_DESC_BIND_OFFSET_PTR* = 24 - SQL_DESC_BIND_TYPE* = 25 - SQL_DESC_CASE_SENSITIVE* = SQL_COLUMN_CASE_SENSITIVE - SQL_DESC_CATALOG_NAME* = SQL_COLUMN_QUALIFIER_NAME - SQL_DESC_CONCISE_TYPE* = SQL_COLUMN_TYPE - SQL_DESC_DATETIME_INTERVAL_PRECISION* = 26 - SQL_DESC_DISPLAY_SIZE* = SQL_COLUMN_DISPLAY_SIZE - SQL_DESC_FIXED_PREC_SCALE* = SQL_COLUMN_MONEY - SQL_DESC_LABEL* = SQL_COLUMN_LABEL - SQL_DESC_LITERAL_PREFIX* = 27 - SQL_DESC_LITERAL_SUFFIX* = 28 - SQL_DESC_LOCAL_TYPE_NAME* = 29 - SQL_DESC_MAXIMUM_SCALE* = 30 - SQL_DESC_MINIMUM_SCALE* = 31 - SQL_DESC_NUM_PREC_RADIX* = 32 - SQL_DESC_PARAMETER_TYPE* = 33 - SQL_DESC_ROWS_PROCESSED_PTR* = 34 - SQL_DESC_SCHEMA_NAME* = SQL_COLUMN_OWNER_NAME - SQL_DESC_SEARCHABLE* = SQL_COLUMN_SEARCHABLE - SQL_DESC_TYPE_NAME* = SQL_COLUMN_TYPE_NAME - SQL_DESC_TABLE_NAME* = SQL_COLUMN_TABLE_NAME - SQL_DESC_UNSIGNED* = SQL_COLUMN_UNSIGNED - SQL_DESC_UPDATABLE* = SQL_COLUMN_UPDATABLE #* SQLEndTran() options */ - SQL_COMMIT* = 0 - SQL_ROLLBACK* = 1 - SQL_ATTR_ROW_ARRAY_SIZE* = 27 #* SQLConfigDataSource() options */ - ODBC_ADD_DSN* = 1 - ODBC_CONFIG_DSN* = 2 - ODBC_REMOVE_DSN* = 3 - ODBC_ADD_SYS_DSN* = 4 - ODBC_CONFIG_SYS_DSN* = 5 - ODBC_REMOVE_SYS_DSN* = 6 - - SQL_ACTIVE_CONNECTIONS* = 0 # SQLGetInfo - SQL_DATA_SOURCE_NAME* = 2 - SQL_DATA_SOURCE_READ_ONLY* = 25 - SQL_DATABASE_NAME* = 2 - SQL_DBMS_NAME* = 17 - SQL_DBMS_VERSION* = 18 - SQL_DRIVER_HDBC* = 3 - SQL_DRIVER_HENV* = 4 - SQL_DRIVER_HSTMT* = 5 - SQL_DRIVER_NAME* = 6 - SQL_DRIVER_VER* = 7 - SQL_FETCH_DIRECTION* = 8 - SQL_ODBC_VER* = 10 - SQL_DRIVER_ODBC_VER* = 77 - SQL_SERVER_NAME* = 13 - SQL_ACTIVE_ENVIRONMENTS* = 116 - SQL_ACTIVE_STATEMENTS* = 1 - SQL_SQL_CONFORMANCE* = 118 - SQL_DATETIME_LITERALS* = 119 - SQL_ASYNC_MODE* = 10021 - SQL_BATCH_ROW_COUNT* = 120 - SQL_BATCH_SUPPORT* = 121 - SQL_CATALOG_LOCATION* = 114 - #SQL_CATALOG_NAME* = 10003 - SQL_CATALOG_NAME_SEPARATOR* = 41 - SQL_CATALOG_TERM* = 42 - SQL_CATALOG_USAGE* = 92 - #SQL_COLLATION_SEQ* = 10004 - SQL_COLUMN_ALIAS* = 87 - #SQL_USER_NAME* = 47 - -proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle, - OutputHandlePtr: var SqlHandle): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: TSqlInteger, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: SqlPointer, BufferLength: TSqlInteger, - StringLength: PSQLINTEGER): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLFreeHandle*(HandleType: TSqlSmallInt, Handle: SqlHandle): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetDiagRec*(HandleType: TSqlSmallInt, Handle: SqlHandle, - RecNumber: TSqlSmallInt, Sqlstate: PSQLCHAR, - NativeError: var TSqlInteger, MessageText: PSQLCHAR, - BufferLength: TSqlSmallInt, TextLength: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetDiagField*(HandleType: TSqlSmallInt, Handle: SqlHandle, - RecNumber: TSqlSmallInt, DiagIdentifier: TSqlSmallInt, - DiagInfoPtr: SqlPointer, BufferLength: TSqlSmallInt, - StringLengthPtr: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLConnect*(ConnectionHandle: SqlHDBC, ServerName: PSQLCHAR, - NameLength1: TSqlSmallInt, UserName: PSQLCHAR, - NameLength2: TSqlSmallInt, Authentication: PSQLCHAR, - NameLength3: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLDisconnect*(ConnectionHandle: SqlHDBC): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLDriverConnect*(hdbc: SqlHDBC, hwnd: SqlHWND, szCsin: cstring, - szCLen: TSqlSmallInt, szCsout: cstring, - cbCSMax: TSqlSmallInt, cbCsOut: var TSqlSmallInt, - f: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLBrowseConnect*(hdbc: SqlHDBC, szConnStrIn: PSQLCHAR, - cbConnStrIn: TSqlSmallInt, szConnStrOut: PSQLCHAR, - cbConnStrOutMax: TSqlSmallInt, - cbConnStrOut: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLExecDirect*(StatementHandle: SqlHStmt, StatementText: PSQLCHAR, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLExecDirectW*(StatementHandle: SqlHStmt, StatementText: WideCString, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLPrepare*(StatementHandle: SqlHStmt, StatementText: PSQLCHAR, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLPrepareW*(StatementHandle: SqlHStmt, StatementText: WideCString, - TextLength: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLCloseCursor*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLExecute*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLFetch*(StatementHandle: SqlHStmt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLNumResultCols*(StatementHandle: SqlHStmt, ColumnCount: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLDescribeCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - ColumnName: PSQLCHAR, BufferLength: TSqlSmallInt, - NameLength: var TSqlSmallInt, DataType: var TSqlSmallInt, - ColumnSize: var SqlUInteger, - DecimalDigits: var TSqlSmallInt, Nullable: var TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLFetchScroll*(StatementHandle: SqlHStmt, FetchOrientation: TSqlSmallInt, - FetchOffset: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLExtendedFetch*(hstmt: SqlHStmt, fFetchType: SqlUSmallInt, - irow: TSqlInteger, pcrow: PSQLUINTEGER, - rgfRowStatus: PSQLUSMALLINT): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLGetData*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - TargetType: TSqlSmallInt, TargetValue: SqlPointer, - BufferLength: TSqlInteger, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetStmtAttr*(StatementHandle: SqlHStmt, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetStmtAttr*(StatementHandle: SqlHStmt, Attribute: TSqlInteger, - Value: SqlPointer, BufferLength: TSqlInteger, - StringLength: PSQLINTEGER): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLGetInfo*(ConnectionHandle: SqlHDBC, InfoType: SqlUSmallInt, - InfoValue: SqlPointer, BufferLength: TSqlSmallInt, - StringLength: PSQLSMALLINT): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLBulkOperations*(StatementHandle: SqlHStmt, Operation: TSqlSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLPutData*(StatementHandle: SqlHStmt, Data: SqlPointer, - StrLen_or_Ind: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLBindCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - TargetType: TSqlSmallInt, TargetValue: SqlPointer, - BufferLength: TSqlInteger, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetPos*(hstmt: SqlHStmt, irow: SqlUSmallInt, fOption: SqlUSmallInt, - fLock: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLDataSources*(EnvironmentHandle: SqlHEnv, Direction: SqlUSmallInt, - ServerName: PSQLCHAR, BufferLength1: TSqlSmallInt, - NameLength1: PSQLSMALLINT, Description: PSQLCHAR, - BufferLength2: TSqlSmallInt, NameLength2: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLDrivers*(EnvironmentHandle: SqlHEnv, Direction: SqlUSmallInt, - DriverDescription: PSQLCHAR, BufferLength1: TSqlSmallInt, - DescriptionLength1: PSQLSMALLINT, DriverAttributes: PSQLCHAR, - BufferLength2: TSqlSmallInt, AttributesLength2: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetConnectAttr*(ConnectionHandle: SqlHDBC, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLGetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR, - BufferLength: TSqlSmallInt, NameLength: PSQLSMALLINT): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLSetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR, - NameLength: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLRowCount*(StatementHandle: SqlHStmt, RowCount: var TSqlInteger): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLBindParameter*(hstmt: SqlHStmt, ipar: SqlUSmallInt, - fParamType: TSqlSmallInt, fCType: TSqlSmallInt, - fSqlType: TSqlSmallInt, cbColDef: SqlUInteger, - ibScale: TSqlSmallInt, rgbValue: SqlPointer, - cbValueMax: TSqlInteger, pcbValue: PSQLINTEGER): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLFreeStmt*(StatementHandle: SqlHStmt, Option: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLColAttribute*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt, - FieldIdentifier: SqlUSmallInt, - CharacterAttribute: PSQLCHAR, BufferLength: TSqlSmallInt, - StringLength: PSQLSMALLINT, - NumericAttribute: SqlPointer): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLEndTran*(HandleType: TSqlSmallInt, Handle: SqlHandle, - CompletionType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLTables*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt, szTableType: PSQLCHAR, - cbTableType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLColumns*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt, szColumnName: PSQLCHAR, - cbColumnName: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.} -proc SQLSpecialColumns*(StatementHandle: SqlHStmt, IdentifierType: SqlUSmallInt, - CatalogName: PSQLCHAR, NameLength1: TSqlSmallInt, - SchemaName: PSQLCHAR, NameLength2: TSqlSmallInt, - TableName: PSQLCHAR, NameLength3: TSqlSmallInt, - Scope: SqlUSmallInt, - Nullable: SqlUSmallInt): TSqlSmallInt{. - dynlib: odbclib, importc.} -proc SQLProcedures*(hstmt: SqlHStmt, szTableQualifier: PSQLCHAR, - cbTableQualifier: TSqlSmallInt, szTableOwner: PSQLCHAR, - cbTableOwner: TSqlSmallInt, szTableName: PSQLCHAR, - cbTableName: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLPrimaryKeys*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, TableName: PSQLCHAR, - NameLength3: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLProcedureColumns*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, ProcName: PSQLCHAR, - NameLength3: TSqlSmallInt, ColumnName: PSQLCHAR, - NameLength4: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib, - importc.} -proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, - NameLength1: TSqlSmallInt, SchemaName: PSQLCHAR, - NameLength2: TSqlSmallInt, TableName: PSQLCHAR, - NameLength3: TSqlSmallInt, Unique: SqlUSmallInt, - Reserved: SqlUSmallInt): TSqlSmallInt {. - dynlib: odbclib, importc.} -proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt, - szSqlState, pfNativeError, szErrorMsg: PSQLCHAR, - cbErrorMsgMax: TSqlSmallInt, - pcbErrorMsg: PSQLSMALLINT): TSqlSmallInt {. - dynlib: odbclib, importc: "SQLError".} - -{.pop.} -when defined(nimHasStyleChecks): - {.pop.} diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 4a53fba80..9921b7ffd 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -7,67 +7,88 @@ # distribution, for details about the copyright. # -## OpenSSL support +## OpenSSL wrapper. Supports OpenSSL >= 1.1.0 dynamically (as default) or statically linked +## using `--dynlibOverride:ssl`. ## -## When OpenSSL is dynamically linked, the wrapper provides partial forward and backward -## compatibility for OpenSSL versions above and below 1.1.0 +## `-d:sslVersion=1.2.3` can be used to force an SSL version. +## This version must be included in the library name. +## `-d:useOpenssl3` may be set for OpenSSL 3 instead. ## -## OpenSSL can also be statically linked using ``--dynlibOverride:ssl`` for OpenSSL >= 1.1.0. -## If you want to statically link against OpenSSL 1.0.x, you now have to -## define the ``openssl10`` symbol via ``-d:openssl10``. +## There is also limited support for OpenSSL 1.0.x which may require `-d:openssl10`. ## ## Build and test examples: ## -## .. code-block:: +## ```cmd +## ./bin/nim c -d:ssl -p:. -r tests/stdlib/tssl.nim +## ./bin/nim c -d:ssl --threads:on -p:. -r tests/stdlib/thttpclient_ssl.nim ## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim ## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passl:-lcrypto --passl:-lssl -r tests/untestable/tssl.nim +## ./bin/nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:ssl -p:testament/lib --threads:on tests/untestable/thttpclient_ssl_remotenetwork.nim +## ``` + +# https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html +# +from std/strutils import startsWith + +when defined(nimPreviewSlimSystem): + import std/syncio -{.deadCodeElim: on.} # dce option deprecated when defined(nimHasStyleChecks): {.push styleChecks: off.} -const useWinVersion = defined(Windows) or defined(nimdoc) +const useWinVersion = defined(windows) or defined(nimdoc) -# To force openSSL version use -d:sslVersion=1.0.0 +# To force openSSL version use -d:sslVersion=1.2.3 # See: #10281, #10230 # General issue: # Other dynamic libraries (like libpg) load different openSSL version then what nim loads. # Having two different openSSL loaded version causes a crash. # Use this compile time define to force the openSSL version that your other dynamic libraries want. const sslVersion {.strdefine.}: string = "" +const useOpenssl3* {.booldefine.} = sslVersion.startsWith('3') when sslVersion != "": when defined(macosx): const DLLSSLName* = "libssl." & sslVersion & ".dylib" DLLUtilName* = "libcrypto." & sslVersion & ".dylib" - from posix import SocketHandle + from std/posix import SocketHandle elif defined(windows): const DLLSSLName* = "libssl-" & sslVersion & ".dll" DLLUtilName* = "libcrypto-" & sslVersion & ".dll" - from winlean import SocketHandle + from std/winlean import SocketHandle else: const DLLSSLName* = "libssl.so." & sslVersion DLLUtilName* = "libcrypto.so." & sslVersion - from posix import SocketHandle + from std/posix import SocketHandle elif useWinVersion: - when not defined(nimOldDlls) and defined(cpu64): + when defined(openssl10) or defined(nimOldDlls): + when defined(cpu64): + const + DLLSSLName* = "(ssleay32|ssleay64).dll" + DLLUtilName* = "(libeay32|libeay64).dll" + else: + const + DLLSSLName* = "ssleay32.dll" + DLLUtilName* = "libeay32.dll" + elif defined(cpu64): const DLLSSLName* = "(libssl-1_1-x64|ssleay64|libssl64).dll" DLLUtilName* = "(libcrypto-1_1-x64|libeay64).dll" else: const DLLSSLName* = "(libssl-1_1|ssleay32|libssl32).dll" - DLLUtilName* = "(libcrypto-1_1|libeay32).dll" + DLLUtilName* = "(libcrypto-1_1|libeay32).dll" - from winlean import SocketHandle + from std/winlean import SocketHandle else: + # same list of versions but ordered differently? when defined(osx): - const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" + const versions = "(.3|.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.48|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)" else: - const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.47|.46|.45|.44|.43|.41|.39|.38|.10|)" + const versions = "(.3|.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.48|.47|.46|.45|.44|.43|.41|.39|.38|.10|)" when defined(macosx): const @@ -81,9 +102,12 @@ else: const DLLSSLName* = "libssl.so" & versions DLLUtilName* = "libcrypto.so" & versions - from posix import SocketHandle + from std/posix import SocketHandle -import dynlib +import std/dynlib + +{.pragma: lcrypto, cdecl, dynlib: DLLUtilName, importc.} +{.pragma: lssl, cdecl, dynlib: DLLSSLName, importc.} type SslStruct {.final, pure.} = object @@ -91,9 +115,9 @@ type PSslPtr* = ptr SslPtr SslCtx* = SslPtr PSSL_METHOD* = SslPtr + PSTACK* = SslPtr PX509* = SslPtr PX509_NAME* = SslPtr - PEVP_MD* = SslPtr PBIO_METHOD* = SslPtr BIO* = SslPtr EVP_PKEY* = SslPtr @@ -116,6 +140,15 @@ type pem_password_cb* = proc(buf: cstring, size, rwflag: cint, userdata: pointer): cint {.cdecl.} + PaddingType* = enum + RSA_PKCS1_PADDING = 1.cint, + RSA_SSLV23_PADDING = 2.cint, + RSA_NO_PADDING = 3.cint, + RSA_PKCS1_OAEP_PADDING = 4.cint, + RSA_X931_PADDING = 5.cint, + RSA_PKCS1_PSS_PADDING = 6.cint + + const SSL_SENT_SHUTDOWN* = 1 SSL_RECEIVED_SHUTDOWN* = 2 @@ -172,6 +205,7 @@ const SSL_CTRL_SET_TLSEXT_SERVERNAME_CB = 53 SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG = 54 SSL_CTRL_SET_TLSEXT_HOSTNAME = 55 + SSL_CTRL_SET_ECDH_AUTO* = 94 TLSEXT_NAMETYPE_host_name* = 0 SSL_TLSEXT_ERR_OK* = 0 SSL_TLSEXT_ERR_ALERT_WARNING* = 1 @@ -192,6 +226,9 @@ const SSL_OP_ALL* = 0x000FFFFF SSL_VERIFY_NONE* = 0x00000000 SSL_VERIFY_PEER* = 0x00000001 + SSL_ST_CONNECT* = 0x1000 + SSL_ST_ACCEPT* = 0x2000 + SSL_ST_INIT* = SSL_ST_CONNECT or SSL_ST_ACCEPT OPENSSL_DES_DECRYPT* = 0 OPENSSL_DES_ENCRYPT* = 1 X509_V_OK* = 0 @@ -244,13 +281,21 @@ proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.} # and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2 # SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0 -when compileOption("dynlibOverride", "ssl"): +const useStaticLink = compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks) + +when useStaticLink: # Static linking when defined(openssl10): proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc SSLeay(): culong {.cdecl, dynlib: DLLUtilName, importc.} + + proc getOpenSSLVersion*(): culong = + SSLeay() + + proc ERR_load_BIO_strings*() {.cdecl, dynlib: DLLUtilName, importc.} else: proc OPENSSL_init_ssl*(opts: uint64, settings: uint8): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.} proc SSL_library_init*(): cint {.discardable.} = @@ -261,7 +306,7 @@ when compileOption("dynlibOverride", "ssl"): proc SSLv23_method*(): PSSL_METHOD = TLS_method() - proc OpenSSL_version_num(): culong {.cdecl, dynlib: DLLSSLName, importc.} + proc OpenSSL_version_num(): culong {.cdecl, dynlib: DLLUtilName, importc.} proc getOpenSSLVersion*(): culong = ## Return OpenSSL version as unsigned long @@ -273,52 +318,105 @@ when compileOption("dynlibOverride", "ssl"): # Static linking against OpenSSL < 1.1.0 is not supported discard + proc ERR_load_BIO_strings*() = + discard + + when defined(libressl) or defined(openssl10): + proc SSL_state(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_in_init*(ssl: SslPtr): cint {.inline.} = + SSL_state(ssl) and SSL_ST_INIT + else: + proc SSL_in_init*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + template OpenSSL_add_all_algorithms*() = discard proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.} + proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} else: - # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some + # Here we're trying to stay compatible between openssl versions. Some # symbols are loaded dynamically and we don't use them if not found. proc thisModule(): LibHandle {.inline.} = var thisMod {.global.}: LibHandle if thisMod.isNil: thisMod = loadLib() + result = thisMod - proc sslModule(): LibHandle {.inline.} = + proc sslModule(): LibHandle {.inline, raises: [LibraryError], tags:[RootEffect].} = var sslMod {.global.}: LibHandle - if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + try: + if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName) + except: + raise newException(LibraryError, "Could not load SSL using " & DLLSSLName) + result = sslMod - proc sslSym(name: string): pointer = - var dl = thisModule() - if not dl.isNil: - result = symAddr(dl, name) + proc utilModule(): LibHandle {.inline.} = + var utilMod {.global.}: LibHandle + if utilMod.isNil: utilMod = loadLibPattern(DLLUtilName) + + result = utilMod + + proc symNullable(dll: LibHandle, name: string, alternativeName = ""): pointer = + # Load from DLL. + if not dll.isNil: + result = symAddr(dll, name) + if result.isNil and alternativeName.len > 0: + result = symAddr(dll, alternativeName) + + # Attempt to load from current exe. if result.isNil: - dl = sslModule() - if not dl.isNil: - result = symAddr(dl, name) + let thisDynlib = thisModule() + if thisDynlib.isNil: return nil + result = symAddr(thisDynlib, name) + if result.isNil and alternativeName.len > 0: + result = symAddr(thisDynlib, alternativeName) + + proc sslSymNullable(name: string, alternativeName = ""): pointer {.raises: [LibraryError], tags:[RootEffect].} = + sslModule().symNullable(name, alternativeName) + + proc sslSymThrows(name: string, alternativeName = ""): pointer {.raises: [LibraryError].} = + result = sslSymNullable(name, alternativeName) + if result.isNil: raiseInvalidLibrary(name) - proc loadPSSLMethod(method1, method2: string): PSSL_METHOD = + proc utilSymNullable(name: string, alternativeName = ""): pointer = + utilModule().symNullable(name, alternativeName) + + proc loadPSSLMethod(method1, method2: string): PSSL_METHOD {.raises: [LibraryError], tags:[RootEffect].} = ## Load <method1> from OpenSSL if available, otherwise <method2> - let m1 = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method1)) - if not m1.isNil: - return m1() - cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method2))() + ## + let methodSym = sslSymNullable(method1, method2) + if methodSym.isNil: + raise newException(LibraryError, "Could not load " & method1 & " nor " & method2) + + let method2Proc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe, raises: [].}](methodSym) + return method2Proc() + + proc CRYPTO_set_mem_functions(a,b,c: pointer) = + let theProc = cast[proc(a,b,c: pointer) {.cdecl.}](utilModule().symNullable("CRYPTO_set_mem_functions")) + if not theProc.isNil: theProc(a, b, c) proc SSL_library_init*(): cint {.discardable.} = ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise ## SSL_library_init - let theProc = cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](sslSym("OPENSSL_init_ssl")) - if not theProc.isNil: - return theProc(0, 0) - let olderProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init")) + let newInitSym = sslSymNullable("OPENSSL_init_ssl") + if not newInitSym.isNil: + let newInitProc = + cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](newInitSym) + return newInitProc(0, 0) + let olderProc = cast[proc(): cint {.cdecl.}](sslSymThrows("SSL_library_init")) if not olderProc.isNil: result = olderProc() proc SSL_load_error_strings*() = - let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings")) + # TODO: Are we ignoring this on purpose? SSL GitHub CI fails otherwise. + let theProc = cast[proc() {.cdecl.}](sslSymNullable("SSL_load_error_strings")) + if not theProc.isNil: theProc() + + proc ERR_load_BIO_strings*() = + let theProc = cast[proc() {.cdecl.}](utilModule().symNullable("ERR_load_BIO_strings")) if not theProc.isNil: theProc() proc SSLv23_client_method*(): PSSL_METHOD = @@ -343,22 +441,45 @@ else: loadPSSLMethod("TLS_server_method", "SSLv23_server_method") proc OpenSSL_add_all_algorithms*() = - let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf")) + # TODO: Are we ignoring this on purpose? SSL GitHub CI fails otherwise. + let theProc = cast[proc() {.cdecl.}](sslSymNullable("OPENSSL_add_all_algorithms_conf")) if not theProc.isNil: theProc() proc getOpenSSLVersion*(): culong = ## Return OpenSSL version as unsigned long or 0 if not available - let theProc = cast[proc(): culong {.cdecl.}](sslSym("OpenSSL_version_num")) + let theProc = cast[proc(): culong {.cdecl, gcsafe.}](utilSymNullable("OpenSSL_version_num", "SSLeay")) result = if theProc.isNil: 0.culong else: theProc() -proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.} + proc SSL_in_init*(ssl: SslPtr): cint = + # A compatibility wrapper for `SSL_in_init()` for OpenSSL 1.0, 1.1 and LibreSSL + const MainProc = "SSL_in_init" + let + theProc {.global.} = cast[proc(ssl: SslPtr): cint {.cdecl, gcsafe.}](sslSymNullable(MainProc)) + # Fallback + sslState {.global.} = cast[proc(ssl: SslPtr): cint {.cdecl, gcsafe.}](sslSymNullable("SSL_state")) + + if not theProc.isNil: + result = theProc(ssl) + elif not sslState.isNil: + result = sslState(ssl) and SSL_ST_INIT + else: + raiseInvalidLibrary MainProc + + proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint = + var theProc {.global.}: proc(ctx: SslCtx, str: cstring): cint {.cdecl, gcsafe.} + if theProc.isNil: + theProc = cast[typeof(theProc)](sslSymThrows("SSL_CTX_set_ciphersuites")) + result = theProc(ctx, str) proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} proc SSL_get_SSL_CTX*(ssl: SslPtr): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} proc SSL_set_SSL_CTX*(ssl: SslPtr, ctx: SslCtx): SslCtx {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_session_id_context*(context: SslCtx, sid_ctx: string, sid_ctx_len: int){.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_verified_chain*(ssl: SslPtr): PSTACK {.cdecl, dynlib: DLLSSLName, + importc.} proc SSL_CTX_new*(meth: PSSL_METHOD): SslCtx{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_load_verify_locations*(ctx: SslCtx, CAfile: cstring, @@ -395,11 +516,11 @@ proc SSL_accept*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_pending*(ssl: SslPtr): cint{.cdecl, dynlib: DLLSSLName, importc.} proc BIO_new_mem_buf*(data: pointer, len: cint): BIO{.cdecl, - dynlib: DLLSSLName, importc.} + dynlib: DLLUtilName, importc.} proc BIO_new_ssl_connect*(ctx: SslCtx): BIO{.cdecl, dynlib: DLLSSLName, importc.} proc BIO_ctrl*(bio: BIO, cmd: cint, larg: int, arg: cstring): int{.cdecl, - dynlib: DLLSSLName, importc.} + dynlib: DLLUtilName, importc.} proc BIO_get_ssl*(bio: BIO, ssl: ptr SslPtr): int = return BIO_ctrl(bio, BIO_C_GET_SSL, 0, cast[cstring](ssl)) proc BIO_set_conn_hostname*(bio: BIO, name: cstring): int = @@ -409,27 +530,53 @@ proc BIO_do_handshake*(bio: BIO): int = proc BIO_do_connect*(bio: BIO): int = return BIO_do_handshake(bio) -when not defined(nimfix): - proc BIO_read*(b: BIO, data: cstring, length: cint): cint{.cdecl, - dynlib: DLLUtilName, importc.} - proc BIO_write*(b: BIO, data: cstring, length: cint): cint{.cdecl, - dynlib: DLLUtilName, importc.} +proc BIO_read*(b: BIO, data: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc.} +proc BIO_write*(b: BIO, data: cstring, length: cint): cint{.cdecl, dynlib: DLLUtilName, importc.} proc BIO_free*(b: BIO): cint{.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_print_errors_fp*(fp: File){.cdecl, dynlib: DLLSSLName, importc.} +proc ERR_print_errors_fp*(fp: File){.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_error_string*(e: cint, buf: cstring): cstring{.cdecl, +proc ERR_error_string*(e: culong, buf: cstring): cstring{.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_get_error*(): cint{.cdecl, dynlib: DLLUtilName, importc.} -proc ERR_peek_last_error*(): cint{.cdecl, dynlib: DLLUtilName, importc.} +proc ERR_get_error*(): culong{.cdecl, dynlib: DLLUtilName, importc.} +proc ERR_peek_last_error*(): culong{.cdecl, dynlib: DLLUtilName, importc.} + +proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLUtilName, importc.} + +proc OPENSSL_sk_num*(stack: PSTACK): int {.cdecl, dynlib: DLLSSLName, importc.} -proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} +proc OPENSSL_sk_value*(stack: PSTACK, index: int): pointer {.cdecl, + dynlib: DLLSSLName, importc.} + +proc d2i_X509*(px: ptr PX509, i: ptr ptr uint8, len: cint): PX509 {.cdecl, + dynlib: DLLUtilName, importc.} -when not useWinVersion and not defined(macosx) and not defined(android) and not defined(nimNoAllocForSSL): - proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, +proc i2d_X509*(cert: PX509; o: ptr ptr uint8): cint {.cdecl, dynlib: DLLUtilName, importc.} +proc d2i_X509*(b: string): PX509 = + ## decode DER/BER bytestring into X.509 certificate struct + var bb = b.cstring + let i = cast[ptr ptr uint8](addr bb) + let ret = d2i_X509(addr result, i, b.len.cint) + if ret.isNil: + raise newException(Exception, "X.509 certificate decoding failed") + +proc i2d_X509*(cert: PX509): string = + ## encode `cert` to DER string + let encoded_length = i2d_X509(cert, nil) + result = newString(encoded_length) + var q = result.cstring + let o = cast[ptr ptr uint8](addr q) + let length = i2d_X509(cert, o) + if length.int <= 0: + raise newException(Exception, "X.509 certificate encoding failed") + +const + useNimsAlloc = not defined(nimNoAllocForSSL) and not defined(gcDestructors) + +when not useWinVersion and not defined(macosx) and not defined(android) and useNimsAlloc: proc allocWrapper(size: int): pointer {.cdecl.} = allocShared(size) proc reallocWrapper(p: pointer; newSize: int): pointer {.cdecl.} = if p == nil: @@ -439,27 +586,28 @@ when not useWinVersion and not defined(macosx) and not defined(android) and not proc deallocWrapper(p: pointer) {.cdecl.} = if p != nil: deallocShared(p) -proc CRYPTO_malloc_init*() = - when not useWinVersion and not defined(macosx) and not defined(android) and not defined(nimNoAllocForSSL): - CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper) + proc CRYPTO_malloc_init*() = + CRYPTO_set_mem_functions(cast[pointer](allocWrapper), cast[pointer](reallocWrapper), cast[pointer](deallocWrapper)) +else: + proc CRYPTO_malloc_init*() = + discard -proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cint, larg: int, parg: pointer): int{. +proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cint, larg: clong, parg: pointer): clong{. cdecl, dynlib: DLLSSLName, importc.} proc SSL_CTX_callback_ctrl(ctx: SslCtx, typ: cint, fp: PFunction): int{. cdecl, dynlib: DLLSSLName, importc.} proc SSLCTXSetMode*(ctx: SslCtx, mode: int): int = - result = SSL_CTX_ctrl(ctx, SSL_CTRL_MODE, mode, nil) + result = SSL_CTX_ctrl(ctx, SSL_CTRL_MODE, clong mode, nil) proc SSL_ctrl*(ssl: SslPtr, cmd: cint, larg: int, parg: pointer): int{. cdecl, dynlib: DLLSSLName, importc.} proc SSL_set_tlsext_host_name*(ssl: SslPtr, name: cstring): int = - result = SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, name) ## Set the SNI server name extension to be used in a client hello. ## Returns 1 if SNI was set, 0 if current SSL configuration doesn't support SNI. - + result = SSL_ctrl(ssl, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name, name) proc SSL_get_servername*(ssl: SslPtr, typ: cint = TLSEXT_NAMETYPE_host_name): cstring {.cdecl, dynlib: DLLSSLName, importc.} ## Retrieve the server name requested in the client hello. This can be used @@ -477,16 +625,16 @@ proc SSL_CTX_set_tlsext_servername_callback*(ctx: SslCtx, cb: proc(ssl: SslPtr, result = SSL_CTX_callback_ctrl(ctx, SSL_CTRL_SET_TLSEXT_SERVERNAME_CB, cast[PFunction](cb)) proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int = - ## Set the pointer to be used in the callback registered to ``SSL_CTX_set_tlsext_servername_callback``. + ## Set the pointer to be used in the callback registered to `SSL_CTX_set_tlsext_servername_callback`. result = SSL_CTX_ctrl(ctx, SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG, 0, arg) type PskClientCallback* = proc (ssl: SslPtr; - hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar; + hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr uint8; max_psk_len: cuint): cuint {.cdecl.} PskServerCallback* = proc (ssl: SslPtr; - identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} + identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.} proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.} ## Set callback called when OpenSSL needs PSK (for client). @@ -500,6 +648,15 @@ proc SSL_CTX_use_psk_identity_hint*(ctx: SslCtx; hint: cstring): cint {.cdecl, d proc SSL_get_psk_identity*(ssl: SslPtr): cstring {.cdecl, dynlib: DLLSSLName, importc.} ## Get PSK identity. +proc SSL_CTX_set_ecdh_auto*(ctx: SslCtx, onoff: cint): cint {.inline.} = + ## Set automatic curve selection. + ## + ## On OpenSSL >= 1.1.0 this is on by default and cannot be disabled. + if getOpenSSLVersion() < 0x010100000 or getOpenSSLVersion() == 0x020000000: + result = cint SSL_CTX_ctrl(ctx, SSL_CTRL_SET_ECDH_AUTO, onoff, nil) + else: + result = 1 + proc bioNew*(b: PBIO_METHOD): BIO{.cdecl, dynlib: DLLUtilName, importc: "BIO_new".} proc bioFreeAll*(b: BIO){.cdecl, dynlib: DLLUtilName, importc: "BIO_free_all".} proc bioSMem*(): PBIO_METHOD{.cdecl, dynlib: DLLUtilName, importc: "BIO_s_mem".} @@ -528,55 +685,78 @@ proc sslDoHandshake*(ssl: SslPtr): cint {.cdecl, dynlib: DLLSSLName, importc: "SSL_do_handshake".} - proc ErrClearError*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_clear_error".} proc ErrFreeStrings*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_free_strings".} proc ErrRemoveState*(pid: cint){.cdecl, dynlib: DLLUtilName, importc: "ERR_remove_state".} proc PEM_read_bio_RSA_PUBKEY*(bp: BIO, x: ptr PRSA, pw: pem_password_cb, u: pointer): PRSA {.cdecl, - dynlib: DLLSSLName, importc.} - + dynlib: DLLUtilName, importc.} +proc PEM_read_RSA_PUBKEY*(fp: pointer; x: ptr PRSA; cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} proc RSA_verify*(kind: cint, origMsg: pointer, origMsgLen: cuint, signature: pointer, - signatureLen: cuint, rsa: PRSA): cint {.cdecl, dynlib: DLLSSLName, importc.} + signatureLen: cuint, rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} +proc PEM_read_RSAPrivateKey*(fp: pointer; x: ptr PRSA; cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_RSAPublicKey*(fp: pointer; x: ptr PRSA; cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_bio_RSAPublicKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc PEM_read_bio_RSAPrivateKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_private_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_public_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_private_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_public_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl, + dynlib: DLLUtilName, importc.} +proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.} +proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} # sha types -proc EVP_md_null*(): EVP_MD {.cdecl, importc.} -proc EVP_md2*(): EVP_MD {.cdecl, importc.} -proc EVP_md4*(): EVP_MD {.cdecl, importc.} -proc EVP_md5*(): EVP_MD {.cdecl, importc.} -proc EVP_sha*(): EVP_MD {.cdecl, importc.} -proc EVP_sha1*(): EVP_MD {.cdecl, importc.} -proc EVP_dss*(): EVP_MD {.cdecl, importc.} -proc EVP_dss1*(): EVP_MD {.cdecl, importc.} -proc EVP_ecdsa*(): EVP_MD {.cdecl, importc.} -proc EVP_sha224*(): EVP_MD {.cdecl, importc.} -proc EVP_sha256*(): EVP_MD {.cdecl, importc.} -proc EVP_sha384*(): EVP_MD {.cdecl, importc.} -proc EVP_sha512*(): EVP_MD {.cdecl, importc.} -proc EVP_mdc2*(): EVP_MD {.cdecl, importc.} -proc EVP_ripemd160*(): EVP_MD {.cdecl, importc.} -proc EVP_whirlpool*(): EVP_MD {.cdecl, importc.} +proc EVP_md_null*(): EVP_MD {.lcrypto.} +proc EVP_md2*(): EVP_MD {.lcrypto.} +proc EVP_md4*(): EVP_MD {.lcrypto.} +proc EVP_md5*(): EVP_MD {.lcrypto.} +proc EVP_sha*(): EVP_MD {.lcrypto.} +proc EVP_sha1*(): EVP_MD {.lcrypto.} +proc EVP_dss*(): EVP_MD {.lcrypto.} +proc EVP_dss1*(): EVP_MD {.lcrypto.} +proc EVP_ecdsa*(): EVP_MD {.lcrypto.} +proc EVP_sha224*(): EVP_MD {.lcrypto.} +proc EVP_sha256*(): EVP_MD {.lcrypto.} +proc EVP_sha384*(): EVP_MD {.lcrypto.} +proc EVP_sha512*(): EVP_MD {.lcrypto.} +proc EVP_mdc2*(): EVP_MD {.lcrypto.} +proc EVP_ripemd160*(): EVP_MD {.lcrypto.} +proc EVP_whirlpool*(): EVP_MD {.lcrypto.} +proc EVP_MD_size*(md: EVP_MD): cint {.lcrypto.} # hmac functions -proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.cdecl, importc.} +proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.lcrypto.} # RSA key functions -proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.cdecl, importc.} -proc EVP_PKEY_free*(p: EVP_PKEY) {.cdecl, importc.} -proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.cdecl, importc.} -proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.cdecl, importc.} -proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.cdecl, importc.} -proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.cdecl, importc.} -proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.cdecl, importc.} -proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.cdecl, importc.} +proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.lcrypto.} +proc EVP_PKEY_free*(p: EVP_PKEY) {.lcrypto.} +proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.lcrypto.} +proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: EVP_MD, engine: SslPtr = nil): cint {.lcrypto.} +proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.lcrypto.} +proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.lcrypto.} +proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.lcrypto.} +proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.lcrypto.} +proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.lcrypto.} +proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.lcrypto.} when defined(macosx) or defined(windows): - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc.} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc.} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.lcrypto.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.lcrypto.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.lcrypto.} else: # some times you will need this instead: - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new".} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free".} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new", dynlib: DLLUtilName.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free", dynlib: DLLUtilName.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup", dynlib: DLLUtilName.} # <openssl/md5.h> type @@ -595,11 +775,11 @@ type proc md5_Init*(c: var MD5_CTX): cint{.importc: "MD5_Init".} proc md5_Update*(c: var MD5_CTX; data: pointer; len: csize_t): cint{.importc: "MD5_Update".} proc md5_Final*(md: cstring; c: var MD5_CTX): cint{.importc: "MD5_Final".} -proc md5*(d: ptr cuchar; n: csize_t; md: ptr cuchar): ptr cuchar{.importc: "MD5".} -proc md5_Transform*(c: var MD5_CTX; b: ptr cuchar){.importc: "MD5_Transform".} +proc md5*(d: ptr uint8; n: csize_t; md: ptr uint8): ptr uint8{.importc: "MD5".} +proc md5_Transform*(c: var MD5_CTX; b: ptr uint8){.importc: "MD5_Transform".} {.pop.} -from strutils import toHex, toLowerAscii +from std/strutils import toHex, toLowerAscii proc hexStr(buf: cstring): string = # turn md5s output into a nice hex str @@ -619,13 +799,13 @@ proc md5_File*(file: string): string {.raises: [IOError,Exception].} = ctx: MD5_CTX discard md5_Init(ctx) - while(let bytes = f.readChars(buf, 0, sz); bytes > 0): + while (let bytes = f.readChars(buf); bytes > 0): discard md5_Update(ctx, buf[0].addr, cast[csize_t](bytes)) - discard md5_Final(buf[0].addr, ctx) + discard md5_Final(cast[cstring](buf[0].addr), ctx) f.close - result = hexStr(addr buf) + result = hexStr(cast[cstring](addr buf)) proc md5_Str*(str: string): string = ## Generate MD5 hash for a string. Result is a 32 character @@ -642,8 +822,88 @@ proc md5_Str*(str: string): string = discard md5_Update(ctx, input[i].addr, cast[csize_t](L)) i += L - discard md5_Final(addr res, ctx) - result = hexStr(addr res) + discard md5_Final(cast[cstring](addr res), ctx) + result = hexStr(cast[cstring](addr res)) when defined(nimHasStyleChecks): {.pop.} + + +# Certificate validation +# On old openSSL version some of these symbols are not available +when not defined(nimDisableCertificateValidation) and not defined(windows): + + # SSL_get_peer_certificate removed in 3.0 + # SSL_get1_peer_certificate added in 3.0 + when useOpenssl3: + proc SSL_get1_peer_certificate*(ssl: SslCtx): PX509 {.cdecl, dynlib: DLLSSLName, importc.} + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 = + SSL_get1_peer_certificate(ssl) + elif useStaticLink: + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 {.cdecl, dynlib: DLLSSLName, importc.} + else: + proc SSL_get_peer_certificate*(ssl: SslCtx): PX509 = + let methodSym = sslSymNullable("SSL_get_peer_certificate", "SSL_get1_peer_certificate") + if methodSym.isNil: + raise newException(LibraryError, "Could not load SSL_get_peer_certificate or SSL_get1_peer_certificate") + let method2Proc = cast[proc(ssl: SslCtx): PX509 {.cdecl, gcsafe, raises: [].}](methodSym) + return method2Proc(ssl) + + proc X509_get_subject_name*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLSSLName, importc.} + + proc X509_get_issuer_name*(a: PX509): PX509_NAME{.cdecl, dynlib: DLLUtilName, importc.} + + proc X509_NAME_oneline*(a: PX509_NAME, buf: cstring, size: cint): cstring {. + cdecl, dynlib:DLLSSLName, importc.} + + proc X509_NAME_get_text_by_NID*(subject:cstring, NID: cint, buf: cstring, size: cint): cint{. + cdecl, dynlib:DLLSSLName, importc.} + + proc X509_check_host*(cert: PX509, name: cstring, namelen: cint, flags:cuint, peername: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.} + + proc X509_free*(cert: PX509) {.cdecl, dynlib: DLLSSLName, importc.} + + # Certificates store + + type PX509_STORE* = SslPtr + type PX509_OBJECT* = SslPtr + + {.push callconv:cdecl, dynlib:DLLUtilName, importc.} + + proc X509_OBJECT_new*(): PX509_OBJECT + proc X509_OBJECT_free*(a: PX509_OBJECT) + + proc X509_STORE_new*(): PX509_STORE + proc X509_STORE_free*(v: PX509_STORE) + proc X509_STORE_lock*(ctx: PX509_STORE): cint + proc X509_STORE_unlock*(ctx: PX509_STORE): cint + proc X509_STORE_up_ref*(v: PX509_STORE): cint + proc X509_STORE_set_flags*(ctx: PX509_STORE; flags: culong): cint + proc X509_STORE_set_purpose*(ctx: PX509_STORE; purpose: cint): cint + proc X509_STORE_set_trust*(ctx: PX509_STORE; trust: cint): cint + proc X509_STORE_add_cert*(ctx: PX509_STORE; x: PX509): cint + + {.pop.} + + when isMainModule: + when defined(nimPreviewSlimSystem): + import std/assertions + # A simple certificate test + let certbytes = readFile("certificate.der") + let cert = d2i_X509(certbytes) + let encoded = cert.i2d_X509() + assert encoded == certbytes + +# Application Layer Protocol Negociation extension (TLS-ALPN, RFC7301) +# Available in at least OpenSSL 1.1.1 and later, not sure if earlier +# --Iced Quinn + +proc SSL_CTX_set_alpn_protos*(ctx: SslCtx; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_alpn_protos*(ssl: SslPtr; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_alpn_select_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_alpn_selected*(ssl: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_protos_advertised_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: ptr cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_proto_select_cb*(ctx: SslCtx; cb: proc(s: SslPtr; out_proto: cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_select_next_proto*(out_proto: ptr cstring; outlen: cstring; server: cstring; server_len: cuint; client: cstring; client_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_next_proto_negotiated*(s: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} + diff --git a/lib/wrappers/pcre.nim b/lib/wrappers/pcre.nim index 7012053b7..9144f6784 100644 --- a/lib/wrappers/pcre.nim +++ b/lib/wrappers/pcre.nim @@ -7,8 +7,6 @@ # distribution, for details about the copyright. # -{.deadCodeElim: on.} # dce option deprecated - # The current PCRE version information. const diff --git a/lib/wrappers/postgres.nim b/lib/wrappers/postgres.nim deleted file mode 100644 index c90116ea2..000000000 --- a/lib/wrappers/postgres.nim +++ /dev/null @@ -1,373 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# This module contains the definitions for structures and externs for -# functions used by frontend postgres applications. It is based on -# Postgresql's libpq-fe.h. -# -# It is for postgreSQL version 7.4 and higher with support for the v3.0 -# connection-protocol. -# - -{.deadCodeElim: on.} # dce option deprecated -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -when defined(windows): - const - dllName = "libpq.dll" -elif defined(macosx): - const - dllName = "libpq.dylib" -else: - const - dllName = "libpq.so(.5|)" -type - POid* = ptr Oid - Oid* = int32 - -const - ERROR_MSG_LENGTH* = 4096 - CMDSTATUS_LEN* = 40 - -type - SockAddr* = array[1..112, int8] - PGresAttDesc*{.pure, final.} = object - name*: cstring - adtid*: Oid - adtsize*: int - - PPGresAttDesc* = ptr PGresAttDesc - PPPGresAttDesc* = ptr PPGresAttDesc - PGresAttValue*{.pure, final.} = object - length*: int32 - value*: cstring - - PPGresAttValue* = ptr PGresAttValue - PPPGresAttValue* = ptr PPGresAttValue - PExecStatusType* = ptr ExecStatusType - ExecStatusType* = enum - PGRES_EMPTY_QUERY = 0, PGRES_COMMAND_OK, PGRES_TUPLES_OK, PGRES_COPY_OUT, - PGRES_COPY_IN, PGRES_BAD_RESPONSE, PGRES_NONFATAL_ERROR, PGRES_FATAL_ERROR - PGlobjfuncs*{.pure, final.} = object - fn_lo_open*: Oid - fn_lo_close*: Oid - fn_lo_creat*: Oid - fn_lo_unlink*: Oid - fn_lo_lseek*: Oid - fn_lo_tell*: Oid - fn_lo_read*: Oid - fn_lo_write*: Oid - - PPGlobjfuncs* = ptr PGlobjfuncs - PConnStatusType* = ptr ConnStatusType - ConnStatusType* = enum - CONNECTION_OK, CONNECTION_BAD, CONNECTION_STARTED, CONNECTION_MADE, - CONNECTION_AWAITING_RESPONSE, CONNECTION_AUTH_OK, CONNECTION_SETENV, - CONNECTION_SSL_STARTUP, CONNECTION_NEEDED - PGconn*{.pure, final.} = object - pghost*: cstring - pgtty*: cstring - pgport*: cstring - pgoptions*: cstring - dbName*: cstring - status*: ConnStatusType - errorMessage*: array[0..(ERROR_MSG_LENGTH) - 1, char] - Pfin*: File - Pfout*: File - Pfdebug*: File - sock*: int32 - laddr*: SockAddr - raddr*: SockAddr - salt*: array[0..(2) - 1, char] - asyncNotifyWaiting*: int32 - notifyList*: pointer - pguser*: cstring - pgpass*: cstring - lobjfuncs*: PPGlobjfuncs - - PPGconn* = ptr PGconn - PGresult*{.pure, final.} = object - ntups*: int32 - numAttributes*: int32 - attDescs*: PPGresAttDesc - tuples*: PPPGresAttValue - tupArrSize*: int32 - resultStatus*: ExecStatusType - cmdStatus*: array[0..(CMDSTATUS_LEN) - 1, char] - binary*: int32 - conn*: PPGconn - - PPGresult* = ptr PGresult - PPostgresPollingStatusType* = ptr PostgresPollingStatusType - PostgresPollingStatusType* = enum - PGRES_POLLING_FAILED = 0, PGRES_POLLING_READING, PGRES_POLLING_WRITING, - PGRES_POLLING_OK, PGRES_POLLING_ACTIVE - PPGTransactionStatusType* = ptr PGTransactionStatusType - PGTransactionStatusType* = enum - PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, - PQTRANS_UNKNOWN - PPGVerbosity* = ptr PGVerbosity - PGVerbosity* = enum - PQERRORS_TERSE, PQERRORS_DEFAULT, PQERRORS_VERBOSE - PPGNotify* = ptr pgNotify - pgNotify*{.pure, final.} = object - relname*: cstring - be_pid*: int32 - extra*: cstring - - PQnoticeReceiver* = proc (arg: pointer, res: PPGresult){.cdecl.} - PQnoticeProcessor* = proc (arg: pointer, message: cstring){.cdecl.} - Ppqbool* = ptr pqbool - pqbool* = char - PPQprintOpt* = ptr PQprintOpt - PQprintOpt*{.pure, final.} = object - header*: pqbool - align*: pqbool - standard*: pqbool - html3*: pqbool - expanded*: pqbool - pager*: pqbool - fieldSep*: cstring - tableOpt*: cstring - caption*: cstring - fieldName*: ptr cstring - - PPQconninfoOption* = ptr PQconninfoOption - PQconninfoOption*{.pure, final.} = object - keyword*: cstring - envvar*: cstring - compiled*: cstring - val*: cstring - label*: cstring - dispchar*: cstring - dispsize*: int32 - - PPQArgBlock* = ptr PQArgBlock - PQArgBlock*{.pure, final.} = object - length*: int32 - isint*: int32 - p*: pointer - -proc pqinitOpenSSL*(do_ssl: int32, do_crypto: int32) {.cdecl, dynlib: dllName, - importc: "PQinitOpenSSL".} -proc pqconnectStart*(conninfo: cstring): PPGconn{.cdecl, dynlib: dllName, - importc: "PQconnectStart".} -proc pqconnectPoll*(conn: PPGconn): PostgresPollingStatusType{.cdecl, - dynlib: dllName, importc: "PQconnectPoll".} -proc pqconnectdb*(conninfo: cstring): PPGconn{.cdecl, dynlib: dllName, - importc: "PQconnectdb".} -proc pqsetdbLogin*(pghost: cstring, pgport: cstring, pgoptions: cstring, - pgtty: cstring, dbName: cstring, login: cstring, pwd: cstring): PPGconn{. - cdecl, dynlib: dllName, importc: "PQsetdbLogin".} -proc pqsetdb*(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME: cstring): PPGconn -proc pqfinish*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQfinish".} -proc pqconndefaults*(): PPQconninfoOption{.cdecl, dynlib: dllName, - importc: "PQconndefaults".} -proc pqconninfoFree*(connOptions: PPQconninfoOption){.cdecl, dynlib: dllName, - importc: "PQconninfoFree".} -proc pqresetStart*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQresetStart".} -proc pqresetPoll*(conn: PPGconn): PostgresPollingStatusType{.cdecl, - dynlib: dllName, importc: "PQresetPoll".} -proc pqreset*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQreset".} -proc pqrequestCancel*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQrequestCancel".} -proc pqdb*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQdb".} -proc pquser*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQuser".} -proc pqpass*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQpass".} -proc pqhost*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQhost".} -proc pqport*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQport".} -proc pqtty*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, importc: "PQtty".} -proc pqoptions*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, - importc: "PQoptions".} -proc pqstatus*(conn: PPGconn): ConnStatusType{.cdecl, dynlib: dllName, - importc: "PQstatus".} -proc pqtransactionStatus*(conn: PPGconn): PGTransactionStatusType{.cdecl, - dynlib: dllName, importc: "PQtransactionStatus".} -proc pqparameterStatus*(conn: PPGconn, paramName: cstring): cstring{.cdecl, - dynlib: dllName, importc: "PQparameterStatus".} -proc pqserverVersion*(conn: PPGconn): int32{.cdecl, - dynlib: dllName, importc: "PQserverVersion".} -proc pqprotocolVersion*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQprotocolVersion".} -proc pqerrorMessage*(conn: PPGconn): cstring{.cdecl, dynlib: dllName, - importc: "PQerrorMessage".} -proc pqsocket*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQsocket".} -proc pqbackendPID*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQbackendPID".} -proc pqconnectionNeedsPassword*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconnectionNeedsPassword".} -proc pqconnectionUsedPassword*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconnectionUsedPassword".} -proc pqclientEncoding*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQclientEncoding".} -proc pqsetClientEncoding*(conn: PPGconn, encoding: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQsetClientEncoding".} -when defined(USE_SSL): - # Get the SSL structure associated with a connection - proc pqgetssl*(conn: PPGconn): PSSL{.cdecl, dynlib: dllName, - importc: "PQgetssl".} -proc pqsetErrorVerbosity*(conn: PPGconn, verbosity: PGVerbosity): PGVerbosity{. - cdecl, dynlib: dllName, importc: "PQsetErrorVerbosity".} -proc pqtrace*(conn: PPGconn, debug_port: File){.cdecl, dynlib: dllName, - importc: "PQtrace".} -proc pquntrace*(conn: PPGconn){.cdecl, dynlib: dllName, importc: "PQuntrace".} -proc pqsetNoticeReceiver*(conn: PPGconn, theProc: PQnoticeReceiver, arg: pointer): PQnoticeReceiver{. - cdecl, dynlib: dllName, importc: "PQsetNoticeReceiver".} -proc pqsetNoticeProcessor*(conn: PPGconn, theProc: PQnoticeProcessor, - arg: pointer): PQnoticeProcessor{.cdecl, - dynlib: dllName, importc: "PQsetNoticeProcessor".} -proc pqexec*(conn: PPGconn, query: cstring): PPGresult{.cdecl, dynlib: dllName, - importc: "PQexec".} -proc pqexecParams*(conn: PPGconn, command: cstring, nParams: int32, - paramTypes: POid, paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQexecParams".} -proc pqprepare*(conn: PPGconn, stmtName, query: cstring, nParams: int32, - paramTypes: POid): PPGresult{.cdecl, dynlib: dllName, importc: "PQprepare".} -proc pqexecPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, - paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, resultFormat: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQexecPrepared".} -proc pqsendQuery*(conn: PPGconn, query: cstring): int32{.cdecl, dynlib: dllName, - importc: "PQsendQuery".} -proc pqsendQueryParams*(conn: PPGconn, command: cstring, nParams: int32, - paramTypes: POid, paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, - resultFormat: int32): int32{.cdecl, dynlib: dllName, - importc: "PQsendQueryParams".} -proc pqsendQueryPrepared*(conn: PPGconn, stmtName: cstring, nParams: int32, - paramValues: cstringArray, - paramLengths, paramFormats: ptr int32, - resultFormat: int32): int32{.cdecl, dynlib: dllName, - importc: "PQsendQueryPrepared".} -proc pqgetResult*(conn: PPGconn): PPGresult{.cdecl, dynlib: dllName, - importc: "PQgetResult".} -proc pqisBusy*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQisBusy".} -proc pqconsumeInput*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQconsumeInput".} -proc pqnotifies*(conn: PPGconn): PPGNotify{.cdecl, dynlib: dllName, - importc: "PQnotifies".} -proc pqputCopyData*(conn: PPGconn, buffer: cstring, nbytes: int32): int32{. - cdecl, dynlib: dllName, importc: "PQputCopyData".} -proc pqputCopyEnd*(conn: PPGconn, errormsg: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQputCopyEnd".} -proc pqgetCopyData*(conn: PPGconn, buffer: cstringArray, async: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetCopyData".} -proc pqgetline*(conn: PPGconn, str: cstring, len: int32): int32{.cdecl, - dynlib: dllName, importc: "PQgetline".} -proc pqputline*(conn: PPGconn, str: cstring): int32{.cdecl, dynlib: dllName, - importc: "PQputline".} -proc pqgetlineAsync*(conn: PPGconn, buffer: cstring, bufsize: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetlineAsync".} -proc pqputnbytes*(conn: PPGconn, buffer: cstring, nbytes: int32): int32{.cdecl, - dynlib: dllName, importc: "PQputnbytes".} -proc pqendcopy*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQendcopy".} -proc pqsetnonblocking*(conn: PPGconn, arg: int32): int32{.cdecl, - dynlib: dllName, importc: "PQsetnonblocking".} -proc pqisnonblocking*(conn: PPGconn): int32{.cdecl, dynlib: dllName, - importc: "PQisnonblocking".} -proc pqflush*(conn: PPGconn): int32{.cdecl, dynlib: dllName, importc: "PQflush".} -proc pqfn*(conn: PPGconn, fnid: int32, result_buf, result_len: ptr int32, - result_is_int: int32, args: PPQArgBlock, nargs: int32): PPGresult{. - cdecl, dynlib: dllName, importc: "PQfn".} -proc pqresultStatus*(res: PPGresult): ExecStatusType{.cdecl, dynlib: dllName, - importc: "PQresultStatus".} -proc pqresStatus*(status: ExecStatusType): cstring{.cdecl, dynlib: dllName, - importc: "PQresStatus".} -proc pqresultErrorMessage*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQresultErrorMessage".} -proc pqresultErrorField*(res: PPGresult, fieldcode: int32): cstring{.cdecl, - dynlib: dllName, importc: "PQresultErrorField".} -proc pqntuples*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQntuples".} -proc pqnfields*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQnfields".} -proc pqbinaryTuples*(res: PPGresult): int32{.cdecl, dynlib: dllName, - importc: "PQbinaryTuples".} -proc pqfname*(res: PPGresult, field_num: int32): cstring{.cdecl, - dynlib: dllName, importc: "PQfname".} -proc pqfnumber*(res: PPGresult, field_name: cstring): int32{.cdecl, - dynlib: dllName, importc: "PQfnumber".} -proc pqftable*(res: PPGresult, field_num: int32): Oid{.cdecl, dynlib: dllName, - importc: "PQftable".} -proc pqftablecol*(res: PPGresult, field_num: int32): int32{.cdecl, - dynlib: dllName, importc: "PQftablecol".} -proc pqfformat*(res: PPGresult, field_num: int32): int32{.cdecl, - dynlib: dllName, importc: "PQfformat".} -proc pqftype*(res: PPGresult, field_num: int32): Oid{.cdecl, dynlib: dllName, - importc: "PQftype".} -proc pqfsize*(res: PPGresult, field_num: int32): int32{.cdecl, dynlib: dllName, - importc: "PQfsize".} -proc pqfmod*(res: PPGresult, field_num: int32): int32{.cdecl, dynlib: dllName, - importc: "PQfmod".} -proc pqcmdStatus*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQcmdStatus".} -proc pqoidStatus*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQoidStatus".} -proc pqoidValue*(res: PPGresult): Oid{.cdecl, dynlib: dllName, - importc: "PQoidValue".} -proc pqcmdTuples*(res: PPGresult): cstring{.cdecl, dynlib: dllName, - importc: "PQcmdTuples".} -proc pqgetvalue*(res: PPGresult, tup_num: int32, field_num: int32): cstring{. - cdecl, dynlib: dllName, importc: "PQgetvalue".} -proc pqgetlength*(res: PPGresult, tup_num: int32, field_num: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetlength".} -proc pqgetisnull*(res: PPGresult, tup_num: int32, field_num: int32): int32{. - cdecl, dynlib: dllName, importc: "PQgetisnull".} -proc pqclear*(res: PPGresult){.cdecl, dynlib: dllName, importc: "PQclear".} -proc pqfreemem*(p: pointer){.cdecl, dynlib: dllName, importc: "PQfreemem".} -proc pqmakeEmptyPGresult*(conn: PPGconn, status: ExecStatusType): PPGresult{. - cdecl, dynlib: dllName, importc: "PQmakeEmptyPGresult".} -proc pqescapeString*(till, `from`: cstring, len: int): int{.cdecl, - dynlib: dllName, importc: "PQescapeString".} -proc pqescapeBytea*(bintext: cstring, binlen: int, bytealen: var int): cstring{. - cdecl, dynlib: dllName, importc: "PQescapeBytea".} -proc pqunescapeBytea*(strtext: cstring, retbuflen: var int): cstring{.cdecl, - dynlib: dllName, importc: "PQunescapeBytea".} -proc pqprint*(fout: File, res: PPGresult, ps: PPQprintOpt){.cdecl, - dynlib: dllName, importc: "PQprint".} -proc pqdisplayTuples*(res: PPGresult, fp: File, fillAlign: int32, - fieldSep: cstring, printHeader: int32, quiet: int32){. - cdecl, dynlib: dllName, importc: "PQdisplayTuples".} -proc pqprintTuples*(res: PPGresult, fout: File, printAttName: int32, - terseOutput: int32, width: int32){.cdecl, dynlib: dllName, - importc: "PQprintTuples".} -proc lo_open*(conn: PPGconn, lobjId: Oid, mode: int32): int32{.cdecl, - dynlib: dllName, importc: "lo_open".} -proc lo_close*(conn: PPGconn, fd: int32): int32{.cdecl, dynlib: dllName, - importc: "lo_close".} -proc lo_read*(conn: PPGconn, fd: int32, buf: cstring, length: int): int32{. - cdecl, dynlib: dllName, importc: "lo_read".} -proc lo_write*(conn: PPGconn, fd: int32, buf: cstring, length: int): int32{. - cdecl, dynlib: dllName, importc: "lo_write".} -proc lo_lseek*(conn: PPGconn, fd: int32, offset: int32, whence: int32): int32{. - cdecl, dynlib: dllName, importc: "lo_lseek".} -proc lo_creat*(conn: PPGconn, mode: int32): Oid{.cdecl, dynlib: dllName, - importc: "lo_creat".} -proc lo_tell*(conn: PPGconn, fd: int32): int32{.cdecl, dynlib: dllName, - importc: "lo_tell".} -proc lo_unlink*(conn: PPGconn, lobjId: Oid): int32{.cdecl, dynlib: dllName, - importc: "lo_unlink".} -proc lo_import*(conn: PPGconn, filename: cstring): Oid{.cdecl, dynlib: dllName, - importc: "lo_import".} -proc lo_export*(conn: PPGconn, lobjId: Oid, filename: cstring): int32{.cdecl, - dynlib: dllName, importc: "lo_export".} -proc pqmblen*(s: cstring, encoding: int32): int32{.cdecl, dynlib: dllName, - importc: "PQmblen".} -proc pqenv2encoding*(): int32{.cdecl, dynlib: dllName, importc: "PQenv2encoding".} -proc pqsetdb(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME: cstring): PPGconn = - result = pqsetdbLogin(M_PGHOST, M_PGPORT, M_PGOPT, M_PGTTY, M_DBNAME, "", "") - -when defined(nimHasStyleChecks): - {.pop.} diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim deleted file mode 100644 index b96a70ef3..000000000 --- a/lib/wrappers/sqlite3.nim +++ /dev/null @@ -1,377 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -{.deadCodeElim: on.} # dce option deprecated - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -when defined(windows): - when defined(nimOldDlls): - const Lib = "sqlite3.dll" - elif defined(cpu64): - const Lib = "sqlite3_64.dll" - else: - const Lib = "sqlite3_32.dll" -elif defined(macosx): - const - Lib = "libsqlite3(|.0).dylib" -else: - const - Lib = "libsqlite3.so(|.0)" - -when defined(staticSqlite): - {.pragma: mylib.} - {.compile: "sqlite3.c".} -else: - {.pragma: mylib, dynlib: Lib.} - -const - SQLITE_INTEGER* = 1 - SQLITE_FLOAT* = 2 - SQLITE_BLOB* = 4 - SQLITE_NULL* = 5 - SQLITE_TEXT* = 3 - SQLITE_UTF8* = 1 - SQLITE_UTF16LE* = 2 - SQLITE_UTF16BE* = 3 # Use native byte order - SQLITE_UTF16* = 4 # sqlite3_create_function only - SQLITE_ANY* = 5 #sqlite_exec return values - SQLITE_OK* = 0 - SQLITE_ERROR* = 1 # SQL error or missing database - SQLITE_INTERNAL* = 2 # An internal logic error in SQLite - SQLITE_PERM* = 3 # Access permission denied - SQLITE_ABORT* = 4 # Callback routine requested an abort - SQLITE_BUSY* = 5 # The database file is locked - SQLITE_LOCKED* = 6 # A table in the database is locked - SQLITE_NOMEM* = 7 # A malloc() failed - SQLITE_READONLY* = 8 # Attempt to write a readonly database - SQLITE_INTERRUPT* = 9 # Operation terminated by sqlite3_interrupt() - SQLITE_IOERR* = 10 # Some kind of disk I/O error occurred - SQLITE_CORRUPT* = 11 # The database disk image is malformed - SQLITE_NOTFOUND* = 12 # (Internal Only) Table or record not found - SQLITE_FULL* = 13 # Insertion failed because database is full - SQLITE_CANTOPEN* = 14 # Unable to open the database file - SQLITE_PROTOCOL* = 15 # Database lock protocol error - SQLITE_EMPTY* = 16 # Database is empty - SQLITE_SCHEMA* = 17 # The database schema changed - SQLITE_TOOBIG* = 18 # Too much data for one row of a table - SQLITE_CONSTRAINT* = 19 # Abort due to constraint violation - SQLITE_MISMATCH* = 20 # Data type mismatch - SQLITE_MISUSE* = 21 # Library used incorrectly - SQLITE_NOLFS* = 22 # Uses OS features not supported on host - SQLITE_AUTH* = 23 # Authorization denied - SQLITE_FORMAT* = 24 # Auxiliary database format error - SQLITE_RANGE* = 25 # 2nd parameter to sqlite3_bind out of range - SQLITE_NOTADB* = 26 # File opened that is not a database file - SQLITE_ROW* = 100 # sqlite3_step() has another row ready - SQLITE_DONE* = 101 # sqlite3_step() has finished executing - SQLITE_COPY* = 0 - SQLITE_CREATE_INDEX* = 1 - SQLITE_CREATE_TABLE* = 2 - SQLITE_CREATE_TEMP_INDEX* = 3 - SQLITE_CREATE_TEMP_TABLE* = 4 - SQLITE_CREATE_TEMP_TRIGGER* = 5 - SQLITE_CREATE_TEMP_VIEW* = 6 - SQLITE_CREATE_TRIGGER* = 7 - SQLITE_CREATE_VIEW* = 8 - SQLITE_DELETE* = 9 - SQLITE_DROP_INDEX* = 10 - SQLITE_DROP_TABLE* = 11 - SQLITE_DROP_TEMP_INDEX* = 12 - SQLITE_DROP_TEMP_TABLE* = 13 - SQLITE_DROP_TEMP_TRIGGER* = 14 - SQLITE_DROP_TEMP_VIEW* = 15 - SQLITE_DROP_TRIGGER* = 16 - SQLITE_DROP_VIEW* = 17 - SQLITE_INSERT* = 18 - SQLITE_PRAGMA* = 19 - SQLITE_READ* = 20 - SQLITE_SELECT* = 21 - SQLITE_TRANSACTION* = 22 - SQLITE_UPDATE* = 23 - SQLITE_ATTACH* = 24 - SQLITE_DETACH* = 25 - SQLITE_ALTER_TABLE* = 26 - SQLITE_REINDEX* = 27 - SQLITE_DENY* = 1 - SQLITE_IGNORE* = 2 # Original from sqlite3.h: - #define SQLITE_STATIC ((void(*)(void *))0) - #define SQLITE_TRANSIENT ((void(*)(void *))-1) - SQLITE_DETERMINISTIC* = 0x800 - -type - Sqlite3 {.pure, final.} = object - PSqlite3* = ptr Sqlite3 - PPSqlite3* = ptr PSqlite3 - Context{.pure, final.} = object - Pcontext* = ptr Context - TStmt{.pure, final.} = object - PStmt* = ptr TStmt - Value{.pure, final.} = object - PValue* = ptr Value - PValueArg* = array[0..127, PValue] - - Callback* = proc (para1: pointer, para2: int32, para3, - para4: cstringArray): int32{.cdecl.} - Tbind_destructor_func* = proc (para1: pointer){.cdecl, locks: 0, tags: [], gcsafe.} - Create_function_step_func* = proc (para1: Pcontext, para2: int32, - para3: PValueArg){.cdecl.} - Create_function_func_func* = proc (para1: Pcontext, para2: int32, - para3: PValueArg){.cdecl.} - Create_function_final_func* = proc (para1: Pcontext){.cdecl.} - Result_func* = proc (para1: pointer){.cdecl.} - Create_collation_func* = proc (para1: pointer, para2: int32, para3: pointer, - para4: int32, para5: pointer): int32{.cdecl.} - Collation_needed_func* = proc (para1: pointer, para2: PSqlite3, eTextRep: int32, - para4: cstring){.cdecl.} - -const - SQLITE_STATIC* = nil - SQLITE_TRANSIENT* = cast[Tbind_destructor_func](-1) - -proc close*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_close".} -proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer, - errmsg: var cstring): int32{.cdecl, mylib, - importc: "sqlite3_exec".} -proc last_insert_rowid*(para1: PSqlite3): int64{.cdecl, mylib, - importc: "sqlite3_last_insert_rowid".} -proc changes*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_changes".} -proc total_changes*(para1: PSqlite3): int32{.cdecl, mylib, - importc: "sqlite3_total_changes".} -proc interrupt*(para1: PSqlite3){.cdecl, mylib, importc: "sqlite3_interrupt".} -proc complete*(sql: cstring): int32{.cdecl, mylib, - importc: "sqlite3_complete".} -proc complete16*(sql: pointer): int32{.cdecl, mylib, - importc: "sqlite3_complete16".} -proc busy_handler*(para1: PSqlite3, - para2: proc (para1: pointer, para2: int32): int32{.cdecl.}, - para3: pointer): int32{.cdecl, mylib, - importc: "sqlite3_busy_handler".} -proc busy_timeout*(para1: PSqlite3, ms: int32): int32{.cdecl, mylib, - importc: "sqlite3_busy_timeout".} -proc get_table*(para1: PSqlite3, sql: cstring, resultp: var cstringArray, - nrow, ncolumn: var cint, errmsg: ptr cstring): int32{.cdecl, - mylib, importc: "sqlite3_get_table".} -proc free_table*(result: cstringArray){.cdecl, mylib, - importc: "sqlite3_free_table".} - # Todo: see how translate sqlite3_mprintf, sqlite3_vmprintf, sqlite3_snprintf - # function sqlite3_mprintf(_para1:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_mprintf'; -proc mprintf*(para1: cstring): cstring{.cdecl, varargs, mylib, - importc: "sqlite3_mprintf".} - #function sqlite3_vmprintf(_para1:Pchar; _para2:va_list):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_vmprintf'; -proc free*(z: cstring){.cdecl, mylib, importc: "sqlite3_free".} - #function sqlite3_snprintf(_para1:longint; _para2:Pchar; _para3:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_snprintf'; -proc snprintf*(para1: int32, para2: cstring, para3: cstring): cstring{.cdecl, - mylib, varargs, importc: "sqlite3_snprintf".} -proc set_authorizer*(para1: PSqlite3, xAuth: proc (para1: pointer, para2: int32, - para3: cstring, para4: cstring, para5: cstring, para6: cstring): int32{. - cdecl.}, pUserData: pointer): int32{.cdecl, mylib, - importc: "sqlite3_set_authorizer".} -proc trace*(para1: PSqlite3, xTrace: proc (para1: pointer, para2: cstring){.cdecl.}, - para3: pointer): pointer{.cdecl, mylib, - importc: "sqlite3_trace".} -proc progress_handler*(para1: PSqlite3, para2: int32, - para3: proc (para1: pointer): int32{.cdecl.}, - para4: pointer){.cdecl, mylib, - importc: "sqlite3_progress_handler".} -proc commit_hook*(para1: PSqlite3, para2: proc (para1: pointer): int32{.cdecl.}, - para3: pointer): pointer{.cdecl, mylib, - importc: "sqlite3_commit_hook".} -proc open*(filename: cstring, ppDb: var PSqlite3): int32{.cdecl, mylib, - importc: "sqlite3_open".} -proc open16*(filename: pointer, ppDb: var PSqlite3): int32{.cdecl, mylib, - importc: "sqlite3_open16".} -proc errcode*(db: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_errcode".} -proc errmsg*(para1: PSqlite3): cstring{.cdecl, mylib, importc: "sqlite3_errmsg".} -proc errmsg16*(para1: PSqlite3): pointer{.cdecl, mylib, - importc: "sqlite3_errmsg16".} -proc prepare*(db: PSqlite3, zSql: cstring, nBytes: int32, ppStmt: var PStmt, - pzTail: ptr cstring): int32{.cdecl, mylib, - importc: "sqlite3_prepare".} - -proc prepare_v2*(db: PSqlite3, zSql: cstring, nByte: cint, ppStmt: var PStmt, - pzTail: ptr cstring): cint {. - importc: "sqlite3_prepare_v2", cdecl, mylib.} - -proc prepare16*(db: PSqlite3, zSql: pointer, nBytes: int32, ppStmt: var PStmt, - pzTail: var pointer): int32{.cdecl, mylib, - importc: "sqlite3_prepare16".} -proc bind_blob*(para1: PStmt, para2: int32, para3: pointer, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, mylib, - importc: "sqlite3_bind_blob".} -proc bind_double*(para1: PStmt, para2: int32, para3: float64): int32{.cdecl, - mylib, importc: "sqlite3_bind_double".} -proc bind_int*(para1: PStmt, para2: int32, para3: int32): int32{.cdecl, - mylib, importc: "sqlite3_bind_int".} -proc bind_int64*(para1: PStmt, para2: int32, para3: int64): int32{.cdecl, - mylib, importc: "sqlite3_bind_int64".} -proc bind_null*(para1: PStmt, para2: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_null".} -proc bind_text*(para1: PStmt, para2: int32, para3: cstring, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, mylib, - importc: "sqlite3_bind_text".} -proc bind_text16*(para1: PStmt, para2: int32, para3: pointer, para4: int32, - para5: Tbind_destructor_func): int32{.cdecl, mylib, - importc: "sqlite3_bind_text16".} - #function sqlite3_bind_value(_para1:Psqlite3_stmt; _para2:longint; _para3:Psqlite3_value):longint;cdecl; external Sqlite3Lib name 'sqlite3_bind_value'; - #These overloaded functions were introduced to allow the use of SQLITE_STATIC and SQLITE_TRANSIENT - #It's the c world man ;-) -proc bind_blob*(para1: PStmt, para2: int32, para3: pointer, n: int32, - para5: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_blob".} -proc bind_text*(para1: PStmt, para2: int32, para3: cstring, n: int32, - para5: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_text".} -proc bind_text16*(para1: PStmt, para2: int32, para3: pointer, para4: int32, - para5: int32): int32{.cdecl, mylib, - importc: "sqlite3_bind_text16".} -proc bind_parameter_count*(para1: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_bind_parameter_count".} -proc bind_parameter_name*(para1: PStmt, para2: int32): cstring{.cdecl, - mylib, importc: "sqlite3_bind_parameter_name".} -proc bind_parameter_index*(para1: PStmt, zName: cstring): int32{.cdecl, - mylib, importc: "sqlite3_bind_parameter_index".} -proc clear_bindings*(para1: PStmt): int32 {.cdecl, - mylib, importc: "sqlite3_clear_bindings".} -proc column_count*(PStmt: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_column_count".} -proc column_name*(para1: PStmt, para2: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_name".} -proc column_table_name*(para1: PStmt; para2: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_table_name".} -proc column_name16*(para1: PStmt, para2: int32): pointer{.cdecl, mylib, - importc: "sqlite3_column_name16".} -proc column_decltype*(para1: PStmt, i: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_decltype".} -proc column_decltype16*(para1: PStmt, para2: int32): pointer{.cdecl, - mylib, importc: "sqlite3_column_decltype16".} -proc step*(para1: PStmt): int32{.cdecl, mylib, importc: "sqlite3_step".} -proc data_count*(PStmt: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_data_count".} -proc column_blob*(para1: PStmt, iCol: int32): pointer{.cdecl, mylib, - importc: "sqlite3_column_blob".} -proc column_bytes*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_bytes".} -proc column_bytes16*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_bytes16".} -proc column_double*(para1: PStmt, iCol: int32): float64{.cdecl, mylib, - importc: "sqlite3_column_double".} -proc column_int*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_int".} -proc column_int64*(para1: PStmt, iCol: int32): int64{.cdecl, mylib, - importc: "sqlite3_column_int64".} -proc column_text*(para1: PStmt, iCol: int32): cstring{.cdecl, mylib, - importc: "sqlite3_column_text".} -proc column_text16*(para1: PStmt, iCol: int32): pointer{.cdecl, mylib, - importc: "sqlite3_column_text16".} -proc column_type*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, - importc: "sqlite3_column_type".} -proc finalize*(PStmt: PStmt): int32{.cdecl, mylib, - importc: "sqlite3_finalize".} -proc reset*(PStmt: PStmt): int32{.cdecl, mylib, importc: "sqlite3_reset".} -proc create_function*(para1: PSqlite3, zFunctionName: cstring, nArg: int32, - eTextRep: int32, para5: pointer, - xFunc: Create_function_func_func, - xStep: Create_function_step_func, - xFinal: Create_function_final_func): int32{.cdecl, - mylib, importc: "sqlite3_create_function".} -proc create_function16*(para1: PSqlite3, zFunctionName: pointer, nArg: int32, - eTextRep: int32, para5: pointer, - xFunc: Create_function_func_func, - xStep: Create_function_step_func, - xFinal: Create_function_final_func): int32{.cdecl, - mylib, importc: "sqlite3_create_function16".} -proc aggregate_count*(para1: Pcontext): int32{.cdecl, mylib, - importc: "sqlite3_aggregate_count".} -proc value_blob*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_blob".} -proc value_bytes*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_bytes".} -proc value_bytes16*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_bytes16".} -proc value_double*(para1: PValue): float64{.cdecl, mylib, - importc: "sqlite3_value_double".} -proc value_int*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_int".} -proc value_int64*(para1: PValue): int64{.cdecl, mylib, - importc: "sqlite3_value_int64".} -proc value_text*(para1: PValue): cstring{.cdecl, mylib, - importc: "sqlite3_value_text".} -proc value_text16*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_text16".} -proc value_text16le*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_text16le".} -proc value_text16be*(para1: PValue): pointer{.cdecl, mylib, - importc: "sqlite3_value_text16be".} -proc value_type*(para1: PValue): int32{.cdecl, mylib, - importc: "sqlite3_value_type".} -proc aggregate_context*(para1: Pcontext, nBytes: int32): pointer{.cdecl, - mylib, importc: "sqlite3_aggregate_context".} -proc user_data*(para1: Pcontext): pointer{.cdecl, mylib, - importc: "sqlite3_user_data".} -proc get_auxdata*(para1: Pcontext, para2: int32): pointer{.cdecl, mylib, - importc: "sqlite3_get_auxdata".} -proc set_auxdata*(para1: Pcontext, para2: int32, para3: pointer, - para4: proc (para1: pointer){.cdecl.}){.cdecl, mylib, - importc: "sqlite3_set_auxdata".} -proc result_blob*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_blob".} -proc result_double*(para1: Pcontext, para2: float64){.cdecl, mylib, - importc: "sqlite3_result_double".} -proc result_error*(para1: Pcontext, para2: cstring, para3: int32){.cdecl, - mylib, importc: "sqlite3_result_error".} -proc result_error16*(para1: Pcontext, para2: pointer, para3: int32){.cdecl, - mylib, importc: "sqlite3_result_error16".} -proc result_int*(para1: Pcontext, para2: int32){.cdecl, mylib, - importc: "sqlite3_result_int".} -proc result_int64*(para1: Pcontext, para2: int64){.cdecl, mylib, - importc: "sqlite3_result_int64".} -proc result_null*(para1: Pcontext){.cdecl, mylib, - importc: "sqlite3_result_null".} -proc result_text*(para1: Pcontext, para2: cstring, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text".} -proc result_text16*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text16".} -proc result_text16le*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text16le".} -proc result_text16be*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, mylib, - importc: "sqlite3_result_text16be".} -proc result_value*(para1: Pcontext, para2: PValue){.cdecl, mylib, - importc: "sqlite3_result_value".} -proc create_collation*(para1: PSqlite3, zName: cstring, eTextRep: int32, - para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, mylib, importc: "sqlite3_create_collation".} -proc create_collation16*(para1: PSqlite3, zName: cstring, eTextRep: int32, - para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, mylib, importc: "sqlite3_create_collation16".} -proc collation_needed*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, mylib, importc: "sqlite3_collation_needed".} -proc collation_needed16*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, mylib, importc: "sqlite3_collation_needed16".} -proc libversion*(): cstring{.cdecl, mylib, importc: "sqlite3_libversion".} - #Alias for allowing better code portability (win32 is not working with external variables) -proc version*(): cstring{.cdecl, mylib, importc: "sqlite3_libversion".} - # Not published functions -proc libversion_number*(): int32{.cdecl, mylib, - importc: "sqlite3_libversion_number".} - #function sqlite3_key(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_key'; - #function sqlite3_rekey(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_rekey'; - #function sqlite3_sleep(_para1:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_sleep'; - #function sqlite3_expired(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_expired'; - #function sqlite3_global_recover:longint;cdecl; external Sqlite3Lib name 'sqlite3_global_recover'; -# implementation - -when defined(nimHasStyleChecks): - {.pop.} |