diff options
Diffstat (limited to 'lib/pure')
32 files changed, 535 insertions, 294 deletions
diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index a15442109..6330a6ba9 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1228,14 +1228,19 @@ else: let newLength = max(len(curList), InitCallbackListSize) var newList = newSeqOfCap[Callback](newLength) + var eventsExtinguished = false for cb in curList: + if eventsExtinguished: + newList.add(cb) + continue if not cb(fd): # Callback wants to be called again. newList.add(cb) # This callback has returned with EAGAIN, so we don't need to # call any other callbacks as they are all waiting for the same event # on the same fd. - break + # We do need to ensure they are called again though. + eventsExtinguished = true withData(selector, fd.int, fdData) do: # Descriptor is still present in the queue. @@ -1601,11 +1606,16 @@ else: p.selector.registerEvent(SelectEvent(ev), data) proc drain*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` + ## Waits for completion of **all** events and processes them. Raises ``ValueError`` ## if there are no pending operations. In contrast to ``poll`` this - ## processes as many events as are available. - if runOnce(timeout) or hasPendingOperations(): - while hasPendingOperations() and runOnce(timeout): discard + ## processes as many events as are available until the timeout has elapsed. + var curTimeout = timeout + let start = now() + while hasPendingOperations(): + discard runOnce(curTimeout) + curTimeout -= (now() - start).inMilliseconds.int + if curTimeout < 0: + break proc poll*(timeout = 500) = ## Waits for completion events and processes them. Raises ``ValueError`` @@ -1635,16 +1645,6 @@ proc createAsyncNativeSocket*(domain: Domain = Domain.AF_INET, inheritable = defined(nimInheritHandles)): AsyncFD = createAsyncNativeSocketImpl(domain, sockType, protocol, inheritable) -proc newAsyncNativeSocket*(domain: cint, sockType: cint, - protocol: cint): AsyncFD {.deprecated: "use createAsyncNativeSocket instead".} = - createAsyncNativeSocketImpl(domain, sockType, protocol) - -proc newAsyncNativeSocket*(domain: Domain = Domain.AF_INET, - sockType: SockType = SOCK_STREAM, - protocol: Protocol = IPPROTO_TCP): AsyncFD - {.deprecated: "use createAsyncNativeSocket instead".} = - createAsyncNativeSocketImpl(domain, sockType, protocol) - when defined(windows) or defined(nimdoc): proc bindToDomain(handle: SocketHandle, domain: Domain) = # Extracted into a separate proc, because connect() on Windows requires diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index 132682b23..4e62ecd97 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -134,7 +134,7 @@ proc expectReply(ftp: AsyncFtpClient): Future[TaintedString] {.async.} = var line = await ftp.csock.recvLine() result = TaintedString(line) var count = 0 - while line[3] == '-': + while line.len > 3 and line[3] == '-': ## Multi-line reply. line = await ftp.csock.recvLine() string(result).add("\n" & line) @@ -231,7 +231,7 @@ proc listDirs*(ftp: AsyncFtpClient, dir = ""): Future[seq[string]] {.async.} = result = splitLines(await ftp.getLines()) -proc existsFile*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = +proc fileExists*(ftp: AsyncFtpClient, file: string): Future[bool] {.async.} = ## Determines whether ``file`` exists. var files = await ftp.listDirs() for f in items(files): diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index 621a4b00c..219ef6c67 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -180,8 +180,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # Extract the documentation comment from the original procedure declaration. # Note that we're not removing it from the body in order not to make this # transformation even more complex. - if prc.body.len > 1 and prc.body[0].kind == nnkCommentStmt: - outerProcBody.add(prc.body[0]) + let body2 = extractDocCommentsAndRunnables(prc.body) # -> var retFuture = newFuture[T]() var retFutureSym = genSym(nskVar, "retFuture") @@ -276,9 +275,10 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = (cast[type(f)](internalTmpFuture)).read() if procBody.kind != nnkEmpty: - result.body = quote: + body2.add quote do: `awaitDefinition` `outerProcBody` + result.body = body2 #echo(treeRepr(result)) #if prcName == "recvLineInto": diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 3fc1a0177..410310e29 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -853,40 +853,49 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, else: raise newException(IOError, "Couldn't resolve address: " & address) -proc recvFrom*(socket: AsyncSocket, size: int, - flags = {SocketFlag.SafeDisconn}): - owned(Future[tuple[data: string, address: string, port: Port]]) +proc recvFrom*(socket: AsyncSocket, data: FutureVar[string], size: int, + address: FutureVar[string], port: FutureVar[Port], + flags = {SocketFlag.SafeDisconn}): owned(Future[int]) {.async, since: (1, 3).} = - ## Receives a datagram data from ``socket``, which must be at least of size - ## ``size``. Returned future will complete once one datagram has been received - ## and will return tuple with: data of packet received; and address and port - ## of datagram's sender. - ## + ## Receives a datagram data from ``socket`` into ``data``, which must be at + ## least of size ``size``. The address and port of datagram's sender will be + ## stored into ``address`` and ``port``, respectively. Returned future will + ## complete once one datagram has been received, and will return size of + ## packet received. + ## ## If an error occurs an OSError exception will be raised. ## ## This proc is normally used with connectionless sockets (UDP sockets). + ## + ## **Notes** + ## * ``data`` must be initialized to the length of ``size``. + ## * ``address`` must be initialized to 46 in length. template adaptRecvFromToDomain(domain: Domain) = var lAddr = sizeof(sAddr).SockLen - let fut = await recvFromInto(AsyncFD(getFd(socket)), cstring(data), size, - cast[ptr SockAddr](addr sAddr), addr lAddr, - flags) + result = await recvFromInto(AsyncFD(getFd(socket)), cstring(data.mget()), size, + cast[ptr SockAddr](addr sAddr), addr lAddr, + flags) - data.setLen(fut) + data.mget().setLen(result) + data.complete() + + getAddrString(cast[ptr SockAddr](addr sAddr), address.mget()) - result.data = data - result.address = getAddrString(cast[ptr SockAddr](addr sAddr)) + address.complete() when domain == AF_INET6: - result.port = ntohs(sAddr.sin6_port).Port + port.complete(ntohs(sAddr.sin6_port).Port) else: - result.port = ntohs(sAddr.sin_port).Port + port.complete(ntohs(sAddr.sin_port).Port) assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket. Use `recv` or `recvInto` instead") assert(not socket.closed, "Cannot `recvFrom` on a closed socket") - - var data = newString(size) + assert(size == len(data.mget()), + "`date` was not initialized correctly. `size` != `len(data.mget())`") + assert(46 == len(address.mget()), + "`address` was not initialized correctly. 46 != `len(address.mget())`") case socket.domain of AF_INET6: @@ -898,6 +907,30 @@ proc recvFrom*(socket: AsyncSocket, size: int, else: raise newException(ValueError, "Unknown socket address family") +proc recvFrom*(socket: AsyncSocket, size: int, + flags = {SocketFlag.SafeDisconn}): + owned(Future[tuple[data: string, address: string, port: Port]]) + {.async, since: (1, 3).} = + ## Receives a datagram data from ``socket``, which must be at least of size + ## ``size``. Returned future will complete once one datagram has been received + ## and will return tuple with: data of packet received; and address and port + ## of datagram's sender. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This proc is normally used with connectionless sockets (UDP sockets). + var + data = newFutureVar[string]() + address = newFutureVar[string]() + port = newFutureVar[Port]() + + data.mget().setLen(size) + address.mget().setLen(46) + + let read = await recvFrom(socket, data, size, address, port, flags) + + result = (data.mget(), address.mget(), port.mget()) + when not defined(testing) and isMainModule: type TestCases = enum diff --git a/lib/pure/asyncstreams.nim b/lib/pure/asyncstreams.nim index 44e73003e..393262c4f 100644 --- a/lib/pure/asyncstreams.nim +++ b/lib/pure/asyncstreams.nim @@ -96,7 +96,7 @@ proc read*[T](future: FutureStream[T]): owned(Future[(bool, T)]) = if resFut.finished: return # We don't want this callback called again. - future.cb = nil + #future.cb = nil # The return value depends on whether the FutureStream has finished. var res: (bool, T) diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index dd7c4d477..734ab9171 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -32,12 +32,7 @@ import strutils, os, strtabs, cookies, uri export uri.encodeUrl, uri.decodeUrl -proc handleHexChar(c: char, x: var int) {.inline.} = - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: assert(false) +include includes/decode_helpers proc addXmlChar(dest: var string, c: char) {.inline.} = case c @@ -93,40 +88,27 @@ proc getEncodedData(allowedMethods: set[RequestMethod]): string = iterator decodeData*(data: string): tuple[key, value: TaintedString] = ## Reads and decodes CGI data and yields the (name, value) pairs the ## data consists of. + proc parseData(data: string, i: int, field: var string): int = + result = i + while result < data.len: + case data[result] + of '%': add(field, decodePercent(data, result)) + of '+': add(field, ' ') + of '=', '&': break + else: add(field, data[result]) + inc(result) + var i = 0 var name = "" var value = "" # decode everything in one pass: while i < data.len: setLen(name, 0) # reuse memory - while i < data.len: - case data[i] - of '%': - var x = 0 - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) - inc(i, 2) - add(name, chr(x)) - of '+': add(name, ' ') - of '=', '&': break - else: add(name, data[i]) - inc(i) + i = parseData(data, i, name) if i >= data.len or data[i] != '=': cgiError("'=' expected") inc(i) # skip '=' setLen(value, 0) # reuse memory - while i < data.len: - case data[i] - of '%': - var x = 0 - if i+2 < data.len: - handleHexChar(data[i+1], x) - handleHexChar(data[i+2], x) - inc(i, 2) - add(value, chr(x)) - of '+': add(value, ' ') - of '&', '\0': break - else: add(value, data[i]) - inc(i) + i = parseData(data, i, value) yield (name.TaintedString, value.TaintedString) if i < data.len: if data[i] == '&': inc(i) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index d096874a3..8150563cc 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -67,9 +67,9 @@ const defaultInitialSize* = 4 template initImpl(result: typed, initialSize: int) = - assert isPowerOfTwo(initialSize) - result.mask = initialSize-1 - newSeq(result.data, initialSize) + let correctSize = nextPowerOfTwo(initialSize) + result.mask = correctSize-1 + newSeq(result.data, correctSize) template checkIfInitialized(deq: typed) = when compiles(defaultInitialSize): @@ -82,11 +82,6 @@ proc initDeque*[T](initialSize: int = 4): Deque[T] = ## Optionally, the initial capacity can be reserved via `initialSize` ## as a performance optimization. ## The length of a newly created deque will still be 0. - ## - ## ``initialSize`` must be a power of two (default: 4). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_. result.initImpl(initialSize) proc len*[T](deq: Deque[T]): int {.inline.} = diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index e998145e7..336dbc07d 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -30,17 +30,23 @@ proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = (h + 1) and maxHash proc mustRehash[T](t: T): bool {.inline.} = + # If this is changed, make sure to synchronize it with `slotsNeeded` below assert(t.dataLen > t.counter) result = (t.dataLen * 2 < t.counter * 3) or (t.dataLen - t.counter < 4) -proc rightSize*(count: Natural): int {.inline.} = +proc slotsNeeded(count: Natural): int {.inline.} = + # Make sure to synchronize with `mustRehash` above + result = nextPowerOfTwo(count * 3 div 2 + 4) + +proc rightSize*(count: Natural): int {.inline, deprecated: "Deprecated since 1.4.0".} = + ## **Deprecated since Nim v1.4.0**, it is not needed anymore + ## because picking the correct size is done internally. + ## ## Return the value of ``initialSize`` to support ``count`` items. ## ## If more items are expected to be added, simply add that ## expected extra amount to the parameter before calling this. - # - # Make sure to synchronize with `mustRehash` - result = nextPowerOfTwo(count * 3 div 2 + 4) + result = count template rawGetKnownHCImpl() {.dirty.} = if t.dataLen == 0: diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index f101d508e..51d8ade85 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -527,7 +527,8 @@ proc insert*[T](dest: var seq[T], src: openArray[T], pos = 0) = assert dest == outcome var j = len(dest) - 1 - var i = len(dest) + len(src) - 1 + var i = j + len(src) + if i == j: return dest.setLen(i + 1) # Move items after `pos` to the end of the sequence. diff --git a/lib/pure/collections/setimpl.nim b/lib/pure/collections/setimpl.nim index d798cbcb3..20da6b6c2 100644 --- a/lib/pure/collections/setimpl.nim +++ b/lib/pure/collections/setimpl.nim @@ -16,12 +16,12 @@ template dataLen(t): untyped = len(t.data) include hashcommon template initImpl(s: typed, size: int) = - assert isPowerOfTwo(size) + let correctSize = slotsNeeded(size) when s is OrderedSet: s.first = -1 s.last = -1 s.counter = 0 - newSeq(s.data, size) + newSeq(s.data, correctSize) template rawInsertImpl() {.dirty.} = if data.len == 0: diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 2b270c2cb..b019da2a7 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -94,11 +94,6 @@ include setimpl proc init*[A](s: var HashSet[A], initialSize = defaultInitialSize) = ## Initializes a hash set. ## - ## The `initialSize` parameter needs to be a power of two (default: 64). - ## If you need to accept runtime values for this, you can use - ## `math.nextPowerOfTwo proc <math.html#nextPowerOfTwo,int>`_ or - ## `rightSize proc <#rightSize,Natural>`_ from this module. - ## ## Starting from Nim v0.20, sets are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -222,7 +217,7 @@ proc toHashSet*[A](keys: openArray[A]): HashSet[A] = assert len(b) == 5 ## b == {'a', 'b', 'c', 'd', 'r'} - result = initHashSet[A](rightSize(keys.len)) + result = initHashSet[A](keys.len) for key in items(keys): result.incl(key) iterator items*[A](s: HashSet[A]): A = @@ -628,11 +623,6 @@ template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = proc init*[A](s: var OrderedSet[A], initialSize = defaultInitialSize) = ## Initializes an ordered hash set. ## - ## The `initialSize` parameter needs to be a power of two (default: 64). - ## If you need to accept runtime values for this, you can use - ## `math.nextPowerOfTwo proc <math.html#nextPowerOfTwo,int>`_ or - ## `rightSize proc <#rightSize,Natural>`_ from this module. - ## ## Starting from Nim v0.20, sets are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -685,7 +675,7 @@ proc toOrderedSet*[A](keys: openArray[A]): OrderedSet[A] = assert len(b) == 5 ## b == {'a', 'b', 'r', 'c', 'd'} # different than in HashSet - result = initOrderedSet[A](rightSize(keys.len)) + result = initOrderedSet[A](keys.len) for key in items(keys): result.incl(key) proc contains*[A](s: OrderedSet[A], key: A): bool = @@ -980,7 +970,7 @@ when isMainModule and not defined(release): block toSeqAndString: var a = toHashSet([2, 7, 5]) - var b = initHashSet[int](rightSize(a.len)) + var b = initHashSet[int](a.len) for x in [2, 7, 5]: b.incl(x) assert($a == $b) #echo a @@ -1098,20 +1088,6 @@ when isMainModule and not defined(release): b.incl(2) assert b.len == 1 - block: - type FakeTable = object - dataLen: int - counter: int - countDeleted: int - - var t: FakeTable - for i in 0 .. 32: - var s = rightSize(i) - t.dataLen = s - t.counter = i - doAssert s > i and not mustRehash(t), - "performance issue: rightSize() will not elide enlarge() at: " & $i - block missingOrExcl: var s = toOrderedSet([2, 3, 6, 7]) assert s.missingOrExcl(4) == true diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index f9182acce..790529b79 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -94,10 +94,4 @@ proc deinitSharedList*[A](t: var SharedList[A]) = clear(t) deinitLock t.lock -proc initSharedList*[A](): SharedList[A] {.deprecated: "use 'init' instead".} = - ## This is not posix compliant, may introduce undefined behavior. - initLock result.lock - result.head = nil - result.tail = nil - {.pop.} diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 23b653c82..cbd922db7 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -207,15 +207,11 @@ proc len*[A, B](t: var SharedTable[A, B]): int = withLock t: result = t.counter -proc init*[A, B](t: var SharedTable[A, B], initialSize = 64) = +proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = ## creates a new hash table that is empty. ## ## This proc must be called before any other usage of `t`. - ## - ## `initialSize` needs to be a power of two. If you need to accept runtime - ## values for this you could use the ``nextPowerOfTwo`` proc from the - ## `math <math.html>`_ module or the ``rightSize`` proc from this module. - assert isPowerOfTwo(initialSize) + let initialSize = slotsNeeded(initialSize) t.counter = 0 t.dataLen = initialSize t.data = cast[KeyValuePairSeq[A, B]](allocShared0( @@ -225,13 +221,3 @@ proc init*[A, B](t: var SharedTable[A, B], initialSize = 64) = proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) = deallocShared(t.data) deinitLock t.lock - -proc initSharedTable*[A, B](initialSize = 64): SharedTable[A, B] {.deprecated: - "use 'init' instead".} = - ## This is not posix compliant, may introduce undefined behavior. - assert isPowerOfTwo(initialSize) - result.counter = 0 - result.dataLen = initialSize - result.data = cast[KeyValuePairSeq[A, B]](allocShared0( - sizeof(KeyValuePair[A, B]) * initialSize)) - initLock result.lock diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index 47c14af93..b9d7c70d9 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -121,12 +121,12 @@ template ctAnd(a, b): bool = else: false template initImpl(result: typed, size: int) = + let correctSize = slotsNeeded(size) when ctAnd(declared(SharedTable), type(result) is SharedTable): - init(result, size) + init(result, correctSize) else: - assert isPowerOfTwo(size) result.counter = 0 - newSeq(result.data, size) + newSeq(result.data, correctSize) when compiles(result.first): result.first = -1 result.last = -1 diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2ea58ce1f..cf864c640 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -239,7 +239,7 @@ type ## <#newTable,int>`_. const - defaultInitialSize* = 64 + defaultInitialSize* = 32 # ------------------------------ helpers --------------------------------- @@ -288,12 +288,6 @@ proc enlarge[A, B](t: var Table[A, B]) = proc initTable*[A, B](initialSize = defaultInitialSize): Table[A, B] = ## Creates a new hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## Starting from Nim v0.20, tables are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -335,7 +329,7 @@ proc toTable*[A, B](pairs: openArray[(A, B)]): Table[A, B] = let b = toTable(a) assert b == {'a': 5, 'b': 9}.toTable - result = initTable[A, B](rightSize(pairs.len)) + result = initTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: Table[A, B], key: A): B = @@ -780,12 +774,6 @@ iterator allValues*[A, B](t: Table[A, B]; key: A): B = proc newTable*[A, B](initialSize = defaultInitialSize): <//>TableRef[A, B] = ## Creates a new ref hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## See also: ## * `newTable proc<#newTable,openArray[]>`_ for creating a `TableRef` ## from a collection of `(key, value)` pairs @@ -1260,12 +1248,6 @@ template forAllOrderedPairs(yieldStmt: untyped) {.dirty.} = proc initOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTable[A, B] = ## Creates a new ordered hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## Starting from Nim v0.20, tables are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -1309,7 +1291,7 @@ proc toOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTable[A, B] = let b = toOrderedTable(a) assert b == {'a': 5, 'b': 9}.toOrderedTable - result = initOrderedTable[A, B](rightSize(pairs.len)) + result = initOrderedTable[A, B](pairs.len) for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTable[A, B], key: A): B = @@ -1771,12 +1753,6 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B = proc newOrderedTable*[A, B](initialSize = defaultInitialSize): <//>OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## See also: ## * `newOrderedTable proc<#newOrderedTable,openArray[]>`_ for creating ## an `OrderedTableRef` from a collection of `(key, value)` pairs @@ -1803,7 +1779,7 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B] let b = newOrderedTable(a) assert b == {'a': 5, 'b': 9}.newOrderedTable - result = newOrderedTable[A, B](rightSize(pairs.len)) + result = newOrderedTable[A, B](pairs.len) for key, val in items(pairs): result.add(key, val) @@ -2251,12 +2227,6 @@ proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = ## Creates a new count table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## Starting from Nim v0.20, tables are initialized by default and it is ## not necessary to call this function explicitly. ## @@ -2269,7 +2239,7 @@ proc initCountTable*[A](initialSize = defaultInitialSize): CountTable[A] = proc toCountTable*[A](keys: openArray[A]): CountTable[A] = ## Creates a new count table with every member of a container ``keys`` ## having a count of how many times it occurs in that container. - result = initCountTable[A](rightSize(keys.len)) + result = initCountTable[A](keys.len) for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTable[A], key: A): int = @@ -2617,12 +2587,6 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) proc newCountTable*[A](initialSize = defaultInitialSize): <//>CountTableRef[A] = ## Creates a new ref count table that is empty. ## - ## ``initialSize`` must be a power of two (default: 64). - ## If you need to accept runtime values for this you could use the - ## `nextPowerOfTwo proc<math.html#nextPowerOfTwo,int>`_ from the - ## `math module<math.html>`_ or the `rightSize proc<#rightSize,Natural>`_ - ## from this module. - ## ## See also: ## * `newCountTable proc<#newCountTable,openArray[A]>`_ for creating ## a `CountTableRef` from a collection @@ -2634,7 +2598,7 @@ proc newCountTable*[A](initialSize = defaultInitialSize): <//>CountTableRef[A] = proc newCountTable*[A](keys: openArray[A]): <//>CountTableRef[A] = ## Creates a new ref count table with every member of a container ``keys`` ## having a count of how many times it occurs in that container. - result = newCountTable[A](rightSize(keys.len)) + result = newCountTable[A](keys.len) for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = @@ -2670,14 +2634,14 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = doAssert a == newCountTable("aaabbbbbbbbbbb") t[].inc(key, val) -proc smallest*[A](t: CountTableRef[A]): (A, int) = +proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n) ## ## See also: ## * `largest proc<#largest,CountTableRef[A]>`_ t[].smallest -proc largest*[A](t: CountTableRef[A]): (A, int) = +proc largest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the ``(key, value)`` pair with the largest ``val``. Efficiency: O(n) ## ## See also: diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index d57e309e9..dbcb1b4a2 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -452,7 +452,7 @@ proc parseColor*(name: string): Color = assert parseColor(b) == Color(0x01_79_fc) doAssertRaises(ValueError): discard parseColor(c) - if name[0] == '#': + if name.len > 0 and name[0] == '#': result = Color(parseHexInt(name)) else: var idx = binarySearch(colorNames, name, colorNameCmp) @@ -472,6 +472,7 @@ proc isColor*(name: string): bool = assert b.isColor assert not c.isColor + if name.len == 0: return false if name[0] == '#': for i in 1 .. name.len-1: if name[i] notin {'0'..'9', 'a'..'f', 'A'..'F'}: return false diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 2abcafb80..f0269f526 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -208,7 +208,7 @@ proc finished(fv: var FlowVarBaseObj) = # the worker thread waits for "data" to be set to nil before shutting down owner.data = nil -proc `=destroy`[T](fv: var FlowVarObj[T]) = +proc `=destroy`[T](fv: var FlowVarObj[T]) = finished(fv) `=destroy`(fv.blob) @@ -321,15 +321,15 @@ var currentPoolSize: int maxPoolSize = MaxThreadPoolSize minPoolSize = 4 - gSomeReady : Semaphore + gSomeReady: Semaphore readyWorker: ptr Worker # A workaround for recursion deadlock issue # https://github.com/nim-lang/Nim/issues/4597 var numSlavesLock: Lock - numSlavesRunning {.guard: numSlavesLock}: int - numSlavesWaiting {.guard: numSlavesLock}: int + numSlavesRunning {.guard: numSlavesLock.}: int + numSlavesWaiting {.guard: numSlavesLock.}: int isSlave {.threadvar.}: bool numSlavesLock.initLock @@ -464,7 +464,7 @@ proc pinnedSpawn*(id: ThreadId; call: sink typed): void {.magic: "Spawn".} ## ``call`` has to be proc call ``p(...)`` where ``p`` is gcsafe and has a ## return type that is either ``void`` or compatible with ``FlowVar[T]``. -template spawnX*(call): void = +template spawnX*(call) = ## Spawns a new task if a CPU core is ready, otherwise executes the ## call in the calling thread. ## diff --git a/lib/pure/includes/decode_helpers.nim b/lib/pure/includes/decode_helpers.nim new file mode 100644 index 000000000..74fe37d07 --- /dev/null +++ b/lib/pure/includes/decode_helpers.nim @@ -0,0 +1,24 @@ +# Include file that implements 'decodePercent' and friends. Do not import it! + +proc handleHexChar(c: char, x: var int, f: var bool) {.inline.} = + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: f = true + +proc decodePercent(s: string, i: var int): char = + ## Converts `%xx` hexadecimal to the charracter with ordinal number `xx`. + ## + ## If `xx` is not a valid hexadecimal value, it is left intact: only the + ## leading `%` is returned as-is, and `xx` characters will be processed in the + ## next step (e.g. in `uri.decodeUrl`) as regular characters. + result = '%' + if i+2 < s.len: + var x = 0 + var failed = false + handleHexChar(s[i+1], x, failed) + handleHexChar(s[i+2], x, failed) + if not failed: + result = chr(x) + inc(i, 2) diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 2637fdf9d..fa6285a04 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -287,6 +287,7 @@ proc substituteLog*(frmt: string, level: Level, runnableExamples: doAssert substituteLog(defaultFmtStr, lvlInfo, "a message") == "INFO a message" doAssert substituteLog("$levelid - ", lvlError, "an error") == "E - an error" + doAssert substituteLog("$levelid", lvlDebug, "error") == "Derror" var msgLen = 0 for arg in args: msgLen += arg.len @@ -300,7 +301,7 @@ proc substituteLog*(frmt: string, level: Level, inc(i) var v = "" let app = when defined(js): "" else: getAppFilename() - while frmt[i] in IdentChars: + while i < frmt.len and frmt[i] in IdentChars: v.add(toLowerAscii(frmt[i])) inc(i) case v @@ -364,7 +365,12 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = let ln = substituteLog(logger.fmtStr, level, args) when defined(js): let cln: cstring = ln - {.emit: "console.log(`cln`);".} + case level + of lvlDebug: {.emit: "console.debug(`cln`);".} + of lvlInfo: {.emit: "console.info(`cln`);".} + of lvlWarn: {.emit: "console.warn(`cln`);".} + of lvlError: {.emit: "console.error(`cln`);".} + else: {.emit: "console.log(`cln`);".} else: try: var handle = stdout @@ -504,7 +510,6 @@ when not defined(js): # ------ proc countLogLines(logger: RollingFileLogger): int = - result = 0 let fp = open(logger.baseName, fmRead) for line in fp.lines(): result.inc() @@ -531,7 +536,7 @@ when not defined(js): mode: FileMode = fmReadWrite, levelThreshold = lvlAll, fmtStr = defaultFmtStr, - maxLines = 1000, + maxLines: Positive = 1000, bufSize: int = -1): RollingFileLogger = ## Creates a new `RollingFileLogger<#RollingFileLogger>`_. ## diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 67e24eedc..49a1192df 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -461,6 +461,40 @@ proc getAddrString*(sockAddr: ptr SockAddr): string = return "unix" raise newException(IOError, "Unknown socket family in getAddrString") +proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) = + ## Stores in ``strAddress`` the string representation of the address inside + ## ``sockAddr`` + ## + ## **Note** + ## * ``strAddress`` must be initialized to 46 in length. + assert(46 == len(strAddress), + "`strAddress` was not initialized correctly. 46 != `len(strAddress)`") + if sockAddr.sa_family.cint == nativeAfInet: + let addr4 = addr cast[ptr Sockaddr_in](sockAddr).sin_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET, addr4, addr strAddress[0], + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + if winlean.inet_ntop(winlean.AF_INET, addr4, addr strAddress[0], + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + elif sockAddr.sa_family.cint == nativeAfInet6: + let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr + when not useWinVersion: + if posix.inet_ntop(posix.AF_INET6, addr6, addr strAddress[0], + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: + strAddress = strAddress.substr("::ffff:".len) + else: + if winlean.inet_ntop(winlean.AF_INET6, addr6, addr strAddress[0], + strAddress.len.int32) == nil: + raiseOSError(osLastError()) + else: + raise newException(IOError, "Unknown socket family in getAddrString") + setLen(strAddress, len(cstring(strAddress))) + when defined(posix) and not defined(nimdoc): proc makeUnixAddr*(path: string): Sockaddr_un = result.sun_family = AF_UNIX.TSa_Family @@ -622,7 +656,7 @@ proc pruneSocketSet(s: var seq[SocketHandle], fd: var TFdSet) = proc selectRead*(readfds: var seq[SocketHandle], timeout = 500): int = ## When a socket in ``readfds`` is ready to be read from then a non-zero ## value will be returned specifying the count of the sockets which can be - ## read from. The sockets which can be read from will also be removed + ## read from. The sockets which cannot be read from will also be removed ## from ``readfds``. ## ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for @@ -644,7 +678,7 @@ proc selectWrite*(writefds: var seq[SocketHandle], timeout = 500): int {.tags: [ReadIOEffect].} = ## When a socket in ``writefds`` is ready to be written to then a non-zero ## value will be returned specifying the count of the sockets which can be - ## written to. The sockets which can be written to will also be removed + ## written to. The sockets which cannot be written to will also be removed ## from ``writefds``. ## ## ``timeout`` is specified in milliseconds and ``-1`` can be specified for diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c1896dc1c..7aeffbc35 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -513,10 +513,10 @@ when defineSsl: # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html proc loadCertificates(ctx: SslCtx, certFile, keyFile: string) = - if certFile != "" and not existsFile(certFile): + if certFile != "" and not fileExists(certFile): raise newException(system.IOError, "Certificate file could not be found: " & certFile) - if keyFile != "" and not existsFile(keyFile): + if keyFile != "" and not fileExists(keyFile): raise newException(system.IOError, "Key file could not be found: " & keyFile) if certFile != "": @@ -614,7 +614,7 @@ when defineSsl: if verifyMode != CVerifyNone: # Use the caDir and caFile parameters if set if caDir != "" or caFile != "": - if newCTX.SSL_CTX_load_verify_locations(caDir, caFile) != 0: + if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != 0: raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).") else: diff --git a/lib/pure/os.nim b/lib/pure/os.nim index dcb63458b..8bbceddeb 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1097,14 +1097,14 @@ when defined(windows) and not weirdTarget: result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or f.cFileName[1].int == dot and f.cFileName[2].int == 0) -proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", +proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], noNimJs.} = ## Returns true if `filename` exists and is a regular file or symlink. ## ## Directories, device files, named pipes and sockets return false. ## ## See also: - ## * `existsDir proc <#existsDir,string>`_ + ## * `dirExists proc <#dirExists,string>`_ ## * `symlinkExists proc <#symlinkExists,string>`_ when defined(windows): when useWinUnicode: @@ -1117,13 +1117,13 @@ proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", var res: Stat return stat(filename, res) >= 0'i32 and S_ISREG(res.st_mode) -proc existsDir*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], +proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], noNimJs.} = ## Returns true if the directory `dir` exists. If `dir` is a file, false ## is returned. Follows symlinks. ## ## See also: - ## * `existsFile proc <#existsFile,string>`_ + ## * `fileExists proc <#fileExists,string>`_ ## * `symlinkExists proc <#symlinkExists,string>`_ when defined(windows): when useWinUnicode: @@ -1143,8 +1143,8 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", ## regardless of whether the link points to a directory or file. ## ## See also: - ## * `existsFile proc <#existsFile,string>`_ - ## * `existsDir proc <#existsDir,string>`_ + ## * `fileExists proc <#fileExists,string>`_ + ## * `dirExists proc <#dirExists,string>`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, link) @@ -1156,21 +1156,14 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", var res: Stat return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) -proc fileExists*(filename: string): bool {.inline, noNimJs.} = - ## Alias for `existsFile proc <#existsFile,string>`_. - ## - ## See also: - ## * `existsDir proc <#existsDir,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - existsFile(filename) -proc dirExists*(dir: string): bool {.inline, noNimJs.} = - ## Alias for `existsDir proc <#existsDir,string>`_. - ## - ## See also: - ## * `existsFile proc <#existsFile,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ - existsDir(dir) +when not defined(nimscript): + when not defined(js): # `noNimJs` doesn't work with templates, this should improve. + template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = + fileExists(args) + template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = + dirExists(args) + # {.deprecated: [existsFile: fileExists].} # pending bug #14819; this would avoid above mentioned issue when not defined(windows) and not weirdTarget: proc checkSymlink(path: string): bool = @@ -1200,7 +1193,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; template checkCurrentDir() = for ext in extensions: result = addFileExt(exe, ext) - if existsFile(result): return + if fileExists(result): return when defined(posix): if '/' in exe: checkCurrentDir() else: @@ -1216,7 +1209,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; var x = expandTilde(candidate) / exe for ext in extensions: var x = addFileExt(x, ext) - if existsFile(x): + if fileExists(x): when not defined(windows): while followSymlinks: # doubles as if here if x.checkSymlink: @@ -2027,7 +2020,7 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", # way of retrieving the true filename for x in walkFiles(result): result = x - if not existsFile(result) and not existsDir(result): + if not fileExists(result) and not dirExists(result): # consider using: `raiseOSError(osLastError(), result)` raise newException(OSError, "file '" & result & "' does not exist") else: @@ -2324,7 +2317,7 @@ proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", result = not rawCreateDir(dir) if result: # path already exists - need to check that it is indeed a directory - if not existsDir(dir): + if not dirExists(dir): raise newException(IOError, "Failed to create '" & dir & "'") proc createDir*(dir: string) {.rtl, extern: "nos$1", @@ -2893,7 +2886,7 @@ when not weirdTarget and defined(openbsd): # search in path for p in split(string(getEnv("PATH")), {PathSep}): var x = joinPath(p, exePath) - if existsFile(x): + if fileExists(x): return expandFilename(x) else: result = "" @@ -2908,7 +2901,7 @@ when not (defined(windows) or defined(macosx) or weirdTarget): # iterate over any path in the $PATH environment variable for p in split(string(getEnv("PATH")), {PathSep}): var x = joinPath(p, result) - if existsFile(x): return x + if fileExists(x): return x else: result = "" diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 3e3391d52..5113695d8 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -18,7 +18,8 @@ include "system/inclrtl" import - strutils, os, strtabs, streams, cpuinfo + strutils, os, strtabs, streams, cpuinfo, streamwrapper, + std/private/since export quoteShell, quoteShellWindows, quoteShellPosix @@ -237,6 +238,10 @@ proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s output stream for reading from. ## + ## You cannot perform peek/write/setOption operations to this stream. + ## Use `peekableOutputStream proc <#peekableOutputStream,Process>`_ + ## if you need to peek stream. + ## ## **WARNING**: The returned `Stream` should not be closed manually as it ## is closed when closing the Process ``p``. ## @@ -247,6 +252,10 @@ proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} proc errorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s error stream for reading from. ## + ## You cannot perform peek/write/setOption operations to this stream. + ## Use `peekableErrorStream proc <#peekableErrorStream,Process>`_ + ## if you need to peek stream. + ## ## **WARNING**: The returned `Stream` should not be closed manually as it ## is closed when closing the Process ``p``. ## @@ -254,6 +263,30 @@ proc errorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## * `inputStream proc <#inputStream,Process>`_ ## * `outputStream proc <#outputStream,Process>`_ +proc peekableOutputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], since: (1, 3).} + ## Returns ``p``'s output stream for reading from. + ## + ## You can peek returned stream. + ## + ## **WARNING**: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `outputStream proc <#outputStream,Process>`_ + ## * `peekableErrorStream proc <#peekableErrorStream,Process>`_ + +proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], since: (1, 3).} + ## Returns ``p``'s error stream for reading from. + ## + ## You can run peek operation to returned stream. + ## + ## **WARNING**: The returned `Stream` should not be closed manually as it + ## is closed when closing the Process ``p``. + ## + ## See also: + ## * `errorStream proc <#errorStream,Process>`_ + ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ + proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. @@ -737,6 +770,18 @@ when defined(Windows) and not defined(useNimRtl): p.errStream = newFileHandleStream(p.errHandle) result = p.errStream + proc peekableOutputStream(p: Process): Stream = + streamAccess(p) + if p.outStream == nil: + p.outStream = newFileHandleStream(p.outHandle).newPipeOutStream + result = p.outStream + + proc peekableErrorStream(p: Process): Stream = + streamAccess(p) + if p.errStream == nil: + p.errStream = newFileHandleStream(p.errHandle).newPipeOutStream + result = p.errStream + proc execCmd(command: string): int = var si: STARTUPINFO @@ -1360,28 +1405,40 @@ elif not defined(useNimRtl): p.exitStatus = status result = exitStatusLikeShell(status) - proc createStream(stream: var owned(Stream), handle: var FileHandle, - fileMode: FileMode) = + proc createStream(handle: var FileHandle, + fileMode: FileMode): owned FileStream = var f: File if not open(f, handle, fileMode): raiseOSError(osLastError()) - stream = newFileStream(f) + return newFileStream(f) proc inputStream(p: Process): Stream = streamAccess(p) if p.inStream == nil: - createStream(p.inStream, p.inHandle, fmWrite) + p.inStream = createStream(p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = streamAccess(p) if p.outStream == nil: - createStream(p.outStream, p.outHandle, fmRead) + p.outStream = createStream(p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = streamAccess(p) if p.errStream == nil: - createStream(p.errStream, p.errHandle, fmRead) + p.errStream = createStream(p.errHandle, fmRead) + return p.errStream + + proc peekableOutputStream(p: Process): Stream = + streamAccess(p) + if p.outStream == nil: + p.outStream = createStream(p.outHandle, fmRead).newPipeOutStream + return p.outStream + + proc peekableErrorStream(p: Process): Stream = + streamAccess(p) + if p.errStream == nil: + p.errStream = createStream(p.errHandle, fmRead).newPipeOutStream return p.errStream proc csystem(cmd: cstring): cint {.nodecl, importc: "system", @@ -1446,6 +1503,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## A convenience proc that runs the `command`, and returns its `output` and ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. ## If `input.len > 0`, it is passed as stdin. + ## ## Note: this could block if `input.len` is greater than your OS's maximum ## pipe buffer size. ## @@ -1456,15 +1514,17 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { ## * `execProcess proc ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## - runnableExamples: - var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") - import strutils, strtabs - stripLineEnd(result[0]) ## portable way to remove trailing newline, if any - doAssert result == ("12", 0) - doAssert execCmdEx("ls --nonexistant").exitCode != 0 - when defined(posix): - assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) - assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) + ## Example: + ## + ## .. code-block:: Nim + ## var result = execCmdEx("nim r --hints:off -", options = {}, input = "echo 3*4") + ## import strutils, strtabs + ## stripLineEnd(result[0]) ## portable way to remove trailing newline, if any + ## doAssert result == ("12", 0) + ## doAssert execCmdEx("ls --nonexistant").exitCode != 0 + ## when defined(posix): + ## assert execCmdEx("echo $FO", env = newStringTable({"FO": "B"})) == ("B\n", 0) + ## assert execCmdEx("echo $PWD", workingDir = "/") == ("/\n", 0) when (NimMajor, NimMinor, NimPatch) < (1, 3, 5): doAssert input.len == 0 diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index a95a5b48d..94df5ea40 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -280,8 +280,8 @@ proc handleShortOption(p: var OptParser; cmd: string) = while i < cmd.len and cmd[i] in {'\t', ' '}: inc(i) p.inShortState = false - if i < cmd.len and cmd[i] in {':', '='} or - card(p.shortNoVal) > 0 and p.key.string[0] notin p.shortNoVal: + if i < cmd.len and (cmd[i] in {':', '='} or + card(p.shortNoVal) > 0 and p.key.string[0] notin p.shortNoVal): if i < cmd.len and cmd[i] in {':', '='}: inc(i) p.inShortState = false diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim index 806316b02..d04244307 100644 --- a/lib/pure/ssl_certs.nim +++ b/lib/pure/ssl_certs.nim @@ -79,9 +79,9 @@ iterator scanSSLCertificates*(useEnvVars = false): string = when not defined(haiku): for p in certificate_paths: if p.endsWith(".pem") or p.endsWith(".crt"): - if existsFile(p): + if fileExists(p): yield p - elif existsDir(p): + elif dirExists(p): for fn in joinPath(p, "*").walkFiles(): yield fn else: diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim new file mode 100644 index 000000000..b99982f1b --- /dev/null +++ b/lib/pure/streamwrapper.nim @@ -0,0 +1,117 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements stream wrapper. +## +## **Since** version 1.2. + +import deques, streams + +type + PipeOutStream*[T] = ref object of T + # When stream peek operation is called, it reads from base stream + # type using `baseReadDataImpl` and stores the content to this buffer. + # Next stream read operation returns data in the buffer so that previus peek + # operation looks like didn't changed read positon. + # When stream read operation that returns N byte data is called and the size is smaller than buffer size, + # first N elements are removed from buffer. + # Deque type can do such operation more efficiently than seq type. + buffer: Deque[char] + baseReadLineImpl: typeof(StreamObj.readLineImpl) + baseReadDataImpl: typeof(StreamObj.readDataImpl) + +proc posReadLine[T](s: Stream, line: var TaintedString): bool = + var s = PipeOutStream[T](s) + assert s.baseReadLineImpl != nil + + let n = s.buffer.len + line.string.setLen(0) + for i in 0..<n: + var c = s.buffer.popFirst + if c == '\c': + c = readChar(s) + return true + elif c == '\L': return true + elif c == '\0': + return line.len > 0 + line.string.add(c) + + var line2: string + result = s.baseReadLineImpl(s, line2) + line.add line2 + +proc posReadData[T](s: Stream, buffer: pointer, bufLen: int): int = + var s = PipeOutStream[T](s) + assert s.baseReadDataImpl != nil + + let + dest = cast[ptr UncheckedArray[char]](buffer) + n = min(s.buffer.len, bufLen) + result = n + for i in 0..<n: + dest[i] = s.buffer.popFirst + if bufLen > n: + result += s.baseReadDataImpl(s, addr dest[n], bufLen - n) + +proc posReadDataStr[T](s: Stream, buffer: var string, slice: Slice[int]): int = + posReadData[T](s, addr buffer[slice.a], slice.len) + +proc posPeekData[T](s: Stream, buffer: pointer, bufLen: int): int = + var s = PipeOutStream[T](s) + assert s.baseReadDataImpl != nil + + let + dest = cast[ptr UncheckedArray[char]](buffer) + n = min(s.buffer.len, bufLen) + + result = n + for i in 0..<n: + dest[i] = s.buffer[i] + + if bufLen > n: + let + newDataNeeded = bufLen - n + numRead = s.baseReadDataImpl(s, addr dest[n], newDataNeeded) + result += numRead + for i in 0..<numRead: + s.buffer.addLast dest[n + i] + +proc newPipeOutStream*[T](s: sink (ref T)): owned PipeOutStream[T] = + ## Wrap pipe for reading with PipeOutStream so that you can use peek* procs and generate runtime error + ## when setPosition/getPosition is called or write operation is performed. + ## + ## Example: + ## + ## .. code-block:: Nim + ## import osproc, streamwrapper + ## var + ## p = startProcess(exePath) + ## outStream = p.outputStream().newPipeOutStream() + ## echo outStream.peekChar + ## p.close() + + assert s.readDataImpl != nil + + new(result) + for dest, src in fields((ref T)(result)[], s[]): + dest = src + wasMoved(s[]) + if result.readLineImpl != nil: + result.baseReadLineImpl = result.readLineImpl + result.readLineImpl = posReadLine[T] + result.baseReadDataImpl = result.readDataImpl + result.readDataImpl = posReadData[T] + result.readDataStrImpl = posReadDataStr[T] + result.peekDataImpl = posPeekData[T] + + # Set nil to anything you may not call. + result.setPositionImpl = nil + result.getPositionImpl = nil + result.writeDataImpl = nil + result.flushImpl = nil diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 5f7ef380d..c37b6b1c0 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -63,7 +63,6 @@ Formatting floats .. code-block:: nim import strformat - doAssert fmt"{-12345:08}" == "-0012345" doAssert fmt"{-1:3}" == " -1" doAssert fmt"{-1:03}" == "-01" @@ -81,6 +80,52 @@ Formatting floats doAssert fmt"{123.456:13e}" == " 1.234560e+02" +Debugging strings +================= + +``fmt"{expr=}"`` expands to ``fmt"expr={expr}"`` namely the text of the expression, +an equal sign and the results of evaluated expression. + +.. code-block:: nim + + import strformat + doAssert fmt"{123.456=}" == "123.456=123.456" + doAssert fmt"{123.456=:>9.3f}" == "123.456= 123.456" + + let x = "hello" + doAssert fmt"{x=}" == "x=hello" + doAssert fmt"{x =}" == "x =hello" + + let y = 3.1415926 + doAssert fmt"{y=:.2f}" == fmt"y={y:.2f}" + doAssert fmt"{y=}" == fmt"y={y}" + doAssert fmt"{y = : <8}" == fmt"y = 3.14159 " + + proc hello(a: string, b: float): int = 12 + let a = "hello" + let b = 3.1415926 + doAssert fmt"{hello(x, y) = }" == "hello(x, y) = 12" + doAssert fmt"{x.hello(y) = }" == "x.hello(y) = 12" + doAssert fmt"{hello x, y = }" == "hello x, y = 12" + + +Note that it is space sensitive: + +.. code-block:: nim + + import strformat + let x = "12" + doAssert fmt"{x=}" == "x=12" + doAssert fmt"{x =:}" == "x =12" + doAssert fmt"{x =}" == "x =12" + doAssert fmt"{x= :}" == "x= 12" + doAssert fmt"{x= }" == "x= 12" + doAssert fmt"{x = :}" == "x = 12" + doAssert fmt"{x = }" == "x = 12" + doAssert fmt"{x = :}" == "x = 12" + doAssert fmt"{x = }" == "x = 12" + + Implementation details ====================== @@ -552,8 +597,18 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = var subexpr = "" while i < f.len and f[i] != closeChar and f[i] != ':': - subexpr.add f[i] - inc i + if f[i] == '=': + let start = i + inc i + i += f.skipWhitespace(i) + if f[i] == closeChar or f[i] == ':': + result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) + else: + subexpr.add f[start ..< i] + else: + subexpr.add f[i] + inc i + var x: NimNode try: x = parseExpr(subexpr) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 1ffc8bf22..32d6ede1e 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2935,11 +2935,6 @@ proc isEmptyOrWhitespace*(s: string): bool {.noSideEffect, rtl, ## Checks if `s` is empty or consists entirely of whitespace characters. result = s.allCharsInSet(Whitespace) -proc isNilOrWhitespace*(s: string): bool {.noSideEffect, rtl, - extern: "nsuIsNilOrWhitespace", - deprecated: "use isEmptyOrWhitespace instead".} = - ## Alias for isEmptyOrWhitespace - result = isEmptyOrWhitespace(s) when isMainModule: proc nonStaticTests = diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index e798bbaf1..99a333cc6 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -37,7 +37,7 @@ type var gTerm {.threadvar.}: owned(PTerminal) -proc newTerminal(): owned(PTerminal) {.gcsafe.} +proc newTerminal(): owned(PTerminal) {.gcsafe, raises: [].} proc getTerminal(): PTerminal {.inline.} = if isNil(gTerm): diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 317376405..0e23077ac 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -93,6 +93,18 @@ since (1, 1): type StaticParam*[value: static type] = object ## used to wrap a static value in `genericParams` +since (1, 3, 5): + template elementType*(a: untyped): typedesc = + ## return element type of `a`, which can be any iterable (over which you + ## can iterate) + runnableExamples: + iterator myiter(n: int): auto = + for i in 0..<n: yield i + doAssert elementType(@[1,2]) is int + doAssert elementType("asdf") is char + doAssert elementType(myiter(3)) is int + typeof(block: (for ai in a: ai)) + import std/macros macro genericParamsImpl(T: typedesc): untyped = @@ -109,6 +121,9 @@ macro genericParamsImpl(T: typedesc): untyped = of nnkTypeDef: impl = impl[2] continue + of nnkTypeOfExpr: + impl = getTypeInst(impl[0]) + continue of nnkBracketExpr: for i in 1..<impl.len: let ai = impl[i] @@ -118,8 +133,32 @@ macro genericParamsImpl(T: typedesc): untyped = ret = ai of ntyStatic: doAssert false else: - since (1, 1): - ret = newTree(nnkBracketExpr, @[bindSym"StaticParam", ai]) + # getType from a resolved symbol might return a typedesc symbol. + # If so, use it directly instead of wrapping it in StaticParam. + if (ai.kind == nnkSym and ai.symKind == nskType) or + (ai.kind == nnkBracketExpr and ai[0].kind == nnkSym and + ai[0].symKind == nskType): + ret = ai + elif ai.kind == nnkInfix and ai[0].kind == nnkIdent and + ai[0].strVal == "..": + # For built-in array types, the "2" is translated to "0..1" then + # automagically translated to "range[0..1]". However this is not + # reflected in the AST, thus requiring manual transformation here. + # + # We will also be losing some context here: + # var a: array[10, int] + # will be translated to: + # var a: array[0..9, int] + # after typecheck. This means that we can't get the exact + # definition as typed by the user, which will cause confusion for + # users expecting: + # genericParams(typeof(a)) is (StaticParam(10), int) + # to be true while in fact the result will be: + # genericParams(typeof(a)) is (range[0..9], int) + ret = newTree(nnkBracketExpr, @[bindSym"range", ai]) + else: + since (1, 1): + ret = newTree(nnkBracketExpr, @[bindSym"StaticParam", ai]) result.add ret break else: @@ -129,32 +168,20 @@ since (1, 1): template genericParams*(T: typedesc): untyped = ## return tuple of generic params for generic `T` runnableExamples: - type Foo[T1, T2]=object + type Foo[T1, T2] = object doAssert genericParams(Foo[float, string]) is (float, string) type Bar[N: static float, T] = object doAssert genericParams(Bar[1.0, string]) is (StaticParam[1.0], string) doAssert genericParams(Bar[1.0, string]).get(0).value == 1.0 + doAssert genericParams(seq[Bar[2.0, string]]).get(0) is Bar[2.0, string] + var s: seq[Bar[3.0, string]] + doAssert genericParams(typeof(s)) is (Bar[3.0, string],) + + # NOTE: For the builtin array type, the index generic param will + # **always** become a range type after it's bound to a variable. + doAssert genericParams(array[10, int]) is (StaticParam[10], int) + var a: array[10, int] + doAssert genericParams(typeof(a)) is (range[0..9], int) type T2 = T genericParamsImpl(T2) - -when isMainModule: - static: - doAssert $type(42) == "int" - doAssert int.name == "int" - - const a1 = name(int) - const a2 = $(int) - const a3 = $int - doAssert a1 == "int" - doAssert a2 == "int" - doAssert a3 == "int" - - proc fun[T: typedesc](t: T) = - const a1 = name(t) - const a2 = $(t) - const a3 = $t - doAssert a1 == "int" - doAssert a2 == "int" - doAssert a3 == "int" - fun(int) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index bea7d9c44..98be959e3 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -94,6 +94,7 @@ ## echo "suite teardown: run once after the tests" import std/private/since +import std/exitprocs import macros, strutils, streams, times, sets, sequtils @@ -498,7 +499,7 @@ template test*(name, body) {.dirty.} = ## .. code-block:: ## ## [OK] roses are red - bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName + bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded, exceptionTypeName, setProgramResult ensureInitialized() @@ -524,7 +525,7 @@ template test*(name, body) {.dirty.} = finally: if testStatusIMPL == TestStatus.FAILED: - programResult = 1 + setProgramResult 1 let testResult = TestResult( suiteName: when declared(testSuiteName): testSuiteName else: "", testName: name, @@ -560,12 +561,11 @@ template fail* = ## fail() ## ## outputs "Checkpoint A" before quitting. - bind ensureInitialized - + bind ensureInitialized, setProgramResult when declared(testStatusIMPL): testStatusIMPL = TestStatus.FAILED else: - programResult = 1 + setProgramResult 1 ensureInitialized() @@ -576,8 +576,7 @@ template fail* = else: formatter.failureOccurred(checkpoints, "") - when declared(programResult): - if abortOnError: quit(programResult) + if abortOnError: quit(1) checkpoints = @[] @@ -679,7 +678,7 @@ macro check*(conditions: untyped): untyped = result = newNimNode(nnkStmtList) for node in checked: if node.kind != nnkCommentStmt: - result.add(newCall(!"check", node)) + result.add(newCall(newIdentNode("check"), node)) else: let lineinfo = newStrLitNode(checked.lineInfo) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index b163a2ab4..04a9d97bd 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -47,6 +47,8 @@ import std/private/since import strutils, parseutils, base64 +include includes/decode_helpers + type Url* = distinct string @@ -90,6 +92,7 @@ proc decodeUrl*(s: string, decodePlus = true): string = ## This means that any ``%xx`` (where ``xx`` denotes a hexadecimal ## value) are converted to the character with ordinal number ``xx``, ## and every other character is carried over. + ## If ``xx`` is not a valid hexadecimal value, it is left intact. ## ## As a special rule, when the value of ``decodePlus`` is true, ``+`` ## characters are converted to a space. @@ -101,12 +104,7 @@ proc decodeUrl*(s: string, decodePlus = true): string = assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test") == "https://nim-lang.org/this is a test" assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test", false) == "https://nim-lang.org/this is a test" - proc handleHexChar(c: char, x: var int) {.inline.} = - case c - of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) - of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) - of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) - else: assert(false) + assert decodeUrl("abc%xyz") == "abc%xyz" result = newString(s.len) var i = 0 @@ -114,11 +112,7 @@ proc decodeUrl*(s: string, decodePlus = true): string = while i < s.len: case s[i] of '%': - var x = 0 - handleHexChar(s[i+1], x) - handleHexChar(s[i+2], x) - inc(i, 2) - result[j] = chr(x) + result[j] = decodePercent(s, i) of '+': if decodePlus: result[j] = ' ' |