diff options
Diffstat (limited to 'lib/pure')
31 files changed, 1760 insertions, 1233 deletions
diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index d410f8ce1..0ea8ef43b 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -9,7 +9,7 @@ include "system/inclrtl" -import os, oids, tables, strutils, macros +import os, oids, tables, strutils, macros, times import rawsockets, net @@ -27,12 +27,12 @@ export TPort, TSocketFlags ## **Note:** This module is still largely experimental. -# TODO: Discarded void PFutures need to be checked for exception. # TODO: ``except`` statement (without `try`) does not work. # TODO: Multiple exception names in a ``except`` don't work. # TODO: The effect system (raises: []) has trouble with my try transformation. # TODO: Can't await in a 'except' body # TODO: getCurrentException(Msg) don't work +# TODO: Check if yielded future is nil and throw a more meaningful exception # -- Futures @@ -41,27 +41,41 @@ type cb: proc () {.closure,gcsafe.} finished: bool error*: ref EBase - stackTrace: string ## For debugging purposes only. + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string PFuture*[T] = ref object of PFutureBase value: T -proc newFuture*[T](): PFuture[T] = +var currentID* = 0 +proc newFuture*[T](fromProc: string = "unspecified"): PFuture[T] = ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. new(result) result.finished = false - result.stackTrace = getStackTrace() + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() proc checkFinished[T](future: PFuture[T]) = - if future.finished: - echo("<----->") - echo(future.stackTrace) - echo("-----") - when T is string: - echo("Contents: ", future.value.repr) - echo("<----->") - echo("Future already finished, cannot finish twice.") - assert false + when not defined(release): + if future.finished: + echo("<-----> ", future.id, " ", future.fromProc) + echo(future.stackTrace) + echo("-----") + when T is string: + echo("Contents: ", future.value.repr) + echo("<----->") + echo("Future already finished, cannot finish twice.") + echo getStackTrace() + assert false proc complete*[T](future: PFuture[T], val: T) = ## Completes ``future`` with value ``val``. @@ -88,6 +102,8 @@ proc fail*[T](future: PFuture[T], error: ref EBase) = checkFinished(future) future.finished = true future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) if future.cb != nil: future.cb() else: @@ -115,13 +131,24 @@ proc `callback=`*[T](future: PFuture[T], ## If future has already completed then ``cb`` will be called immediately. future.callback = proc () = cb(future) +proc echoOriginalStackTrace[T](future: PFuture[T]) = + # TODO: Come up with something better. + when not defined(release): + echo("Original stack trace in ", future.fromProc, ":") + if not future.errorStackTrace.isNil() and future.errorStackTrace != "": + echo(future.errorStackTrace) + else: + echo("Empty or nil stack trace.") + proc read*[T](future: PFuture[T]): T = ## Retrieves the value of ``future``. Future must be finished otherwise ## this function will fail with a ``EInvalidValue`` exception. ## ## If the result of the future is an error then that error will be raised. if future.finished: - if future.error != nil: raise future.error + if future.error != nil: + echoOriginalStackTrace(future) + raise future.error when T isnot void: return future.value else: @@ -150,7 +177,44 @@ proc asyncCheck*[T](future: PFuture[T]) = ## This should be used instead of ``discard`` to discard void futures. future.callback = proc () = - if future.failed: raise future.error + if future.failed: + echoOriginalStackTrace(future) + raise future.error + +proc `and`*[T, Y](fut1: PFuture[T], fut2: PFuture[Y]): PFuture[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: PFuture[T], fut2: PFuture[Y]): PFuture[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb() = + if not retFuture.finished: retFuture.complete() + fut1.callback = cb + fut2.callback = cb + return retFuture + +type + PDispatcherBase = ref object of PObject + timers: seq[tuple[finishAt: float, fut: PFuture[void]]] + +proc processTimers(p: PDispatcherBase) = + var oldTimers = p.timers + p.timers = @[] + for t in oldTimers: + if epochTime() >= t.finishAt: + t.fut.complete() + else: + p.timers.add(t) when defined(windows) or defined(nimdoc): import winlean, sets, hashes @@ -162,7 +226,7 @@ when defined(windows) or defined(nimdoc): cb: proc (sock: TAsyncFD, bytesTransferred: DWORD, errcode: TOSErrorCode) {.closure,gcsafe.} - PDispatcher* = ref object + PDispatcher* = ref object of PDispatcherBase ioPort: THandle handles: TSet[TAsyncFD] @@ -181,6 +245,7 @@ when defined(windows) or defined(nimdoc): new result result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) result.handles = initSet[TAsyncFD]() + result.timers = @[] var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -207,8 +272,9 @@ when defined(windows) or defined(nimdoc): proc poll*(timeout = 500) = ## Waits for completion events and processes them. let p = getGlobalDispatcher() - if p.handles.len == 0: - raise newException(EInvalidValue, "No handles registered in dispatcher.") + if p.handles.len == 0 and p.timers.len == 0: + raise newException(EInvalidValue, + "No handles or timers registered in dispatcher.") let llTimeout = if timeout == -1: winlean.INFINITE @@ -242,6 +308,9 @@ when defined(windows) or defined(nimdoc): discard else: osError(errCode) + # Timer processing. + processTimers(p) + var connectExPtr: pointer = nil var acceptExPtr: pointer = nil var getAcceptExSockAddrsPtr: pointer = nil @@ -314,7 +383,7 @@ when defined(windows) or defined(nimdoc): ## Returns a ``PFuture`` which will complete when the connection succeeds ## or an error occurs. verifyPresence(socket) - var retFuture = newFuture[void]() + var retFuture = newFuture[void]("connect") # Apparently ``ConnectEx`` expects the socket to be initially bound: var saddr: Tsockaddr_in saddr.sin_family = int16(toInt(af)) @@ -384,7 +453,7 @@ when defined(windows) or defined(nimdoc): # '\0' in the message currently signifies a socket disconnect. Who # knows what will happen when someone sends that to our socket. verifyPresence(socket) - var retFuture = newFuture[string]() + var retFuture = newFuture[string]("recv") var dataBuf: TWSABuf dataBuf.buf = cast[cstring](alloc0(size)) dataBuf.len = size @@ -405,7 +474,10 @@ when defined(windows) or defined(nimdoc): copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) retFuture.complete($data) else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) + if flags.isDisconnectionError(errcode): + retFuture.complete("") + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) if dataBuf.buf != nil: dealloc dataBuf.buf dataBuf.buf = nil @@ -459,7 +531,7 @@ when defined(windows) or defined(nimdoc): ## Sends ``data`` to ``socket``. The returned future will complete once all ## data has been sent. verifyPresence(socket) - var retFuture = newFuture[void]() + var retFuture = newFuture[void]("send") var dataBuf: TWSABuf dataBuf.buf = data # since this is not used in a callback, this is fine @@ -474,7 +546,10 @@ when defined(windows) or defined(nimdoc): if errcode == TOSErrorCode(-1): retFuture.complete() else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) + if flags.isDisconnectionError(errcode): + retFuture.complete() + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) ) let ret = WSASend(socket.TSocketHandle, addr dataBuf, 1, addr bytesReceived, @@ -494,15 +569,21 @@ when defined(windows) or defined(nimdoc): # free ``ol``. return retFuture - proc acceptAddr*(socket: TAsyncFD): + proc acceptAddr*(socket: TAsyncFD, flags = {TSocketFlags.SafeDisconn}): PFuture[tuple[address: string, client: TAsyncFD]] = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. ## - ## The resulting client socket is automatically registered to dispatcher. + ## The resulting client socket is automatically registered to the + ## dispatcher. + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. verifyPresence(socket) - var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]() + var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]("acceptAddr") var clientSock = newRawSocket() if clientSock == osInvalidSocket: osError(osLastError()) @@ -534,6 +615,18 @@ when defined(windows) or defined(nimdoc): client: clientSock.TAsyncFD) ) + template failAccept(errcode): stmt = + if flags.isDisconnectionError(errcode): + var newAcceptFut = acceptAddr(socket, flags) + newAcceptFut.callback = + proc () = + if newAcceptFut.failed: + retFuture.fail(newAcceptFut.readError) + else: + retFuture.complete(newAcceptFut.read) + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + var ol = PCustomOverlapped() GC_ref(ol) ol.data = TCompletionData(sock: socket, cb: @@ -542,7 +635,7 @@ when defined(windows) or defined(nimdoc): if errcode == TOSErrorCode(-1): completeAccept() else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) + failAccept(errcode) ) # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx @@ -555,7 +648,7 @@ when defined(windows) or defined(nimdoc): if not ret: let err = osLastError() if err.int32 != ERROR_IO_PENDING: - retFuture.fail(newException(EOS, osErrorMsg(err))) + failAccept(err) GC_unref(ol) else: completeAccept() @@ -606,7 +699,7 @@ else: readCBs: seq[TCallback] writeCBs: seq[TCallback] - PDispatcher* = ref object + PDispatcher* = ref object of PDispatcherBase selector: PSelector proc `==`*(x, y: TAsyncFD): bool {.borrow.} @@ -614,6 +707,7 @@ else: proc newDispatcher*(): PDispatcher = new result result.selector = newSelector() + result.timers = @[] var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -693,10 +787,12 @@ else: else: # FD no longer a part of the selector. Likely been closed # (e.g. socket disconnected). + + processTimers(p) proc connect*(socket: TAsyncFD, address: string, port: TPort, af = AF_INET): PFuture[void] = - var retFuture = newFuture[void]() + var retFuture = newFuture[void]("connect") proc cb(sock: TAsyncFD): bool = # We have connected. @@ -731,7 +827,7 @@ else: proc recv*(socket: TAsyncFD, size: int, flags = {TSocketFlags.SafeDisconn}): PFuture[string] = - var retFuture = newFuture[string]() + var retFuture = newFuture[string]("recv") var readBuffer = newString(size) @@ -762,7 +858,7 @@ else: proc send*(socket: TAsyncFD, data: string, flags = {TSocketFlags.SafeDisconn}): PFuture[void] = - var retFuture = newFuture[void]() + var retFuture = newFuture[void]("send") var written = 0 @@ -792,9 +888,10 @@ else: addWrite(socket, cb) return retFuture - proc acceptAddr*(socket: TAsyncFD): + proc acceptAddr*(socket: TAsyncFD, flags = {TSocketFlags.SafeDisconn}): PFuture[tuple[address: string, client: TAsyncFD]] = - var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]() + var retFuture = newFuture[tuple[address: string, + client: TAsyncFD]]("acceptAddr") proc cb(sock: TAsyncFD): bool = result = true var sockAddress: Tsockaddr_in @@ -807,19 +904,31 @@ else: if lastError.int32 == EINTR: return false else: - retFuture.fail(newException(EOS, osErrorMsg(lastError))) + if flags.isDisconnectionError(lastError): + return false + else: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) else: register(client.TAsyncFD) retFuture.complete(($inet_ntoa(sockAddress.sin_addr), client.TAsyncFD)) addRead(socket, cb) return retFuture -proc accept*(socket: TAsyncFD): PFuture[TAsyncFD] = +proc sleepAsync*(ms: int): PFuture[void] = + ## Suspends the execution of the current async procedure for the next + ## ``ms`` miliseconds. + var retFuture = newFuture[void]("sleepAsync") + let p = getGlobalDispatcher() + p.timers.add((epochTime() + (ms / 1000), retFuture)) + return retFuture + +proc accept*(socket: TAsyncFD, + flags = {TSocketFlags.SafeDisconn}): PFuture[TAsyncFD] = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection. ## The future will complete when the connection is successfully accepted. - var retFut = newFuture[TAsyncFD]() - var fut = acceptAddr(socket) + var retFut = newFuture[TAsyncFD]("accept") + var fut = acceptAddr(socket, flags) fut.callback = proc (future: PFuture[tuple[address: string, client: TAsyncFD]]) = assert future.finished @@ -845,11 +954,16 @@ template createCb*(retFutureSym, iteratorNameSym, else: next.callback = cb except: - retFutureSym.fail(getCurrentException()) + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) cb() #{.pop.} proc generateExceptionCheck(futSym, - exceptBranch, rootReceiver: PNimrodNode): PNimrodNode {.compileTime.} = + exceptBranch, rootReceiver, fromNode: PNimrodNode): PNimrodNode {.compileTime.} = if exceptBranch == nil: result = rootReceiver else: @@ -869,20 +983,21 @@ proc generateExceptionCheck(futSym, ) ) ) - let elseNode = newNimNode(nnkElse) - elseNode.add newNimNode(nnkStmtList) + let elseNode = newNimNode(nnkElse, fromNode) + elseNode.add newNimNode(nnkStmtList, fromNode) elseNode[0].add rootReceiver result.add elseNode template createVar(result: var PNimrodNode, futSymName: string, asyncProc: PNimrodNode, - valueReceiver, rootReceiver: expr) = - result = newNimNode(nnkStmtList) + valueReceiver, rootReceiver: expr, + fromNode: PNimrodNode) = + result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - result.add newNimNode(nnkYieldStmt).add(futSym) # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future<x> valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read - result.add generateExceptionCheck(futSym, exceptBranch, rootReceiver) + result.add generateExceptionCheck(futSym, exceptBranch, rootReceiver, fromNode) proc processBody(node, retFutureSym: PNimrodNode, subTypeIsVoid: bool, @@ -891,7 +1006,7 @@ proc processBody(node, retFutureSym: PNimrodNode, result = node case node.kind of nnkReturnStmt: - result = newNimNode(nnkStmtList) + result = newNimNode(nnkStmtList, node) if node[0].kind == nnkEmpty: if not subtypeIsVoid: result.add newCall(newIdentNode("complete"), retFutureSym, @@ -902,36 +1017,36 @@ proc processBody(node, retFutureSym: PNimrodNode, result.add newCall(newIdentNode("complete"), retFutureSym, node[0].processBody(retFutureSym, subtypeIsVoid, exceptBranch)) - result.add newNimNode(nnkReturnStmt).add(newNilLit()) + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt - of nnkCommand: + of nnkCommand, nnkCall: if node[0].kind == nnkIdent and node[0].ident == !"await": case node[1].kind - of nnkIdent: + of nnkIdent, nnkInfix: # await x - result = newNimNode(nnkYieldStmt).add(node[1]) # -> yield x - of nnkCall: + result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + of nnkCall, nnkCommand: # await foo(p, x) var futureValue: PNimrodNode result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, - futureValue) + futureValue, node) else: error("Invalid node kind in 'await', got: " & $node[1].kind) - elif node[1].kind == nnkCommand and node[1][0].kind == nnkIdent and - node[1][0].ident == !"await": + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": # foo await x var newCommand = node result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], - newCommand) + newCommand, node) of nnkVarSection, nnkLetSection: case node[0][2].kind of nnkCommand: - if node[0][2][0].ident == !"await": + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": # var x = await y var newVarSection = node # TODO: Should this use copyNimNode? result.createVar("future" & $node[0][0].ident, node[0][2][1], - newVarSection[0][2], newVarSection) + newVarSection[0][2], newVarSection, node) else: discard of nnkAsgn: case node[1].kind @@ -939,7 +1054,7 @@ proc processBody(node, retFutureSym: PNimrodNode, if node[1][0].ident == !"await": # x = await y var newAsgn = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn) + result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) else: discard of nnkDiscardStmt: # discard await x @@ -947,10 +1062,10 @@ proc processBody(node, retFutureSym: PNimrodNode, node[0][0].ident == !"await": var newDiscard = node result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], - newDiscard[0], newDiscard) + newDiscard[0], newDiscard, node) of nnkTryStmt: # try: await x; except: ... - result = newNimNode(nnkStmtList) + result = newNimNode(nnkStmtList, node) proc processForTry(n: PNimrodNode, i: var int, res: PNimrodNode): bool {.compileTime.} = result = false @@ -1009,7 +1124,7 @@ macro async*(prc: stmt): stmt {.immediate.} = (returnType.kind == nnkBracketExpr and returnType[1].kind == nnkIdent and returnType[1].ident == !"void") - var outerProcBody = newNimNode(nnkStmtList) + var outerProcBody = newNimNode(nnkStmtList, prc[6]) # -> var retFuture = newFuture[T]() var retFutureSym = genSym(nskVar, "retFuture") @@ -1019,9 +1134,10 @@ macro async*(prc: stmt): stmt {.immediate.} = outerProcBody.add( newVarStmt(retFutureSym, newCall( - newNimNode(nnkBracketExpr).add( + newNimNode(nnkBracketExpr, prc[6]).add( newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. - subRetType)))) # Get type from return type of this proc + subRetType), + newLit(prc[0].getName)))) # Get type from return type of this proc # -> iterator nameIter(): PFutureBase {.closure.} = # -> var result: T @@ -1030,7 +1146,7 @@ macro async*(prc: stmt): stmt {.immediate.} = var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkVarSection).add( + procBody.insert(0, newNimNode(nnkVarSection, prc[6]).add( newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T procBody.add( newCall(newIdentNode("complete"), @@ -1041,7 +1157,7 @@ macro async*(prc: stmt): stmt {.immediate.} = var closureIterator = newProc(iteratorNameSym, [newIdentNode("PFutureBase")], procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma).add(newIdentNode("closure")) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) outerProcBody.add(closureIterator) # -> createCb(retFuture) @@ -1051,7 +1167,7 @@ macro async*(prc: stmt): stmt {.immediate.} = outerProcBody.add procCb # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt).add(retFutureSym) + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) result = prc @@ -1068,8 +1184,8 @@ macro async*(prc: stmt): stmt {.immediate.} = result[6] = outerProcBody #echo(treeRepr(result)) - #if prc[0].getName == "routeReq": - #echo(toStrLit(result)) + #if prc[0].getName == "getFile": + # echo(toStrLit(result)) proc recvLine*(socket: TAsyncFD): PFuture[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once @@ -1110,3 +1226,11 @@ proc runForever*() = ## Begins a never ending global dispatcher poll loop. while true: poll() + +proc waitFor*[T](fut: PFuture[T]) = + ## **Blocks** the current thread until the specified future completes. + while not fut.finished: + poll() + + if fut.failed: + raise fut.error diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim new file mode 100644 index 000000000..f1b1d1400 --- /dev/null +++ b/lib/pure/asyncftpclient.nim @@ -0,0 +1,295 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import asyncdispatch, asyncnet, strutils, parseutils, os, times + +from ftpclient import TFtpBase, EInvalidReply, TFtpEvent +from net import bufferSize + +type + TAsyncFtpClient* = TFtpBase[PAsyncSocket] + PAsyncFtpClient* = ref TAsyncFtpClient + + ProgressChangedProc* = + proc (total, progress: BiggestInt, speed: float): + PFuture[void] {.closure, gcsafe.} + +proc expectReply(ftp: PAsyncFtpClient): PFuture[TaintedString] = + result = ftp.csock.recvLine() + +proc send*(ftp: PAsyncFtpClient, m: string): PFuture[TaintedString] {.async.} = + ## Send a message to the server, and wait for a primary reply. + ## ``\c\L`` is added for you. + await ftp.csock.send(m & "\c\L") + return await ftp.expectReply() + +proc assertReply(received: TaintedString, expected: varargs[string]) = + for i in items(expected): + if received.string.startsWith(i): return + raise newException(EInvalidReply, + "Expected reply '$1' got: $2" % + [expected.join("' or '"), received.string]) + +proc pasv(ftp: PAsyncFtpClient) {.async.} = + ## Negotiate a data connection. + ftp.dsock = newAsyncSocket() + + var pasvMsg = (await ftp.send("PASV")).string.strip.TaintedString + assertReply(pasvMsg, "227") + var betweenParens = captureBetween(pasvMsg.string, '(', ')') + var nums = betweenParens.split(',') + var ip = nums[0.. -3] + var port = nums[-2.. -1] + var properPort = port[0].parseInt()*256+port[1].parseInt() + await ftp.dsock.connect(ip.join("."), TPort(properPort.toU16)) + ftp.dsockConnected = True + +proc normalizePathSep(path: string): string = + return replace(path, '\\', '/') + +proc connect*(ftp: PAsyncFtpClient) {.async.} = + ## Connect to the FTP server specified by ``ftp``. + await ftp.csock.connect(ftp.address, ftp.port) + + var reply = await ftp.expectReply() + if reply.startsWith("120"): + # 120 Service ready in nnn minutes. + # We wait until we receive 220. + reply = await ftp.expectReply() + assertReply(reply, "220") + + if ftp.user != "": + assertReply(await(ftp.send("USER " & ftp.user)), "230", "331") + + if ftp.pass != "": + assertReply(await(ftp.send("PASS " & ftp.pass)), "230") + +proc pwd*(ftp: PAsyncFtpClient): PFuture[TaintedString] {.async.} = + ## Returns the current working directory. + let wd = await ftp.send("PWD") + assertReply wd, "257" + return wd.string.captureBetween('"').TaintedString # " + +proc cd*(ftp: PAsyncFtpClient, dir: string) {.async.} = + ## Changes the current directory on the remote FTP server to ``dir``. + assertReply(await(ftp.send("CWD " & dir.normalizePathSep)), "250") + +proc cdup*(ftp: PAsyncFtpClient) {.async.} = + ## Changes the current directory to the parent of the current directory. + assertReply(await(ftp.send("CDUP")), "200") + +proc getLines(ftp: PAsyncFtpClient): PFuture[string] {.async.} = + ## Downloads text data in ASCII mode + result = "" + assert ftp.dsockConnected + while ftp.dsockConnected: + let r = await ftp.dsock.recvLine() + if r.string == "": + ftp.dsockConnected = false + else: + result.add(r.string & "\n") + + assertReply(await(ftp.expectReply()), "226") + +proc listDirs*(ftp: PAsyncFtpClient, dir = ""): PFuture[seq[string]] {.async.} = + ## Returns a list of filenames in the given directory. If ``dir`` is "", + ## the current directory is used. If ``async`` is true, this + ## function will return immediately and it will be your job to + ## use asyncio's ``poll`` to progress this operation. + await ftp.pasv() + + assertReply(await(ftp.send("NLST " & dir.normalizePathSep)), ["125", "150"]) + + result = splitLines(await ftp.getLines()) + +proc existsFile*(ftp: PAsyncFtpClient, file: string): PFuture[bool] {.async.} = + ## Determines whether ``file`` exists. + var files = await ftp.listDirs() + for f in items(files): + if f.normalizePathSep == file.normalizePathSep: return true + +proc createDir*(ftp: PAsyncFtpClient, dir: string, recursive = false){.async.} = + ## Creates a directory ``dir``. If ``recursive`` is true, the topmost + ## subdirectory of ``dir`` will be created first, following the secondmost... + ## etc. this allows you to give a full path as the ``dir`` without worrying + ## about subdirectories not existing. + if not recursive: + assertReply(await(ftp.send("MKD " & dir.normalizePathSep)), "257") + else: + var reply = TaintedString"" + var previousDirs = "" + for p in split(dir, {os.dirSep, os.altSep}): + if p != "": + previousDirs.add(p) + reply = await ftp.send("MKD " & previousDirs) + previousDirs.add('/') + assertReply reply, "257" + +proc chmod*(ftp: PAsyncFtpClient, path: string, + permissions: set[TFilePermission]) {.async.} = + ## Changes permission of ``path`` to ``permissions``. + var userOctal = 0 + var groupOctal = 0 + var otherOctal = 0 + for i in items(permissions): + case i + of fpUserExec: userOctal.inc(1) + of fpUserWrite: userOctal.inc(2) + of fpUserRead: userOctal.inc(4) + of fpGroupExec: groupOctal.inc(1) + of fpGroupWrite: groupOctal.inc(2) + of fpGroupRead: groupOctal.inc(4) + of fpOthersExec: otherOctal.inc(1) + of fpOthersWrite: otherOctal.inc(2) + of fpOthersRead: otherOctal.inc(4) + + var perm = $userOctal & $groupOctal & $otherOctal + assertReply(await(ftp.send("SITE CHMOD " & perm & + " " & path.normalizePathSep)), "200") + +proc list*(ftp: PAsyncFtpClient, dir = ""): PFuture[string] {.async.} = + ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current + ## working directory. + await ftp.pasv() + + let reply = await ftp.send("LIST" & " " & dir.normalizePathSep) + assertReply(reply, ["125", "150"]) + + result = await ftp.getLines() + +proc retrText*(ftp: PAsyncFtpClient, file: string): PFuture[string] {.async.} = + ## Retrieves ``file``. File must be ASCII text. + await ftp.pasv() + let reply = await ftp.send("RETR " & file.normalizePathSep) + assertReply(reply, ["125", "150"]) + + result = await ftp.getLines() + +proc getFile(ftp: PAsyncFtpClient, file: TFile, total: BiggestInt, + onProgressChanged: ProgressChangedProc) {.async.} = + assert ftp.dsockConnected + var progress = 0 + var progressInSecond = 0 + var countdownFut = sleepAsync(1000) + var dataFut = ftp.dsock.recv(bufferSize) + while ftp.dsockConnected: + await dataFut or countdownFut + if countdownFut.finished: + asyncCheck onProgressChanged(total, progress, + progressInSecond.float) + progressInSecond = 0 + countdownFut = sleepAsync(1000) + + if dataFut.finished: + let data = dataFut.read + if data != "": + progress.inc(data.len) + progressInSecond.inc(data.len) + file.write(data) + dataFut = ftp.dsock.recv(bufferSize) + else: + ftp.dsockConnected = False + + assertReply(await(ftp.expectReply()), "226") + +proc defaultOnProgressChanged*(total, progress: BiggestInt, + speed: float): PFuture[void] {.nimcall,gcsafe.} = + ## Default FTP ``onProgressChanged`` handler. Does nothing. + result = newFuture[void]() + #echo(total, " ", progress, " ", speed) + result.complete() + +proc retrFile*(ftp: PAsyncFtpClient, file, dest: string, + onProgressChanged = defaultOnProgressChanged) {.async.} = + ## Downloads ``file`` and saves it to ``dest``. + ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function + ## when the download is finished. The event's ``filename`` field will be equal + ## to ``file``. + var destFile = open(dest, mode = fmWrite) + await ftp.pasv() + var reply = await ftp.send("RETR " & file.normalizePathSep) + assertReply reply, ["125", "150"] + if {'(', ')'} notin reply.string: + raise newException(EInvalidReply, "Reply has no file size.") + var fileSize: biggestInt + if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0: + raise newException(EInvalidReply, "Reply has no file size.") + + await getFile(ftp, destFile, fileSize, onProgressChanged) + +proc doUpload(ftp: PAsyncFtpClient, file: TFile, + onProgressChanged: ProgressChangedProc) {.async.} = + assert ftp.dsockConnected + + let total = file.getFileSize() + var data = newStringOfCap(4000) + var progress = 0 + var progressInSecond = 0 + var countdownFut = sleepAsync(1000) + var sendFut: PFuture[void] = nil + while ftp.dsockConnected: + if sendFut == nil or sendFut.finished: + progress.inc(data.len) + progressInSecond.inc(data.len) + # TODO: Async file reading. + let len = file.readBuffer(addr(data[0]), 4000) + setLen(data, len) + if len == 0: + # File finished uploading. + ftp.dsock.close() + ftp.dsockConnected = false + + assertReply(await(ftp.expectReply()), "226") + else: + sendFut = ftp.dsock.send(data) + + if countdownFut.finished: + asyncCheck onProgressChanged(total, progress, progressInSecond.float) + progressInSecond = 0 + countdownFut = sleepAsync(1000) + + await countdownFut or sendFut + +proc storeFile*(ftp: PAsyncFtpClient, file, dest: string, + onProgressChanged = defaultOnProgressChanged) {.async.} = + ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this + ## function asynchronously is recommended to view the progress of + ## the download. + ## The ``EvStore`` event is passed to the specified ``handleEvent`` function + ## when the upload is finished, and the ``filename`` field will be + ## equal to ``file``. + var destFile = open(file) + await ftp.pasv() + + let reply = await ftp.send("STOR " & dest.normalizePathSep) + assertReply reply, ["125", "150"] + + await doUpload(ftp, destFile, onProgressChanged) + +proc newAsyncFtpClient*(address: string, port = TPort(21), + user, pass = ""): PAsyncFtpClient = + ## Creates a new ``PAsyncFtpClient`` object. + new result + result.user = user + result.pass = pass + result.address = address + result.port = port + result.dsockConnected = false + result.csock = newAsyncSocket() + +when isMainModule: + var ftp = newAsyncFtpClient("example.com", user = "test", pass = "test") + proc main(ftp: PAsyncFtpClient) {.async.} = + await ftp.connect() + echo await ftp.pwd() + echo await ftp.listDirs() + await ftp.storeFile("payload.jpg", "payload.jpg") + await ftp.retrFile("payload.jpg", "payload2.jpg") + echo("Finished") + + waitFor main(ftp) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index ee6658fd1..c8bd5cfc1 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -11,14 +11,14 @@ ## ## **Note:** This module is still largely experimental. -import strtabs, asyncnet, asyncdispatch, parseutils, parseurl, strutils +import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils type TRequest* = object client*: PAsyncSocket # TODO: Separate this into a Response object? reqMethod*: string headers*: PStringTable protocol*: tuple[orig: string, major, minor: int] - url*: TURL + url*: TUri hostname*: string ## The hostname of the client that made the request. body*: string @@ -97,7 +97,8 @@ proc sendStatus(client: PAsyncSocket, status: string): PFuture[void] = client.send("HTTP/1.1 " & status & "\c\L") proc processClient(client: PAsyncSocket, address: string, - callback: proc (request: TRequest): PFuture[void]) {.async.} = + callback: proc (request: TRequest): + PFuture[void] {.closure, gcsafe.}) {.async.} = while true: # GET /path HTTP/1.1 # Header: val @@ -135,7 +136,7 @@ proc processClient(client: PAsyncSocket, address: string, request.headers[kv.key] = kv.value request.reqMethod = reqMethod - request.url = parseUrl(path) + request.url = parseUri(path) try: request.protocol = protocol.parseProtocol() except EInvalidValue: @@ -184,7 +185,7 @@ proc processClient(client: PAsyncSocket, address: string, break proc serve*(server: PAsyncHttpServer, port: TPort, - callback: proc (request: TRequest): PFuture[void], + callback: proc (request: TRequest): PFuture[void] {.closure,gcsafe.}, address = "") {.async.} = ## Starts the process of listening for incoming HTTP connections on the ## specified address and port. @@ -199,19 +200,23 @@ proc serve*(server: PAsyncHttpServer, port: TPort, #var (address, client) = await server.socket.acceptAddr() var fut = await server.socket.acceptAddr() asyncCheck processClient(fut.client, fut.address, callback) + #echo(f.isNil) + #echo(f.repr) proc close*(server: PAsyncHttpServer) = ## Terminates the async http server instance. server.socket.close() when isMainModule: - var server = newAsyncHttpServer() - proc cb(req: TRequest) {.async.} = - #echo(req.reqMethod, " ", req.url) - #echo(req.headers) - let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", - "Content-type": "text/plain; charset=utf-8"} - await req.respond(Http200, "Hello World", headers.newStringTable()) - - asyncCheck server.serve(TPort(5555), cb) - runForever() + proc main = + var server = newAsyncHttpServer() + proc cb(req: TRequest) {.async.} = + #echo(req.reqMethod, " ", req.url) + #echo(req.headers) + let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", + "Content-type": "text/plain; charset=utf-8"} + await req.respond(Http200, "Hello World", headers.newStringTable()) + + asyncCheck server.serve(TPort(5555), cb) + runForever() + main() diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index c68ca4350..6b67bf4b5 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -671,25 +671,26 @@ when isMainModule: testRead(s, 2) disp.register(client) - var d = newDispatcher() - - var s = AsyncSocket() - s.connect("amber.tenthbit.net", TPort(6667)) - s.handleConnect = - proc (s: PAsyncSocket) = - testConnect(s, 1) - s.handleRead = - proc (s: PAsyncSocket) = - testRead(s, 1) - d.register(s) - - var server = AsyncSocket() - server.handleAccept = - proc (s: PAsyncSocket) = - testAccept(s, d, 78) - server.bindAddr(TPort(5555)) - server.listen() - d.register(server) - - while d.poll(-1): discard + proc main = + var d = newDispatcher() + + var s = AsyncSocket() + s.connect("amber.tenthbit.net", TPort(6667)) + s.handleConnect = + proc (s: PAsyncSocket) = + testConnect(s, 1) + s.handleRead = + proc (s: PAsyncSocket) = + testRead(s, 1) + d.register(s) + + var server = AsyncSocket() + server.handleAccept = + proc (s: PAsyncSocket) = + testAccept(s, d, 78) + server.bindAddr(TPort(5555)) + server.listen() + d.register(server) + while d.poll(-1): discard + main() diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 374ac77e3..5095d9461 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -36,9 +36,9 @@ ## let client = await server.accept() ## clients.add client ## -## processClient(client) +## asyncCheck processClient(client) ## -## serve() +## asyncCheck serve() ## runForever() ## ## @@ -135,13 +135,13 @@ proc send*(socket: PAsyncSocket, data: string, assert socket != nil result = send(socket.fd.TAsyncFD, data, flags) -proc acceptAddr*(socket: PAsyncSocket): +proc acceptAddr*(socket: PAsyncSocket, flags = {TSocketFlags.SafeDisconn}): PFuture[tuple[address: string, client: PAsyncSocket]] = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. - var retFuture = newFuture[tuple[address: string, client: PAsyncSocket]]() - var fut = acceptAddr(socket.fd.TAsyncFD) + var retFuture = newFuture[tuple[address: string, client: PAsyncSocket]]("asyncnet.acceptAddr") + var fut = acceptAddr(socket.fd.TAsyncFD, flags) fut.callback = proc (future: PFuture[tuple[address: string, client: TAsyncFD]]) = assert future.finished @@ -153,12 +153,13 @@ proc acceptAddr*(socket: PAsyncSocket): retFuture.complete(resultTup) return retFuture -proc accept*(socket: PAsyncSocket): PFuture[PAsyncSocket] = +proc accept*(socket: PAsyncSocket, + flags = {TSocketFlags.SafeDisconn}): PFuture[PAsyncSocket] = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection. ## The future will complete when the connection is successfully accepted. - var retFut = newFuture[PAsyncSocket]() - var fut = acceptAddr(socket) + var retFut = newFuture[PAsyncSocket]("asyncnet.accept") + var fut = acceptAddr(socket, flags) fut.callback = proc (future: PFuture[tuple[address: string, client: PAsyncSocket]]) = assert future.finished diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index c50c4165b..2629e9f40 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -409,6 +409,23 @@ template mapIt*(varSeq, pred: expr) = let it {.inject.} = varSeq[i] varSeq[i] = pred +template newSeqWith*(len: int, init: expr): expr = + ## creates a new sequence, calling `init` to initialize each value. Example: + ## + ## .. code-block:: nimrod + ## var seq2D = newSeqWith(20, newSeq[bool](10)) + ## seq2D[0][0] = true + ## seq2D[1][0] = true + ## seq2D[0][1] = true + ## + ## import math + ## var seqRand = newSeqWith(20, random(10)) + ## echo seqRand + var result {.gensym.} = newSeq[type(init)](len) + for i in 0 .. <len: + result[i] = init + result + when isMainModule: import strutils block: # concat test @@ -557,4 +574,11 @@ when isMainModule: doAssert b.distribute(5, true)[4].len == 5 doAssert b.distribute(5, false)[4].len == 2 + block: # newSeqWith tests + var seq2D = newSeqWith(4, newSeq[bool](2)) + seq2D[0][0] = true + seq2D[1][0] = true + seq2D[0][1] = true + doAssert seq2D == @[@[true, true], @[true, false], @[false, false], @[false, false]] + echo "Finished doc tests" diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 42cdc682f..ce901963e 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -128,7 +128,7 @@ proc mget*[A](s: var TSet[A], key: A): var A = ## for sharing. assert s.isValid, "The set needs to be initialized." var index = rawGet(s, key) - if index >= 0: result = t.data[index].key + if index >= 0: result = s.data[index].key else: raise newException(EInvalidKey, "key not found: " & $key) proc contains*[A](s: TSet[A], key: A): bool = @@ -713,6 +713,24 @@ proc `$`*[A](s: TOrderedSet[A]): string = assert s.isValid, "The set needs to be initialized." dollarImpl() +proc `==`*[A](s, t: TOrderedSet[A]): bool = + ## Equality for ordered sets. + if s.counter != t.counter: return false + var h = s.first + var g = s.first + var compared = 0 + while h >= 0 and g >= 0: + var nxh = s.data[h].next + var nxg = t.data[g].next + if s.data[h].slot == seFilled and s.data[g].slot == seFilled: + if s.data[h].key == s.data[g].key: + inc compared + else: + return false + h = nxh + g = nxg + result = compared == s.counter + proc testModule() = ## Internal micro test to validate docstrings and such. block isValidTest: @@ -858,7 +876,7 @@ proc testModule() = var b = initOrderedSet[int]() for x in [2, 4, 5]: b.incl(x) assert($a == $b) - # assert(a == b) # https://github.com/Araq/Nimrod/issues/1413 + assert(a == b) # https://github.com/Araq/Nimrod/issues/1413 block initBlocks: var a: TOrderedSet[int] diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index dfa819f64..8d7f28f8e 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -18,15 +18,24 @@ when not defined(windows): when defined(linux): import linux + +when defined(freebsd) or defined(macosx): + {.emit:"#include <sys/types.h>".} + +when defined(openbsd) or defined(netbsd): + {.emit:"#include <sys/param.h>".} when defined(macosx) or defined(bsd): + # we HAVE to emit param.h before sysctl.h so we cannot use .header here + # either. The amount of archaic bullshit in Poonix based OSes is just insane. + {.emit:"#include <sys/sysctl.h>".} const CTL_HW = 6 HW_AVAILCPU = 25 HW_NCPU = 3 proc sysctl(x: ptr array[0..3, cint], y: cint, z: pointer, a: var csize, b: pointer, c: int): cint {. - importc: "sysctl", header: "<sys/sysctl.h>".} + importc: "sysctl", nodecl.} proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## returns the numer of the processors/cores the machine has. diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index fd1041918..f46822d94 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -9,6 +9,9 @@ ## Implements Nimrod's 'spawn'. +when not compileOption("threads"): + {.error: "Threadpool requires --threads:on option.".} + import cpuinfo, cpuload, locks {.push stackTrace:off.} @@ -92,7 +95,7 @@ type FlowVarBase* = ref FlowVarBaseObj ## untyped base class for 'FlowVar[T]' FlowVarBaseObj = object of TObject - ready, usesCondVar: bool + ready, usesCondVar, awaited: bool cv: CondVar #\ # for 'awaitAny' support ai: ptr AwaitInfo @@ -126,15 +129,15 @@ type proc await*(fv: FlowVarBase) = ## waits until the value for the flowVar arrives. Usually it is not necessary ## to call this explicitly. - if fv.usesCondVar: - fv.usesCondVar = false + if fv.usesCondVar and not fv.awaited: + fv.awaited = true await(fv.cv) destroyCondVar(fv.cv) proc finished(fv: FlowVarBase) = doAssert fv.ai.isNil, "flowVar is still attached to an 'awaitAny'" # we have to protect against the rare cases where the owner of the flowVar - # simply disregards the flowVar and yet the "flowVarr" has not yet written + # simply disregards the flowVar and yet the "flowVar" has not yet written # anything to it: await(fv) if fv.data.isNil: return @@ -207,6 +210,7 @@ proc `^`*[T](fv: FlowVar[T]): T = ## blocks until the value is available and then returns this value. await(fv) when T is string or T is seq: + # XXX closures? deepCopy? result = cast[T](fv.data) else: result = fv.blob @@ -264,6 +268,10 @@ proc slave(w: ptr Worker) {.thread.} = w.shutdown = false atomicDec currentPoolSize +var + workers: array[MaxThreadPoolSize, TThread[ptr Worker]] + workersData: array[MaxThreadPoolSize, Worker] + proc setMinPoolSize*(size: range[1..MaxThreadPoolSize]) = ## sets the minimal thread pool size. The default value of this is 4. minPoolSize = size @@ -272,10 +280,10 @@ proc setMaxPoolSize*(size: range[1..MaxThreadPoolSize]) = ## sets the minimal thread pool size. The default value of this ## is ``MaxThreadPoolSize``. maxPoolSize = size - -var - workers: array[MaxThreadPoolSize, TThread[ptr Worker]] - workersData: array[MaxThreadPoolSize, Worker] + if currentPoolSize > maxPoolSize: + for i in maxPoolSize..currentPoolSize-1: + let w = addr(workersData[i]) + w.shutdown = true proc activateThread(i: int) {.noinline.} = workersData[i].taskArrived = createCondVar() diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index d1cf36a87..49bf92980 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -28,8 +28,9 @@ proc parseCookies*(s: string): PStringTable = if s[i] == '\0': break inc(i) # skip ';' -proc setCookie*(key, value: string, domain = "", path = "", - expires = "", noName = false): string = +proc setCookie*(key, value: string, domain = "", path = "", + expires = "", noName = false, + secure = false, httpOnly = false): string = ## Creates a command in the format of ## ``Set-Cookie: key=value; Domain=...; ...`` result = "" @@ -38,16 +39,20 @@ proc setCookie*(key, value: string, domain = "", path = "", if domain != "": result.add("; Domain=" & domain) if path != "": result.add("; Path=" & path) if expires != "": result.add("; Expires=" & expires) + if secure: result.add("; secure") + if httpOnly: result.add("; HttpOnly") proc setCookie*(key, value: string, expires: TTimeInfo, - domain = "", path = "", noName = false): string = + domain = "", path = "", noName = false, + secure = false, httpOnly = false): string = ## Creates a command in the format of ## ``Set-Cookie: key=value; Domain=...; ...`` ## ## **Note:** UTC is assumed as the timezone for ``expires``. return setCookie(key, value, domain, path, - format(expires, "ddd',' dd MMM yyyy HH:mm:ss 'UTC'"), noname) + format(expires, "ddd',' dd MMM yyyy HH:mm:ss 'UTC'"), + noname, secure, httpOnly) when isMainModule: var tim = TTime(int(getTime()) + 76 * (60 * 60 * 24)) @@ -55,5 +60,3 @@ when isMainModule: echo(setCookie("test", "value", tim.getGMTime())) echo parseCookies("uid=1; kp=2") - - \ No newline at end of file diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index 53f6688b9..e8d3f762e 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -10,6 +10,10 @@ include "system/inclrtl" import sockets, strutils, parseutils, times, os, asyncio +from asyncnet import nil +from rawsockets import nil +from asyncdispatch import PFuture + ## This module **partially** implements an FTP client as specified ## by `RFC 959 <http://tools.ietf.org/html/rfc959>`_. ## @@ -34,34 +38,32 @@ import sockets, strutils, parseutils, times, os, asyncio type - TFTPClient* = object of TObject - case isAsync: bool - of false: - csock: TSocket # Command connection socket - dsock: TSocket # Data connection socket - else: - dummyA, dummyB: pointer # workaround a Nimrod API issue - asyncCSock: PAsyncSocket - asyncDSock: PAsyncSocket + PFtpBase*[SockType] = ref TFtpBase[SockType] + TFtpBase*[SockType] = object + csock*: SockType + dsock*: SockType + when SockType is asyncio.PAsyncSocket: handleEvent*: proc (ftp: PAsyncFTPClient, ev: TFTPEvent){.closure,gcsafe.} disp: PDispatcher asyncDSockID: PDelegate - user, pass: string - address: string - port: TPort + user*, pass*: string + address*: string + when SockType is asyncnet.PAsyncSocket: + port*: rawsockets.TPort + else: + port*: TPort - jobInProgress: bool - job: ref TFTPJob + jobInProgress*: bool + job*: PFTPJob[SockType] - dsockConnected: bool - - PFTPClient* = ref TFTPClient + dsockConnected*: bool FTPJobType* = enum JRetrText, JRetr, JStore - TFTPJob = object - prc: proc (ftp: PFTPClient, async: bool): bool {.nimcall, gcsafe.} + PFtpJob[T] = ref TFtpJob[T] + TFTPJob[T] = object + prc: proc (ftp: PFTPBase[T], async: bool): bool {.nimcall, gcsafe.} case typ*: FTPJobType of JRetrText: lines: string @@ -75,8 +77,11 @@ type toStore: string # Data left to upload (Only used with async) else: nil + TFtpClient* = TFtpBase[TSocket] + PFtpClient* = ref TFTPClient + PAsyncFTPClient* = ref TAsyncFTPClient ## Async alternative to TFTPClient. - TAsyncFTPClient* = object of TFTPClient + TAsyncFTPClient* = TFtpBase[asyncio.PAsyncSocket] FTPEventType* = enum EvTransferProgress, EvLines, EvRetr, EvStore @@ -106,30 +111,30 @@ proc ftpClient*(address: string, port = TPort(21), result.address = address result.port = port - result.isAsync = false result.dsockConnected = false result.csock = socket() if result.csock == InvalidSocket: osError(osLastError()) -proc getDSock(ftp: PFTPClient): TSocket = - if ftp.isAsync: return ftp.asyncDSock else: return ftp.dsock +proc getDSock[T](ftp: PFTPBase[T]): TSocket = + return ftp.dsock -proc getCSock(ftp: PFTPClient): TSocket = - if ftp.isAsync: return ftp.asyncCSock else: return ftp.csock +proc getCSock[T](ftp: PFTPBase[T]): TSocket = + return ftp.csock template blockingOperation(sock: TSocket, body: stmt) {.immediate.} = - if ftp.isAsync: - sock.setBlocking(true) body - if ftp.isAsync: - sock.setBlocking(false) -proc expectReply(ftp: PFTPClient): TaintedString = +template blockingOperation(sock: asyncio.PAsyncSocket, body: stmt) {.immediate.} = + sock.setBlocking(true) + body + sock.setBlocking(false) + +proc expectReply[T](ftp: PFtpBase[T]): TaintedString = result = TaintedString"" blockingOperation(ftp.getCSock()): ftp.getCSock().readLine(result) -proc send*(ftp: PFTPClient, m: string): TaintedString = +proc send*[T](ftp: PFtpBase[T], m: string): TaintedString = ## Send a message to the server, and wait for a primary reply. ## ``\c\L`` is added for you. blockingOperation(ftp.getCSock()): @@ -149,8 +154,8 @@ proc assertReply(received: TaintedString, expected: varargs[string]) = "Expected reply '$1' got: $2" % [expected.join("' or '"), received.string]) -proc createJob(ftp: PFTPClient, - prc: proc (ftp: PFTPClient, async: bool): bool {. +proc createJob[T](ftp: PFtpBase[T], + prc: proc (ftp: PFtpBase[T], async: bool): bool {. nimcall,gcsafe.}, cmd: FTPJobType) = if ftp.jobInProgress: @@ -165,7 +170,7 @@ proc createJob(ftp: PFTPClient, of JRetr, JStore: ftp.job.toStore = "" -proc deleteJob(ftp: PFTPClient) = +proc deleteJob[T](ftp: PFtpBase[T]) = assert ftp.jobInProgress ftp.jobInProgress = false case ftp.job.typ @@ -173,12 +178,9 @@ proc deleteJob(ftp: PFTPClient) = ftp.job.lines = "" of JRetr, JStore: ftp.job.file.close() - if ftp.isAsync: - ftp.asyncDSock.close() - else: - ftp.dsock.close() + ftp.dsock.close() -proc handleTask(s: PAsyncSocket, ftp: PFTPClient) = +proc handleTask(s: PAsyncSocket, ftp: PAsyncFTPClient) = if ftp.jobInProgress: if ftp.job.typ in {JRetr, JStore}: if epochTime() - ftp.job.lastProgressReport >= 1.0: @@ -193,12 +195,12 @@ proc handleTask(s: PAsyncSocket, ftp: PFTPClient) = ftp.job.oneSecond = 0 ftp.handleEvent(PAsyncFTPClient(ftp), r) -proc handleWrite(s: PAsyncSocket, ftp: PFTPClient) = +proc handleWrite(s: PAsyncSocket, ftp: PAsyncFTPClient) = if ftp.jobInProgress: if ftp.job.typ == JStore: assert (not ftp.job.prc(ftp, true)) -proc handleConnect(s: PAsyncSocket, ftp: PFTPClient) = +proc handleConnect(s: PAsyncSocket, ftp: PAsyncFTPClient) = ftp.dsockConnected = true assert(ftp.jobInProgress) if ftp.job.typ == JStore: @@ -206,30 +208,32 @@ proc handleConnect(s: PAsyncSocket, ftp: PFTPClient) = else: s.delHandleWrite() -proc handleRead(s: PAsyncSocket, ftp: PFTPClient) = +proc handleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = assert ftp.jobInProgress assert ftp.job.typ != JStore # This can never return true, because it shouldn't check for code # 226 from csock. assert(not ftp.job.prc(ftp, true)) -proc pasv(ftp: PFTPClient) = +proc pasv[T](ftp: PFtpBase[T]) = ## Negotiate a data connection. - if not ftp.isAsync: + when T is TSocket: ftp.dsock = socket() if ftp.dsock == InvalidSocket: osError(osLastError()) - else: - ftp.asyncDSock = AsyncSocket() - ftp.asyncDSock.handleRead = + elif T is PAsyncSocket: + ftp.dsock = AsyncSocket() + ftp.dsock.handleRead = proc (s: PAsyncSocket) = handleRead(s, ftp) - ftp.asyncDSock.handleConnect = + ftp.dsock.handleConnect = proc (s: PAsyncSocket) = handleConnect(s, ftp) - ftp.asyncDSock.handleTask = + ftp.dsock.handleTask = proc (s: PAsyncSocket) = handleTask(s, ftp) - ftp.disp.register(ftp.asyncDSock) + ftp.disp.register(ftp.dsock) + else: + {.fatal: "Incorrect socket instantiation".} var pasvMsg = ftp.send("PASV").string.strip.TaintedString assertReply(pasvMsg, "227") @@ -238,23 +242,24 @@ proc pasv(ftp: PFTPClient) = var ip = nums[0.. -3] var port = nums[-2.. -1] var properPort = port[0].parseInt()*256+port[1].parseInt() - if ftp.isAsync: - ftp.asyncDSock.connect(ip.join("."), TPort(properPort.toU16)) + ftp.dsock.connect(ip.join("."), TPort(properPort.toU16)) + when T is PAsyncSocket: ftp.dsockConnected = False else: - ftp.dsock.connect(ip.join("."), TPort(properPort.toU16)) ftp.dsockConnected = True proc normalizePathSep(path: string): string = return replace(path, '\\', '/') -proc connect*(ftp: PFTPClient) = +proc connect*[T](ftp: PFtpBase[T]) = ## Connect to the FTP server specified by ``ftp``. - if ftp.isAsync: - blockingOperation(ftp.asyncCSock): - ftp.asyncCSock.connect(ftp.address, ftp.port) - else: + when T is PAsyncSocket: + blockingOperation(ftp.csock): + ftp.csock.connect(ftp.address, ftp.port) + elif T is TSocket: ftp.csock.connect(ftp.address, ftp.port) + else: + {.fatal: "Incorrect socket instantiation".} # TODO: Handle 120? or let user handle it. assertReply ftp.expectReply(), "220" @@ -279,25 +284,27 @@ proc cdup*(ftp: PFTPClient) = ## Changes the current directory to the parent of the current directory. assertReply ftp.send("CDUP"), "200" -proc getLines(ftp: PFTPClient, async: bool = false): bool = +proc getLines[T](ftp: PFtpBase[T], async: bool = false): bool = ## Downloads text data in ASCII mode ## Returns true if the download is complete. ## It doesn't if `async` is true, because it doesn't check for 226 then. if ftp.dsockConnected: var r = TaintedString"" - if ftp.isAsync: + when T is PAsyncSocket: if ftp.asyncDSock.readLine(r): if r.string == "": ftp.dsockConnected = false else: ftp.job.lines.add(r.string & "\n") - else: + elif T is TSocket: assert(not async) ftp.dsock.readLine(r) if r.string == "": ftp.dsockConnected = false else: ftp.job.lines.add(r.string & "\n") + else: + {.fatal: "Incorrect socket instantiation".} if not async: var readSocks: seq[TSocket] = @[ftp.getCSock()] @@ -307,14 +314,14 @@ proc getLines(ftp: PFTPClient, async: bool = false): bool = assertReply ftp.expectReply(), "226" return true -proc listDirs*(ftp: PFTPClient, dir: string = "", +proc listDirs*[T](ftp: PFtpBase[T], dir: string = "", async = false): seq[string] = ## Returns a list of filenames in the given directory. If ``dir`` is "", ## the current directory is used. If ``async`` is true, this ## function will return immediately and it will be your job to ## use asyncio's ``poll`` to progress this operation. - ftp.createJob(getLines, JRetrText) + ftp.createJob(getLines[T], JRetrText) ftp.pasv() assertReply ftp.send("NLST " & dir.normalizePathSep), ["125", "150"] @@ -384,12 +391,12 @@ proc chmod*(ftp: PFTPClient, path: string, assertReply ftp.send("SITE CHMOD " & perm & " " & path.normalizePathSep), "200" -proc list*(ftp: PFTPClient, dir: string = "", async = false): string = +proc list*[T](ftp: PFtpBase[T], dir: string = "", async = false): string = ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current ## working directory. If ``async`` is true, this function will return ## immediately and it will be your job to call asyncio's ## ``poll`` to progress this operation. - ftp.createJob(getLines, JRetrText) + ftp.createJob(getLines[T], JRetrText) ftp.pasv() assertReply(ftp.send("LIST" & " " & dir.normalizePathSep), ["125", "150"]) @@ -401,11 +408,11 @@ proc list*(ftp: PFTPClient, dir: string = "", async = false): string = else: return "" -proc retrText*(ftp: PFTPClient, file: string, async = false): string = +proc retrText*[T](ftp: PFtpBase[T], file: string, async = false): string = ## Retrieves ``file``. File must be ASCII text. ## If ``async`` is true, this function will return immediately and ## it will be your job to call asyncio's ``poll`` to progress this operation. - ftp.createJob(getLines, JRetrText) + ftp.createJob(getLines[T], JRetrText) ftp.pasv() assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"] @@ -416,15 +423,17 @@ proc retrText*(ftp: PFTPClient, file: string, async = false): string = else: return "" -proc getFile(ftp: PFTPClient, async = false): bool = +proc getFile[T](ftp: PFtpBase[T], async = false): bool = if ftp.dsockConnected: var r = "".TaintedString var bytesRead = 0 var returned = false if async: - if not ftp.isAsync: raise newException(EFTP, "FTPClient must be async.") - bytesRead = ftp.AsyncDSock.recvAsync(r, BufferSize) - returned = bytesRead != -1 + when T is TSocket: + raise newException(EFTP, "FTPClient must be async.") + else: + bytesRead = ftp.dsock.recvAsync(r, BufferSize) + returned = bytesRead != -1 else: bytesRead = getDSock(ftp).recv(r, BufferSize) returned = true @@ -443,13 +452,13 @@ proc getFile(ftp: PFTPClient, async = false): bool = assertReply ftp.expectReply(), "226" return true -proc retrFile*(ftp: PFTPClient, file, dest: string, async = false) = +proc retrFile*[T](ftp: PFtpBase[T], file, dest: string, async = false) = ## Downloads ``file`` and saves it to ``dest``. Usage of this function ## asynchronously is recommended to view the progress of the download. ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function ## when the download is finished, and the ``filename`` field will be equal ## to ``file``. - ftp.createJob(getFile, JRetr) + ftp.createJob(getFile[T], JRetr) ftp.job.file = open(dest, mode = fmWrite) ftp.pasv() var reply = ftp.send("RETR " & file.normalizePathSep) @@ -468,11 +477,11 @@ proc retrFile*(ftp: PFTPClient, file, dest: string, async = false) = while not ftp.job.prc(ftp, false): discard ftp.deleteJob() -proc doUpload(ftp: PFTPClient, async = false): bool = +proc doUpload[T](ftp: PFtpBase[T], async = false): bool = if ftp.dsockConnected: if ftp.job.toStore.len() > 0: assert(async) - let bytesSent = ftp.asyncDSock.sendAsync(ftp.job.toStore) + let bytesSent = ftp.dsock.sendAsync(ftp.job.toStore) if bytesSent == ftp.job.toStore.len: ftp.job.toStore = "" elif bytesSent != ftp.job.toStore.len and bytesSent != 0: @@ -485,7 +494,7 @@ proc doUpload(ftp: PFTPClient, async = false): bool = setLen(s, len) if len == 0: # File finished uploading. - if ftp.isAsync: ftp.asyncDSock.close() else: ftp.dsock.close() + ftp.dsock.close() ftp.dsockConnected = false if not async: @@ -496,7 +505,7 @@ proc doUpload(ftp: PFTPClient, async = false): bool = if not async: getDSock(ftp).send(s) else: - let bytesSent = ftp.asyncDSock.sendAsync(s) + let bytesSent = ftp.dsock.sendAsync(s) if bytesSent == 0: ftp.job.toStore.add(s) elif bytesSent != s.len: @@ -506,14 +515,14 @@ proc doUpload(ftp: PFTPClient, async = false): bool = ftp.job.progress.inc(len) ftp.job.oneSecond.inc(len) -proc store*(ftp: PFTPClient, file, dest: string, async = false) = +proc store*[T](ftp: PFtpBase[T], file, dest: string, async = false) = ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this ## function asynchronously is recommended to view the progress of ## the download. ## The ``EvStore`` event is passed to the specified ``handleEvent`` function ## when the upload is finished, and the ``filename`` field will be ## equal to ``file``. - ftp.createJob(doUpload, JStore) + ftp.createJob(doUpload[T], JStore) ftp.job.file = open(file) ftp.job.total = ftp.job.file.getFileSize() ftp.job.lastProgressReport = epochTime() @@ -526,16 +535,12 @@ proc store*(ftp: PFTPClient, file, dest: string, async = false) = while not ftp.job.prc(ftp, false): discard ftp.deleteJob() -proc close*(ftp: PFTPClient) = +proc close*[T](ftp: PFTPBase[T]) = ## Terminates the connection to the server. assertReply ftp.send("QUIT"), "221" if ftp.jobInProgress: ftp.deleteJob() - if ftp.isAsync: - ftp.asyncCSock.close() - ftp.asyncDSock.close() - else: - ftp.csock.close() - ftp.dsock.close() + ftp.csock.close() + ftp.dsock.close() proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = if ftp.jobInProgress: @@ -572,66 +577,65 @@ proc asyncFTPClient*(address: string, port = TPort(21), dres.pass = pass dres.address = address dres.port = port - dres.isAsync = true dres.dsockConnected = false dres.handleEvent = handleEvent - dres.asyncCSock = AsyncSocket() - dres.asyncCSock.handleRead = + dres.csock = AsyncSocket() + dres.csock.handleRead = proc (s: PAsyncSocket) = csockHandleRead(s, dres) result = dres proc register*(d: PDispatcher, ftp: PAsyncFTPClient): PDelegate {.discardable.} = ## Registers ``ftp`` with dispatcher ``d``. - assert ftp.isAsync ftp.disp = d - return ftp.disp.register(ftp.asyncCSock) + return ftp.disp.register(ftp.csock) when isMainModule: - var d = newDispatcher() - let hev = - proc (ftp: PAsyncFTPClient, event: TFTPEvent) = - case event.typ - of EvStore: - echo("Upload finished!") - ftp.retrFile("payload.JPG", "payload2.JPG", async = true) - of EvTransferProgress: - var time: int64 = -1 - if event.speed != 0: - time = (event.bytesTotal - event.bytesFinished) div event.speed - echo(event.currentJob) - echo(event.speed div 1000, " kb/s. - ", - event.bytesFinished, "/", event.bytesTotal, - " - ", time, " seconds") - echo(d.len) - of EvRetr: - echo("Download finished!") - ftp.close() - echo d.len - else: assert(false) - var ftp = asyncFTPClient("picheta.me", user = "test", pass = "asf", handleEvent = hev) - - d.register(ftp) - d.len.echo() - ftp.connect() - echo "connected" - ftp.store("payload.JPG", "payload.JPG", async = true) - d.len.echo() - echo "uploading..." - while true: - if not d.poll(): break - + proc main = + var d = newDispatcher() + let hev = + proc (ftp: PAsyncFTPClient, event: TFTPEvent) = + case event.typ + of EvStore: + echo("Upload finished!") + ftp.retrFile("payload.jpg", "payload2.jpg", async = true) + of EvTransferProgress: + var time: int64 = -1 + if event.speed != 0: + time = (event.bytesTotal - event.bytesFinished) div event.speed + echo(event.currentJob) + echo(event.speed div 1000, " kb/s. - ", + event.bytesFinished, "/", event.bytesTotal, + " - ", time, " seconds") + echo(d.len) + of EvRetr: + echo("Download finished!") + ftp.close() + echo d.len + else: assert(false) + var ftp = asyncFTPClient("example.com", user = "foo", pass = "bar", handleEvent = hev) + + d.register(ftp) + d.len.echo() + ftp.connect() + echo "connected" + ftp.store("payload.jpg", "payload.jpg", async = true) + d.len.echo() + echo "uploading..." + while true: + if not d.poll(): break + main() when isMainModule and false: - var ftp = ftpClient("picheta.me", user = "asdasd", pass = "asfwq") + var ftp = ftpClient("example.com", user = "foo", pass = "bar") ftp.connect() echo ftp.pwd() echo ftp.list() echo("uploading") - ftp.store("payload.JPG", "payload.JPG", async = false) + ftp.store("payload.jpg", "payload.jpg", async = false) echo("Upload complete") - ftp.retrFile("payload.JPG", "payload2.JPG", async = false) + ftp.retrFile("payload.jpg", "payload2.jpg", async = false) echo("Download complete") sleep(5000) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 9bacc80d6..4db6ac6ed 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -654,8 +654,7 @@ when isMainModule: resp = await client.request("http://nimrod-lang.org/download.html") echo("Got response: ", resp.status) - asyncCheck main() - runForever() + waitFor main() else: #downloadFile("http://force7.de/nimrod/index.html", "nimrodindex.html") diff --git a/lib/pure/irc.nim b/lib/pure/irc.nim deleted file mode 100644 index 49d9a9a34..000000000 --- a/lib/pure/irc.nim +++ /dev/null @@ -1,503 +0,0 @@ -# -# -# Nimrod's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements an asynchronous IRC client. -## -## Currently this module requires at least some knowledge of the IRC protocol. -## It provides a function for sending raw messages to the IRC server, together -## with some basic functions like sending a message to a channel. -## It automizes the process of keeping the connection alive, so you don't -## need to reply to PING messages. In fact, the server is also PING'ed to check -## the amount of lag. -## -## .. code-block:: Nimrod -## -## var client = irc("picheta.me", joinChans = @["#bots"]) -## client.connect() -## while True: -## var event: TIRCEvent -## if client.poll(event): -## case event.typ -## of EvConnected: nil -## of EvDisconnected: -## client.reconnect() -## of EvMsg: -## # Write your message reading code here. -## -## **Warning:** The API of this module is unstable, and therefore is subject -## to change. - -include "system/inclrtl" - -import sockets, strutils, parseutils, times, asyncio, os - -type - TIRC* = object of TObject - address: string - port: TPort - nick, user, realname, serverPass: string - case isAsync: bool - of true: - handleEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure, gcsafe.} - asyncSock: PAsyncSocket - myDispatcher: PDispatcher - of false: - dummyA: pointer - dummyB: pointer # workaround a Nimrod API issue - dummyC: pointer - sock: TSocket - status: TInfo - lastPing: float - lastPong: float - lag: float - channelsToJoin: seq[string] - msgLimit: bool - messageBuffer: seq[tuple[timeToSend: float, m: string]] - lastReconnect: float - - PIRC* = ref TIRC - - PAsyncIRC* = ref TAsyncIRC - TAsyncIRC* = object of TIRC - - TIRCMType* = enum - MUnknown, - MNumeric, - MPrivMsg, - MJoin, - MPart, - MMode, - MTopic, - MInvite, - MKick, - MQuit, - MNick, - MNotice, - MPing, - MPong, - MError - - TIRCEventType* = enum - EvMsg, EvConnected, EvDisconnected - TIRCEvent* = object ## IRC Event - case typ*: TIRCEventType - of EvConnected: - ## Connected to server. - ## Only occurs with AsyncIRC. - nil - of EvDisconnected: - ## Disconnected from the server - nil - of EvMsg: ## Message from the server - cmd*: TIRCMType ## Command (e.g. PRIVMSG) - nick*, user*, host*, servername*: string - numeric*: string ## Only applies to ``MNumeric`` - params*: seq[string] ## Parameters of the IRC message - origin*: string ## The channel/user that this msg originated from - raw*: string ## Raw IRC message - timestamp*: TTime ## UNIX epoch time the message was received - -proc send*(irc: PIRC, message: string, sendImmediately = false) = - ## Sends ``message`` as a raw command. It adds ``\c\L`` for you. - var sendMsg = true - if irc.msgLimit and not sendImmediately: - var timeToSend = epochTime() - if irc.messageBuffer.len() >= 3: - timeToSend = (irc.messageBuffer[irc.messageBuffer.len()-1][0] + 2.0) - - irc.messageBuffer.add((timeToSend, message)) - sendMsg = false - - if sendMsg: - try: - if irc.isAsync: - irc.asyncSock.send(message & "\c\L") - else: - irc.sock.send(message & "\c\L") - except EOS: - # Assuming disconnection of every EOS could be bad, - # but I can't exactly check for EBrokenPipe. - irc.status = SockClosed - -proc privmsg*(irc: PIRC, target, message: string) = - ## Sends ``message`` to ``target``. ``Target`` can be a channel, or a user. - irc.send("PRIVMSG $1 :$2" % [target, message]) - -proc notice*(irc: PIRC, target, message: string) = - ## Sends ``notice`` to ``target``. ``Target`` can be a channel, or a user. - irc.send("NOTICE $1 :$2" % [target, message]) - -proc join*(irc: PIRC, channel: string, key = "") = - ## Joins ``channel``. - ## - ## If key is not ``""``, then channel is assumed to be key protected and this - ## function will join the channel using ``key``. - if key == "": - irc.send("JOIN " & channel) - else: - irc.send("JOIN " & channel & " " & key) - -proc part*(irc: PIRC, channel, message: string) = - ## Leaves ``channel`` with ``message``. - irc.send("PART " & channel & " :" & message) - -proc close*(irc: PIRC) = - ## Closes connection to an IRC server. - ## - ## **Warning:** This procedure does not send a ``QUIT`` message to the server. - irc.status = SockClosed - if irc.isAsync: - irc.asyncSock.close() - else: - irc.sock.close() - -proc isNumber(s: string): bool = - ## Checks if `s` contains only numbers. - var i = 0 - while s[i] in {'0'..'9'}: inc(i) - result = i == s.len and s.len > 0 - -proc parseMessage(msg: string): TIRCEvent = - result.typ = EvMsg - result.cmd = MUnknown - result.raw = msg - result.timestamp = times.getTime() - var i = 0 - # Process the prefix - if msg[i] == ':': - inc(i) # Skip `:` - var nick = "" - i.inc msg.parseUntil(nick, {'!', ' '}, i) - result.nick = "" - result.serverName = "" - if msg[i] == '!': - result.nick = nick - inc(i) # Skip `!` - i.inc msg.parseUntil(result.user, {'@'}, i) - inc(i) # Skip `@` - i.inc msg.parseUntil(result.host, {' '}, i) - inc(i) # Skip ` ` - else: - result.serverName = nick - inc(i) # Skip ` ` - - # Process command - var cmd = "" - i.inc msg.parseUntil(cmd, {' '}, i) - - if cmd.isNumber: - result.cmd = MNumeric - result.numeric = cmd - else: - case cmd - of "PRIVMSG": result.cmd = MPrivMsg - of "JOIN": result.cmd = MJoin - of "PART": result.cmd = MPart - of "PONG": result.cmd = MPong - of "PING": result.cmd = MPing - of "MODE": result.cmd = MMode - of "TOPIC": result.cmd = MTopic - of "INVITE": result.cmd = MInvite - of "KICK": result.cmd = MKick - of "QUIT": result.cmd = MQuit - of "NICK": result.cmd = MNick - of "NOTICE": result.cmd = MNotice - of "ERROR": result.cmd = MError - else: result.cmd = MUnknown - - # Don't skip space here. It is skipped in the following While loop. - - # Params - result.params = @[] - var param = "" - while msg[i] != '\0' and msg[i] != ':': - inc(i) # Skip ` `. - i.inc msg.parseUntil(param, {' ', ':', '\0'}, i) - if param != "": - result.params.add(param) - param.setlen(0) - - if msg[i] == ':': - inc(i) # Skip `:`. - result.params.add(msg[i..msg.len-1]) - -proc connect*(irc: PIRC) = - ## Connects to an IRC server as specified by ``irc``. - assert(irc.address != "") - assert(irc.port != TPort(0)) - - irc.sock.connect(irc.address, irc.port) - - irc.status = SockConnected - - # Greet the server :) - if irc.serverPass != "": irc.send("PASS " & irc.serverPass, true) - irc.send("NICK " & irc.nick, true) - irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) - -proc reconnect*(irc: PIRC, timeout = 5000) = - ## Reconnects to an IRC server. - ## - ## ``Timeout`` specifies the time to wait in miliseconds between multiple - ## consecutive reconnections. - ## - ## This should be used when an ``EvDisconnected`` event occurs. - let secSinceReconnect = int(epochTime() - irc.lastReconnect) - if secSinceReconnect < timeout: - sleep(timeout - secSinceReconnect) - irc.sock = socket() - if irc.sock == InvalidSocket: osError(osLastError()) - irc.connect() - irc.lastReconnect = epochTime() - -proc irc*(address: string, port: TPort = 6667.TPort, - nick = "NimrodBot", - user = "NimrodBot", - realname = "NimrodBot", serverPass = "", - joinChans: seq[string] = @[], - msgLimit: bool = true): PIRC = - ## Creates a ``TIRC`` object. - new(result) - result.address = address - result.port = port - result.nick = nick - result.user = user - result.realname = realname - result.serverPass = serverPass - result.lastPing = epochTime() - result.lastPong = -1.0 - result.lag = -1.0 - result.channelsToJoin = joinChans - result.msgLimit = msgLimit - result.messageBuffer = @[] - result.status = SockIdle - result.sock = socket() - if result.sock == InvalidSocket: osError(osLastError()) - -proc processLine(irc: PIRC, line: string): TIRCEvent = - if line.len == 0: - irc.close() - result.typ = EvDisconnected - else: - result = parseMessage(line) - # Get the origin - result.origin = result.params[0] - if result.origin == irc.nick and - result.nick != "": result.origin = result.nick - - if result.cmd == MError: - irc.close() - result.typ = EvDisconnected - return - - if result.cmd == MPing: - irc.send("PONG " & result.params[0]) - if result.cmd == MPong: - irc.lag = epochTime() - parseFloat(result.params[result.params.high]) - irc.lastPong = epochTime() - if result.cmd == MNumeric: - if result.numeric == "001": - # Check the nickname. - if irc.nick != result.params[0]: - assert ' ' notin result.params[0] - irc.nick = result.params[0] - for chan in items(irc.channelsToJoin): - irc.join(chan) - if result.cmd == MNick: - if result.nick == irc.nick: - irc.nick = result.params[0] - -proc processOther(irc: PIRC, ev: var TIRCEvent): bool = - result = false - if epochTime() - irc.lastPing >= 20.0: - irc.lastPing = epochTime() - irc.send("PING :" & formatFloat(irc.lastPing), true) - - if epochTime() - irc.lastPong >= 120.0 and irc.lastPong != -1.0: - irc.close() - ev.typ = EvDisconnected # TODO: EvTimeout? - return true - - for i in 0..irc.messageBuffer.len-1: - if epochTime() >= irc.messageBuffer[0][0]: - irc.send(irc.messageBuffer[0].m, true) - irc.messageBuffer.delete(0) - else: - break # messageBuffer is guaranteed to be from the quickest to the - # later-est. - -proc poll*(irc: PIRC, ev: var TIRCEvent, - timeout: int = 500): bool = - ## This function parses a single message from the IRC server and returns - ## a TIRCEvent. - ## - ## This function should be called often as it also handles pinging - ## the server. - ## - ## This function provides a somewhat asynchronous IRC implementation, although - ## it should only be used for simple things for example an IRC bot which does - ## not need to be running many time critical tasks in the background. If you - ## require this, use the asyncio implementation. - - if not (irc.status == SockConnected): - # Do not close the socket here, it is already closed! - ev.typ = EvDisconnected - var line = TaintedString"" - var socks = @[irc.sock] - var ret = socks.select(timeout) - if ret == -1: osError(osLastError()) - if socks.len() != 0 and ret != 0: - irc.sock.readLine(line) - ev = irc.processLine(line.string) - result = true - - if processOther(irc, ev): result = true - -proc getLag*(irc: PIRC): float = - ## Returns the latency between this client and the IRC server in seconds. - ## - ## If latency is unknown, returns -1.0. - return irc.lag - -proc isConnected*(irc: PIRC): bool = - ## Returns whether this IRC client is connected to an IRC server. - return irc.status == SockConnected - -proc getNick*(irc: PIRC): string = - ## Returns the current nickname of the client. - return irc.nick - -# -- Asyncio dispatcher - -proc handleConnect(s: PAsyncSocket, irc: PAsyncIRC) = - # Greet the server :) - if irc.serverPass != "": irc.send("PASS " & irc.serverPass, true) - irc.send("NICK " & irc.nick, true) - irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) - irc.status = SockConnected - - var ev: TIRCEvent - ev.typ = EvConnected - irc.handleEvent(irc, ev) - -proc handleRead(s: PAsyncSocket, irc: PAsyncIRC) = - var line = "".TaintedString - var ret = s.readLine(line) - if ret: - if line == "": - var ev: TIRCEvent - irc.close() - ev.typ = EvDisconnected - irc.handleEvent(irc, ev) - else: - var ev = irc.processLine(line.string) - irc.handleEvent(irc, ev) - -proc handleTask(s: PAsyncSocket, irc: PAsyncIRC) = - var ev: TIRCEvent - if irc.processOther(ev): - irc.handleEvent(irc, ev) - -proc register*(d: PDispatcher, irc: PAsyncIRC) = - ## Registers ``irc`` with dispatcher ``d``. - irc.asyncSock.handleConnect = - proc (s: PAsyncSocket) = - handleConnect(s, irc) - irc.asyncSock.handleRead = - proc (s: PAsyncSocket) = - handleRead(s, irc) - irc.asyncSock.handleTask = - proc (s: PAsyncSocket) = - handleTask(s, irc) - d.register(irc.asyncSock) - irc.myDispatcher = d - -proc connect*(irc: PAsyncIRC) = - ## Equivalent of connect for ``TIRC`` but specifically created for asyncio. - assert(irc.address != "") - assert(irc.port != TPort(0)) - - irc.asyncSock.connect(irc.address, irc.port) - -proc reconnect*(irc: PAsyncIRC, timeout = 5000) = - ## Reconnects to an IRC server. - ## - ## ``Timeout`` specifies the time to wait in miliseconds between multiple - ## consecutive reconnections. - ## - ## This should be used when an ``EvDisconnected`` event occurs. - ## - ## When successfully reconnected an ``EvConnected`` event will occur. - let secSinceReconnect = int(epochTime() - irc.lastReconnect) - if secSinceReconnect < timeout: - sleep(timeout - secSinceReconnect) - irc.asyncSock = AsyncSocket() - irc.myDispatcher.register(irc) - irc.connect() - irc.lastReconnect = epochTime() - -proc asyncIRC*(address: string, port: TPort = 6667.TPort, - nick = "NimrodBot", - user = "NimrodBot", - realname = "NimrodBot", serverPass = "", - joinChans: seq[string] = @[], - msgLimit: bool = true, - ircEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure,gcsafe.} - ): PAsyncIRC = - ## Use this function if you want to use asyncio's dispatcher. - ## - ## **Note:** Do **NOT** use this if you're writing a simple IRC bot which only - ## requires one task to be run, i.e. this should not be used if you want a - ## synchronous IRC client implementation, use ``irc`` for that. - - new(result) - result.isAsync = true - result.address = address - result.port = port - result.nick = nick - result.user = user - result.realname = realname - result.serverPass = serverPass - result.lastPing = epochTime() - result.lastPong = -1.0 - result.lag = -1.0 - result.channelsToJoin = joinChans - result.msgLimit = msgLimit - result.messageBuffer = @[] - result.handleEvent = ircEvent - result.asyncSock = AsyncSocket() - -when isMainModule: - #var m = parseMessage("ERROR :Closing Link: dom96.co.cc (Ping timeout: 252 seconds)") - #echo(repr(m)) - - - - var client = irc("amber.tenthbit.net", nick="TestBot1234", - joinChans = @["#flood"]) - client.connect() - while true: - var event: TIRCEvent - if client.poll(event): - case event.typ - of EvConnected: - discard - of EvDisconnected: - break - of EvMsg: - if event.cmd == MPrivMsg: - var msg = event.params[event.params.high] - if msg == "|test": client.privmsg(event.origin, "hello") - if msg == "|excessFlood": - for i in 0..10: - client.privmsg(event.origin, "TEST" & $i) - - #echo( repr(event) ) - #echo("Lag: ", formatFloat(client.getLag())) - - diff --git a/lib/pure/json.nim b/lib/pure/json.nim index a45900f29..5d51c2d87 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -7,47 +7,71 @@ # distribution, for details about the copyright. # -## This module implements a simple high performance `JSON`:idx: -## parser. JSON (JavaScript Object Notation) is a lightweight -## data-interchange format that is easy for humans to read and write -## (unlike XML). It is easy for machines to parse and generate. -## JSON is based on a subset of the JavaScript Programming Language, -## Standard ECMA-262 3rd Edition - December 1999. +## This module implements a simple high performance `JSON`:idx: parser. `JSON +## (JavaScript Object Notation) <http://www.json.org>`_ is a lightweight +## data-interchange format that is easy for humans to read and write (unlike +## XML). It is easy for machines to parse and generate. JSON is based on a +## subset of the JavaScript Programming Language, `Standard ECMA-262 3rd +## Edition - December 1999 +## <http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf>`_. ## -## Usage example: +## Parsing small values quickly can be done with the convenience `parseJson() +## <#parseJson,string>`_ proc which returns the whole JSON tree. If you are +## parsing very big JSON inputs or want to skip most of the items in them you +## can initialize your own `TJsonParser <#TJsonParser>`_ with the `open() +## <#open>`_ proc and call `next() <#next>`_ in a loop to process the +## individual parsing events. +## +## If you need to create JSON objects from your Nimrod types you can call procs +## like `newJObject() <#newJObject>`_ (or their equivalent `%() +## <#%,openArray[tuple[string,PJsonNode]]>`_ generic constructor). For +## consistency you can provide your own ``%`` operators for custom object +## types: ## ## .. code-block:: nimrod -## let -## small_json = """{"test": 1.3, "key2": true}""" -## jobj = parseJson(small_json) -## assert (jobj.kind == JObject) -## echo($jobj["test"].fnum) -## echo($jobj["key2"].bval) +## type +## Person = object ## Generic person record. +## age: int ## The age of the person. +## name: string ## The name of the person. ## -## Results in: +## proc `%`(p: Person): PJsonNode = +## ## Converts a Person into a PJsonNode. +## result = %[("age", %p.age), ("name", %p.name)] ## -## .. code-block:: nimrod +## proc test() = +## # Tests making some jsons. +## var p: Person +## p.age = 24 +## p.name = "Minah" +## echo(%p) # { "age": 24, "name": "Minah"} +## +## p.age = 33 +## p.name = "Sojin" +## echo(%p) # { "age": 33, "name": "Sojin"} ## -## 1.3000000000000000e+00 -## true +## If you don't need special logic in your Nimrod objects' serialization code +## you can also use the `marshal module <marshal.html>`_ which converts objects +## directly to JSON. import hashes, strutils, lexbase, streams, unicode type - TJsonEventKind* = enum ## enumeration of all events that may occur when parsing - jsonError, ## an error ocurred during parsing - jsonEof, ## end of file reached - jsonString, ## a string literal - jsonInt, ## an integer literal - jsonFloat, ## a float literal - jsonTrue, ## the value ``true`` - jsonFalse, ## the value ``false`` - jsonNull, ## the value ``null`` - jsonObjectStart, ## start of an object: the ``{`` token - jsonObjectEnd, ## end of an object: the ``}`` token - jsonArrayStart, ## start of an array: the ``[`` token - jsonArrayEnd ## start of an array: the ``]`` token + TJsonEventKind* = enum ## Events that may occur when parsing. \ + ## + ## You compare these values agains the result of the `kind() proc <#kind>`_. + jsonError, ## An error ocurred during parsing. + jsonEof, ## End of file reached. + jsonString, ## A string literal. + jsonInt, ## An integer literal. + jsonFloat, ## A float literal. + jsonTrue, ## The value ``true``. + jsonFalse, ## The value ``false``. + jsonNull, ## The value ``null``. + jsonObjectStart, ## Start of an object: the ``{`` token. + jsonObjectEnd, ## End of an object: the ``}`` token. + jsonArrayStart, ## Start of an array: the ``[`` token. + jsonArrayEnd ## Start of an array: the ``]`` token. TTokKind = enum # must be synchronized with TJsonEventKind! tkError, @@ -65,7 +89,7 @@ type tkColon, tkComma - TJsonError* = enum ## enumeration that lists all errors that can occur + TJsonError = enum ## enumeration that lists all errors that can occur errNone, ## no error errInvalidToken, ## invalid token errStringExpected, ## string expected @@ -82,7 +106,9 @@ type stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma, stateExpectObjectComma, stateExpectColon, stateExpectValue - TJsonParser* = object of TBaseLexer ## the parser object. + TJsonParser* = object of TBaseLexer ## The JSON parser object. \ + ## + ## Create a variable of this type and use `open() <#open>`_ on it. a: string tok: TTokKind kind: TJsonEventKind @@ -117,59 +143,129 @@ const ] proc open*(my: var TJsonParser, input: PStream, filename: string) = - ## initializes the parser with an input stream. `Filename` is only used - ## for nice error messages. - lexbase.open(my, input) + ## Initializes the JSON parser with an `input stream <streams.html>`_. + ## + ## The `filename` parameter is not strictly required and is used only for + ## nice error messages. You can pass ``nil`` as long as you never use procs + ## like `errorMsg() <#errorMsg>`_ or `errorMsgExpected() + ## <#errorMsgExpected>`_ but passing a dummy filename like ``<input string>`` + ## is safer and more user friendly. Example: + ## + ## .. code-block:: nimrod + ## import json, streams + ## + ## var + ## s = newStringStream("some valid json") + ## p: TJsonParser + ## p.open(s, "<input string>") + ## + ## Once opened, you can process JSON parsing events with the `next() + ## <#next>`_ proc. my.filename = filename my.state = @[stateStart] my.kind = jsonError my.a = "" -proc close*(my: var TJsonParser) {.inline.} = - ## closes the parser `my` and its associated input stream. +proc close*(my: var TJsonParser) {.inline.} = + ## Closes the parser `my` and its associated input stream. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var + ## s = newStringStream("some valid json") + ## p: TJsonParser + ## p.open(s, "<input string>") + ## finally: p.close + ## # write here parsing of input lexbase.close(my) proc str*(my: TJsonParser): string {.inline.} = - ## returns the character data for the events: ``jsonInt``, ``jsonFloat``, - ## ``jsonString`` + ## Returns the character data for the `events <#TJsonEventKind>`_ + ## ``jsonInt``, ``jsonFloat`` and ``jsonString``. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds when used + ## with other event types. See `next() <#next>`_ for an usage example. assert(my.kind in {jsonInt, jsonFloat, jsonString}) return my.a proc getInt*(my: TJsonParser): BiggestInt {.inline.} = - ## returns the number for the event: ``jsonInt`` + ## Returns the number for the `jsonInt <#TJsonEventKind>`_ event. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds when used + ## with other event types. See `next() <#next>`_ for an usage example. assert(my.kind == jsonInt) return parseBiggestInt(my.a) proc getFloat*(my: TJsonParser): float {.inline.} = - ## returns the number for the event: ``jsonFloat`` + ## Returns the number for the `jsonFloat <#TJsonEventKind>`_ event. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds when used + ## with other event types. See `next() <#next>`_ for an usage example. assert(my.kind == jsonFloat) return parseFloat(my.a) proc kind*(my: TJsonParser): TJsonEventKind {.inline.} = - ## returns the current event type for the JSON parser + ## Returns the current event type for the `JSON parser <#TJsonParser>`_. + ## + ## Call this proc just after `next() <#next>`_ to act on the new event. return my.kind proc getColumn*(my: TJsonParser): int {.inline.} = - ## get the current column the parser has arrived at. + ## Get the current column the parser has arrived at. + ## + ## While this is mostly used by procs like `errorMsg() <#errorMsg>`_ you can + ## use it as well to show user warnings if you are validating JSON values + ## during parsing. See `next() <#next>`_ for the full example: + ## + ## .. code-block:: nimrod + ## case parser.kind + ## ... + ## of jsonString: + ## let inputValue = parser.str + ## if previousValues.contains(inputValue): + ## echo "$1($2, $3) Warning: repeated value '$4'" % [ + ## parser.getFilename, $parser.getLine, $parser.getColumn, + ## inputValue] + ## ... result = getColNumber(my, my.bufpos) proc getLine*(my: TJsonParser): int {.inline.} = - ## get the current line the parser has arrived at. + ## Get the current line the parser has arrived at. + ## + ## While this is mostly used by procs like `errorMsg() <#errorMsg>`_ you can + ## use it as well to indicate user warnings if you are validating JSON values + ## during parsing. See `next() <#next>`_ and `getColumn() <#getColumn>`_ for + ## examples. result = my.lineNumber proc getFilename*(my: TJsonParser): string {.inline.} = - ## get the filename of the file that the parser processes. + ## Get the filename of the file that the parser is processing. + ## + ## This is the value you pass to the `open() <#open>`_ proc. While this is + ## mostly used by procs like `errorMsg() <#errorMsg>`_ you can use it as well + ## to indicate user warnings if you are validating JSON values during + ## parsing. See `next() <#next>`_ and `getColumn() <#getColumn>`_ for + ## examples. result = my.filename proc errorMsg*(my: TJsonParser): string = - ## returns a helpful error message for the event ``jsonError`` + ## Returns a helpful error message for the `jsonError <#TJsonEventKind>`_ + ## event. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds when used + ## with other event types. See `next() <#next>`_ for an usage example. assert(my.kind == jsonError) result = "$1($2, $3) Error: $4" % [ my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] proc errorMsgExpected*(my: TJsonParser, e: string): string = - ## returns an error message "`e` expected" in the same format as the - ## other error messages + ## Returns an error message "`e` expected". + ## + ## The message is in the same format as the other error messages which + ## include the parser filename, line and column values. This is used by + ## `raiseParseErr() <#raiseParseErr>`_ to raise an `EJsonParsingError + ## <#EJsonParsingError>`_. result = "$1($2, $3) Error: $4" % [ my.filename, $getLine(my), $getColumn(my), e & " expected"] @@ -382,7 +478,32 @@ proc getTok(my: var TJsonParser): TTokKind = my.tok = result proc next*(my: var TJsonParser) = - ## retrieves the first/next event. This controls the parser. + ## Retrieves the first/next event for the `JSON parser <#TJsonParser>`_. + ## + ## You are meant to call this method inside an infinite loop. After each + ## call, check the result of the `kind() <#kind>`_ proc to know what has to + ## be done next (eg. break out due to end of file). Here is a basic example + ## which simply echoes all found elements by the parser: + ## + ## .. code-block:: nimrod + ## parser.open(stream, "<input string>") + ## while true: + ## parser.next + ## case parser.kind + ## of jsonError: + ## echo parser.errorMsg + ## break + ## of jsonEof: break + ## of jsonString: echo parser.str + ## of jsonInt: echo parser.getInt + ## of jsonFloat: echo parser.getFloat + ## of jsonTrue: echo "true" + ## of jsonFalse: echo "false" + ## of jsonNull: echo "null" + ## of jsonObjectStart: echo "{" + ## of jsonObjectEnd: echo "}" + ## of jsonArrayStart: echo "[" + ## of jsonArrayEnd: echo "]" var tk = getTok(my) var i = my.state.len-1 # the following code is a state machine. If we had proper coroutines, @@ -502,7 +623,16 @@ proc next*(my: var TJsonParser) = # ------------- higher level interface --------------------------------------- type - TJsonNodeKind* = enum ## possible JSON node types + TJsonNodeKind* = enum ## Possible `JSON node <#TJsonNodeKind>`_ types. \ + ## + ## To build nodes use the helper procs + ## `newJNull() <#newJNull>`_, + ## `newJBool() <#newJBool>`_, + ## `newJInt() <#newJInt>`_, + ## `newJFloat() <#newJFloat>`_, + ## `newJString() <#newJString>`_, + ## `newJObject() <#newJObject>`_ and + ## `newJArray() <#newJArray>`_. JNull, JBool, JInt, @@ -511,8 +641,9 @@ type JObject, JArray - PJsonNode* = ref TJsonNode ## JSON node - TJsonNode* {.final, pure, acyclic.} = object + PJsonNode* = ref TJsonNode ## Reference to a `JSON node <#TJsonNode>`_. + TJsonNode* {.final, pure, acyclic.} = object ## `Object variant \ + ## <manual.html#object-variants>`_ wrapping all possible JSON types. case kind*: TJsonNodeKind of JString: str*: string @@ -529,14 +660,36 @@ type of JArray: elems*: seq[PJsonNode] - EJsonParsingError* = object of EInvalidValue ## is raised for a JSON error + EJsonParsingError* = object of EInvalidValue ## Raised during JSON parsing. \ + ## + ## Example: + ## + ## .. code-block:: nimrod + ## let smallJson = """{"test: 1.3, "key2": true}""" + ## try: + ## discard parseJson(smallJson) + ## # --> Bad JSON! input(1, 18) Error: : expected + ## except EJsonParsingError: + ## echo "Bad JSON! " & getCurrentExceptionMsg() proc raiseParseErr*(p: TJsonParser, msg: string) {.noinline, noreturn.} = - ## raises an `EJsonParsingError` exception. + ## Raises an `EJsonParsingError <#EJsonParsingError>`_ exception. + ## + ## The message for the exception will be built passing the `msg` parameter to + ## the `errorMsgExpected() <#errorMsgExpected>`_ proc. raise newException(EJsonParsingError, errorMsgExpected(p, msg)) proc newJString*(s: string): PJsonNode = - ## Creates a new `JString PJsonNode`. + ## Creates a new `JString PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = newJString("A string") + ## echo node + ## # --> "A string" + ## + ## Or you can use the shorter `%() proc <#%,string>`_. new(result) result.kind = JString result.str = s @@ -547,80 +700,206 @@ proc newJStringMove(s: string): PJsonNode = shallowCopy(result.str, s) proc newJInt*(n: BiggestInt): PJsonNode = - ## Creates a new `JInt PJsonNode`. + ## Creates a new `JInt PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = newJInt(900_100_200_300) + ## echo node + ## # --> 900100200300 + ## + ## Or you can use the shorter `%() proc <#%,BiggestInt>`_. new(result) result.kind = JInt result.num = n proc newJFloat*(n: float): PJsonNode = - ## Creates a new `JFloat PJsonNode`. + ## Creates a new `JFloat PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = newJFloat(3.14) + ## echo node + ## # --> 3.14 + ## + ## Or you can use the shorter `%() proc <#%,float>`_. new(result) result.kind = JFloat result.fnum = n proc newJBool*(b: bool): PJsonNode = - ## Creates a new `JBool PJsonNode`. + ## Creates a new `JBool PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = newJBool(true) + ## echo node + ## # --> true + ## + ## Or you can use the shorter `%() proc <#%,bool>`_. new(result) result.kind = JBool result.bval = b proc newJNull*(): PJsonNode = - ## Creates a new `JNull PJsonNode`. + ## Creates a new `JNull PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = newJNull() + ## echo node + ## # --> null new(result) proc newJObject*(): PJsonNode = - ## Creates a new `JObject PJsonNode` + ## Creates a new `JObject PJsonNode <#TJsonNodeKind>`_. + ## + ## The `PJsonNode <#PJsonNode>`_ will be initialized with an empty ``fields`` + ## sequence to which you can add new elements. Example: + ## + ## .. code-block:: nimrod + ## var node = newJObject() + ## node.add("age", newJInt(24)) + ## node.add("name", newJString("Minah")) + ## echo node + ## # --> { "age": 24, "name": "Minah"} + ## + ## Or you can use the shorter `%() proc + ## <#%,openArray[tuple[string,PJsonNode]]>`_. new(result) result.kind = JObject result.fields = @[] proc newJArray*(): PJsonNode = - ## Creates a new `JArray PJsonNode` + ## Creates a new `JArray PJsonNode <#TJsonNodeKind>`_. + ## + ## The `PJsonNode <#PJsonNode>`_ will be initialized with an empty ``elems`` + ## sequence to which you can add new elements. Example: + ## + ## .. code-block:: nimrod + ## var node = newJArray() + ## node.add(newJString("Mixing types")) + ## node.add(newJInt(42)) + ## node.add(newJString("is madness")) + ## node.add(newJFloat(3.14)) + ## echo node + ## # --> [ "Mixing types", 42, "is madness", 3.14] + ## + ## Or you can use the shorter `%() proc <#%,openArray[PJsonNode]>`_. new(result) result.kind = JArray result.elems = @[] proc `%`*(s: string): PJsonNode = - ## Generic constructor for JSON data. Creates a new `JString PJsonNode`. + ## Creates a new `JString PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = %"A string" + ## echo node + ## # --> "A string" + ## + ## This generic constructor is equivalent to the `newJString() + ## <#newJString>`_ proc. new(result) result.kind = JString result.str = s proc `%`*(n: BiggestInt): PJsonNode = - ## Generic constructor for JSON data. Creates a new `JInt PJsonNode`. + ## Creates a new `JInt PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = %900_100_200_300 + ## echo node + ## # --> 900100200300 + ## + ## This generic constructor is equivalent to the `newJInt() <#newJInt>`_ + ## proc. new(result) result.kind = JInt result.num = n proc `%`*(n: float): PJsonNode = - ## Generic constructor for JSON data. Creates a new `JFloat PJsonNode`. + ## Creates a new `JFloat PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = %3.14 + ## echo node + ## # --> 3.14 + ## + ## This generic constructor is equivalent to the `newJFloat() <#newJFloat>`_ + ## proc. new(result) result.kind = JFloat result.fnum = n proc `%`*(b: bool): PJsonNode = - ## Generic constructor for JSON data. Creates a new `JBool PJsonNode`. + ## Creates a new `JBool PJsonNode <#TJsonNodeKind>`_. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## var node = %true + ## echo node + ## # --> true + ## + ## This generic constructor is equivalent to the `newJBool() <#newJBool>`_ + ## proc. new(result) result.kind = JBool result.bval = b proc `%`*(keyVals: openArray[tuple[key: string, val: PJsonNode]]): PJsonNode = - ## Generic constructor for JSON data. Creates a new `JObject PJsonNode` + ## Creates a new `JObject PJsonNode <#TJsonNodeKind>`_. + ## + ## Unlike the `newJObject() <#newJObject>`_ proc, which returns an object + ## that has to be further manipulated, you can use this generic constructor + ## to create JSON objects with all their fields in one go. Example: + ## + ## .. code-block:: nimrod + ## let node = %[("age", %24), ("name", %"Minah")] + ## echo node + ## # --> { "age": 24, "name": "Minah"} new(result) result.kind = JObject newSeq(result.fields, keyVals.len) for i, p in pairs(keyVals): result.fields[i] = p proc `%`*(elements: openArray[PJsonNode]): PJsonNode = - ## Generic constructor for JSON data. Creates a new `JArray PJsonNode` + ## Creates a new `JArray PJsonNode <#TJsonNodeKind>`_. + ## + ## Unlike the `newJArray() <#newJArray>`_ proc, which returns an object + ## that has to be further manipulated, you can use this generic constructor + ## to create JSON arrays with all their values in one go. Example: + ## + ## .. code-block:: nimrod + ## let node = %[%"Mixing types", %42, + ## %"is madness", %3.14,] + ## echo node + ## # --> [ "Mixing types", 42, "is madness", 3.14] new(result) result.kind = JArray newSeq(result.elems, elements.len) for i, p in pairs(elements): result.elems[i] = p proc `==`* (a,b: PJsonNode): bool = - ## Check two nodes for equality + ## Check two `PJsonNode <#PJsonNode>`_ nodes for equality. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## assert(%1 == %1) + ## assert(%1 != %2) if a.isNil: if b.isNil: return true return false @@ -644,7 +923,21 @@ proc `==`* (a,b: PJsonNode): bool = a.fields == b.fields proc hash* (n:PJsonNode): THash = - ## Compute the hash for a JSON node + ## Computes the hash for a JSON node. + ## + ## The `THash <hashes.html#THash>`_ allows JSON nodes to be used as keys for + ## `sets <sets.html>`_ or `tables <tables.html>`_. Example: + ## + ## .. code-block:: nimrod + ## import json, sets + ## + ## var + ## uniqueValues = initSet[PJsonNode]() + ## values = %[%1, %2, %1, %2, %3] + ## for value in values.elems: + ## discard uniqueValues.containsOrIncl(value) + ## echo uniqueValues + ## # --> {1, 2, 3} case n.kind of JArray: result = hash(n.elems) @@ -662,17 +955,40 @@ proc hash* (n:PJsonNode): THash = result = hash(0) proc len*(n: PJsonNode): int = - ## If `n` is a `JArray`, it returns the number of elements. - ## If `n` is a `JObject`, it returns the number of pairs. - ## Else it returns 0. + ## Returns the number of children items for this `PJsonNode <#PJsonNode>`_. + ## + ## If `n` is a `JArray <#TJsonNodeKind>`_, it will return the number of + ## elements. If `n` is a `JObject <#TJsonNodeKind>`_, it will return the + ## number of key-value pairs. For all other types this proc returns zero. + ## Example: + ## + ## .. code-block:: nimrod + ## let + ## n1 = %[("age", %33), ("name", %"Sojin")] + ## n2 = %[%1, %2, %3, %4, %5, %6, %7] + ## n3 = %"Some odd string we have here" + ## echo n1.len # --> 2 + ## echo n2.len # --> 7 + ## echo n3.len # --> 0 + ## case n.kind of JArray: result = n.elems.len of JObject: result = n.fields.len else: discard proc `[]`*(node: PJsonNode, name: string): PJsonNode = - ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## Gets a named field from a `JObject <#TJsonNodeKind>`_ `PJsonNode + ## <#PJsonNode>`_. + ## + ## Returns the value for `name` or nil if `node` doesn't contain such a + ## field. This proc will `assert <system.html#assert>`_ in debug builds if + ## `name` is ``nil`` or `node` is not a ``JObject``. On release builds it + ## will likely crash. Example: + ## + ## .. code-block:: nimrod + ## let node = %[("age", %40), ("name", %"Britney")] + ## echo node["name"] + ## # --> "Britney" assert(not isNil(node)) assert(node.kind == JObject) for key, item in items(node.fields): @@ -681,35 +997,92 @@ proc `[]`*(node: PJsonNode, name: string): PJsonNode = return nil proc `[]`*(node: PJsonNode, index: int): PJsonNode = - ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## Gets the `index` item from a `JArray <#TJsonNodeKind>`_ `PJsonNode + ## <#PJsonNode>`_. + ## + ## Returns the specified item. Result is undefined if `index` is out of + ## bounds. This proc will `assert <system.html#assert>`_ in debug builds if + ## `node` is ``nil`` or not a ``JArray``. Example: + ## + ## .. code-block:: nimrod + ## let node = %[%"Mixing types", %42, + ## %"is madness", %3.14,] + ## echo node[2] + ## # --> "is madness" assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] proc hasKey*(node: PJsonNode, key: string): bool = - ## Checks if `key` exists in `node`. + ## Returns `true` if `key` exists in a `JObject <#TJsonNodeKind>`_ `PJsonNode + ## <#PJsonNode>`_. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds if `node` is + ## not a ``JObject``. On release builds it will likely crash. Example: + ## + ## .. code-block:: nimrod + ## let node = %[("age", %40), ("name", %"Britney")] + ## echo node.hasKey("email") + ## # --> false assert(node.kind == JObject) for k, item in items(node.fields): if k == key: return true proc existsKey*(node: PJsonNode, key: string): bool {.deprecated.} = node.hasKey(key) - ## Deprecated for `hasKey` + ## Deprecated for `hasKey() <#hasKey>`_. proc add*(father, child: PJsonNode) = - ## Adds `child` to a JArray node `father`. + ## Adds `child` to a `JArray <#TJsonNodeKind>`_ `PJsonNode <#PJsonNode>`_ + ## `father` node. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds if `node` is + ## not a ``JArray``. On release builds it will likely crash. Example: + ## + ## .. code-block:: nimrod + ## var node = %[%"Mixing types", %42] + ## node.add(%"is madness") + ## echo node + ## # --> false assert father.kind == JArray father.elems.add(child) proc add*(obj: PJsonNode, key: string, val: PJsonNode) = - ## Adds ``(key, val)`` pair to the JObject node `obj`. For speed - ## reasons no check for duplicate keys is performed! - ## But ``[]=`` performs the check. + ## Adds ``(key, val)`` pair to a `JObject <#TJsonNodeKind>`_ `PJsonNode + ## <#PJsonNode>`_ `obj` node. + ## + ## For speed reasons no check for duplicate keys is performed! But ``[]=`` + ## performs the check. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds if `node` is + ## not a ``JObject``. On release builds it will likely crash. Example: + ## + ## .. code-block:: nimrod + ## var node = newJObject() + ## node.add("age", newJInt(12)) + ## # This is wrong! But we need speed… + ## node.add("age", newJInt(24)) + ## echo node + ## # --> { "age": 12, "age": 24} assert obj.kind == JObject obj.fields.add((key, val)) proc `[]=`*(obj: PJsonNode, key: string, val: PJsonNode) = - ## Sets a field from a `JObject`. Performs a check for duplicate keys. + ## Sets a field from a `JObject <#TJsonNodeKind>`_ `PJsonNode + ## <#PJsonNode>`_ `obj` node. + ## + ## Unlike the `add() <#add,PJsonNode,string,PJsonNode>`_ proc this will + ## perform a check for duplicate keys and replace existing values. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds if `node` is + ## not a ``JObject``. On release builds it will likely crash. Example: + ## + ## .. code-block:: nimrod + ## var node = newJObject() + ## node["age"] = %12 + ## # The new value replaces the previous one. + ## node["age"] = %24 + ## echo node + ## # --> { "age": 24} assert(obj.kind == JObject) for i in 0..obj.fields.len-1: if obj.fields[i].key == key: @@ -736,6 +1109,18 @@ proc `{}=`*(node: PJsonNode, names: varargs[string], value: PJsonNode) = proc delete*(obj: PJsonNode, key: string) = ## Deletes ``obj[key]`` preserving the order of the other (key, value)-pairs. + ## + ## If `key` doesn't exist in `obj` ``EInvalidIndex`` will be raised. This + ## proc will `assert <system.html#assert>`_ in debug builds if `node` is not + ## a ``JObject``. On release builds it will likely crash. Example: + ## + ## .. code-block:: nimrod + ## var node = %[("age", %37), ("name", %"Chris"), ("male", %false)] + ## echo node + ## # --> { "age": 37, "name": "Chris", "male": false} + ## node.delete("age") + ## echo node + ## # --> { "name": "Chris", "male": false} assert(obj.kind == JObject) for i in 0..obj.fields.len-1: if obj.fields[i].key == key: @@ -744,7 +1129,9 @@ proc delete*(obj: PJsonNode, key: string) = raise newException(EInvalidIndex, "key not in object") proc copy*(p: PJsonNode): PJsonNode = - ## Performs a deep copy of `a`. + ## Performs a deep copy of `p`. + ## + ## Modifications to the copy won't affect the original. case p.kind of JString: result = newJString(p.str) @@ -779,6 +1166,12 @@ proc nl(s: var string, ml: bool) = proc escapeJson*(s: string): string = ## Converts a string `s` to its JSON representation. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## echo """name: "Torbjørn"""".escapeJson + ## # --> "name: \"Torbj\u00F8rn\"" result = newStringOfCap(s.len + s.len shr 3) result.add("\"") for x in runes(s): @@ -850,24 +1243,58 @@ proc toPretty(result: var string, node: PJsonNode, indent = 2, ml = true, result.add("null") proc pretty*(node: PJsonNode, indent = 2): string = - ## Converts `node` to its JSON Representation, with indentation and - ## on multiple lines. + ## Converts `node` to a pretty JSON representation. + ## + ## The representation will have indentation use multiple lines. Example: + ## + ## .. code-block:: nimrod + ## let node = %[("age", %33), ("name", %"Sojin")] + ## echo node + ## # --> { "age": 33, "name": "Sojin"} + ## echo node.pretty + ## # --> { + ## # "age": 33, + ## # "name": "Sojin" + ## # } result = "" toPretty(result, node, indent) proc `$`*(node: PJsonNode): string = - ## Converts `node` to its JSON Representation on one line. + ## Converts `node` to its JSON representation on one line. result = "" toPretty(result, node, 1, false) iterator items*(node: PJsonNode): PJsonNode = - ## Iterator for the items of `node`. `node` has to be a JArray. + ## Iterator for the items of `node`. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds if `node` is + ## not a `JArray <#TJsonNodeKind>`_. On release builds it will likely crash. + ## Example: + ## + ## .. code-block:: nimrod + ## let numbers = %[%1, %2, %3] + ## for n in numbers.items: + ## echo "Number ", n + ## ## --> Number 1 + ## ## Number 2 + ## ## Number 3 assert node.kind == JArray for i in items(node.elems): yield i iterator pairs*(node: PJsonNode): tuple[key: string, val: PJsonNode] = - ## Iterator for the child elements of `node`. `node` has to be a JObject. + ## Iterator for the child elements of `node`. + ## + ## This proc will `assert <system.html#assert>`_ in debug builds if `node` is + ## not a `JObject <#TJsonNodeKind>`_. On release builds it will likely crash. + ## Example: + ## + ## .. code-block:: nimrod + ## var node = %[("age", %37), ("name", %"Chris")] + ## for key, value in node.pairs: + ## echo "Key: ", key, ", value: ", value + ## # --> Key: age, value: 37 + ## # Key: name, value: "Chris" assert node.kind == JObject for key, val in items(node.fields): yield (key, val) @@ -926,8 +1353,12 @@ proc parseJson(p: var TJsonParser): PJsonNode = when not defined(js): proc parseJson*(s: PStream, filename: string): PJsonNode = - ## Parses from a stream `s` into a `PJsonNode`. `filename` is only needed - ## for nice error messages. + ## Generic convenience proc to parse stream `s` into a `PJsonNode`. + ## + ## This wraps around `open() <#open>`_ and `next() <#next>`_ to return the + ## full JSON DOM. Errors will be raised as exceptions, this requires the + ## `filename` parameter to not be ``nil`` to avoid crashes. + assert(not isNil(filename)) var p: TJsonParser p.open(s, filename) discard getTok(p) # read first token @@ -936,10 +1367,28 @@ when not defined(js): proc parseJson*(buffer: string): PJsonNode = ## Parses JSON from `buffer`. + ## + ## Specialized version around `parseJson(PStream, string) + ## <#parseJson,PStream,string>`_. Example: + ## + ## .. code-block:: nimrod + ## let + ## smallJson = """{"test": 1.3, "key2": true}""" + ## jobj = parseJson(smallJson) + ## assert jobj.kind == JObject + ## + ## assert jobj["test"].kind == JFloat + ## echo jobj["test"].fnum # --> 1.3 + ## + ## assert jobj["key2"].kind == JBool + ## echo jobj["key2"].bval # --> true result = parseJson(newStringStream(buffer), "input") proc parseFile*(filename: string): PJsonNode = ## Parses `file` into a `PJsonNode`. + ## + ## Specialized version around `parseJson(PStream, string) + ## <#parseJson,PStream,string>`_. var stream = newFileStream(filename, fmRead) if stream == nil: raise newException(EIO, "cannot read from file: " & filename) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 2f7a696b9..97c7b0e05 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -40,17 +40,6 @@ const ## after the decimal point ## for Nimrod's ``float`` type. -type - TFloatClass* = enum ## describes the class a floating point value belongs to. - ## This is the type that is returned by `classify`. - fcNormal, ## value is an ordinary nonzero floating point value - fcSubnormal, ## value is a subnormal (a very small) floating point value - fcZero, ## value is zero - fcNegZero, ## value is the negative zero - fcNan, ## value is Not-A-Number (NAN) - fcInf, ## value is positive infinity - fcNegInf ## value is negative infinity - proc classify*(x: float): TFloatClass = ## classifies a floating point value. Returns `x`'s class as specified by ## `TFloatClass`. @@ -219,7 +208,7 @@ when not defined(JS): proc randomize(seed: int) = srand(cint(seed)) - when defined(srand48): srand48(seed) + when declared(srand48): srand48(seed) proc random(max: int): int = result = int(rand()) mod max @@ -279,8 +268,13 @@ proc `mod`*(x, y: float): float = result = if y == 0.0: x else: x - y * (x/y).floor proc random*[T](x: TSlice[T]): T = + ## For a slice `a .. b` returns a value in the range `a .. b-1`. result = random(x.b - x.a) + x.a - + +proc random[T](a: openarray[T]): T = + ## returns a random element from the openarray `a`. + result = a[random(a.low..a.len)] + type TRunningStat* {.pure,final.} = object ## an accumulator for statistical data n*: int ## number of pushed data diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 31fefc6c8..ffeb0beff 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -54,7 +54,7 @@ proc mapMem*(m: var TMemFile, mode: TFileMode = fmRead, nil, mappedSize, if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - if readonly: MAP_PRIVATE else: MAP_SHARED, + if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE), m.handle, offset) if result == cast[pointer](MAP_FAILED): osError(osLastError()) @@ -207,7 +207,7 @@ proc open*(filename: string, mode: TFileMode = fmRead, nil, result.size, if readonly: PROT_READ else: PROT_READ or PROT_WRITE, - if readonly: MAP_PRIVATE else: MAP_SHARED, + if readonly: (MAP_PRIVATE or MAP_POPULATE) else: (MAP_SHARED or MAP_POPULATE), result.handle, offset) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ddc2bbe2d..696527467 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -11,292 +11,10 @@ {.deadCodeElim: on.} import rawsockets, os, strutils, unsigned, parseutils, times -export TPort, `$` +export TPort, `$`, `==` const useWinVersion = defined(Windows) or defined(nimdoc) -type - IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address - IPv6, ## IPv6 address - IPv4 ## IPv4 address - - TIpAddress* = object ## stores an arbitrary IP address - case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) - of IpAddressFamily.IPv6: - address_v6*: array[0..15, uint8] ## Contains the IP address in bytes in - ## case of IPv6 - of IpAddressFamily.IPv4: - address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in - ## case of IPv4 - -proc IPv4_any*(): TIpAddress = - ## Returns the IPv4 any address, which can be used to listen on all available - ## network adapters - result = TIpAddress( - family: IpAddressFamily.IPv4, - address_v4: [0'u8, 0, 0, 0]) - -proc IPv4_loopback*(): TIpAddress = - ## Returns the IPv4 loopback address (127.0.0.1) - result = TIpAddress( - family: IpAddressFamily.IPv4, - address_v4: [127'u8, 0, 0, 1]) - -proc IPv4_broadcast*(): TIpAddress = - ## Returns the IPv4 broadcast address (255.255.255.255) - result = TIpAddress( - family: IpAddressFamily.IPv4, - address_v4: [255'u8, 255, 255, 255]) - -proc IPv6_any*(): TIpAddress = - ## Returns the IPv6 any address (::0), which can be used - ## to listen on all available network adapters - result = TIpAddress( - family: IpAddressFamily.IPv6, - address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - -proc IPv6_loopback*(): TIpAddress = - ## Returns the IPv6 loopback address (::1) - result = TIpAddress( - family: IpAddressFamily.IPv6, - address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) - -proc `==`*(lhs, rhs: TIpAddress): bool = - ## Compares two IpAddresses for Equality. Returns two if the addresses are equal - if lhs.family != rhs.family: return false - if lhs.family == IpAddressFamily.IPv4: - for i in low(lhs.address_v4) .. high(lhs.address_v4): - if lhs.address_v4[i] != rhs.address_v4[i]: return false - else: # IPv6 - for i in low(lhs.address_v6) .. high(lhs.address_v6): - if lhs.address_v6[i] != rhs.address_v6[i]: return false - return true - -proc `$`*(address: TIpAddress): string = - ## Converts an TIpAddress into the textual representation - result = "" - case address.family - of IpAddressFamily.IPv4: - for i in 0 .. 3: - if i != 0: - result.add('.') - result.add($address.address_v4[i]) - of IpAddressFamily.IPv6: - var - currentZeroStart = -1 - currentZeroCount = 0 - biggestZeroStart = -1 - biggestZeroCount = 0 - # Look for the largest block of zeros - for i in 0..7: - var isZero = address.address_v6[i*2] == 0 and address.address_v6[i*2+1] == 0 - if isZero: - if currentZeroStart == -1: - currentZeroStart = i - currentZeroCount = 1 - else: - currentZeroCount.inc() - if currentZeroCount > biggestZeroCount: - biggestZeroCount = currentZeroCount - biggestZeroStart = currentZeroStart - else: - currentZeroStart = -1 - - if biggestZeroCount == 8: # Special case ::0 - result.add("::") - else: # Print address - var printedLastGroup = false - for i in 0..7: - var word:uint16 = (cast[uint16](address.address_v6[i*2])) shl 8 - word = word or cast[uint16](address.address_v6[i*2+1]) - - if biggestZeroCount != 0 and # Check if group is in skip group - (i >= biggestZeroStart and i < (biggestZeroStart + biggestZeroCount)): - if i == biggestZeroStart: # skip start - result.add("::") - printedLastGroup = false - else: - if printedLastGroup: - result.add(':') - var - afterLeadingZeros = false - mask = 0xF000'u16 - for j in 0'u16..3'u16: - var val = (mask and word) shr (4'u16*(3'u16-j)) - if val != 0 or afterLeadingZeros: - if val < 0xA: - result.add(chr(uint16(ord('0'))+val)) - else: # val >= 0xA - result.add(chr(uint16(ord('a'))+val-0xA)) - afterLeadingZeros = true - mask = mask shr 4 - printedLastGroup = true - -proc parseIPv4Address(address_str: string): TIpAddress = - ## Parses IPv4 adresses - ## Raises EInvalidValue on errors - var - byteCount = 0 - currentByte:uint16 = 0 - seperatorValid = false - - result.family = IpAddressFamily.IPv4 - - for i in 0 .. high(address_str): - if address_str[i] in strutils.Digits: # Character is a number - currentByte = currentByte * 10 + - cast[uint16](ord(address_str[i]) - ord('0')) - if currentByte > 255'u16: - raise newException(EInvalidValue, - "Invalid IP Address. Value is out of range") - seperatorValid = true - elif address_str[i] == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: - raise newException(EInvalidValue, - "Invalid IP Address. The address consists of too many groups") - result.address_v4[byteCount] = cast[uint8](currentByte) - currentByte = 0 - byteCount.inc - seperatorValid = false - else: - raise newException(EInvalidValue, - "Invalid IP Address. Address contains an invalid character") - - if byteCount != 3 or not seperatorValid: - raise newException(EInvalidValue, "Invalid IP Address") - result.address_v4[byteCount] = cast[uint8](currentByte) - -proc parseIPv6Address(address_str: string): TIpAddress = - ## Parses IPv6 adresses - ## Raises EInvalidValue on errors - result.family = IpAddressFamily.IPv6 - if address_str.len < 2: - raise newException(EInvalidValue, "Invalid IP Address") - - var - groupCount = 0 - currentGroupStart = 0 - currentShort:uint32 = 0 - seperatorValid = true - dualColonGroup = -1 - lastWasColon = false - v4StartPos = -1 - byteCount = 0 - - for i,c in address_str: - if c == ':': - if not seperatorValid: - raise newException(EInvalidValue, - "Invalid IP Address. Address contains an invalid seperator") - if lastWasColon: - if dualColonGroup != -1: - raise newException(EInvalidValue, - "Invalid IP Address. Address contains more than one \"::\" seperator") - dualColonGroup = groupCount - seperatorValid = false - elif i != 0 and i != high(address_str): - if groupCount >= 8: - raise newException(EInvalidValue, - "Invalid IP Address. The address consists of too many groups") - result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) - result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) - currentShort = 0 - groupCount.inc() - if dualColonGroup != -1: seperatorValid = false - elif i == 0: # only valid if address starts with :: - if address_str[1] != ':': - raise newException(EInvalidValue, - "Invalid IP Address. Address may not start with \":\"") - else: # i == high(address_str) - only valid if address ends with :: - if address_str[high(address_str)-1] != ':': - raise newException(EInvalidValue, - "Invalid IP Address. Address may not end with \":\"") - lastWasColon = true - currentGroupStart = i + 1 - elif c == '.': # Switch to parse IPv4 mode - if i < 3 or not seperatorValid or groupCount >= 7: - raise newException(EInvalidValue, "Invalid IP Address") - v4StartPos = currentGroupStart - currentShort = 0 - seperatorValid = false - break - elif c in strutils.HexDigits: - if c in strutils.Digits: # Normal digit - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) - elif c >= 'a' and c <= 'f': # Lower case hex - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 - else: # Upper case hex - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 - if currentShort > 65535'u32: - raise newException(EInvalidValue, - "Invalid IP Address. Value is out of range") - lastWasColon = false - seperatorValid = true - else: - raise newException(EInvalidValue, - "Invalid IP Address. Address contains an invalid character") - - - if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff - if seperatorValid: # Copy remaining data - if groupCount >= 8: - raise newException(EInvalidValue, - "Invalid IP Address. The address consists of too many groups") - result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) - result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) - groupCount.inc() - else: # Must parse IPv4 address - for i,c in address_str[v4StartPos..high(address_str)]: - if c in strutils.Digits: # Character is a number - currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: - raise newException(EInvalidValue, - "Invalid IP Address. Value is out of range") - seperatorValid = true - elif c == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: - raise newException(EInvalidValue, "Invalid IP Address") - result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) - currentShort = 0 - byteCount.inc() - seperatorValid = false - else: # Invalid character - raise newException(EInvalidValue, - "Invalid IP Address. Address contains an invalid character") - - if byteCount != 3 or not seperatorValid: - raise newException(EInvalidValue, "Invalid IP Address") - result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) - groupCount += 2 - - # Shift and fill zeros in case of :: - if groupCount > 8: - raise newException(EInvalidValue, - "Invalid IP Address. The address consists of too many groups") - elif groupCount < 8: # must fill - if dualColonGroup == -1: - raise newException(EInvalidValue, - "Invalid IP Address. The address consists of too few groups") - var toFill = 8 - groupCount # The number of groups to fill - var toShift = groupCount - dualColonGroup # Nr of known groups after :: - for i in 0..2*toShift-1: # shift - result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] - for i in 0..2*toFill-1: # fill with 0s - result.address_v6[dualColonGroup*2+i] = 0 - elif dualColonGroup != -1: - raise newException(EInvalidValue, - "Invalid IP Address. The address consists of too many groups") - -proc parseIpAddress*(address_str: string): TIpAddress = - ## Parses an IP address - ## Raises EInvalidValue on error - if address_str == nil: - raise newException(EInvalidValue, "IP Address string is nil") - if address_str.contains(':'): - return parseIPv6Address(address_str) - else: - return parseIPv4Address(address_str) - when defined(ssl): import openssl @@ -361,7 +79,7 @@ proc isDisconnectionError*(flags: set[TSocketFlags], when useWinVersion: TSocketFlags.SafeDisconn in flags and lastError.int32 in {WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, - WSAEDISCON} + WSAEDISCON, ERROR_NETNAME_DELETED} else: TSocketFlags.SafeDisconn in flags and lastError.int32 in {ECONNRESET, EPIPE, ENETRESET} @@ -569,8 +287,8 @@ proc bindAddr*(socket: PSocket, port = TPort(0), address = "") {. osError(osLastError()) dealloc(aiList) -proc acceptAddr*(server: PSocket, client: var PSocket, address: var string) {. - tags: [FReadIO].} = +proc acceptAddr*(server: PSocket, client: var PSocket, address: var string, + flags = {TSocketFlags.SafeDisconn}) {.tags: [FReadIO].} = ## Blocks until a connection is being made from a client. When a connection ## is made sets ``client`` to the client socket and ``address`` to the address ## of the connecting client. @@ -581,6 +299,11 @@ proc acceptAddr*(server: PSocket, client: var PSocket, address: var string) {. ## ## **Note**: ``client`` must be initialised (with ``new``), this function ## makes no effort to initialise the ``client`` variable. + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. assert(client != nil) var sockAddress: Tsockaddr_in var addrLen = sizeof(sockAddress).TSocklen @@ -589,6 +312,8 @@ proc acceptAddr*(server: PSocket, client: var PSocket, address: var string) {. if sock == osInvalidSocket: let err = osLastError() + if flags.isDisconnectionError(err): + acceptAddr(server, client, address, flags) osError(err) else: client.fd = sock @@ -658,15 +383,20 @@ when false: #defined(ssl): acceptAddrPlain(AcceptNoClient, AcceptSuccess): doHandshake() -proc accept*(server: PSocket, client: var PSocket) {.tags: [FReadIO].} = +proc accept*(server: PSocket, client: var PSocket, + flags = {TSocketFlags.SafeDisconn}) {.tags: [FReadIO].} = ## Equivalent to ``acceptAddr`` but doesn't return the address, only the ## socket. ## ## **Note**: ``client`` must be initialised (with ``new``), this function ## makes no effort to initialise the ``client`` variable. - + ## + ## The ``accept`` call may result in an error if the connecting socket + ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` + ## flag is specified then this error will not be raised and instead + ## accept will be called again. var addrDummy = "" - acceptAddr(server, client, addrDummy) + acceptAddr(server, client, addrDummy, flags) proc close*(socket: PSocket) = ## Closes a socket. @@ -1173,3 +903,285 @@ proc isSSL*(socket: PSocket): bool = return socket.isSSL proc getFD*(socket: PSocket): TSocketHandle = return socket.fd ## Returns the socket's file descriptor + +type + IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address + IPv6, ## IPv6 address + IPv4 ## IPv4 address + + TIpAddress* = object ## stores an arbitrary IP address + case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) + of IpAddressFamily.IPv6: + address_v6*: array[0..15, uint8] ## Contains the IP address in bytes in + ## case of IPv6 + of IpAddressFamily.IPv4: + address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in + ## case of IPv4 + +proc IPv4_any*(): TIpAddress = + ## Returns the IPv4 any address, which can be used to listen on all available + ## network adapters + result = TIpAddress( + family: IpAddressFamily.IPv4, + address_v4: [0'u8, 0, 0, 0]) + +proc IPv4_loopback*(): TIpAddress = + ## Returns the IPv4 loopback address (127.0.0.1) + result = TIpAddress( + family: IpAddressFamily.IPv4, + address_v4: [127'u8, 0, 0, 1]) + +proc IPv4_broadcast*(): TIpAddress = + ## Returns the IPv4 broadcast address (255.255.255.255) + result = TIpAddress( + family: IpAddressFamily.IPv4, + address_v4: [255'u8, 255, 255, 255]) + +proc IPv6_any*(): TIpAddress = + ## Returns the IPv6 any address (::0), which can be used + ## to listen on all available network adapters + result = TIpAddress( + family: IpAddressFamily.IPv6, + address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + +proc IPv6_loopback*(): TIpAddress = + ## Returns the IPv6 loopback address (::1) + result = TIpAddress( + family: IpAddressFamily.IPv6, + address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) + +proc `==`*(lhs, rhs: TIpAddress): bool = + ## Compares two IpAddresses for Equality. Returns two if the addresses are equal + if lhs.family != rhs.family: return false + if lhs.family == IpAddressFamily.IPv4: + for i in low(lhs.address_v4) .. high(lhs.address_v4): + if lhs.address_v4[i] != rhs.address_v4[i]: return false + else: # IPv6 + for i in low(lhs.address_v6) .. high(lhs.address_v6): + if lhs.address_v6[i] != rhs.address_v6[i]: return false + return true + +proc `$`*(address: TIpAddress): string = + ## Converts an TIpAddress into the textual representation + result = "" + case address.family + of IpAddressFamily.IPv4: + for i in 0 .. 3: + if i != 0: + result.add('.') + result.add($address.address_v4[i]) + of IpAddressFamily.IPv6: + var + currentZeroStart = -1 + currentZeroCount = 0 + biggestZeroStart = -1 + biggestZeroCount = 0 + # Look for the largest block of zeros + for i in 0..7: + var isZero = address.address_v6[i*2] == 0 and address.address_v6[i*2+1] == 0 + if isZero: + if currentZeroStart == -1: + currentZeroStart = i + currentZeroCount = 1 + else: + currentZeroCount.inc() + if currentZeroCount > biggestZeroCount: + biggestZeroCount = currentZeroCount + biggestZeroStart = currentZeroStart + else: + currentZeroStart = -1 + + if biggestZeroCount == 8: # Special case ::0 + result.add("::") + else: # Print address + var printedLastGroup = false + for i in 0..7: + var word:uint16 = (cast[uint16](address.address_v6[i*2])) shl 8 + word = word or cast[uint16](address.address_v6[i*2+1]) + + if biggestZeroCount != 0 and # Check if group is in skip group + (i >= biggestZeroStart and i < (biggestZeroStart + biggestZeroCount)): + if i == biggestZeroStart: # skip start + result.add("::") + printedLastGroup = false + else: + if printedLastGroup: + result.add(':') + var + afterLeadingZeros = false + mask = 0xF000'u16 + for j in 0'u16..3'u16: + var val = (mask and word) shr (4'u16*(3'u16-j)) + if val != 0 or afterLeadingZeros: + if val < 0xA: + result.add(chr(uint16(ord('0'))+val)) + else: # val >= 0xA + result.add(chr(uint16(ord('a'))+val-0xA)) + afterLeadingZeros = true + mask = mask shr 4 + printedLastGroup = true + +proc parseIPv4Address(address_str: string): TIpAddress = + ## Parses IPv4 adresses + ## Raises EInvalidValue on errors + var + byteCount = 0 + currentByte:uint16 = 0 + seperatorValid = false + + result.family = IpAddressFamily.IPv4 + + for i in 0 .. high(address_str): + if address_str[i] in strutils.Digits: # Character is a number + currentByte = currentByte * 10 + + cast[uint16](ord(address_str[i]) - ord('0')) + if currentByte > 255'u16: + raise newException(EInvalidValue, + "Invalid IP Address. Value is out of range") + seperatorValid = true + elif address_str[i] == '.': # IPv4 address separator + if not seperatorValid or byteCount >= 3: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + result.address_v4[byteCount] = cast[uint8](currentByte) + currentByte = 0 + byteCount.inc + seperatorValid = false + else: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not seperatorValid: + raise newException(EInvalidValue, "Invalid IP Address") + result.address_v4[byteCount] = cast[uint8](currentByte) + +proc parseIPv6Address(address_str: string): TIpAddress = + ## Parses IPv6 adresses + ## Raises EInvalidValue on errors + result.family = IpAddressFamily.IPv6 + if address_str.len < 2: + raise newException(EInvalidValue, "Invalid IP Address") + + var + groupCount = 0 + currentGroupStart = 0 + currentShort:uint32 = 0 + seperatorValid = true + dualColonGroup = -1 + lastWasColon = false + v4StartPos = -1 + byteCount = 0 + + for i,c in address_str: + if c == ':': + if not seperatorValid: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid seperator") + if lastWasColon: + if dualColonGroup != -1: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains more than one \"::\" seperator") + dualColonGroup = groupCount + seperatorValid = false + elif i != 0 and i != high(address_str): + if groupCount >= 8: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + currentShort = 0 + groupCount.inc() + if dualColonGroup != -1: seperatorValid = false + elif i == 0: # only valid if address starts with :: + if address_str[1] != ':': + raise newException(EInvalidValue, + "Invalid IP Address. Address may not start with \":\"") + else: # i == high(address_str) - only valid if address ends with :: + if address_str[high(address_str)-1] != ':': + raise newException(EInvalidValue, + "Invalid IP Address. Address may not end with \":\"") + lastWasColon = true + currentGroupStart = i + 1 + elif c == '.': # Switch to parse IPv4 mode + if i < 3 or not seperatorValid or groupCount >= 7: + raise newException(EInvalidValue, "Invalid IP Address") + v4StartPos = currentGroupStart + currentShort = 0 + seperatorValid = false + break + elif c in strutils.HexDigits: + if c in strutils.Digits: # Normal digit + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) + elif c >= 'a' and c <= 'f': # Lower case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 + else: # Upper case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 + if currentShort > 65535'u32: + raise newException(EInvalidValue, + "Invalid IP Address. Value is out of range") + lastWasColon = false + seperatorValid = true + else: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid character") + + + if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff + if seperatorValid: # Copy remaining data + if groupCount >= 8: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + groupCount.inc() + else: # Must parse IPv4 address + for i,c in address_str[v4StartPos..high(address_str)]: + if c in strutils.Digits: # Character is a number + currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) + if currentShort > 255'u32: + raise newException(EInvalidValue, + "Invalid IP Address. Value is out of range") + seperatorValid = true + elif c == '.': # IPv4 address separator + if not seperatorValid or byteCount >= 3: + raise newException(EInvalidValue, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + currentShort = 0 + byteCount.inc() + seperatorValid = false + else: # Invalid character + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not seperatorValid: + raise newException(EInvalidValue, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + groupCount += 2 + + # Shift and fill zeros in case of :: + if groupCount > 8: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + elif groupCount < 8: # must fill + if dualColonGroup == -1: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too few groups") + var toFill = 8 - groupCount # The number of groups to fill + var toShift = groupCount - dualColonGroup # Nr of known groups after :: + for i in 0..2*toShift-1: # shift + result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] + for i in 0..2*toFill-1: # fill with 0s + result.address_v6[dualColonGroup*2+i] = 0 + elif dualColonGroup != -1: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + +proc parseIpAddress*(address_str: string): TIpAddress = + ## Parses an IP address + ## Raises EInvalidValue on error + if address_str == nil: + raise newException(EInvalidValue, "IP Address string is nil") + if address_str.contains(':'): + return parseIPv6Address(address_str) + else: + return parseIPv4Address(address_str) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index ab7cd1944..6f94d0656 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -26,7 +26,7 @@ const withThreads = compileOption("threads") tickCountCorrection = 50_000 -when not defined(system.TStackTrace): +when not declared(system.TStackTrace): type TStackTrace = array [0..20, cstring] # We use a simple hash table of bounded size to keep track of the stack traces: @@ -146,7 +146,7 @@ proc `//`(a, b: int): string = result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDefault, 2)) proc writeProfile() {.noconv.} = - when defined(system.TStackTrace): + when declared(system.TStackTrace): system.profilerHook = nil const filename = "profile_results.txt" echo "writing " & filename & "..." @@ -189,16 +189,16 @@ var disabled: int proc disableProfiling*() = - when defined(system.TStackTrace): + when declared(system.TStackTrace): atomicDec disabled system.profilerHook = nil proc enableProfiling*() = - when defined(system.TStackTrace): + when declared(system.TStackTrace): if atomicInc(disabled) >= 0: system.profilerHook = hook -when defined(system.TStackTrace): +when declared(system.TStackTrace): system.profilerHook = hook addQuitProc(writeProfile) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a70bfa7f1..71089494f 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -365,8 +365,9 @@ when defined(windows): template getFilename(f: expr): expr = $f.cFilename proc skipFindData(f: TWIN32_FIND_DATA): bool {.inline.} = + # Note - takes advantage of null delimiter in the cstring const dot = ord('.') - result = f.cFileName[0].int == dot and(f.cFileName[1].int == 0 or + result = f.cFileName[0].int == dot and (f.cFileName[1].int == 0 or f.cFileName[1].int == dot and f.cFileName[2].int == 0) proc existsFile*(filename: string): bool {.rtl, extern: "nos$1", @@ -997,7 +998,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", if c_rename(source, dest) != 0'i32: raise newException(EOS, $strerror(errno)) -when not defined(ENOENT) and not defined(Windows): +when not declared(ENOENT) and not defined(Windows): when NoFakeVars: const ENOENT = cint(2) # 2 on most systems including Solaris else: @@ -1331,10 +1332,10 @@ proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ proc rawCreateDir(dir: string) = when defined(solaris): - if mkdir(dir, 0o711) != 0'i32 and errno != EEXIST and errno != ENOSYS: + if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST and errno != ENOSYS: osError(osLastError()) elif defined(unix): - if mkdir(dir, 0o711) != 0'i32 and errno != EEXIST: + if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST: osError(osLastError()) else: when useWinUnicode: @@ -1457,7 +1458,8 @@ proc parseCmdLine*(c: string): seq[string] {. var a = "" while true: setLen(a, 0) - while c[i] == ' ' or c[i] == '\t': inc(i) + # eat all delimiting whitespace + while c[i] == ' ' or c[i] == '\t' or c [i] == '\l' or c [i] == '\r' : inc(i) when defined(windows): # parse a single argument according to the above rules: if c[i] == '\0': break @@ -1614,11 +1616,11 @@ when defined(nimdoc): ## ## **Availability**: On Posix there is no portable way to get the command ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `defined() <system.html#defined>`_. + ## can test for its availability with `declared() <system.html#declared>`_. ## Example: ## ## .. code-block:: nimrod - ## when defined(paramCount): + ## when declared(paramCount): ## # Use paramCount() here ## else: ## # Do something else! @@ -1637,11 +1639,11 @@ when defined(nimdoc): ## ## **Availability**: On Posix there is no portable way to get the command ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `defined() <system.html#defined>`_. + ## can test for its availability with `declared() <system.html#declared>`_. ## Example: ## ## .. code-block:: nimrod - ## when defined(paramStr): + ## when declared(paramStr): ## # Use paramStr() here ## else: ## # Do something else! @@ -1681,7 +1683,7 @@ elif not defined(createNimRtl): # Docstring in nimdoc block. result = cmdCount-1 -when defined(paramCount) or defined(nimdoc): +when declared(paramCount) or defined(nimdoc): proc commandLineParams*(): seq[TaintedString] = ## Convenience proc which returns the command line parameters. ## @@ -1690,11 +1692,11 @@ when defined(paramCount) or defined(nimdoc): ## ## **Availability**: On Posix there is no portable way to get the command ## line from a DLL and thus the proc isn't defined in this environment. You - ## can test for its availability with `defined() <system.html#defined>`_. + ## can test for its availability with `declared() <system.html#declared>`_. ## Example: ## ## .. code-block:: nimrod - ## when defined(commandLineParams): + ## when declared(commandLineParams): ## # Use commandLineParams() here ## else: ## # Do something else! @@ -1713,7 +1715,7 @@ when defined(linux) or defined(solaris) or defined(bsd) or defined(aix): when not (defined(windows) or defined(macosx)): proc getApplHeuristic(): string = - when defined(paramStr): + when declared(paramStr): result = string(paramStr(0)) # POSIX guaranties that this contains the executable # as it has been executed by the calling process @@ -1860,12 +1862,12 @@ proc expandTilde*(path: string): string = when defined(Windows): type - DeviceId = int32 - FileId = int64 + DeviceId* = int32 + FileId* = int64 else: type - DeviceId = TDev - FileId = TIno + DeviceId* = TDev + FileId* = TIno type FileInfo* = object @@ -1908,6 +1910,7 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: formalInfo.kind = succ(result.kind) + else: template checkAndIncludeMode(rawMode, formalMode: expr) = if (rawInfo.st_mode and rawMode) != 0'i32: @@ -1994,4 +1997,26 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo = osError(osLastError()) rawToFormalFileInfo(rawInfo, result) +proc isHidden*(path: string): bool = + ## Determines whether a given path is hidden or not. Returns false if the + ## file doesn't exist. The given path must be accessible from the current + ## working directory of the program. + ## + ## On Windows, a file is hidden if the file's 'hidden' attribute is set. + ## On Unix-like systems, a file is hidden if it starts with a '.' (period) + ## and is not *just* '.' or '..' ' ." + when defined(Windows): + wrapUnary(attributes, getFileAttributesW, path) + if attributes != -1'i32: + result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 + else: + if fileExists(path): + let + fileName = extractFilename(path) + nameLen = len(fileName) + if nameLen == 2: + result = (fileName[0] == '.') and (fileName[1] != '.') + elif nameLen > 2: + result = (fileName[0] == '.') and (fileName[3] != '.') + {.pop.} diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index c74fa1ceb..3c181bf53 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -643,7 +643,7 @@ elif not defined(useNimRtl): data.workingDir = workingDir - when defined(posix_spawn) and not defined(useFork) and + when declared(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux): pid = startProcessAuxSpawn(data) else: diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 68ae537c7..f43853fe6 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -37,7 +37,7 @@ type ## or the argument, ``value`` is not "" if ## the option was given a value -when defined(os.paramCount): +when declared(os.paramCount): # we cannot provide this for NimRtl creation on Posix, because we can't # access the command line arguments then! @@ -127,7 +127,7 @@ proc cmdLineRest*(p: TOptParser): TaintedString {. ## retrieves the rest of the command line that has not been parsed yet. result = strip(substr(p.cmd, p.pos, len(p.cmd) - 1)).TaintedString -when defined(initOptParser): +when declared(initOptParser): iterator getopt*(): tuple[kind: TCmdLineKind, key, val: TaintedString] = ## This is an convenience iterator for iterating over the command line. diff --git a/lib/pure/parseopt2.nim b/lib/pure/parseopt2.nim index 5e79d8a18..7638171d1 100644 --- a/lib/pure/parseopt2.nim +++ b/lib/pure/parseopt2.nim @@ -119,7 +119,7 @@ proc cmdLineRest*(p: TOptParser): TaintedString {.rtl, extern: "npo$1", deprecat type TGetoptResult* = tuple[kind: TCmdLineKind, key, val: TaintedString] -when defined(paramCount): +when declared(paramCount): iterator getopt*(): TGetoptResult = ## This is an convenience iterator for iterating over the command line. ## This uses the TOptParser object. Example: diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 68b1ab223..efe169c1d 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -870,7 +870,7 @@ template `=~`*(s: string, pattern: TPeg): bool = ## echo("syntax error") ## bind maxSubpatterns - when not definedInScope(matches): + when not declaredInScope(matches): var matches {.inject.}: array[0..MaxSubpatterns-1, string] match(s, pattern, matches) diff --git a/lib/pure/rawsockets.nim b/lib/pure/rawsockets.nim index d96741846..fea09dfa2 100644 --- a/lib/pure/rawsockets.nim +++ b/lib/pure/rawsockets.nim @@ -22,7 +22,7 @@ const useWinVersion = defined(Windows) or defined(nimdoc) when useWinVersion: import winlean export WSAEWOULDBLOCK, WSAECONNRESET, WSAECONNABORTED, WSAENETRESET, - WSAEDISCON + WSAEDISCON, ERROR_NETNAME_DELETED else: import posix export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 4a6c3f530..eb3792bce 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -58,8 +58,8 @@ proc newRope(data: string): PRope = result.data = data var - cache: PRope # the root of the cache tree - N: PRope # dummy rope needed for splay algorithm + cache {.threadvar.}: PRope # the root of the cache tree + N {.threadvar.}: PRope # dummy rope needed for splay algorithm when countCacheMisses: var misses, hits: int diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 6f8924d83..1d17de233 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -803,6 +803,36 @@ proc rfind*(s, sub: string, start: int = -1): int {.noSideEffect.} = if result != -1: return return -1 +proc count*(s: string, sub: string, overlapping: bool = false): int {.noSideEffect, + rtl, extern: "nsuCountString".} = + ## Count the occurences of a substring `sub` in the string `s`. + ## Overlapping occurences of `sub` only count when `overlapping` + ## is set to true. + var i = 0 + while true: + i = s.find(sub, i) + if i < 0: + break + if overlapping: + inc i + else: + i += sub.len + inc result + +proc count*(s: string, sub: char): int {.noSideEffect, + rtl, extern: "nsuCountChar".} = + ## Count the occurences of the character `sub` in the string `s`. + for c in s: + if c == sub: + inc result + +proc count*(s: string, subs: set[char]): int {.noSideEffect, + rtl, extern: "nsuCountCharSet".} = + ## Count the occurences of the group of character `subs` in the string `s`. + for c in s: + if c in subs: + inc result + proc quoteIfContainsWhite*(s: string): string {.deprecated.} = ## Returns ``'"' & s & '"'`` if `s` contains a space and does not ## start with a quote, else returns `s`. @@ -1354,3 +1384,8 @@ when isMainModule: doAssert parseEnum[TMyEnum]("enu_D") == enuD doAssert parseEnum("invalid enum value", enC) == enC + + doAssert count("foofoofoo", "foofoo") == 1 + doAssert count("foofoofoo", "foofoo", overlapping = true) == 2 + doAssert count("foofoofoo", 'f') == 3 + doAssert count("foofoofoobar", {'f','b'}) == 4 diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 92797744a..ed87610d6 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -84,7 +84,8 @@ proc getFormatArg(p: var TFormatParser, a: openArray[string]): int = if result >=% a.len: raiseInvalidFormat("index out of bounds: " & $result) p.i = i -proc scanDollar(p: var TFormatParser, a: openarray[string], s: var string) +proc scanDollar(p: var TFormatParser, a: openarray[string], s: var string) {. + noSideEffect.} proc emitChar(p: var TFormatParser, x: var string, ch: char) {.inline.} = x.add(ch) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 498511899..8b33d2c73 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -63,44 +63,44 @@ elif defined(windows): elif defined(JS): type TTime* {.final, importc.} = object - getDay: proc (): int {.tags: [], raises: [].} - getFullYear: proc (): int {.tags: [], raises: [].} - getHours: proc (): int {.tags: [], raises: [].} - getMilliseconds: proc (): int {.tags: [], raises: [].} - getMinutes: proc (): int {.tags: [], raises: [].} - getMonth: proc (): int {.tags: [], raises: [].} - getSeconds: proc (): int {.tags: [], raises: [].} - getTime: proc (): int {.tags: [], raises: [].} - getTimezoneOffset: proc (): int {.tags: [], raises: [].} - getDate: proc (): int {.tags: [], raises: [].} - getUTCDate: proc (): int {.tags: [], raises: [].} - getUTCFullYear: proc (): int {.tags: [], raises: [].} - getUTCHours: proc (): int {.tags: [], raises: [].} - getUTCMilliseconds: proc (): int {.tags: [], raises: [].} - getUTCMinutes: proc (): int {.tags: [], raises: [].} - getUTCMonth: proc (): int {.tags: [], raises: [].} - getUTCSeconds: proc (): int {.tags: [], raises: [].} - getUTCDay: proc (): int {.tags: [], raises: [].} - getYear: proc (): int {.tags: [], raises: [].} - parse: proc (s: cstring): TTime {.tags: [], raises: [].} - setDate: proc (x: int) {.tags: [], raises: [].} - setFullYear: proc (x: int) {.tags: [], raises: [].} - setHours: proc (x: int) {.tags: [], raises: [].} - setMilliseconds: proc (x: int) {.tags: [], raises: [].} - setMinutes: proc (x: int) {.tags: [], raises: [].} - setMonth: proc (x: int) {.tags: [], raises: [].} - setSeconds: proc (x: int) {.tags: [], raises: [].} - setTime: proc (x: int) {.tags: [], raises: [].} - setUTCDate: proc (x: int) {.tags: [], raises: [].} - setUTCFullYear: proc (x: int) {.tags: [], raises: [].} - setUTCHours: proc (x: int) {.tags: [], raises: [].} - setUTCMilliseconds: proc (x: int) {.tags: [], raises: [].} - setUTCMinutes: proc (x: int) {.tags: [], raises: [].} - setUTCMonth: proc (x: int) {.tags: [], raises: [].} - setUTCSeconds: proc (x: int) {.tags: [], raises: [].} - setYear: proc (x: int) {.tags: [], raises: [].} - toGMTString: proc (): cstring {.tags: [], raises: [].} - toLocaleString: proc (): cstring {.tags: [], raises: [].} + getDay: proc (): int {.tags: [], raises: [], gcsafe.} + getFullYear: proc (): int {.tags: [], raises: [], gcsafe.} + getHours: proc (): int {.tags: [], raises: [], gcsafe.} + getMilliseconds: proc (): int {.tags: [], raises: [], gcsafe.} + getMinutes: proc (): int {.tags: [], raises: [], gcsafe.} + getMonth: proc (): int {.tags: [], raises: [], gcsafe.} + getSeconds: proc (): int {.tags: [], raises: [], gcsafe.} + getTime: proc (): int {.tags: [], raises: [], gcsafe.} + getTimezoneOffset: proc (): int {.tags: [], raises: [], gcsafe.} + getDate: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCDate: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCFullYear: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCHours: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCMilliseconds: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCMinutes: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCMonth: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCSeconds: proc (): int {.tags: [], raises: [], gcsafe.} + getUTCDay: proc (): int {.tags: [], raises: [], gcsafe.} + getYear: proc (): int {.tags: [], raises: [], gcsafe.} + parse: proc (s: cstring): TTime {.tags: [], raises: [], gcsafe.} + setDate: proc (x: int) {.tags: [], raises: [], gcsafe.} + setFullYear: proc (x: int) {.tags: [], raises: [], gcsafe.} + setHours: proc (x: int) {.tags: [], raises: [], gcsafe.} + setMilliseconds: proc (x: int) {.tags: [], raises: [], gcsafe.} + setMinutes: proc (x: int) {.tags: [], raises: [], gcsafe.} + setMonth: proc (x: int) {.tags: [], raises: [], gcsafe.} + setSeconds: proc (x: int) {.tags: [], raises: [], gcsafe.} + setTime: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCDate: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCFullYear: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCHours: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCMilliseconds: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCMinutes: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCMonth: proc (x: int) {.tags: [], raises: [], gcsafe.} + setUTCSeconds: proc (x: int) {.tags: [], raises: [], gcsafe.} + setYear: proc (x: int) {.tags: [], raises: [], gcsafe.} + toGMTString: proc (): cstring {.tags: [], raises: [], gcsafe.} + toLocaleString: proc (): cstring {.tags: [], raises: [], gcsafe.} type TTimeInfo* = object of TObject ## represents a time in different parts @@ -513,7 +513,7 @@ elif defined(JS): result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - proc `$`(timeInfo: TTimeInfo): string = return $(TimeInfoToTIme(timeInfo)) + proc `$`(timeInfo: TTimeInfo): string = return $(timeInfoToTime(timeInfo)) proc `$`(time: TTime): string = return $time.toLocaleString() proc `-` (a, b: TTime): int64 = diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index e7bd363cf..3203ee699 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -11,7 +11,26 @@ ## working with types proc name*(t: typedesc): string {.magic: "TypeTrait".} - ## Returns the name of the given type + ## Returns the name of the given type. + ## + ## Example: + ## + ## .. code-block:: + ## + ## import typetraits + ## + ## proc `$`*[T](some:typedesc[T]): string = name(T) + ## + ## template test(x): stmt = + ## echo "type: ", type(x), ", value: ", x + ## + ## test 42 + ## # --> type: int, value: 42 + ## test "Foo" + ## # --> type: string, value: Foo + ## test(@['A','B']) + ## # --> type: seq[char], value: @[A, B] + proc arity*(t: typedesc): int {.magic: "TypeTrait".} - ## Returns the arity of the given type \ No newline at end of file + ## Returns the arity of the given type diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index f5640a1b4..7cc95f0ad 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -19,7 +19,7 @@ import macros -when defined(stdout): +when declared(stdout): import os when not defined(ECMAScript): @@ -99,7 +99,7 @@ template fail* = when not defined(ECMAScript): if AbortOnError: quit(1) - when defined(TestStatusIMPL): + when declared(TestStatusIMPL): TestStatusIMPL = FAILED else: program_result += 1 @@ -188,7 +188,7 @@ macro expect*(exceptions: varargs[expr], body: stmt): stmt {.immediate.} = result = getAst(expectBody(errorTypes, exp.lineinfo, body)) -when defined(stdout): +when declared(stdout): ## Reading settings var envOutLvl = os.getEnv("NIMTEST_OUTPUT_LVL").string diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 95b48a850..1af7db7d5 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -151,6 +151,8 @@ proc addEscaped*(result: var string, s: string) = of '>': result.add(">") of '&': result.add("&") of '"': result.add(""") + of '\'': result.add("'") + of '/': result.add("/") else: result.add(c) proc escape*(s: string): string = @@ -164,6 +166,8 @@ proc escape*(s: string): string = ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` + ## ``'`` ``'`` + ## ``/`` ``/`` ## ------------ ------------------- result = newStringOfCap(s.len) addEscaped(result, s) |