summary refs log tree commit diff stats
path: root/lib/pure
diff options
context:
space:
mode:
Diffstat (limited to 'lib/pure')
-rw-r--r--lib/pure/asyncdispatch.nim4
-rw-r--r--lib/pure/asyncftpclient.nim13
-rw-r--r--lib/pure/asynchttpserver.nim30
-rw-r--r--lib/pure/asyncnet.nim30
-rw-r--r--lib/pure/httpclient.nim229
-rw-r--r--lib/pure/httpcore.nim8
-rw-r--r--lib/pure/ioselectors.nim36
-rw-r--r--lib/pure/ioselects/ioselectors_kqueue.nim41
-rw-r--r--lib/pure/net.nim22
9 files changed, 327 insertions, 86 deletions
diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index f9085de55..b4e28d9bc 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -1646,7 +1646,7 @@ proc send*(socket: AsyncFD, data: string,
 # -- Await Macro
 include asyncmacro
 
-proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
+proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
   ## Reads a line of data from ``socket``. Returned future will complete once
   ## a full line is read or an error occurs.
   ##
@@ -1664,6 +1664,8 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
   ##
   ## **Note**: This procedure is mostly used for testing. You likely want to
   ## use ``asyncnet.recvLine`` instead.
+  ##
+  ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead.
 
   template addNLIfEmpty(): stmt =
     if result.len == 0:
diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim
index 3087c4536..019a18f55 100644
--- a/lib/pure/asyncftpclient.nim
+++ b/lib/pure/asyncftpclient.nim
@@ -354,6 +354,16 @@ proc store*(ftp: AsyncFtpClient, file, dest: string,
 
   await doUpload(ftp, destFile, onProgressChanged)
 
+proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} =
+  ## Rename a file or directory on the remote FTP Server from current name
+  ## ``name_from`` to new name ``name_to``
+  assertReply(await ftp.send("RNFR " & name_from), "350")
+  assertReply(await ftp.send("RNTO " & name_to), "250")
+
+proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} =
+  ## Delete a file ``filename`` on the remote FTP server
+  assertReply(await ftp.send("DELE " & filename), "250")
+
 proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} =
   ## Delete a directory ``dir`` on the remote FTP server
   assertReply(await ftp.send("RMD " & dir), "250")
@@ -377,6 +387,9 @@ when not defined(testing) and isMainModule:
     echo await ftp.listDirs()
     await ftp.store("payload.jpg", "payload.jpg")
     await ftp.retrFile("payload.jpg", "payload2.jpg")
+    await ftp.rename("payload.jpg", "payload_renamed.jpg")
+    await ftp.store("payload.jpg", "payload_remove.jpg")
+    await ftp.removeFile("payload_remove.jpg")
     await ftp.createDir("deleteme")
     await ftp.removeDir("deleteme")
     echo("Finished")
diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim
index 6a7326e83..a658097f9 100644
--- a/lib/pure/asynchttpserver.nim
+++ b/lib/pure/asynchttpserver.nim
@@ -9,6 +9,12 @@
 
 ## This module implements a high performance asynchronous HTTP server.
 ##
+## This HTTP server has not been designed to be used in production, but
+## for testing applications locally. Because of this, when deploying your
+## application you should use a reverse proxy (for example nginx) instead of
+## allowing users to connect directly to this server.
+##
+##
 ## Examples
 ## --------
 ##
@@ -38,7 +44,7 @@ export httpcore except parseHeader
 type
   Request* = object
     client*: AsyncSocket # TODO: Separate this into a Response object?
-    reqMethod*: string
+    reqMethod*: HttpMethod
     headers*: HttpHeaders
     protocol*: tuple[orig: string, major, minor: int]
     url*: Uri
@@ -127,7 +133,14 @@ proc processClient(client: AsyncSocket, address: string,
     var i = 0
     for linePart in lineFut.mget.split(' '):
       case i
-      of 0: request.reqMethod.shallowCopy(linePart.normalize)
+      of 0:
+        try:
+          # TODO: this is likely slow.
+          request.reqMethod = parseEnum[HttpMethod]("http" & linePart)
+        except ValueError:
+          asyncCheck request.respond(Http400, "Invalid request method. Got: " &
+                                     linePart)
+          continue
       of 1: parseUri(linePart, request.url)
       of 2:
         try:
@@ -159,7 +172,7 @@ proc processClient(client: AsyncSocket, address: string,
         request.client.close()
         return
 
-    if request.reqMethod == "post":
+    if request.reqMethod == HttpPost:
       # Check for Expect header
       if request.headers.hasKey("Expect"):
         if "100-continue" in request.headers["Expect"]:
@@ -178,17 +191,12 @@ proc processClient(client: AsyncSocket, address: string,
       else:
         request.body = await client.recv(contentLength)
         assert request.body.len == contentLength
-    elif request.reqMethod == "post":
+    elif request.reqMethod == HttpPost:
       await request.respond(Http400, "Bad Request. No Content-Length.")
       continue
 
-    case request.reqMethod
-    of "get", "post", "head", "put", "delete", "trace", "options",
-       "connect", "patch":
-      await callback(request)
-    else:
-      await request.respond(Http400, "Invalid request method. Got: " &
-        request.reqMethod)
+    # Call the user's callback.
+    await callback(request)
 
     if "upgrade" in request.headers.getOrDefault("connection"):
       return
diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim
index 334f95baa..14ebde4a2 100644
--- a/lib/pure/asyncnet.nim
+++ b/lib/pure/asyncnet.nim
@@ -388,7 +388,7 @@ proc accept*(socket: AsyncSocket,
   return retFut
 
 proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
-    flags = {SocketFlag.SafeDisconn}) {.async.} =
+    flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {.async.} =
   ## Reads a line of data from ``socket`` into ``resString``.
   ##
   ## If a full line is read ``\r\L`` is not
@@ -401,13 +401,14 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
   ## is read) then line will be set to ``""``.
   ## The partial line **will be lost**.
   ##
+  ## The ``maxLength`` parameter determines the maximum amount of characters
+  ## that can be read before a ``ValueError`` is raised. This prevents Denial
+  ## of Service (DOS) attacks.
+  ##
   ## **Warning**: The ``Peek`` flag is not yet implemented.
   ##
   ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the
   ## protocol uses ``\r\L`` to delimit a new line.
-  ##
-  ## **Warning**: ``recvLineInto`` currently uses a raw pointer to a string for
-  ## performance reasons. This will likely change soon to use FutureVars.
   assert SocketFlag.Peek notin flags ## TODO:
   assert(not resString.mget.isNil(),
          "String inside resString future needs to be initialised")
@@ -454,6 +455,12 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
         else:
           resString.mget.add socket.buffer[socket.currPos]
       socket.currPos.inc()
+
+      # Verify that this isn't a DOS attack: #3847.
+      if resString.mget.len > maxLength:
+        let msg = "recvLine received more than the specified `maxLength` " &
+                  "allowed."
+        raise newException(ValueError, msg)
   else:
     var c = ""
     while true:
@@ -475,10 +482,17 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
         resString.complete()
         return
       resString.mget.add c
+
+      # Verify that this isn't a DOS attack: #3847.
+      if resString.mget.len > maxLength:
+        let msg = "recvLine received more than the specified `maxLength` " &
+                  "allowed."
+        raise newException(ValueError, msg)
   resString.complete()
 
 proc recvLine*(socket: AsyncSocket,
-    flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} =
+    flags = {SocketFlag.SafeDisconn},
+    maxLength = MaxLineLength): Future[string] {.async.} =
   ## Reads a line of data from ``socket``. Returned future will complete once
   ## a full line is read or an error occurs.
   ##
@@ -492,6 +506,10 @@ proc recvLine*(socket: AsyncSocket,
   ## is read) then line will be set to ``""``.
   ## The partial line **will be lost**.
   ##
+  ## The ``maxLength`` parameter determines the maximum amount of characters
+  ## that can be read before a ``ValueError`` is raised. This prevents Denial
+  ## of Service (DOS) attacks.
+  ##
   ## **Warning**: The ``Peek`` flag is not yet implemented.
   ##
   ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol
@@ -501,7 +519,7 @@ proc recvLine*(socket: AsyncSocket,
   # TODO: Optimise this
   var resString = newFutureVar[string]("asyncnet.recvLine")
   resString.mget() = ""
-  await socket.recvLineInto(resString, flags)
+  await socket.recvLineInto(resString, flags, maxLength)
   result = resString.mget()
 
 proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} =
diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim
index 27b3b46be..4404a9426 100644
--- a/lib/pure/httpclient.nim
+++ b/lib/pure/httpclient.nim
@@ -8,79 +8,102 @@
 #
 
 ## This module implements a simple HTTP client that can be used to retrieve
-## webpages/other data.
-##
-##
-## **Note**: This module is not ideal, connection is not kept alive so sites with
-## many redirects are expensive. As such in the future this module may change,
-## and the current procedures will be deprecated.
+## webpages and other data.
 ##
 ## Retrieving a website
 ## ====================
 ##
 ## This example uses HTTP GET to retrieve
-## ``http://google.com``
+## ``http://google.com``:
 ##
 ## .. code-block:: Nim
+##   var client = newHttpClient()
 ##   echo(getContent("http://google.com"))
 ##
+## The same action can also be performed asynchronously, simply use the
+## ``AsyncHttpClient``:
+##
+## .. code-block:: Nim
+##   var client = newAsyncHttpClient()
+##   echo(await getContent("http://google.com"))
+##
+## The functionality implemented by ``HttpClient`` and ``AsyncHttpClient``
+## is the same, so you can use whichever one suits you best in the examples
+## shown here.
+##
+## **Note:** You will need to run asynchronous examples in an async proc
+## otherwise you will get an ``Undeclared identifier: 'await'`` error.
+##
 ## Using HTTP POST
 ## ===============
 ##
 ## This example demonstrates the usage of the W3 HTML Validator, it
-## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to
-## the server.
+## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to be
+## validated to the server.
 ##
 ## .. code-block:: Nim
+##   var client = newHttpClient()
 ##   var data = newMultipartData()
 ##   data["output"] = "soap12"
 ##   data["uploaded_file"] = ("test.html", "text/html",
 ##     "<html><head></head><body><p>test</p></body></html>")
 ##
-##   echo postContent("http://validator.w3.org/check", multipart=data)
+##   echo client.postContent("http://validator.w3.org/check", multipart=data)
 ##
-## Asynchronous HTTP requests
-## ==========================
+## Progress reporting
+## ==================
 ##
-## You simply have to create a new instance of the ``AsyncHttpClient`` object.
-## You may then use ``await`` on the functions defined for that object.
-## Keep in mind that the following code needs to be inside an asynchronous
-## procedure.
-##
-## .. code-block::nim
+## You may specify a callback procedure to be called during an HTTP request.
+## This callback will be executed every second with information about the
+## progress of the HTTP request.
 ##
+## .. code-block:: Nim
 ##    var client = newAsyncHttpClient()
-##    var resp = await client.request("http://google.com")
+##    proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} =
+##      echo("Downloaded ", progress, " of ", total)
+##      echo("Current rate: ", speed div 1000, "kb/s")
+##    client.onProgressChanged = onProgressChanged
+##    discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test")
+##
+## If you would like to remove the callback simply set it to ``nil``.
+##
+## .. code-block:: Nim
+##   client.onProgressChanged = nil
 ##
 ## SSL/TLS support
 ## ===============
 ## This requires the OpenSSL library, fortunately it's widely used and installed
 ## on many operating systems. httpclient will use SSL automatically if you give
 ## any of the functions a url with the ``https`` schema, for example:
-## ``https://github.com/``, you also have to compile with ``ssl`` defined like so:
+## ``https://github.com/``.
+##
+## You will also have to compile with ``ssl`` defined like so:
 ## ``nim c -d:ssl ...``.
 ##
 ## Timeouts
 ## ========
-## Currently all functions support an optional timeout, by default the timeout is set to
-## `-1` which means that the function will never time out. The timeout is
+##
+## Currently only the synchronous functions support a timeout.
+## The timeout is
 ## measured in milliseconds, once it is set any call on a socket which may
-## block will be susceptible to this timeout, however please remember that the
+## block will be susceptible to this timeout.
+##
+## It may be surprising but the
 ## function as a whole can take longer than the specified timeout, only
 ## individual internal calls on the socket are affected. In practice this means
 ## that as long as the server is sending data an exception will not be raised,
-## if however data does not reach client within the specified timeout an ETimeout
-## exception will then be raised.
+## if however data does not reach the client within the specified timeout a
+## ``TimeoutError`` exception will be raised.
 ##
 ## Proxy
 ## =====
 ##
-## A proxy can be specified as a param to any of these procedures, the ``newProxy``
-## constructor should be used for this purpose. However,
-## currently only basic authentication is supported.
+## A proxy can be specified as a param to any of the procedures defined in
+## this module. To do this, use the ``newProxy`` constructor. Unfortunately,
+## only basic authentication is supported at the moment.
 
 import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes,
-  math, random, httpcore
+  math, random, httpcore, times
 import asyncnet, asyncdispatch
 import nativesockets
 
@@ -379,15 +402,18 @@ proc format(p: MultipartData): tuple[header, body: string] =
 
 proc request*(url: string, httpMethod: string, extraHeaders = "",
               body = "", sslContext = defaultSSLContext, timeout = -1,
-              userAgent = defUserAgent, proxy: Proxy = nil): Response =
+              userAgent = defUserAgent, proxy: Proxy = nil): Response
+              {.deprecated.} =
   ## | Requests ``url`` with the custom method string specified by the
   ## | ``httpMethod`` parameter.
   ## | Extra headers can be specified and must be separated by ``\c\L``
   ## | An optional timeout can be specified in milliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
+  ##
+  ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead.
   var r = if proxy == nil: parseUri(url) else: proxy.url
   var hostUrl = if proxy == nil: r else: parseUri(url)
-  var headers = substr(httpMethod, len("http")).toUpper()
+  var headers = httpMethod.toUpper()
   # TODO: Use generateHeaders further down once it supports proxies.
 
   var s = newSocket()
@@ -471,15 +497,18 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
   if body != "":
     s.send(body)
 
-  result = parseResponse(s, httpMethod != "httpHEAD", timeout)
+  result = parseResponse(s, httpMethod != "HEAD", timeout)
 
 proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
               body = "", sslContext = defaultSSLContext, timeout = -1,
-              userAgent = defUserAgent, proxy: Proxy = nil): Response =
+              userAgent = defUserAgent, proxy: Proxy = nil): Response
+              {.deprecated.} =
   ## | Requests ``url`` with the specified ``httpMethod``.
   ## | Extra headers can be specified and must be separated by ``\c\L``
   ## | An optional timeout can be specified in milliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
+  ##
+  ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead.
   result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout,
                    userAgent, proxy)
 
@@ -502,12 +531,14 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string =
 proc get*(url: string, extraHeaders = "", maxRedirects = 5,
           sslContext: SSLContext = defaultSSLContext,
           timeout = -1, userAgent = defUserAgent,
-          proxy: Proxy = nil): Response =
+          proxy: Proxy = nil): Response {.deprecated.} =
   ## | GETs the ``url`` and returns a ``Response`` object
   ## | This proc also handles redirection
   ## | Extra headers can be specified and must be separated by ``\c\L``.
   ## | An optional timeout can be specified in milliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
+  ##
+  ## ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead.
   result = request(url, httpGET, extraHeaders, "", sslContext, timeout,
                    userAgent, proxy)
   var lastURL = url
@@ -521,12 +552,14 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5,
 proc getContent*(url: string, extraHeaders = "", maxRedirects = 5,
                  sslContext: SSLContext = defaultSSLContext,
                  timeout = -1, userAgent = defUserAgent,
-                 proxy: Proxy = nil): string =
+                 proxy: Proxy = nil): string {.deprecated.} =
   ## | GETs the body and returns it as a string.
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
   ## | Extra headers can be specified and must be separated by ``\c\L``.
   ## | An optional timeout can be specified in milliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
+  ##
+  ## **Deprecated since version 0.15.0**: use ``HttpClient.getContent`` instead.
   var r = get(url, extraHeaders, maxRedirects, sslContext, timeout, userAgent,
               proxy)
   if r.status[0] in {'4','5'}:
@@ -539,7 +572,7 @@ proc post*(url: string, extraHeaders = "", body = "",
            sslContext: SSLContext = defaultSSLContext,
            timeout = -1, userAgent = defUserAgent,
            proxy: Proxy = nil,
-           multipart: MultipartData = nil): Response =
+           multipart: MultipartData = nil): Response {.deprecated.} =
   ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object.
   ## | This proc adds the necessary Content-Length header.
   ## | This proc also handles redirection.
@@ -548,6 +581,8 @@ proc post*(url: string, extraHeaders = "", body = "",
   ## server takes longer than specified an ETimeout exception will be raised.
   ## | The optional ``multipart`` parameter can be used to create
   ## ``multipart/form-data`` POSTs comfortably.
+  ##
+  ## **Deprecated since version 0.15.0**: use ``HttpClient.post`` instead.
   let (mpHeaders, mpBody) = format(multipart)
 
   template withNewLine(x): expr =
@@ -577,7 +612,8 @@ proc postContent*(url: string, extraHeaders = "", body = "",
                   sslContext: SSLContext = defaultSSLContext,
                   timeout = -1, userAgent = defUserAgent,
                   proxy: Proxy = nil,
-                  multipart: MultipartData = nil): string =
+                  multipart: MultipartData = nil): string
+                  {.deprecated.} =
   ## | POSTs ``body`` to ``url`` and returns the response's body as a string
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
   ## | Extra headers can be specified and must be separated by ``\c\L``.
@@ -585,6 +621,9 @@ proc postContent*(url: string, extraHeaders = "", body = "",
   ## server takes longer than specified an ETimeout exception will be raised.
   ## | The optional ``multipart`` parameter can be used to create
   ## ``multipart/form-data`` POSTs comfortably.
+  ##
+  ## **Deprecated since version 0.15.0**: use ``HttpClient.postContent``
+  ## instead.
   var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout,
                userAgent, proxy, multipart)
   if r.status[0] in {'4','5'}:
@@ -610,7 +649,7 @@ proc downloadFile*(url: string, outputFilename: string,
 proc generateHeaders(requestUrl: Uri, httpMethod: string,
                      headers: HttpHeaders, body: string, proxy: Proxy): string =
   # GET
-  result = substr(httpMethod, len("http")).toUpper()
+  result = httpMethod.toUpper()
   result.add ' '
 
   if proxy.isNil:
@@ -653,17 +692,30 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string,
   add(result, "\c\L")
 
 type
+  ProgressChangedProc*[ReturnType] =
+    proc (total, progress, speed: BiggestInt):
+      ReturnType {.closure, gcsafe.}
+
   HttpClientBase*[SocketType] = ref object
     socket: SocketType
     connected: bool
     currentURL: Uri ## Where we are currently connected.
-    headers*: HttpHeaders
+    headers*: HttpHeaders ## Headers to send in requests.
     maxRedirects: int
     userAgent: string
     timeout: int ## Only used for blocking HttpClient for now.
     proxy: Proxy
+    ## ``nil`` or the callback to call when request progress changes.
+    when SocketType is Socket:
+      onProgressChanged*: ProgressChangedProc[void]
+    else:
+      onProgressChanged*: ProgressChangedProc[Future[void]]
     when defined(ssl):
       sslContext: net.SslContext
+    contentTotal: BiggestInt
+    contentProgress: BiggestInt
+    oneSecondProgress: BiggestInt
+    lastProgressReport: float
 
 type
   HttpClient* = HttpClientBase[Socket]
@@ -692,6 +744,7 @@ proc newHttpClient*(userAgent = defUserAgent,
   result.maxRedirects = maxRedirects
   result.proxy = proxy
   result.timeout = timeout
+  result.onProgressChanged = nil
   when defined(ssl):
     result.sslContext = sslContext
 
@@ -721,6 +774,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent,
   result.maxRedirects = maxRedirects
   result.proxy = proxy
   result.timeout = -1 # TODO
+  result.onProgressChanged = nil
   when defined(ssl):
     result.sslContext = sslContext
 
@@ -730,19 +784,37 @@ proc close*(client: HttpClient | AsyncHttpClient) =
     client.socket.close()
     client.connected = false
 
-proc recvFull(socket: Socket | AsyncSocket,
+proc reportProgress(client: HttpClient | AsyncHttpClient,
+                    progress: BiggestInt) {.multisync.} =
+  client.contentProgress += progress
+  client.oneSecondProgress += progress
+  if epochTime() - client.lastProgressReport >= 1.0:
+    if not client.onProgressChanged.isNil:
+      await client.onProgressChanged(client.contentTotal,
+                                     client.contentProgress,
+                                     client.oneSecondProgress)
+      client.oneSecondProgress = 0
+      client.lastProgressReport = epochTime()
+
+proc recvFull(client: HttpClient | AsyncHttpClient,
               size: int, timeout: int): Future[string] {.multisync.} =
   ## Ensures that all the data requested is read and returned.
   result = ""
   while true:
     if size == result.len: break
-    when socket is Socket:
-      let data = socket.recv(size - result.len, timeout)
+
+    let remainingSize = size - result.len
+    let sizeToRecv = min(remainingSize, net.BufferSize)
+
+    when client.socket is Socket:
+      let data = client.socket.recv(sizeToRecv, timeout)
     else:
-      let data = await socket.recv(size - result.len)
+      let data = await client.socket.recv(sizeToRecv)
     if data == "": break # We've been disconnected.
     result.add data
 
+    await reportProgress(client, data.len)
+
 proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string]
                  {.multisync.} =
   result = ""
@@ -770,10 +842,10 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string]
         httpError("Invalid chunk size: " & chunkSizeStr)
       inc(i)
     if chunkSize <= 0:
-      discard await recvFull(client.socket, 2, client.timeout) # Skip \c\L
+      discard await recvFull(client, 2, client.timeout) # Skip \c\L
       break
-    result.add await recvFull(client.socket, chunkSize, client.timeout)
-    discard await recvFull(client.socket, 2, client.timeout) # Skip \c\L
+    result.add await recvFull(client, chunkSize, client.timeout)
+    discard await recvFull(client, 2, client.timeout) # Skip \c\L
     # Trailer headers will only be sent if the request specifies that we want
     # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
 
@@ -781,6 +853,12 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
                headers: HttpHeaders,
                httpVersion: string): Future[string] {.multisync.} =
   result = ""
+  # Reset progress from previous requests.
+  client.contentTotal = 0
+  client.contentProgress = 0
+  client.oneSecondProgress = 0
+  client.lastProgressReport = 0
+
   if headers.getOrDefault"Transfer-Encoding" == "chunked":
     result = await parseChunks(client)
   else:
@@ -789,8 +867,9 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
     var contentLengthHeader = headers.getOrDefault"Content-Length"
     if contentLengthHeader != "":
       var length = contentLengthHeader.parseint()
+      client.contentTotal = length
       if length > 0:
-        result = await client.socket.recvFull(length, client.timeout)
+        result = await client.recvFull(length, client.timeout)
         if result == "":
           httpError("Got disconnected while trying to read body.")
         if result.len != length:
@@ -804,7 +883,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
       if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0":
         var buf = ""
         while true:
-          buf = await client.socket.recvFull(4000, client.timeout)
+          buf = await client.recvFull(4000, client.timeout)
           if buf == "": break
           result.add(buf)
 
@@ -902,8 +981,6 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
   ## Connection will kept alive. Further requests on the same ``client`` to
   ## the same hostname will not require a new connection to be made. The
   ## connection can be closed by using the ``close`` procedure.
-  ##
-  ## The returned future will complete once the request is completed.
   let connectionUrl =
     if client.proxy.isNil: parseUri(url) else: client.proxy.url
   let requestUrl = parseUri(url)
@@ -933,14 +1010,15 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
   if not client.headers.hasKey("user-agent") and client.userAgent != "":
     client.headers["User-Agent"] = client.userAgent
 
-  var headers = generateHeaders(requestUrl, $httpMethod,
+  var headers = generateHeaders(requestUrl, httpMethod,
                                 client.headers, body, client.proxy)
 
   await client.socket.send(headers)
   if body != "":
     await client.socket.send(body)
 
-  result = await parseResponse(client, httpMethod notin {HttpHead, HttpConnect})
+  result = await parseResponse(client,
+                               httpMethod.toLower() notin ["head", "connect"])
 
   # Restore the clients proxy in case it was overwritten.
   client.proxy = savedProxy
@@ -950,11 +1028,12 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
   ## Connects to the hostname specified by the URL and performs a request
   ## using the method specified.
   ##
-  ## Connection will kept alive. Further requests on the same ``client`` to
+  ## Connection will be kept alive. Further requests on the same ``client`` to
   ## the same hostname will not require a new connection to be made. The
   ## connection can be closed by using the ``close`` procedure.
   ##
-  ## The returned future will complete once the request is completed.
+  ## When a request is made to a different hostname, the current connection will
+  ## be closed.
   result = await request(client, url, $httpMethod, body)
 
 proc get*(client: HttpClient | AsyncHttpClient,
@@ -964,6 +1043,8 @@ proc get*(client: HttpClient | AsyncHttpClient,
   ## This procedure will follow redirects up to a maximum number of redirects
   ## specified in ``client.maxRedirects``.
   result = await client.request(url, HttpGET)
+
+  # Handle redirects.
   var lastURL = url
   for i in 1..client.maxRedirects:
     if result.status.redirection():
@@ -971,6 +1052,21 @@ proc get*(client: HttpClient | AsyncHttpClient,
       result = await client.request(redirectTo, HttpGET)
       lastURL = redirectTo
 
+proc getContent*(client: HttpClient | AsyncHttpClient,
+                 url: string): Future[string] {.multisync.} =
+  ## Connects to the hostname specified by the URL and performs a GET request.
+  ##
+  ## This procedure will follow redirects up to a maximum number of redirects
+  ## specified in ``client.maxRedirects``.
+  ##
+  ## A ``HttpRequestError`` will be raised if the server responds with a
+  ## client error (status code 4xx) or a server error (status code 5xx).
+  let resp = await get(client, url)
+  if resp.code.is4xx or resp.code.is5xx:
+    raise newException(HttpRequestError, resp.status)
+  else:
+    return resp.body
+
 proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "",
            multipart: MultipartData = nil): Future[Response] {.multisync.} =
   ## Connects to the hostname specified by the URL and performs a POST request.
@@ -990,3 +1086,28 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "",
   client.headers["Content-Length"] = $len(xb)
 
   result = await client.request(url, HttpPOST, xb)
+  # Handle redirects.
+  var lastURL = url
+  for i in 1..client.maxRedirects:
+    if result.status.redirection():
+      let redirectTo = getNewLocation(lastURL, result.headers)
+      var meth = if result.status != "307": HttpGet else: HttpPost
+      result = await client.request(redirectTo, meth, xb)
+      lastURL = redirectTo
+
+proc postContent*(client: HttpClient | AsyncHttpClient, url: string,
+                  body = "",
+                  multipart: MultipartData = nil): Future[string]
+                  {.multisync.} =
+  ## Connects to the hostname specified by the URL and performs a POST request.
+  ##
+  ## This procedure will follow redirects up to a maximum number of redirects
+  ## specified in ``client.maxRedirects``.
+  ##
+  ## A ``HttpRequestError`` will be raised if the server responds with a
+  ## client error (status code 4xx) or a server error (status code 5xx).
+  let resp = await post(client, url, body, multipart)
+  if resp.code.is4xx or resp.code.is5xx:
+    raise newException(HttpRequestError, resp.status)
+  else:
+    return resp.body
diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim
index ba69c5669..8147f1c50 100644
--- a/lib/pure/httpcore.nim
+++ b/lib/pure/httpcore.nim
@@ -41,8 +41,9 @@ type
                       ## changing in the request.
     HttpOptions,      ## Returns the HTTP methods that the server supports
                       ## for specified address.
-    HttpConnect       ## Converts the request connection to a transparent
+    HttpConnect,      ## Converts the request connection to a transparent
                       ## TCP/IP tunnel, usually used for proxies.
+    HttpPatch         ## Applies partial modifications to a resource.
 
 {.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost,
                httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace,
@@ -224,7 +225,7 @@ proc `$`*(code: HttpCode): string =
   ## For example:
   ##
   ##   .. code-block:: nim
-  ##       doAssert(Http404.status == "404 Not Found")
+  ##       doAssert($Http404 == "404 Not Found")
   case code.int
   of 100: "100 Continue"
   of 101: "101 Switching Protocols"
@@ -296,6 +297,9 @@ proc is5xx*(code: HttpCode): bool =
   ## Determines whether ``code`` is a 5xx HTTP status code.
   return code.int in {500 .. 599}
 
+proc `$`*(httpMethod: HttpMethod): string =
+  return (system.`$`(httpMethod))[4 .. ^1].toUpper()
+
 when isMainModule:
   var test = newHttpHeaders()
   test["Connection"] = @["Upgrade", "Close"]
diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim
index a5d5d2c01..adb3497ac 100644
--- a/lib/pure/ioselectors.nim
+++ b/lib/pure/ioselectors.nim
@@ -44,14 +44,21 @@ when defined(nimdoc):
 
     Event* {.pure.} = enum
       ## An enum which hold event types
-      Read,    ## Descriptor is available for read
-      Write,   ## Descriptor is available for write
-      Timer,   ## Timer descriptor is completed
-      Signal,  ## Signal is raised
-      Process, ## Process is finished
-      Vnode,   ## Currently not supported
-      User,    ## User event is raised
-      Error    ## Error happens while waiting, for descriptor
+      Read,        ## Descriptor is available for read
+      Write,       ## Descriptor is available for write
+      Timer,       ## Timer descriptor is completed
+      Signal,      ## Signal is raised
+      Process,     ## Process is finished
+      Vnode,       ## BSD specific file change happens
+      User,        ## User event is raised
+      Error,       ## Error happens while waiting, for descriptor
+      VnodeWrite,  ## NOTE_WRITE (BSD specific, write to file occured)
+      VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occured)
+      VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended)
+      VnodeAttrib, ## NOTE_ATTRIB (BSD specific, file attributes changed)
+      VnodeLink,   ## NOTE_LINK (BSD specific, file link count changed)
+      VnodeRename, ## NOTE_RENAME (BSD specific, file renamed)
+      VnodeRevoke  ## NOTE_REVOKE (BSD specific, file revoke occured)
 
     ReadyKey*[T] = object
       ## An object which holds result for descriptor
@@ -107,6 +114,15 @@ when defined(nimdoc):
     ## ``data`` application-defined data, which to be passed, when
     ## ``ev`` happens.
 
+  proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event],
+                         data: T) =
+    ## Registers selector BSD/MacOSX specific vnode events for file
+    ## descriptor ``fd`` and events ``events``.
+    ## ``data`` application-defined data, which to be passed, when
+    ## vnode event happens.
+    ##
+    ## This function is supported only by BSD and MacOSX.
+
   proc newSelectEvent*(): SelectEvent =
     ## Creates new event ``SelectEvent``.
 
@@ -194,7 +210,9 @@ else:
       deallocShared(cast[pointer](sa))
   type
     Event* {.pure.} = enum
-      Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot
+      Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot,
+      VnodeWrite, VnodeDelete, VnodeExtend, VnodeAttrib, VnodeLink,
+      VnodeRename, VnodeRevoke
 
     ReadyKey*[T] = object
       fd* : int
diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim
index 3e86f19aa..cdaeeae26 100644
--- a/lib/pure/ioselects/ioselectors_kqueue.nim
+++ b/lib/pure/ioselects/ioselectors_kqueue.nim
@@ -262,6 +262,30 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) =
   modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil)
   inc(s.count)
 
+template processVnodeEvents(events: set[Event]): cuint =
+  var rfflags = 0.cuint
+  if events == {Event.VnodeWrite, Event.VnodeDelete, Event.VnodeExtend,
+                Event.VnodeAttrib, Event.VnodeLink, Event.VnodeRename,
+                Event.VnodeRevoke}:
+    rfflags = NOTE_DELETE or NOTE_WRITE or NOTE_EXTEND or NOTE_ATTRIB or
+              NOTE_LINK or NOTE_RENAME or NOTE_REVOKE
+  else:
+    if Event.VnodeDelete in events: rfflags = rfflags or NOTE_DELETE
+    if Event.VnodeWrite in events: rfflags = rfflags or NOTE_WRITE
+    if Event.VnodeExtend in events: rfflags = rfflags or NOTE_EXTEND
+    if Event.VnodeAttrib in events: rfflags = rfflags or NOTE_ATTRIB
+    if Event.VnodeLink in events: rfflags = rfflags or NOTE_LINK
+    if Event.VnodeRename in events: rfflags = rfflags or NOTE_RENAME
+    if Event.VnodeRevoke in events: rfflags = rfflags or NOTE_REVOKE
+  rfflags
+
+proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], data: T) =
+  let fdi = fd.int
+  setKey(s, fdi, fdi, {Event.Vnode} + events, 0, data)
+  var fflags = processVnodeEvents(events)
+  modifyKQueue(s, fdi.uint, EVFILT_VNODE, EV_ADD or EV_CLEAR, fflags, 0, nil)
+  inc(s.count)
+
 proc unregister*[T](s: Selector[T], fd: int|SocketHandle) =
   let fdi = int(fd)
   s.checkFd(fdi)
@@ -295,6 +319,9 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) =
       discard posix.close(cint(pkey.key.fd))
       modifyKQueue(s, fdi.uint, EVFILT_PROC, EV_DELETE, 0, 0, nil)
       dec(s.count)
+    elif Event.Vnode in pkey.events:
+      modifyKQueue(s, fdi.uint, EVFILT_VNODE, EV_DELETE, 0, 0, nil)
+      dec(s.count)
     elif Event.User in pkey.events:
       modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil)
       dec(s.count)
@@ -392,6 +419,20 @@ proc selectInto*[T](s: Selector[T], timeout: int,
         of EVFILT_VNODE:
           pkey = addr(s.fds[kevent.ident.int])
           pkey.key.events = {Event.Vnode}
+          if (kevent.fflags and NOTE_DELETE) != 0:
+            pkey.key.events.incl(Event.VnodeDelete)
+          if (kevent.fflags and NOTE_WRITE) != 0:
+            pkey.key.events.incl(Event.VnodeWrite)
+          if (kevent.fflags and NOTE_EXTEND) != 0:
+            pkey.key.events.incl(Event.VnodeExtend)
+          if (kevent.fflags and NOTE_ATTRIB) != 0:
+            pkey.key.events.incl(Event.VnodeAttrib)
+          if (kevent.fflags and NOTE_LINK) != 0:
+            pkey.key.events.incl(Event.VnodeLink)
+          if (kevent.fflags and NOTE_RENAME) != 0:
+            pkey.key.events.incl(Event.VnodeRename)
+          if (kevent.fflags and NOTE_REVOKE) != 0:
+            pkey.key.events.incl(Event.VnodeRevoke)
         of EVFILT_SIGNAL:
           pkey = addr(s.fds[cast[int](kevent.udata)])
           pkey.key.events = {Event.Signal}
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index a70f60a8e..d4f239c49 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -112,6 +112,7 @@ else:
 
 const
   BufferSize*: int = 4000 ## size of a buffered socket's buffer
+  MaxLineLength* = 1_000_000
 
 type
   SocketImpl* = object ## socket type
@@ -1006,7 +1007,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} =
     result = recv(socket.fd, addr(c), 1, MSG_PEEK)
 
 proc readLine*(socket: Socket, line: var TaintedString, timeout = -1,
-               flags = {SocketFlag.SafeDisconn}) {.
+               flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {.
   tags: [ReadIOEffect, TimeEffect].} =
   ## Reads a line of data from ``socket``.
   ##
@@ -1021,6 +1022,10 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1,
   ## A timeout can be specified in milliseconds, if data is not received within
   ## the specified time an ETimeout exception will be raised.
   ##
+  ## The ``maxLength`` parameter determines the maximum amount of characters
+  ## that can be read before a ``ValueError`` is raised. This prevents Denial
+  ## of Service (DOS) attacks.
+  ##
   ## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
 
   template addNLIfEmpty() =
@@ -1054,8 +1059,15 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1,
       return
     add(line.string, c)
 
+    # Verify that this isn't a DOS attack: #3847.
+    if line.string.len > maxLength:
+      let msg = "recvLine received more than the specified `maxLength` " &
+                "allowed."
+      raise newException(ValueError, msg)
+
 proc recvLine*(socket: Socket, timeout = -1,
-               flags = {SocketFlag.SafeDisconn}): TaintedString =
+               flags = {SocketFlag.SafeDisconn},
+               maxLength = MaxLineLength): TaintedString =
   ## Reads a line of data from ``socket``.
   ##
   ## If a full line is read ``\r\L`` is not
@@ -1069,9 +1081,13 @@ proc recvLine*(socket: Socket, timeout = -1,
   ## A timeout can be specified in milliseconds, if data is not received within
   ## the specified time an ETimeout exception will be raised.
   ##
+  ## The ``maxLength`` parameter determines the maximum amount of characters
+  ## that can be read before a ``ValueError`` is raised. This prevents Denial
+  ## of Service (DOS) attacks.
+  ##
   ## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
   result = ""
-  readLine(socket, result, timeout, flags)
+  readLine(socket, result, timeout, flags, maxLength)
 
 proc recvFrom*(socket: Socket, data: var string, length: int,
                address: var string, port: var Port, flags = 0'i32): int {.
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-02-10 04:41:37 +0000