diff options
Diffstat (limited to 'lib/pure/httpclient.nim')
| -rw-r--r-- | lib/pure/httpclient.nim | 229 |
1 files changed, 175 insertions, 54 deletions
diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 27b3b46be..4404a9426 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -8,79 +8,102 @@ # ## This module implements a simple HTTP client that can be used to retrieve -## webpages/other data. -## -## -## **Note**: This module is not ideal, connection is not kept alive so sites with -## many redirects are expensive. As such in the future this module may change, -## and the current procedures will be deprecated. +## webpages and other data. ## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve -## ``http://google.com`` +## ``http://google.com``: ## ## .. code-block:: Nim +## var client = newHttpClient() ## echo(getContent("http://google.com")) ## +## The same action can also be performed asynchronously, simply use the +## ``AsyncHttpClient``: +## +## .. code-block:: Nim +## var client = newAsyncHttpClient() +## echo(await getContent("http://google.com")) +## +## The functionality implemented by ``HttpClient`` and ``AsyncHttpClient`` +## is the same, so you can use whichever one suits you best in the examples +## shown here. +## +## **Note:** You will need to run asynchronous examples in an async proc +## otherwise you will get an ``Undeclared identifier: 'await'`` error. +## ## Using HTTP POST ## =============== ## ## This example demonstrates the usage of the W3 HTML Validator, it -## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to -## the server. +## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to be +## validated to the server. ## ## .. code-block:: Nim +## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") ## -## echo postContent("http://validator.w3.org/check", multipart=data) +## echo client.postContent("http://validator.w3.org/check", multipart=data) ## -## Asynchronous HTTP requests -## ========================== +## Progress reporting +## ================== ## -## You simply have to create a new instance of the ``AsyncHttpClient`` object. -## You may then use ``await`` on the functions defined for that object. -## Keep in mind that the following code needs to be inside an asynchronous -## procedure. -## -## .. code-block::nim +## You may specify a callback procedure to be called during an HTTP request. +## This callback will be executed every second with information about the +## progress of the HTTP request. ## +## .. code-block:: Nim ## var client = newAsyncHttpClient() -## var resp = await client.request("http://google.com") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") +## client.onProgressChanged = onProgressChanged +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## If you would like to remove the callback simply set it to ``nil``. +## +## .. code-block:: Nim +## client.onProgressChanged = nil ## ## SSL/TLS support ## =============== ## This requires the OpenSSL library, fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the ``https`` schema, for example: -## ``https://github.com/``, you also have to compile with ``ssl`` defined like so: +## ``https://github.com/``. +## +## You will also have to compile with ``ssl`` defined like so: ## ``nim c -d:ssl ...``. ## ## Timeouts ## ======== -## Currently all functions support an optional timeout, by default the timeout is set to -## `-1` which means that the function will never time out. The timeout is +## +## Currently only the synchronous functions support a timeout. +## The timeout is ## measured in milliseconds, once it is set any call on a socket which may -## block will be susceptible to this timeout, however please remember that the +## block will be susceptible to this timeout. +## +## It may be surprising but the ## function as a whole can take longer than the specified timeout, only ## individual internal calls on the socket are affected. In practice this means ## that as long as the server is sending data an exception will not be raised, -## if however data does not reach client within the specified timeout an ETimeout -## exception will then be raised. +## if however data does not reach the client within the specified timeout a +## ``TimeoutError`` exception will be raised. ## ## Proxy ## ===== ## -## A proxy can be specified as a param to any of these procedures, the ``newProxy`` -## constructor should be used for this purpose. However, -## currently only basic authentication is supported. +## A proxy can be specified as a param to any of the procedures defined in +## this module. To do this, use the ``newProxy`` constructor. Unfortunately, +## only basic authentication is supported at the moment. import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, - math, random, httpcore + math, random, httpcore, times import asyncnet, asyncdispatch import nativesockets @@ -379,15 +402,18 @@ proc format(p: MultipartData): tuple[header, body: string] = proc request*(url: string, httpMethod: string, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = + userAgent = defUserAgent, proxy: Proxy = nil): Response + {.deprecated.} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. var r = if proxy == nil: parseUri(url) else: proxy.url var hostUrl = if proxy == nil: r else: parseUri(url) - var headers = substr(httpMethod, len("http")).toUpper() + var headers = httpMethod.toUpper() # TODO: Use generateHeaders further down once it supports proxies. var s = newSocket() @@ -471,15 +497,18 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", if body != "": s.send(body) - result = parseResponse(s, httpMethod != "httpHEAD", timeout) + result = parseResponse(s, httpMethod != "HEAD", timeout) proc request*(url: string, httpMethod = httpGET, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = + userAgent = defUserAgent, proxy: Proxy = nil): Response + {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout, userAgent, proxy) @@ -502,12 +531,14 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = proc get*(url: string, extraHeaders = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response = + proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object ## | This proc also handles redirection ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. result = request(url, httpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) var lastURL = url @@ -521,12 +552,14 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): string = + proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.getContent`` instead. var r = get(url, extraHeaders, maxRedirects, sslContext, timeout, userAgent, proxy) if r.status[0] in {'4','5'}: @@ -539,7 +572,7 @@ proc post*(url: string, extraHeaders = "", body = "", sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, - multipart: MultipartData = nil): Response = + multipart: MultipartData = nil): Response {.deprecated.} = ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object. ## | This proc adds the necessary Content-Length header. ## | This proc also handles redirection. @@ -548,6 +581,8 @@ proc post*(url: string, extraHeaders = "", body = "", ## server takes longer than specified an ETimeout exception will be raised. ## | The optional ``multipart`` parameter can be used to create ## ``multipart/form-data`` POSTs comfortably. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.post`` instead. let (mpHeaders, mpBody) = format(multipart) template withNewLine(x): expr = @@ -577,7 +612,8 @@ proc postContent*(url: string, extraHeaders = "", body = "", sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, - multipart: MultipartData = nil): string = + multipart: MultipartData = nil): string + {.deprecated.} = ## | POSTs ``body`` to ``url`` and returns the response's body as a string ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. @@ -585,6 +621,9 @@ proc postContent*(url: string, extraHeaders = "", body = "", ## server takes longer than specified an ETimeout exception will be raised. ## | The optional ``multipart`` parameter can be used to create ## ``multipart/form-data`` POSTs comfortably. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.postContent`` + ## instead. var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout, userAgent, proxy, multipart) if r.status[0] in {'4','5'}: @@ -610,7 +649,7 @@ proc downloadFile*(url: string, outputFilename: string, proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, body: string, proxy: Proxy): string = # GET - result = substr(httpMethod, len("http")).toUpper() + result = httpMethod.toUpper() result.add ' ' if proxy.isNil: @@ -653,17 +692,30 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string, add(result, "\c\L") type + ProgressChangedProc*[ReturnType] = + proc (total, progress, speed: BiggestInt): + ReturnType {.closure, gcsafe.} + HttpClientBase*[SocketType] = ref object socket: SocketType connected: bool currentURL: Uri ## Where we are currently connected. - headers*: HttpHeaders + headers*: HttpHeaders ## Headers to send in requests. maxRedirects: int userAgent: string timeout: int ## Only used for blocking HttpClient for now. proxy: Proxy + ## ``nil`` or the callback to call when request progress changes. + when SocketType is Socket: + onProgressChanged*: ProgressChangedProc[void] + else: + onProgressChanged*: ProgressChangedProc[Future[void]] when defined(ssl): sslContext: net.SslContext + contentTotal: BiggestInt + contentProgress: BiggestInt + oneSecondProgress: BiggestInt + lastProgressReport: float type HttpClient* = HttpClientBase[Socket] @@ -692,6 +744,7 @@ proc newHttpClient*(userAgent = defUserAgent, result.maxRedirects = maxRedirects result.proxy = proxy result.timeout = timeout + result.onProgressChanged = nil when defined(ssl): result.sslContext = sslContext @@ -721,6 +774,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.maxRedirects = maxRedirects result.proxy = proxy result.timeout = -1 # TODO + result.onProgressChanged = nil when defined(ssl): result.sslContext = sslContext @@ -730,19 +784,37 @@ proc close*(client: HttpClient | AsyncHttpClient) = client.socket.close() client.connected = false -proc recvFull(socket: Socket | AsyncSocket, +proc reportProgress(client: HttpClient | AsyncHttpClient, + progress: BiggestInt) {.multisync.} = + client.contentProgress += progress + client.oneSecondProgress += progress + if epochTime() - client.lastProgressReport >= 1.0: + if not client.onProgressChanged.isNil: + await client.onProgressChanged(client.contentTotal, + client.contentProgress, + client.oneSecondProgress) + client.oneSecondProgress = 0 + client.lastProgressReport = epochTime() + +proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int): Future[string] {.multisync.} = ## Ensures that all the data requested is read and returned. result = "" while true: if size == result.len: break - when socket is Socket: - let data = socket.recv(size - result.len, timeout) + + let remainingSize = size - result.len + let sizeToRecv = min(remainingSize, net.BufferSize) + + when client.socket is Socket: + let data = client.socket.recv(sizeToRecv, timeout) else: - let data = await socket.recv(size - result.len) + let data = await client.socket.recv(sizeToRecv) if data == "": break # We've been disconnected. result.add data + await reportProgress(client, data.len) + proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string] {.multisync.} = result = "" @@ -770,10 +842,10 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string] httpError("Invalid chunk size: " & chunkSizeStr) inc(i) if chunkSize <= 0: - discard await recvFull(client.socket, 2, client.timeout) # Skip \c\L + discard await recvFull(client, 2, client.timeout) # Skip \c\L break - result.add await recvFull(client.socket, chunkSize, client.timeout) - discard await recvFull(client.socket, 2, client.timeout) # Skip \c\L + result.add await recvFull(client, chunkSize, client.timeout) + discard await recvFull(client, 2, client.timeout) # Skip \c\L # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 @@ -781,6 +853,12 @@ proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpVersion: string): Future[string] {.multisync.} = result = "" + # Reset progress from previous requests. + client.contentTotal = 0 + client.contentProgress = 0 + client.oneSecondProgress = 0 + client.lastProgressReport = 0 + if headers.getOrDefault"Transfer-Encoding" == "chunked": result = await parseChunks(client) else: @@ -789,8 +867,9 @@ proc parseBody(client: HttpClient | AsyncHttpClient, var contentLengthHeader = headers.getOrDefault"Content-Length" if contentLengthHeader != "": var length = contentLengthHeader.parseint() + client.contentTotal = length if length > 0: - result = await client.socket.recvFull(length, client.timeout) + result = await client.recvFull(length, client.timeout) if result == "": httpError("Got disconnected while trying to read body.") if result.len != length: @@ -804,7 +883,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: - buf = await client.socket.recvFull(4000, client.timeout) + buf = await client.recvFull(4000, client.timeout) if buf == "": break result.add(buf) @@ -902,8 +981,6 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## Connection will kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. - ## - ## The returned future will complete once the request is completed. let connectionUrl = if client.proxy.isNil: parseUri(url) else: client.proxy.url let requestUrl = parseUri(url) @@ -933,14 +1010,15 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, if not client.headers.hasKey("user-agent") and client.userAgent != "": client.headers["User-Agent"] = client.userAgent - var headers = generateHeaders(requestUrl, $httpMethod, + var headers = generateHeaders(requestUrl, httpMethod, client.headers, body, client.proxy) await client.socket.send(headers) if body != "": await client.socket.send(body) - result = await parseResponse(client, httpMethod notin {HttpHead, HttpConnect}) + result = await parseResponse(client, + httpMethod.toLower() notin ["head", "connect"]) # Restore the clients proxy in case it was overwritten. client.proxy = savedProxy @@ -950,11 +1028,12 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## Connects to the hostname specified by the URL and performs a request ## using the method specified. ## - ## Connection will kept alive. Further requests on the same ``client`` to + ## Connection will be kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. ## - ## The returned future will complete once the request is completed. + ## When a request is made to a different hostname, the current connection will + ## be closed. result = await request(client, url, $httpMethod, body) proc get*(client: HttpClient | AsyncHttpClient, @@ -964,6 +1043,8 @@ proc get*(client: HttpClient | AsyncHttpClient, ## This procedure will follow redirects up to a maximum number of redirects ## specified in ``client.maxRedirects``. result = await client.request(url, HttpGET) + + # Handle redirects. var lastURL = url for i in 1..client.maxRedirects: if result.status.redirection(): @@ -971,6 +1052,21 @@ proc get*(client: HttpClient | AsyncHttpClient, result = await client.request(redirectTo, HttpGET) lastURL = redirectTo +proc getContent*(client: HttpClient | AsyncHttpClient, + url: string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a GET request. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in ``client.maxRedirects``. + ## + ## A ``HttpRequestError`` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + let resp = await get(client, url) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + else: + return resp.body + proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a POST request. @@ -990,3 +1086,28 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", client.headers["Content-Length"] = $len(xb) result = await client.request(url, HttpPOST, xb) + # Handle redirects. + var lastURL = url + for i in 1..client.maxRedirects: + if result.status.redirection(): + let redirectTo = getNewLocation(lastURL, result.headers) + var meth = if result.status != "307": HttpGet else: HttpPost + result = await client.request(redirectTo, meth, xb) + lastURL = redirectTo + +proc postContent*(client: HttpClient | AsyncHttpClient, url: string, + body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a POST request. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in ``client.maxRedirects``. + ## + ## A ``HttpRequestError`` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + let resp = await post(client, url, body, multipart) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + else: + return resp.body |