diff options
Diffstat (limited to 'lib/pure/httpclient.nim')
| -rw-r--r--[-rwxr-xr-x] | lib/pure/httpclient.nim | 1596 |
1 files changed, 1309 insertions, 287 deletions
diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index c02a7a6e7..08ea99627 100755..100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -1,338 +1,1360 @@ # # -# Nimrod's Runtime Library -# (c) Copyright 2010 Dominik Picheta, Andreas Rumpf +# Nim's Runtime Library +# (c) Copyright 2019 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements a simple HTTP client that can be used to retrieve -## webpages/other data. +## webpages and other data. +## +## .. warning:: Validate untrusted inputs: URI parsers and getters are not detecting malicious URIs. ## ## Retrieving a website ## ==================== -## +## ## This example uses HTTP GET to retrieve -## ``http://google.com`` -## -## .. code-block:: nimrod -## echo(getContent("http://google.com")) -## +## `http://google.com`: +## +## ```Nim +## import std/httpclient +## var client = newHttpClient() +## try: +## echo client.getContent("http://google.com") +## finally: +## client.close() +## ``` +## +## The same action can also be performed asynchronously, simply use the +## `AsyncHttpClient`: +## +## ```Nim +## import std/[asyncdispatch, httpclient] +## +## proc asyncProc(): Future[string] {.async.} = +## var client = newAsyncHttpClient() +## try: +## return await client.getContent("http://google.com") +## finally: +## client.close() +## +## echo waitFor asyncProc() +## ``` +## +## The functionality implemented by `HttpClient` and `AsyncHttpClient` +## is the same, so you can use whichever one suits you best in the examples +## shown here. +## +## **Note:** You need to run asynchronous examples in an async proc +## otherwise you will get an `Undeclared identifier: 'await'` error. +## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. +## ## Using HTTP POST ## =============== -## -## This example demonstrates the usage of the W3 HTML Validator, it -## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to -## the server. -## -## .. code-block:: nimrod -## var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L" -## var body: string = "--xyz\c\L" -## # soap 1.2 output -## body.add("Content-Disposition: form-data; name=\"output\"\c\L") -## body.add("\c\Lsoap12\c\L") -## -## # html -## body.add("--xyz\c\L") -## body.add("Content-Disposition: form-data; name=\"uploaded_file\";" & -## " filename=\"test.html\"\c\L") -## body.add("Content-Type: text/html\c\L") -## body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L") -## body.add("--xyz--") -## -## echo(postContent("http://validator.w3.org/check", headers, body)) - -import sockets, strutils, parseurl, parseutils, strtabs +## +## This example demonstrates the usage of the W3 HTML Validator, it +## uses `multipart/form-data` as the `Content-Type` to send the HTML to be +## validated to the server. +## +## ```Nim +## var client = newHttpClient() +## var data = newMultipartData() +## data["output"] = "soap12" +## data["uploaded_file"] = ("test.html", "text/html", +## "<html><head></head><body><p>test</p></body></html>") +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` +## +## To stream files from disk when performing the request, use `addFiles`. +## +## **Note:** This will allocate a new `Mimetypes` database every time you call +## it, you can pass your own via the `mimeDb` parameter to avoid this. +## +## ```Nim +## let mimes = newMimetypes() +## var client = newHttpClient() +## var data = newMultipartData() +## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes) +## try: +## echo client.postContent("http://validator.w3.org/check", multipart=data) +## finally: +## client.close() +## ``` +## +## You can also make post requests with custom headers. +## This example sets `Content-Type` to `application/json` +## and uses a json object for the body +## +## ```Nim +## import std/[httpclient, json] +## +## let client = newHttpClient() +## client.headers = newHttpHeaders({ "Content-Type": "application/json" }) +## let body = %*{ +## "data": "some text" +## } +## try: +## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body) +## echo response.status +## finally: +## client.close() +## ``` +## +## Progress reporting +## ================== +## +## You may specify a callback procedure to be called during an HTTP request. +## This callback will be executed every second with information about the +## progress of the HTTP request. +## +## ```Nim +## import std/[asyncdispatch, httpclient] +## +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") +## +## proc asyncProc() {.async.} = +## var client = newAsyncHttpClient() +## client.onProgressChanged = onProgressChanged +## try: +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## finally: +## client.close() +## +## waitFor asyncProc() +## ``` +## +## If you would like to remove the callback simply set it to `nil`. +## +## ```Nim +## client.onProgressChanged = nil +## ``` +## +## .. warning:: The `total` reported by httpclient may be 0 in some cases. +## +## +## SSL/TLS support +## =============== +## This requires the OpenSSL library. Fortunately it's widely used and installed +## on many operating systems. httpclient will use SSL automatically if you give +## any of the functions a url with the `https` schema, for example: +## `https://github.com/`. +## +## You will also have to compile with `ssl` defined like so: +## `nim c -d:ssl ...`. +## +## Certificate validation is performed by default. +## +## A set of directories and files from the `ssl_certs <ssl_certs.html>`_ +## module are scanned to locate CA certificates. +## +## Example of setting SSL verification parameters in a new client: +## +## ```Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## ``` +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## +## See `newContext <net.html#newContext.string,string,string,string>`_ to tweak or disable certificate validation. +## +## Timeouts +## ======== +## +## Currently only the synchronous functions support a timeout. +## The timeout is +## measured in milliseconds, once it is set any call on a socket which may +## block will be susceptible to this timeout. +## +## It may be surprising but the +## function as a whole can take longer than the specified timeout, only +## individual internal calls on the socket are affected. In practice this means +## that as long as the server is sending data an exception will not be raised, +## if however data does not reach the client within the specified timeout a +## `TimeoutError` exception will be raised. +## +## Here is how to set a timeout when creating an `HttpClient` instance: +## +## ```Nim +## import std/httpclient +## +## let client = newHttpClient(timeout = 42) +## ``` +## +## Proxy +## ===== +## +## A proxy can be specified as a param to any of the procedures defined in +## this module. To do this, use the `newProxy` constructor. Unfortunately, +## only basic authentication is supported at the moment. +## +## Some examples on how to configure a Proxy for `HttpClient`: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Use proxies with basic authentication: +## +## ```Nim +## import std/httpclient +## +## let myProxy = newProxy("http://myproxy.network", auth="user:password") +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Get Proxy URL from environment variables: +## +## ```Nim +## import std/httpclient +## +## var url = "" +## try: +## if existsEnv("http_proxy"): +## url = getEnv("http_proxy") +## elif existsEnv("https_proxy"): +## url = getEnv("https_proxy") +## except ValueError: +## echo "Unable to parse proxy from environment variables." +## +## let myProxy = newProxy(url = url) +## let client = newHttpClient(proxy = myProxy) +## ``` +## +## Redirects +## ========= +## +## The maximum redirects can be set with the `maxRedirects` of `int` type, +## it specifies the maximum amount of redirects to follow, +## it defaults to `5`, you can set it to `0` to disable redirects. +## +## Here you can see an example about how to set the `maxRedirects` of `HttpClient`: +## +## ```Nim +## import std/httpclient +## +## let client = newHttpClient(maxRedirects = 0) +## ``` +## + +import std/private/since + +import std/[ + net, strutils, uri, parseutils, base64, os, mimetypes, + math, random, httpcore, times, tables, streams, monotimes, + asyncnet, asyncdispatch, asyncfile, nativesockets, +] + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +export httpcore except parseHeader # TODO: The `except` doesn't work type - TResponse* = tuple[ - version: string, - status: string, - headers: PStringTable, - body: string] + Response* = ref object + version*: string + status*: string + headers*: HttpHeaders + body: string + bodyStream*: Stream + + AsyncResponse* = ref object + version*: string + status*: string + headers*: HttpHeaders + body: string + bodyStream*: FutureStream[string] + +proc code*(response: Response | AsyncResponse): HttpCode + {.raises: [ValueError, OverflowDefect].} = + ## Retrieves the specified response's `HttpCode`. + ## + ## Raises a `ValueError` if the response's `status` does not have a + ## corresponding `HttpCode`. + return response.status[0 .. 2].parseInt.HttpCode + +proc contentType*(response: Response | AsyncResponse): string {.inline.} = + ## Retrieves the specified response's content type. + ## + ## This is effectively the value of the "Content-Type" header. + response.headers.getOrDefault("content-type") - EInvalidProtocol* = object of EBase ## exception that is raised when server - ## does not conform to the implemented - ## protocol +proc contentLength*(response: Response | AsyncResponse): int = + ## Retrieves the specified response's content length. + ## + ## This is effectively the value of the "Content-Length" header. + ## + ## A `ValueError` exception will be raised if the value is not an integer. + ## If the Content-Length header is not set in the response, ContentLength is set to the value -1. + var contentLengthHeader = response.headers.getOrDefault("Content-Length", HttpHeaderValues(@["-1"])) + result = contentLengthHeader.parseInt() - EHttpRequestErr* = object of EBase ## Thrown in the ``getContent`` proc - ## and ``postContent`` proc, - ## when the server returns an error +proc lastModified*(response: Response | AsyncResponse): DateTime = + ## Retrieves the specified response's last modified time. + ## + ## This is effectively the value of the "Last-Modified" header. + ## + ## Raises a `ValueError` if the parsing fails or the value is not a correctly + ## formatted time. + var lastModifiedHeader = response.headers.getOrDefault("last-modified") + result = parse(lastModifiedHeader, "ddd, dd MMM yyyy HH:mm:ss 'GMT'", utc()) + +proc body*(response: Response): string = + ## Retrieves the specified response's body. + ## + ## The response's body stream is read synchronously. + if response.body.len == 0: + response.body = response.bodyStream.readAll() + return response.body + +proc body*(response: AsyncResponse): Future[string] {.async.} = + ## Reads the response's body and caches it. The read is performed only + ## once. + if response.body.len == 0: + response.body = await readAll(response.bodyStream) + return response.body + +type + Proxy* = ref object + url*: Uri + auth*: string + + MultipartEntry = object + name, content: string + case isFile: bool + of true: + filename, contentType: string + fileSize: int64 + isStream: bool + else: discard + + MultipartEntries* = openArray[tuple[name, content: string]] + MultipartData* = ref object + content: seq[MultipartEntry] + + ProtocolError* = object of IOError ## exception that is raised when server + ## does not conform to the implemented + ## protocol + + HttpRequestError* = object of IOError ## Thrown in the `getContent` proc + ## and `postContent` proc, + ## when the server returns an error + +const defUserAgent* = "Nim-httpclient/" & NimVersion proc httpError(msg: string) = - var e: ref EInvalidProtocol + var e: ref ProtocolError new(e) e.msg = msg raise e - + proc fileError(msg: string) = - var e: ref EIO + var e: ref IOError new(e) e.msg = msg raise e -proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} = - result = d[i] - while result == '\0': - d = s.recv() - i = 0 - result = d[i] - -proc parseChunks(d: var string, start: int, s: TSocket): string = - # get chunks: - var i = start - result = "" +when not defined(ssl): + type SslContext = ref object +var defaultSslContext {.threadvar.}: SslContext + +proc getDefaultSSL(): SslContext = + result = defaultSslContext + when defined(ssl): + if result == nil: + defaultSslContext = newContext(verifyMode = CVerifyPeer) + result = defaultSslContext + doAssert result != nil, "failure to initialize the SSL context" + +proc newProxy*(url: string; auth = ""): Proxy = + ## Constructs a new `TProxy` object. + result = Proxy(url: parseUri(url), auth: auth) + +proc newProxy*(url: Uri; auth = ""): Proxy = + ## Constructs a new `TProxy` object. + result = Proxy(url: url, auth: auth) + +proc newMultipartData*: MultipartData {.inline.} = + ## Constructs a new `MultipartData` object. + MultipartData() + +proc `$`*(data: MultipartData): string {.since: (1, 1).} = + ## convert MultipartData to string so it's human readable when echo + ## see https://github.com/nim-lang/Nim/issues/11863 + const sep = "-".repeat(30) + for pos, entry in data.content: + result.add(sep & center($pos, 3) & sep) + result.add("\nname=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"\n") + result.add("Content-Type: " & entry.contentType) + result.add("\n\n" & entry.content & "\n") + +proc add*(p: MultipartData, name, content: string, filename: string = "", + contentType: string = "", useStream = true) = + ## Add a value to the multipart data. + ## + ## When `useStream` is `false`, the file will be read into memory. + ## + ## Raises a `ValueError` exception if + ## `name`, `filename` or `contentType` contain newline characters. + if {'\c', '\L'} in name: + raise newException(ValueError, "name contains a newline character") + if {'\c', '\L'} in filename: + raise newException(ValueError, "filename contains a newline character") + if {'\c', '\L'} in contentType: + raise newException(ValueError, "contentType contains a newline character") + + var entry = MultipartEntry( + name: name, + content: content, + isFile: filename.len > 0 + ) + + if entry.isFile: + entry.isStream = useStream + entry.filename = filename + entry.contentType = contentType + + p.content.add(entry) + +proc add*(p: MultipartData, xs: MultipartEntries): MultipartData + {.discardable.} = + ## Add a list of multipart entries to the multipart data `p`. All values are + ## added without a filename and without a content type. + ## + ## ```Nim + ## data.add({"action": "login", "format": "json"}) + ## ``` + for name, content in xs.items: + p.add(name, content) + result = p + +proc newMultipartData*(xs: MultipartEntries): MultipartData = + ## Create a new multipart data object and fill it with the entries `xs` + ## directly. + ## + ## ```Nim + ## var data = newMultipartData({"action": "login", "format": "json"}) + ## ``` + result = MultipartData() + for entry in xs: + result.add(entry.name, entry.content) + +proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]], + mimeDb = newMimetypes(), useStream = true): + MultipartData {.discardable.} = + ## Add files to a multipart data object. The files will be streamed from disk + ## when the request is being made. When `stream` is `false`, the files are + ## instead read into memory, but beware this is very memory ineffecient even + ## for small files. The MIME types will automatically be determined. + ## Raises an `IOError` if the file cannot be opened or reading fails. To + ## manually specify file content, filename and MIME type, use `[]=` instead. + ## + ## ```Nim + ## data.addFiles({"uploaded_file": "public/test.html"}) + ## ``` + for name, file in xs.items: + var contentType: string + let (_, fName, ext) = splitFile(file) + if ext.len > 0: + contentType = mimeDb.getMimetype(ext[1..ext.high], "") + let content = if useStream: file else: readFile(file) + p.add(name, content, fName & ext, contentType, useStream = useStream) + result = p + +proc `[]=`*(p: MultipartData, name, content: string) {.inline.} = + ## Add a multipart entry to the multipart data `p`. The value is added + ## without a filename and without a content type. + ## + ## ```Nim + ## data["username"] = "NimUser" + ## ``` + p.add(name, content) + +proc `[]=`*(p: MultipartData, name: string, + file: tuple[name, contentType, content: string]) {.inline.} = + ## Add a file to the multipart data `p`, specifying filename, contentType + ## and content manually. + ## + ## ```Nim + ## data["uploaded_file"] = ("test.html", "text/html", + ## "<html><head></head><body><p>test</p></body></html>") + ## ``` + p.add(name, file.content, file.name, file.contentType, useStream = false) + +proc getBoundary(p: MultipartData): string = + if p == nil or p.content.len == 0: return + while true: + result = $rand(int.high) + for i, entry in p.content: + if result in entry.content: break + elif i == p.content.high: return + +proc sendFile(socket: Socket | AsyncSocket, + entry: MultipartEntry) {.multisync.} = + const chunkSize = 2^18 + let file = + when socket is AsyncSocket: openAsync(entry.content) + else: newFileStream(entry.content, fmRead) + + var buffer: string + while true: + buffer = + when socket is AsyncSocket: (await read(file, chunkSize)) + else: readStr(file, chunkSize) + if buffer.len == 0: break + await socket.send(buffer) + file.close() + +proc getNewLocation(lastURL: Uri, headers: HttpHeaders): Uri = + let newLocation = headers.getOrDefault"Location" + if newLocation == "": httpError("location header expected") + # Relative URLs. (Not part of the spec, but soon will be.) + let parsedLocation = parseUri(newLocation) + if parsedLocation.hostname == "" and parsedLocation.path != "": + result = lastURL + result.path = parsedLocation.path + result.query = parsedLocation.query + result.anchor = parsedLocation.anchor + else: + result = parsedLocation + +proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeaders, + proxy: Proxy): string = + # GET + result = $httpMethod + result.add ' ' + + if proxy.isNil or requestUrl.scheme == "https": + # /path?query + if not requestUrl.path.startsWith("/"): result.add '/' + result.add(requestUrl.path) + if requestUrl.query.len > 0: + result.add("?" & requestUrl.query) + else: + # Remove the 'http://' from the URL for CONNECT requests for TLS connections. + var modifiedUrl = requestUrl + if requestUrl.scheme == "https": modifiedUrl.scheme = "" + result.add($modifiedUrl) + + # HTTP/1.1\c\l + result.add(" HTTP/1.1" & httpNewLine) + + # Host header. + if not headers.hasKey("Host"): + if requestUrl.port == "": + add(result, "Host: " & requestUrl.hostname & httpNewLine) + else: + add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine) + + # Connection header. + if not headers.hasKey("Connection"): + add(result, "Connection: Keep-Alive" & httpNewLine) + + # Proxy auth header. + if not proxy.isNil and proxy.auth != "": + let auth = base64.encode(proxy.auth) + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) + + for key, val in headers: + add(result, key & ": " & val & httpNewLine) + + add(result, httpNewLine) + +type + ProgressChangedProc*[ReturnType] = + proc (total, progress, speed: BiggestInt): + ReturnType {.closure, gcsafe.} + + HttpClientBase*[SocketType] = ref object + socket: SocketType + connected: bool + currentURL: Uri ## Where we are currently connected. + headers*: HttpHeaders ## Headers to send in requests. + maxRedirects: Natural ## Maximum redirects, set to `0` to disable. + userAgent: string + timeout*: int ## Only used for blocking HttpClient for now. + proxy: Proxy + ## `nil` or the callback to call when request progress changes. + when SocketType is Socket: + onProgressChanged*: ProgressChangedProc[void] + else: + onProgressChanged*: ProgressChangedProc[Future[void]] + when defined(ssl): + sslContext: net.SslContext + contentTotal: BiggestInt + contentProgress: BiggestInt + oneSecondProgress: BiggestInt + lastProgressReport: MonoTime + when SocketType is AsyncSocket: + bodyStream: FutureStream[string] + parseBodyFut: Future[void] + else: + bodyStream: Stream + getBody: bool ## When `false`, the body is never read in requestAux. + +type + HttpClient* = HttpClientBase[Socket] + +proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + timeout = -1, headers = newHttpHeaders()): HttpClient = + ## Creates a new HttpClient instance. + ## + ## `userAgent` specifies the user agent that will be used when making + ## requests. + ## + ## `maxRedirects` specifies the maximum amount of redirects to follow, + ## default is 5. + ## + ## `sslContext` specifies the SSL context to use for HTTPS requests. + ## See `SSL/TLS support <#sslslashtls-support>`_ + ## + ## `proxy` specifies an HTTP proxy to use for this HTTP client's + ## connections. + ## + ## `timeout` specifies the number of milliseconds to allow before a + ## `TimeoutError` is raised. + ## + ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/strutils + + let exampleHtml = newHttpClient().getContent("http://example.com") + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + + new result + result.headers = headers + result.userAgent = userAgent + result.maxRedirects = maxRedirects + result.proxy = proxy + result.timeout = timeout + result.onProgressChanged = nil + result.bodyStream = newStringStream() + result.getBody = true + when defined(ssl): + result.sslContext = sslContext + +type + AsyncHttpClient* = HttpClientBase[AsyncSocket] + +proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, + sslContext = getDefaultSSL(), proxy: Proxy = nil, + headers = newHttpHeaders()): AsyncHttpClient = + ## Creates a new AsyncHttpClient instance. + ## + ## `userAgent` specifies the user agent that will be used when making + ## requests. + ## + ## `maxRedirects` specifies the maximum amount of redirects to follow, + ## default is 5. + ## + ## `sslContext` specifies the SSL context to use for HTTPS requests. + ## + ## `proxy` specifies an HTTP proxy to use for this HTTP client's + ## connections. + ## + ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/[asyncdispatch, strutils] + + proc asyncProc(): Future[string] {.async.} = + let client = newAsyncHttpClient() + result = await client.getContent("http://example.com") + + let exampleHtml = waitFor asyncProc() + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + + new result + result.headers = headers + result.userAgent = userAgent + result.maxRedirects = maxRedirects + result.proxy = proxy + result.timeout = -1 # TODO + result.onProgressChanged = nil + result.bodyStream = newFutureStream[string]("newAsyncHttpClient") + result.getBody = true + when defined(ssl): + result.sslContext = sslContext + +proc close*(client: HttpClient | AsyncHttpClient) = + ## Closes any connections held by the HTTP client. + if client.connected: + client.socket.close() + client.connected = false + +proc getSocket*(client: HttpClient): Socket {.inline.} = + ## Get network socket, useful if you want to find out more details about the connection. + ## + ## This example shows info about local and remote endpoints: + ## + ## ```Nim + ## if client.connected: + ## echo client.getSocket.getLocalAddr + ## echo client.getSocket.getPeerAddr + ## ``` + return client.socket + +proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} = + return client.socket + +proc reportProgress(client: HttpClient | AsyncHttpClient, + progress: BiggestInt) {.multisync.} = + client.contentProgress += progress + client.oneSecondProgress += progress + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: + if not client.onProgressChanged.isNil: + await client.onProgressChanged(client.contentTotal, + client.contentProgress, + client.oneSecondProgress) + client.oneSecondProgress = 0 + client.lastProgressReport = getMonoTime() + +proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int, + keep: bool): Future[int] {.multisync.} = + ## Ensures that all the data requested is read and returned. + var readLen = 0 + while true: + if size == readLen: break + + let remainingSize = size - readLen + let sizeToRecv = min(remainingSize, net.BufferSize) + + when client.socket is Socket: + let data = client.socket.recv(sizeToRecv, timeout) + else: + let data = await client.socket.recv(sizeToRecv) + if data == "": + client.close() + break # We've been disconnected. + + readLen.inc(data.len) + if keep: + await client.bodyStream.write(data) + + await reportProgress(client, data.len) + + return readLen + +proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void] + {.multisync.} = while true: var chunkSize = 0 - var digitFound = false - while true: - case d[i] - of '0'..'9': - digitFound = true - chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('0')) - of 'a'..'f': - digitFound = true - chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('a') + 10) - of 'A'..'F': - digitFound = true - chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('A') + 10) - of '\0': - d = s.recv() - i = -1 - else: break + var chunkSizeStr = await client.socket.recvLine() + var i = 0 + if chunkSizeStr == "": + httpError("Server terminated connection prematurely") + while i < chunkSizeStr.len: + case chunkSizeStr[i] + of '0'..'9': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) + of 'a'..'f': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) + of 'A'..'F': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) + of ';': + # http://tools.ietf.org/html/rfc2616#section-3.6.1 + # We don't care about chunk-extensions. + break + else: + httpError("Invalid chunk size: " & chunkSizeStr) inc(i) - if not digitFound: httpError("Chunksize expected") - if chunkSize <= 0: break - while charAt(d, i, s) notin {'\C', '\L', '\0'}: inc(i) - if charAt(d, i, s) == '\C': inc(i) - if charAt(d, i, s) == '\L': inc(i) - else: httpError("CR-LF after chunksize expected") - - var x = copy(d, i, i+chunkSize-1) - var size = x.len - result.add(x) - inc(i, size) - if size < chunkSize: - # read in the rest: - var missing = chunkSize - size - var L = result.len - setLen(result, L + missing) - while missing > 0: - var bytesRead = s.recv(addr(result[L]), missing) - inc(L, bytesRead) - dec(missing, bytesRead) - # next chunk: - d = s.recv() - i = 0 - # skip trailing CR-LF: - while charAt(d, i, s) in {'\C', '\L'}: inc(i) - -proc parseBody(d: var string, start: int, s: TSocket, - headers: PStringTable): string = - if headers["Transfer-Encoding"] == "chunked": - result = parseChunks(d, start, s) + if chunkSize <= 0: + discard await recvFull(client, 2, client.timeout, false) # Skip \c\L + break + var bytesRead = await recvFull(client, chunkSize, client.timeout, true) + if bytesRead != chunkSize: + httpError("Server terminated connection prematurely") + + bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L + if bytesRead != 2: + httpError("Server terminated connection prematurely") + + # Trailer headers will only be sent if the request specifies that we want + # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 + +proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, + httpVersion: string): Future[void] {.multisync.} = + # Reset progress from previous requests. + client.contentTotal = 0 + client.contentProgress = 0 + client.oneSecondProgress = 0 + client.lastProgressReport = MonoTime() + + when client is AsyncHttpClient: + assert(not client.bodyStream.finished) + + if headers.getOrDefault"Transfer-Encoding" == "chunked": + await parseChunks(client) else: - result = copy(d, start) # -REGION- Content-Length # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3 - var contentLengthHeader = headers["Content-Length"] + var contentLengthHeader = headers.getOrDefault"Content-Length" if contentLengthHeader != "": - var length = contentLengthHeader.parseint() - while result.len() < length: result.add(s.recv()) + var length = contentLengthHeader.parseInt() + client.contentTotal = length + if length > 0: + let recvLen = await client.recvFull(length, client.timeout, true) + if recvLen == 0: + client.close() + httpError("Got disconnected while trying to read body.") + if recvLen != length: + httpError("Received length doesn't match expected length. Wanted " & + $length & " got: " & $recvLen) else: # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO - + # -REGION- Connection: Close # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 - if headers["Connection"] == "close": - while True: - var moreData = recv(s) - if moreData.len == 0: break - result.add(moreData) - -proc parseResponse(s: TSocket): TResponse = - var d = s.recv() - var i = 0 - - # Parse the version - # Parses the first line of the headers - # ``HTTP/1.1`` 200 OK - var L = skipIgnoreCase(d, "HTTP/1.1", i) - if L > 0: - result.version = "1.1" - inc(i, L) + let implicitConnectionClose = + httpVersion == "1.0" or + # This doesn't match the HTTP spec, but it fixes issues for non-conforming servers. + (httpVersion == "1.1" and headers.getOrDefault"Connection" == "") + if headers.getOrDefault"Connection" == "close" or implicitConnectionClose: + while true: + let recvLen = await client.recvFull(4000, client.timeout, true) + if recvLen != 4000: + client.close() + break + + when client is AsyncHttpClient: + client.bodyStream.complete() else: - L = skipIgnoreCase(d, "HTTP/1.0", i) - if L > 0: - result.version = "1.0" - inc(i, L) - else: - httpError("invalid HTTP header") - L = skipWhiteSpace(d, i) - if L <= 0: httpError("invalid HTTP header") - inc(i, L) - - result.status = "" - while d[i] notin {'\C', '\L', '\0'}: - result.status.add(d[i]) - inc(i) - if d[i] == '\C': inc(i) - if d[i] == '\L': inc(i) - else: httpError("invalid HTTP header, CR-LF expected") - - # Parse the headers - # Everything after the first line leading up to the body - # htype: hvalue - result.headers = newStringTable(modeCaseInsensitive) + client.bodyStream.setPosition(0) + + # If the server will close our connection, then no matter the method of + # reading the body, we need to close our socket. + if headers.getOrDefault"Connection" == "close": + client.close() + +proc parseResponse(client: HttpClient | AsyncHttpClient, + getBody: bool): Future[Response | AsyncResponse] + {.multisync.} = + new result + var parsedStatus = false + var linei = 0 + var fullyRead = false + var lastHeaderName = "" + var line = "" + result.headers = newHttpHeaders() while true: - var key = "" - while d[i] != ':': - if d[i] == '\0': httpError("invalid HTTP header, ':' expected") - key.add(d[i]) - inc(i) - inc(i) # skip ':' - if d[i] == ' ': inc(i) # skip if the character is a space - var val = "" - while d[i] notin {'\C', '\L', '\0'}: - val.add(d[i]) - inc(i) - - result.headers[key] = val - - if d[i] == '\C': inc(i) - if d[i] == '\L': inc(i) - else: httpError("invalid HTTP header, CR-LF expected") - - if d[i] == '\C': inc(i) - if d[i] == '\L': - inc(i) + linei = 0 + when client is HttpClient: + line = await client.socket.recvLine(client.timeout) + else: + line = await client.socket.recvLine() + if line == "": + # We've been disconnected. + client.close() break - - result.body = parseBody(d, i, s, result.headers) + if line == httpNewLine: + fullyRead = true + break + if not parsedStatus: + # Parse HTTP version info and status code. + var le = skipIgnoreCase(line, "HTTP/", linei) + if le <= 0: + httpError("invalid http version, `" & line & "`") + inc(linei, le) + le = skipIgnoreCase(line, "1.1", linei) + if le > 0: result.version = "1.1" + else: + le = skipIgnoreCase(line, "1.0", linei) + if le <= 0: httpError("unsupported http version") + result.version = "1.0" + inc(linei, le) + # Status code + linei.inc skipWhitespace(line, linei) + result.status = line[linei .. ^1] + parsedStatus = true + else: + # Parse headers + # There's at least one char because empty lines are handled above (with client.close) + if line[0] in {' ', '\t'}: + # Check if it's a multiline header value, if so, append to the header we're currently parsing + # This works because a line with a header must start with the header name without any leading space + # See https://datatracker.ietf.org/doc/html/rfc7230, section 3.2 and 3.2.4 + # Multiline headers are deprecated in the spec, but it's better to parse them than crash + if lastHeaderName == "": + # Some extra unparsable lines in the HTTP output - we ignore them + discard + else: + result.headers.table[result.headers.toCaseInsensitive(lastHeaderName)][^1].add "\n" & line + else: + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("Invalid headers - received empty header name") + if line.len == le: httpError("Invalid headers - no colon after header name") + inc(linei, le) # Skip the parsed header name + inc(linei) # Skip : + # If we want to be HTTP spec compliant later, error on linei == line.len (for empty header value) + lastHeaderName = name # Remember the header name for the possible multi-line header + result.headers.add(name, line[linei .. ^1].strip()) + if result.headers.len > headerLimit: + httpError("too many headers") -type - THttpMethod* = enum ## the requested HttpMethod - httpHEAD, ## Asks for the response identical to the one that would - ## correspond to a GET request, but without the response - ## body. - httpGET, ## Retrieves the specified resource. - httpPOST, ## Submits data to be processed to the identified - ## resource. The data is included in the body of the - ## request. - httpPUT, ## Uploads a representation of the specified resource. - httpDELETE, ## Deletes the specified resource. - httpTRACE, ## Echoes back the received request, so that a client - ## can see what intermediate servers are adding or - ## changing in the request. - httpOPTIONS, ## Returns the HTTP methods that the server supports - ## for specified address. - httpCONNECT ## Converts the request connection to a transparent - ## TCP/IP tunnel, usually used for proxies. - -proc request*(url: string, httpMethod = httpGET, extraHeaders = "", - body = ""): TResponse = - ## | Requests ``url`` with the specified ``httpMethod``. - ## | Extra headers can be specified and must be seperated by ``\c\L`` - var r = parseUrl(url) - - var headers = copy($httpMethod, len("http")) - if r.path != "": - headers.add(" " & r.path & r.query) - headers.add(" / HTTP/1.1\c\L") - - add(headers, "Host: " & r.hostname & "\c\L") - add(headers, extraHeaders) - add(headers, "\c\L") - - var s = socket() - s.connect(r.hostname, TPort(80)) - s.send(headers) - if body != "": - s.send(body) - - result = parseResponse(s) - s.close() - -proc redirection(status: string): bool = - const redirectionNRs = ["301", "302", "303", "307"] - for i in items(redirectionNRs): - if status.startsWith(i): - return True - -proc get*(url: string, maxRedirects = 5): TResponse = - ## | GET's the ``url`` and returns a ``TResponse`` object - ## | This proc also handles redirection - result = request(url) - for i in 1..maxRedirects: - if result.status.redirection(): - var locationHeader = result.headers["Location"] - if locationHeader == "": httpError("location header expected") - result = request(locationHeader) - -proc getContent*(url: string): string = - ## | GET's the body and returns it as a string. - ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - var r = get(url) - if r.status[0] in {'4','5'}: - raise newException(EHTTPRequestErr, r.status) + if not fullyRead: + httpError("Connection was closed before full request has been made") + + when client is HttpClient: + result.bodyStream = newStringStream() else: - return r.body - -proc post*(url: string, extraHeaders = "", body = "", - maxRedirects = 5): TResponse = - ## | POST's ``body`` to the ``url`` and returns a ``TResponse`` object. - ## | This proc adds the necessary Content-Length header. - ## | This proc also handles redirection. - var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L" - result = request(url, httpPOST, xh, body) - for i in 1..maxRedirects: - if result.status.redirection(): - var locationHeader = result.headers["Location"] - if locationHeader == "": httpError("location header expected") - var meth = if result.status != "307": httpGet else: httpPost - result = request(locationHeader, meth, xh, body) - -proc postContent*(url: string, extraHeaders = "", body = ""): string = - ## | POST's ``body`` to ``url`` and returns the response's body as a string - ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - var r = post(url, extraHeaders, body) - if r.status[0] in {'4','5'}: - raise newException(EHTTPRequestErr, r.status) + result.bodyStream = newFutureStream[string]("parseResponse") + + if getBody and result.code != Http204: + client.bodyStream = result.bodyStream + when client is HttpClient: + parseBody(client, result.headers, result.version) + else: + assert(client.parseBodyFut.isNil or client.parseBodyFut.finished) + # do not wait here for the body request to complete + client.parseBodyFut = parseBody(client, result.headers, result.version) + client.parseBodyFut.addCallback do(): + if client.parseBodyFut.failed: + client.bodyStream.fail(client.parseBodyFut.error) else: - return r.body - -proc downloadFile*(url: string, outputFilename: string) = - ## Downloads ``url`` and saves it to ``outputFilename`` - var f: TFile - if open(f, outputFilename, fmWrite): - f.write(getContent(url)) - f.close() + when client is AsyncHttpClient: + result.bodyStream.complete() + +proc newConnection(client: HttpClient | AsyncHttpClient, + url: Uri) {.multisync.} = + if client.currentURL.hostname != url.hostname or + client.currentURL.scheme != url.scheme or + client.currentURL.port != url.port or + (not client.connected): + # Connect to proxy if specified + let connectionUrl = + if client.proxy.isNil: url else: client.proxy.url + + let isSsl = connectionUrl.scheme.toLowerAscii() == "https" + + if isSsl and not defined(ssl): + raise newException(HttpRequestError, + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") + + if client.connected: + client.close() + client.connected = false + + # TODO: I should be able to write 'net.Port' here... + let port = + if connectionUrl.port == "": + if isSsl: + nativesockets.Port(443) + else: + nativesockets.Port(80) + else: nativesockets.Port(connectionUrl.port.parseInt) + + when client is HttpClient: + client.socket = await net.dial(connectionUrl.hostname, port) + elif client is AsyncHttpClient: + client.socket = await asyncnet.dial(connectionUrl.hostname, port) + else: {.fatal: "Unsupported client type".} + + when defined(ssl): + if isSsl: + try: + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, connectionUrl.hostname) + except: + client.socket.close() + raise getCurrentException() + + # If need to CONNECT through proxy + if url.scheme == "https" and not client.proxy.isNil: + when defined(ssl): + # Pass only host:port for CONNECT + var connectUrl = initUri() + connectUrl.hostname = url.hostname + connectUrl.port = if url.port != "": url.port else: "443" + + let proxyHeaderString = generateHeaders(connectUrl, HttpConnect, + newHttpHeaders(), client.proxy) + await client.socket.send(proxyHeaderString) + let proxyResp = await parseResponse(client, false) + + if not proxyResp.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, url.hostname) + else: + raise newException(HttpRequestError, + "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.") + + # May be connected through proxy but remember actual URL being accessed + client.currentURL = url + client.connected = true + +proc readFileSizes(client: HttpClient | AsyncHttpClient, + multipart: MultipartData) {.multisync.} = + for entry in multipart.content.mitems(): + if not entry.isFile: continue + if not entry.isStream: + entry.fileSize = entry.content.len + continue + + # TODO: look into making getFileSize work with async + let fileSize = getFileSize(entry.content) + entry.fileSize = fileSize + +proc format(entry: MultipartEntry, boundary: string): string = + result = "--" & boundary & httpNewLine + result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"") + if entry.isFile: + result.add("; filename=\"" & entry.filename & "\"" & httpNewLine) + result.add("Content-Type: " & entry.contentType & httpNewLine) else: - fileError("Unable to open file") + result.add(httpNewLine & httpNewLine & entry.content) + +proc format(client: HttpClient | AsyncHttpClient, + multipart: MultipartData): Future[seq[string]] {.multisync.} = + let bound = getBoundary(multipart) + client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound + + await client.readFileSizes(multipart) + + var length: int64 + for entry in multipart.content: + result.add(format(entry, bound) & httpNewLine) + if entry.isFile: + length += entry.fileSize + httpNewLine.len + + result.add "--" & bound & "--" & httpNewLine + + for s in result: length += s.len + client.headers["Content-Length"] = $length + +proc override(fallback, override: HttpHeaders): HttpHeaders = + # Right-biased map union for `HttpHeaders` + + result = newHttpHeaders() + # Copy by value + result.table[] = fallback.table[] + + if override.isNil: + # Return the copy of fallback so it does not get modified + return result + + for k, vs in override.table: + result[k] = vs + +proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, + httpMethod: HttpMethod, body = "", headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + # Helper that actually makes the request. Does not handle redirects. + if url.scheme == "": + raise newException(ValueError, "No uri scheme supplied.") + + when client is AsyncHttpClient: + if not client.parseBodyFut.isNil: + # let the current operation finish before making another request + await client.parseBodyFut + client.parseBodyFut = nil + + await newConnection(client, url) + + var newHeaders: HttpHeaders + + var data: seq[string] + if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 + data = await client.format(multipart) + newHeaders = client.headers.override(headers) + else: + newHeaders = client.headers.override(headers) + # Only change headers if they have not been specified already + if not newHeaders.hasKey("Content-Length"): + if body.len != 0: + newHeaders["Content-Length"] = $body.len + elif httpMethod notin {HttpGet, HttpHead}: + newHeaders["Content-Length"] = "0" + + if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0: + newHeaders["User-Agent"] = client.userAgent + + let headerString = generateHeaders(url, httpMethod, newHeaders, + client.proxy) + await client.socket.send(headerString) + if data.len > 0: + var buffer: string + for i, entry in multipart.content: + buffer.add data[i] + if not entry.isFile: continue + if buffer.len > 0: + await client.socket.send(buffer) + buffer.setLen(0) + if entry.isStream: + await client.socket.sendFile(entry) + else: + await client.socket.send(entry.content) + buffer.add httpNewLine + # send the rest and the last boundary + await client.socket.send(buffer & data[^1]) + elif body.len > 0: + await client.socket.send(body) -when isMainModule: - #downloadFile("http://force7.de/nimrod/index.html", "nimrodindex.html") - #downloadFile("http://www.httpwatch.com/", "ChunkTest.html") - #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com", - # "validator.html") + let getBody = httpMethod notin {HttpHead, HttpConnect} and + client.getBody + result = await parseResponse(client, getBody) - #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& - # charset=%28detect+automatically%29&doctype=Inline&group=0") +proc request*(client: HttpClient | AsyncHttpClient, url: Uri | string, + httpMethod: HttpMethod | string = HttpGet, body = "", + headers: HttpHeaders = nil, + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a request + ## using the custom method string specified by `httpMethod`. + ## + ## Connection will be kept alive. Further requests on the same `client` to + ## the same hostname will not require a new connection to be made. The + ## connection can be closed by using the `close` procedure. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in `client.maxRedirects`. + ## + ## You need to make sure that the `url` doesn't contain any newline + ## characters. Failing to do so will raise `AssertionDefect`. + ## + ## `headers` are HTTP headers that override the `client.headers` for + ## this specific request only and will not be persisted. + ## + ## **Deprecated since v1.5**: use HttpMethod enum instead; string parameter httpMethod is deprecated + when url is string: + doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters") + let url = parseUri(url) + + when httpMethod is string: + {.warning: + "Deprecated since v1.5; use HttpMethod enum instead; string parameter httpMethod is deprecated".} + let httpMethod = case httpMethod + of "HEAD": + HttpHead + of "GET": + HttpGet + of "POST": + HttpPost + of "PUT": + HttpPut + of "DELETE": + HttpDelete + of "TRACE": + HttpTrace + of "OPTIONS": + HttpOptions + of "CONNECT": + HttpConnect + of "PATCH": + HttpPatch + else: + raise newException(ValueError, "Invalid HTTP method name: " & httpMethod) + + result = await client.requestAux(url, httpMethod, body, headers, multipart) + + var lastURL = url + for i in 1..client.maxRedirects: + let statusCode = result.code + + if statusCode notin {Http301, Http302, Http303, Http307, Http308}: + break + + let redirectTo = getNewLocation(lastURL, result.headers) + var redirectMethod: HttpMethod + var redirectBody: string + # For more informations about the redirect methods see: + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections + case statusCode + of Http301, Http302, Http303: + # The method is changed to GET unless it is GET or HEAD (RFC2616) + if httpMethod notin {HttpGet, HttpHead}: + redirectMethod = HttpGet + else: + redirectMethod = httpMethod + # The body is stripped away + redirectBody = "" + # Delete any header value associated with the body + if not headers.isNil(): + headers.del("Content-Length") + headers.del("Content-Type") + headers.del("Transfer-Encoding") + of Http307, Http308: + # The method and the body are unchanged + redirectMethod = httpMethod + redirectBody = body + else: + # Unreachable + doAssert(false) + + # Check if the redirection is to the same domain or a sub-domain (foo.com + # -> sub.foo.com) + if redirectTo.hostname != lastURL.hostname and + not redirectTo.hostname.endsWith("." & lastURL.hostname): + # Perform some cleanup of the header values + if headers != nil: + # Delete the Host header + headers.del("Host") + # Do not send any sensitive info to a unknown host + headers.del("Authorization") + + result = await client.requestAux(redirectTo, redirectMethod, redirectBody, + headers, multipart) + lastURL = redirectTo + +proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} = + ## Returns the content of a response as a string. + ## + ## A `HttpRequestError` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status.move) + else: + return await resp.bodyStream.readAll() + +proc head*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a HEAD request. + ## + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpHead) + +proc get*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a GET request. + ## + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpGet) + +proc getContent*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a GET request. + let resp = await get(client, url) + return await responseContent(resp) + +proc delete*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[Response | AsyncResponse] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a DELETE request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpDelete) + +proc deleteContent*(client: HttpClient | AsyncHttpClient, + url: Uri | string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a DELETE request. + let resp = await delete(client, url) + return await responseContent(resp) + +proc post*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a POST request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPost, body, multipart=multipart) + +proc postContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a POST request. + let resp = await post(client, url, body, multipart) + return await responseContent(resp) + +proc put*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a PUT request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPut, body, multipart=multipart) + +proc putContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL andreturns the content of a PUT request. + let resp = await put(client, url, body, multipart) + return await responseContent(resp) + +proc patch*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[Response | AsyncResponse] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a PATCH request. + ## This procedure uses httpClient values such as `client.maxRedirects`. + result = await client.request(url, HttpPatch, body, multipart=multipart) + +proc patchContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and returns the content of a PATCH request. + let resp = await patch(client, url, body, multipart) + return await responseContent(resp) + +proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) = + ## Downloads `url` and saves it to `filename`. + client.getBody = false + defer: + client.getBody = true + let resp = client.get(url) - var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L" - var body: string = "--xyz\c\L" - # soap 1.2 output - body.add("Content-Disposition: form-data; name=\"output\"\c\L") - body.add("\c\Lsoap12\c\L") + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + + client.bodyStream = newFileStream(filename, fmWrite) + if client.bodyStream.isNil: + fileError("Unable to open file") + parseBody(client, resp.headers, resp.version) + client.bodyStream.close() + +proc downloadFileEx(client: AsyncHttpClient, + url: Uri | string, filename: string): Future[void] {.async.} = + ## Downloads `url` and saves it to `filename`. + client.getBody = false + let resp = await client.get(url) - # html - body.add("--xyz\c\L") - body.add("Content-Disposition: form-data; name=\"uploaded_file\";" & - " filename=\"test.html\"\c\L") - body.add("Content-Type: text/html\c\L") - body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L") - body.add("--xyz--") - - echo(postContent("http://validator.w3.org/check", headers, body)) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + + client.bodyStream = newFutureStream[string]("downloadFile") + var file = openAsync(filename, fmWrite) + defer: file.close() + # Let `parseBody` write response data into client.bodyStream in the + # background. + let parseBodyFut = parseBody(client, resp.headers, resp.version) + parseBodyFut.addCallback do(): + if parseBodyFut.failed: + client.bodyStream.fail(parseBodyFut.error) + # The `writeFromStream` proc will complete once all the data in the + # `bodyStream` has been written to the file. + await file.writeFromStream(client.bodyStream) + +proc downloadFile*(client: AsyncHttpClient, url: Uri | string, + filename: string): Future[void] = + result = newFuture[void]("downloadFile") + try: + result = downloadFileEx(client, url, filename) + except Exception as exc: + result.fail(exc) + finally: + result.addCallback( + proc () = client.getBody = true + ) |