Improvements to httpclient. Refs #4423.

* Adds ability to query HttpCode and compare it with strings.
* Moves HttpMethod to HttpCore module.
* Implements synchronous HttpClient using {.multisync.}.

4 files changed, 198 insertions, 87 deletions

diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim
index 778ca2cbb..adbe3a95f 100644
--- a/lib/pure/httpclient.nim
+++ b/lib/pure/httpclient.nim
@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2010 Dominik Picheta, Andreas Rumpf
+#        (c) Copyright 2016 Dominik Picheta, Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -87,12 +87,20 @@ import nativesockets
 export httpcore except parseHeader # TODO: The ``except`` doesn't work
 
 type
-  Response* = tuple[
-    version: string,
-    status: string,
-    headers: HttpHeaders,
-    body: string]
+  Response* = object
+    version*: string
+    status*: string
+    headers*: HttpHeaders
+    body*: string
+
+proc code*(response: Response): HttpCode {.raises: [ValueError].} =
+  ## Retrieves the specified response's ``HttpCode``.
+  ##
+  ## Raises a ``ValueError`` if the response's ``status`` does not have a
+  ## corresponding ``HttpCode``.
+  return parseEnum[HttpCode](response.status)
 
+type
   Proxy* = ref object
     url*: Uri
     auth*: string
@@ -253,25 +261,6 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response =
   else:
     result.body = ""
 
-type
-  HttpMethod* = enum  ## the requested HttpMethod
-    httpHEAD,         ## Asks for the response identical to the one that would
-                      ## correspond to a GET request, but without the response
-                      ## body.
-    httpGET,          ## Retrieves the specified resource.
-    httpPOST,         ## Submits data to be processed to the identified
-                      ## resource. The data is included in the body of the
-                      ## request.
-    httpPUT,          ## Uploads a representation of the specified resource.
-    httpDELETE,       ## Deletes the specified resource.
-    httpTRACE,        ## Echoes back the received request, so that a client
-                      ## can see what intermediate servers are adding or
-                      ## changing in the request.
-    httpOPTIONS,      ## Returns the HTTP methods that the server supports
-                      ## for specified address.
-    httpCONNECT       ## Converts the request connection to a transparent
-                      ## TCP/IP tunnel, usually used for proxies.
-
 {.deprecated: [THttpMethod: HttpMethod].}
 
 when not defined(ssl):
@@ -397,7 +386,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
   ## server takes longer than specified an ETimeout exception will be raised.
   var r = if proxy == nil: parseUri(url) else: proxy.url
   var hostUrl = if proxy == nil: r else: parseUri(url)
-  var headers = substr(httpMethod, len("http"))
+  var headers = substr(httpMethod, len("http")).toUpper()
   # TODO: Use generateHeaders further down once it supports proxies.
 
   var s = newSocket()
@@ -620,7 +609,7 @@ proc downloadFile*(url: string, outputFilename: string,
 proc generateHeaders(r: Uri, httpMethod: string,
                      headers: StringTableRef, body: string): string =
   # TODO: Use this in the blocking HttpClient once it supports proxies.
-  result = substr(httpMethod, len("http"))
+  result = substr(httpMethod, len("http")).toUpper()
   # TODO: Proxies
   result.add ' '
   if r.path[0] != '/': result.add '/'
@@ -643,8 +632,8 @@ proc generateHeaders(r: Uri, httpMethod: string,
   add(result, "\c\L")
 
 type
-  AsyncHttpClient* = ref object
-    socket: AsyncSocket
+  HttpClientBase*[SocketType] = ref object
+    socket: SocketType
     connected: bool
     currentURL: Uri ## Where we are currently connected.
     headers*: StringTableRef
@@ -653,6 +642,30 @@ type
     when defined(ssl):
       sslContext: net.SslContext
 
+type
+  HttpClient* = HttpClientBase[Socket]
+
+proc newHttpClient*(userAgent = defUserAgent,
+    maxRedirects = 5, sslContext = defaultSslContext): HttpClient =
+  ## Creates a new HttpClient instance.
+  ##
+  ## ``userAgent`` specifies the user agent that will be used when making
+  ## requests.
+  ##
+  ## ``maxRedirects`` specifies the maximum amount of redirects to follow,
+  ## default is 5.
+  ##
+  ## ``sslContext`` specifies the SSL context to use for HTTPS requests.
+  new result
+  result.headers = newStringTable(modeCaseInsensitive)
+  result.userAgent = userAgent
+  result.maxRedirects = maxRedirects
+  when defined(ssl):
+    result.sslContext = sslContext
+
+type
+  AsyncHttpClient* = HttpClientBase[AsyncSocket]
+
 {.deprecated: [PAsyncHttpClient: AsyncHttpClient].}
 
 proc newAsyncHttpClient*(userAgent = defUserAgent,
@@ -673,13 +686,14 @@ proc newAsyncHttpClient*(userAgent = defUserAgent,
   when defined(ssl):
     result.sslContext = sslContext
 
-proc close*(client: AsyncHttpClient) =
+proc close*(client: HttpClient | AsyncHttpClient) =
   ## Closes any connections held by the HTTP client.
   if client.connected:
     client.socket.close()
     client.connected = false
 
-proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} =
+proc recvFull(socket: Socket | AsyncSocket,
+              size: int): Future[string] {.multisync.} =
   ## Ensures that all the data requested is read and returned.
   result = ""
   while true:
@@ -688,7 +702,8 @@ proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} =
     if data == "": break # We've been disconnected.
     result.add data
 
-proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} =
+proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string]
+                 {.multisync.} =
   result = ""
   while true:
     var chunkSize = 0
@@ -721,9 +736,9 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} =
     # Trailer headers will only be sent if the request specifies that we want
     # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
 
-proc parseBody(client: AsyncHttpClient,
+proc parseBody(client: HttpClient | AsyncHttpClient,
                headers: HttpHeaders,
-               httpVersion: string): Future[string] {.async.} =
+               httpVersion: string): Future[string] {.multisync.} =
   result = ""
   if headers.getOrDefault"Transfer-Encoding" == "chunked":
     result = await parseChunks(client)
@@ -752,8 +767,8 @@ proc parseBody(client: AsyncHttpClient,
           if buf == "": break
           result.add(buf)
 
-proc parseResponse(client: AsyncHttpClient,
-                   getBody: bool): Future[Response] {.async.} =
+proc parseResponse(client: HttpClient | AsyncHttpClient,
+                   getBody: bool): Future[Response] {.multisync.} =
   var parsedStatus = false
   var linei = 0
   var fullyRead = false
@@ -803,11 +818,17 @@ proc parseResponse(client: AsyncHttpClient,
   else:
     result.body = ""
 
-proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} =
+proc newConnection(client: HttpClient | AsyncHttpClient,
+                   url: Uri) {.multisync.} =
   if client.currentURL.hostname != url.hostname or
       client.currentURL.scheme != url.scheme:
     if client.connected: client.close()
-    client.socket = newAsyncSocket()
+
+    when client is HttpClient:
+      client.socket = newSocket()
+    elif client is AsyncHttpClient:
+      client.socket = newAsyncSocket()
+    else: {.fatal: "Unsupported client type".}
 
     # TODO: I should be able to write 'net.Port' here...
     let port =
@@ -829,8 +850,8 @@ proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} =
     client.currentURL = url
     client.connected = true
 
-proc request*(client: AsyncHttpClient, url: string, httpMethod: string,
-              body = ""): Future[Response] {.async.} =
+proc request*(client: HttpClient | AsyncHttpClient, url: string,
+              httpMethod: string, body = ""): Future[Response] {.multisync.} =
   ## Connects to the hostname specified by the URL and performs a request
   ## using the custom method string specified by ``httpMethod``.
   ##
@@ -853,8 +874,8 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod: string,
 
   result = await parseResponse(client, httpMethod != "httpHEAD")
 
-proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET,
-              body = ""): Future[Response] =
+proc request*(client: HttpClient | AsyncHttpClient, url: string,
+              httpMethod = HttpGET, body = ""): Future[Response] {.multisync.} =
   ## Connects to the hostname specified by the URL and performs a request
   ## using the method specified.
   ##
@@ -863,9 +884,10 @@ proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET,
   ## connection can be closed by using the ``close`` procedure.
   ##
   ## The returned future will complete once the request is completed.
-  result = request(client, url, $httpMethod, body)
+  result = await request(client, url, $httpMethod, body)
 
-proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} =
+proc get*(client: HttpClient | AsyncHttpClient,
+          url: string): Future[Response] {.multisync.} =
   ## Connects to the hostname specified by the URL and performs a GET request.
   ##
   ## This procedure will follow redirects up to a maximum number of redirects
@@ -878,7 +900,8 @@ proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} =
       result = await client.request(redirectTo, httpGET)
       lastURL = redirectTo
 
-proc post*(client: AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response] {.async.} =
+proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "",
+           multipart: MultipartData = nil): Future[Response] {.multisync.} =
   ## Connects to the hostname specified by the URL and performs a POST request.
   ##
   ## This procedure will follow redirects up to a maximum number of redirects
@@ -895,45 +918,4 @@ proc post*(client: AsyncHttpClient, url: string, body = "", multipart: Multipart
     client.headers["Content-Type"] = mpHeader.split(": ")[1]
   client.headers["Content-Length"] = $len(xb)
 
-  result = await client.request(url, httpPOST, xb)
-
-when not defined(testing) and isMainModule:
-  when true:
-    # Async
-    proc main() {.async.} =
-      var client = newAsyncHttpClient()
-      var resp = await client.request("http://picheta.me")
-
-      echo("Got response: ", resp.status)
-      echo("Body:\n")
-      echo(resp.body)
-
-      resp = await client.request("http://picheta.me/asfas.html")
-      echo("Got response: ", resp.status)
-
-      resp = await client.request("http://picheta.me/aboutme.html")
-      echo("Got response: ", resp.status)
-
-      resp = await client.request("http://nim-lang.org/")
-      echo("Got response: ", resp.status)
-
-      resp = await client.request("http://nim-lang.org/download.html")
-      echo("Got response: ", resp.status)
-
-    waitFor main()
-
-  else:
-    #downloadFile("http://force7.de/nim/index.html", "nimindex.html")
-    #downloadFile("http://www.httpwatch.com/", "ChunkTest.html")
-    #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com",
-    # "validator.html")
-
-    #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
-    #  charset=%28detect+automatically%29&doctype=Inline&group=0")
-
-    var data = newMultipartData()
-    data["output"] = "soap12"
-    data["uploaded_file"] = ("test.html", "text/html",
-      "<html><head></head><body><p>test</p></body></html>")
-
-    echo postContent("http://validator.w3.org/check", multipart=data)
+  result = await client.request(url, HttpPOST, xb)
diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim
index e1be746ed..7c631400f 100644
--- a/lib/pure/httpcore.nim
+++ b/lib/pure/httpcore.nim
@@ -71,6 +71,28 @@ type
     HttpVer11,
     HttpVer10
 
+  HttpMethod* = enum  ## the requested HttpMethod
+    HttpHead,         ## Asks for the response identical to the one that would
+                      ## correspond to a GET request, but without the response
+                      ## body.
+    HttpGet,          ## Retrieves the specified resource.
+    HttpPost,         ## Submits data to be processed to the identified
+                      ## resource. The data is included in the body of the
+                      ## request.
+    HttpPut,          ## Uploads a representation of the specified resource.
+    HttpDelete,       ## Deletes the specified resource.
+    HttpTrace,        ## Echoes back the received request, so that a client
+                      ## can see what intermediate servers are adding or
+                      ## changing in the request.
+    HttpOptions,      ## Returns the HTTP methods that the server supports
+                      ## for specified address.
+    HttpConnect       ## Converts the request connection to a transparent
+                      ## TCP/IP tunnel, usually used for proxies.
+
+{.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost,
+               httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace,
+               httpOptions: HttpOptions, httpConnect: HttpConnect].}
+
 const headerLimit* = 10_000
 
 proc newHttpHeaders*(): HttpHeaders =
@@ -188,6 +210,25 @@ proc `==`*(protocol: tuple[orig: string, major, minor: int],
     of HttpVer10: 0
   result = protocol.major == major and protocol.minor == minor
 
+proc `==`*(rawCode: string, code: HttpCode): bool =
+  return rawCode.toLower() == ($code).toLower()
+
+proc is2xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 2xx HTTP status code.
+  return ($code).startsWith("2")
+
+proc is3xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 3xx HTTP status code.
+  return ($code).startsWith("3")
+
+proc is4xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 4xx HTTP status code.
+  return ($code).startsWith("4")
+
+proc is5xx*(code: HttpCode): bool =
+  ## Determines whether ``code`` is a 5xx HTTP status code.
+  return ($code).startsWith("5")
+
 when isMainModule:
   var test = newHttpHeaders()
   test["Connection"] = @["Upgrade", "Close"]
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 50ae553de..c26511f6a 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -966,6 +966,22 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1,
     socket.socketError(result, lastError = lastError)
   data.setLen(result)
 
+proc recv*(socket: Socket, size: int, timeout = -1,
+           flags = {SocketFlag.SafeDisconn}): string {.inline.} =
+  ## Higher-level version of ``recv`` which returns a string.
+  ##
+  ## When ``""`` is returned the socket's connection has been closed.
+  ##
+  ## This function will throw an EOS exception when an error occurs.
+  ##
+  ## A timeout may be specified in milliseconds, if enough data is not received
+  ## within the time specified an ETimeout exception will be raised.
+  ##
+  ##
+  ## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
+  result = newString(size)
+  discard recv(socket, result, size, timeout, flags)
+
 proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} =
   if socket.isBuffered:
     result = 1
@@ -1035,6 +1051,25 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1,
       return
     add(line.string, c)
 
+proc recvLine*(socket: Socket, timeout = -1,
+               flags = {SocketFlag.SafeDisconn}): TaintedString =
+  ## Reads a line of data from ``socket``.
+  ##
+  ## If a full line is read ``\r\L`` is not
+  ## added to the result, however if solely ``\r\L`` is read then the result
+  ## will be set to it.
+  ##
+  ## If the socket is disconnected, the result will be set to ``""``.
+  ##
+  ## An EOS exception will be raised in the case of a socket error.
+  ##
+  ## A timeout can be specified in milliseconds, if data is not received within
+  ## the specified time an ETimeout exception will be raised.
+  ##
+  ## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
+  result = ""
+  readLine(socket, result, timeout, flags)
+
 proc recvFrom*(socket: Socket, data: var string, length: int,
                address: var string, port: var Port, flags = 0'i32): int {.
                tags: [ReadIOEffect].} =
diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim
new file mode 100644
index 000000000..ced39d9c9
--- /dev/null
+++ b/tests/stdlib/thttpclient.nim
@@ -0,0 +1,53 @@
+import strutils
+
+import httpclient, asyncdispatch
+
+proc asyncTest() {.async.} =
+  var client = newAsyncHttpClient()
+  var resp = await client.request("http://example.com/")
+  doAssert(resp.code.is2xx)
+  doAssert("<title>Example Domain</title>" in resp.body)
+
+  resp = await client.request("http://example.com/404")
+  doAssert(resp.code.is4xx)
+  doAssert(resp.code == Http404)
+  doAssert(resp.status == Http404)
+
+  resp = await client.request("https://google.com/")
+  doAssert(resp.code.is2xx or resp.code.is3xx)
+
+proc syncTest() =
+  var client = newHttpClient()
+  var resp = client.request("http://example.com/")
+  doAssert(resp.code.is2xx)
+  doAssert("<title>Example Domain</title>" in resp.body)
+
+  resp = client.request("http://example.com/404")
+  doAssert(resp.code.is4xx)
+  doAssert(resp.code == Http404)
+  doAssert(resp.status == Http404)
+
+  resp = client.request("https://google.com/")
+  doAssert(resp.code.is2xx or resp.code.is3xx)
+
+syncTest()
+
+waitFor(asyncTest())
+
+#[
+
+  else:
+    #downloadFile("http://force7.de/nim/index.html", "nimindex.html")
+    #downloadFile("http://www.httpwatch.com/", "ChunkTest.html")
+    #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com",
+    # "validator.html")
+
+    #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
+    #  charset=%28detect+automatically%29&doctype=Inline&group=0")
+
+    var data = newMultipartData()
+    data["output"] = "soap12"
+    data["uploaded_file"] = ("test.html", "text/html",
+      "<html><head></head><body><p>test</p></body></html>")
+
+    echo postContent("http://validator.w3.org/check", multipart=data)]#