Move http out of main binary

Now it is (technically) no longer mandatory to link to libcurl.

Also, Chawan is at last completely protocol and network backend
agnostic :)

* Implement multipart requests in local CGI
* Implement simultaneous download of CGI data
* Add REQUEST_HEADERS env var with all headers
* cssparser: add a missing check in consumeEscape

10 files changed, 255 insertions, 51 deletions

diff --git a/adapter/format/gopher2html.nim b/adapter/format/gopher2html.nim
index 33cd564a..1004cc2c 100644
--- a/adapter/format/gopher2html.nim
+++ b/adapter/format/gopher2html.nim
@@ -5,26 +5,9 @@ import std/os
 import std/streams
 import std/strutils
 
-import utils/twtstr
-
 include ../gophertypes
 
-func gopherType(c: char): GopherType =
-  return case c
-  of '0': TEXT_FILE
-  of '1': DIRECTORY
-  of '3': ERROR
-  of '5': DOS_BINARY
-  of '7': SEARCH
-  of 'm': MESSAGE
-  of 's': SOUND
-  of 'g': GIF
-  of 'h': HTML
-  of 'i': INFO
-  of 'I': IMAGE
-  of '9': BINARY
-  of 'p': PNG
-  else: UNKNOWN
+import utils/twtstr
 
 const ControlPercentEncodeSet = {char(0x00)..char(0x1F), char(0x7F)..char(0xFF)}
 const QueryPercentEncodeSet = (ControlPercentEncodeSet + {' ', '"', '#', '<', '>'})
diff --git a/adapter/gophertypes.nim b/adapter/gophertypes.nim
index dbc8d64d..a65b75fe 100644
--- a/adapter/gophertypes.nim
+++ b/adapter/gophertypes.nim
@@ -1,4 +1,4 @@
-type GopherType = enum
+type GopherType* = enum
   UNKNOWN = "unsupported"
   TEXT_FILE = "text file"
   ERROR = "error"
@@ -13,3 +13,20 @@ type GopherType = enum
   IMAGE = "image"
   BINARY = "binary"
   PNG = "png"
+
+func gopherType*(c: char): GopherType =
+  return case c
+  of '0': TEXT_FILE
+  of '1': DIRECTORY
+  of '3': ERROR
+  of '5': DOS_BINARY
+  of '7': SEARCH
+  of 'm': MESSAGE
+  of 's': SOUND
+  of 'g': GIF
+  of 'h': HTML
+  of 'i': INFO
+  of 'I': IMAGE
+  of '9': BINARY
+  of 'p': PNG
+  else: UNKNOWN
diff --git a/adapter/protocol/curlerrors.nim b/adapter/protocol/curlerrors.nim
new file mode 100644
index 00000000..3bb6cf6d
--- /dev/null
+++ b/adapter/protocol/curlerrors.nim
@@ -0,0 +1,17 @@
+import bindings/curl
+import loader/connecterror
+
+func curlErrorToChaError*(res: CURLcode): ConnectErrorCode =
+  return case res
+  of CURLE_OK: CONNECTION_SUCCESS
+  of CURLE_URL_MALFORMAT: ERROR_INVALID_URL #TODO should never occur...
+  of CURLE_COULDNT_CONNECT: ERROR_CONNECTION_REFUSED
+  of CURLE_COULDNT_RESOLVE_PROXY: ERROR_FAILED_TO_RESOLVE_PROXY
+  of CURLE_COULDNT_RESOLVE_HOST: ERROR_FAILED_TO_RESOLVE_HOST
+  of CURLE_PROXY: ERROR_PROXY_REFUSED_TO_CONNECT
+  else: ERROR_INTERNAL
+
+proc getCurlConnectionError*(res: CURLcode): string =
+  let e = $int(curlErrorToChaError(res))
+  let msg = $curl_easy_strerror(res)
+  return "Cha-Control: ConnectionError " & e & " " & msg & "\n"
diff --git a/adapter/protocol/curlwrap.nim b/adapter/protocol/curlwrap.nim
new file mode 100644
index 00000000..7aef4182
--- /dev/null
+++ b/adapter/protocol/curlwrap.nim
@@ -0,0 +1,10 @@
+import bindings/curl
+
+template setopt*(curl: CURL, opt: CURLoption, arg: typed) =
+  discard curl_easy_setopt(curl, opt, arg)
+
+template setopt*(curl: CURL, opt: CURLoption, arg: string) =
+  discard curl_easy_setopt(curl, opt, cstring(arg))
+
+template getinfo*(curl: CURL, info: CURLINFO, arg: typed) =
+  discard curl_easy_getinfo(curl, info, arg)
diff --git a/adapter/protocol/data.nim b/adapter/protocol/data.nim
index 7b022976..4f5da2e8 100644
--- a/adapter/protocol/data.nim
+++ b/adapter/protocol/data.nim
@@ -2,15 +2,17 @@ import std/envvars
 import std/base64
 import std/strutils
 
+import loader/connecterror
 import utils/twtstr
 
 proc main() =
   let str = getEnv("MAPPED_URI_PATH")
   const si = "data:".len # start index
+  const iu = $int(ERROR_INVALID_URL)
   var ct = str.until(',', si)
   for c in ct:
     if c notin AsciiAlphaNumeric and c != '/':
-      stdout.write("Cha-Control: ConnectionError -7 invalid data URL")
+      stdout.write("Cha-Control: ConnectionError " & iu  & " invalid data URL")
       return
   let sd = si + ct.len + 1 # data start
   let body = percentDecode(str, sd)
@@ -21,7 +23,7 @@ proc main() =
       stdout.write("Content-Type: " & ct & "\n\n")
       stdout.write(d)
     except ValueError:
-      stdout.write("Cha-Control: ConnectionError -7 invalid data URL")
+      stdout.write("Cha-Control: ConnectionError " & iu  & " invalid data URL")
   else:
     stdout.write("Content-Type: " & ct & "\n\n")
     stdout.write(body)
diff --git a/adapter/protocol/dirlist.nim b/adapter/protocol/dirlist.nim
new file mode 100644
index 00000000..2328cd55
--- /dev/null
+++ b/adapter/protocol/dirlist.nim
@@ -0,0 +1,60 @@
+import algorithm
+
+import utils/twtstr
+
+type DirlistItemType = enum
+  ITEM_FILE, ITEM_LINK, ITEM_DIR
+
+type DirlistItem* = object
+  name*: string
+  modified*: string
+  case t*: DirlistItemType
+  of ITEM_LINK:
+    linkto*: string
+  of ITEM_FILE:
+    nsize*: int
+  of ITEM_DIR:
+    discard
+
+type NameWidthTuple = tuple[name: string, width: int]
+
+func makeDirlist*(items: seq[DirlistItem]): string =
+  var names: seq[NameWidthTuple]
+  var maxw = 20
+  for item in items:
+    var name = item.name
+    if item.t == ITEM_LINK:
+      name &= '@'
+    elif item.t == ITEM_DIR:
+      name &= '/'
+    let w = name.width()
+    maxw = max(w, maxw)
+    names.add((name, w))
+  names.sort(proc(a, b: NameWidthTuple): int = cmp(a.name, b.name))
+  var outs = "<A HREF=\"../\">[Upper Directory]</A>\n"
+  for i in 0 ..< items.len:
+    let item = items[i]
+    var (name, width) = names[i]
+    var path = percentEncode(item.name, PathPercentEncodeSet)
+    if item.t == ITEM_LINK:
+      if item.linkto.len > 0 and item.linkto[^1] == '/':
+        # If the target is a directory, treat it as a directory. (For FTP.)
+        path &= '/'
+    elif item.t == ITEM_DIR:
+      path &= '/'
+    var line = "<A HREF=\"" & path & "\">" & htmlEscape(name) & "</A>"
+    while width <= maxw:
+      if width mod 2 == 0:
+        line &= ' '
+      else:
+        line &= '.'
+      inc width
+    if line[^1] != ' ':
+      line &= ' '
+    line &= htmlEscape(item.modified)
+    if item.t == ITEM_FILE:
+      line &= ' ' & convert_size(item.nsize)
+    elif item.t == ITEM_LINK:
+      line &= " -> " & htmlEscape(item.linkto)
+    outs &= line & '\n'
+  return outs
diff --git a/adapter/protocol/file.nim b/adapter/protocol/file.nim
index f3ffa93e..168be58b 100644
--- a/adapter/protocol/file.nim
+++ b/adapter/protocol/file.nim
@@ -4,8 +4,9 @@ import std/streams
 import std/times
 import std/envvars
 
+import dirlist
+
 import loader/connecterror
-import loader/dirlist
 import utils/twtstr
 
 proc loadDir(path: string) =
@@ -84,14 +85,14 @@ proc loadFile(istream: Stream) =
   stdout.write("\n")
   let outs = newFileStream(stdout)
   while not istream.atEnd:
-    const bufferSize = 4096
-    var buffer {.noinit.}: array[bufferSize, char]
+    const BufferSize = 16384
+    var buffer {.noinit.}: array[BufferSize, char]
     while true:
-      let n = readData(istream, addr buffer[0], bufferSize)
+      let n = readData(istream, addr buffer[0], BufferSize)
       if n == 0:
         break
       outs.writeData(addr buffer[0], n)
-      if n < bufferSize:
+      if n < BufferSize:
         break
 
 proc main() =
diff --git a/adapter/protocol/ftp.nim b/adapter/protocol/ftp.nim
index ad4a51a3..7d072471 100644
--- a/adapter/protocol/ftp.nim
+++ b/adapter/protocol/ftp.nim
@@ -2,10 +2,12 @@ import std/envvars
 import std/options
 import std/strutils
 
+import curlerrors
+import curlwrap
+import dirlist
+
 import bindings/curl
 import loader/connecterror
-import loader/curlwrap
-import loader/dirlist
 import types/opt
 import types/url
 import utils/twtstr
@@ -185,7 +187,7 @@ proc main() =
   let res = curl_easy_perform(curl)
   if res != CURLE_OK:
     if not op.statusline:
-      stdout.write("Cha-Control: ConnectionError " & $int(res) & "\n")
+      stdout.write(getCurlConnectionError(res))
   elif op.dirmode:
     op.finish()
   curl_easy_cleanup(curl)
diff --git a/adapter/protocol/gopher.nim b/adapter/protocol/gopher.nim
index cf3038f7..91875a1e 100644
--- a/adapter/protocol/gopher.nim
+++ b/adapter/protocol/gopher.nim
@@ -1,38 +1,22 @@
 import std/envvars
 import std/options
 
+import curlwrap
+import curlerrors
+
+import ../gophertypes
+
 import bindings/curl
 import loader/connecterror
-import loader/curlwrap
-import loader/request
 import types/opt
 import types/url
 import utils/twtstr
 
-include ../gophertypes
-
 type GopherHandle = ref object
   curl: CURL
   t: GopherType
   statusline: bool
 
-func gopherType(c: char): GopherType =
-  return case c
-  of '0': TEXT_FILE
-  of '1': DIRECTORY
-  of '3': ERROR
-  of '5': DOS_BINARY
-  of '7': SEARCH
-  of 'm': MESSAGE
-  of 's': SOUND
-  of 'g': GIF
-  of 'h': HTML
-  of 'i': INFO
-  of 'I': IMAGE
-  of '9': BINARY
-  of 'p': PNG
-  else: UNKNOWN
-
 proc onStatusLine(op: GopherHandle) =
   var status: clong
   op.curl.getinfo(CURLINFO_RESPONSE_CODE, addr status)
@@ -107,7 +91,7 @@ proc main() =
     curl.setopt(CURLOPT_PROXY, proxy)
   let res = curl_easy_perform(curl)
   if res != CURLE_OK and not op.statusline:
-    stdout.write("Cha-Control: ConnectionError " & $int(res) & "\n")
+    stdout.write(getCurlConnectionError(res))
   curl_easy_cleanup(curl)
 
 main()
diff --git a/adapter/protocol/http.nim b/adapter/protocol/http.nim
new file mode 100644
index 00000000..10b0c060
--- /dev/null
+++ b/adapter/protocol/http.nim
@@ -0,0 +1,128 @@
+import std/envvars
+import std/options
+import std/strutils
+
+import curlerrors
+import curlwrap
+
+import bindings/curl
+import types/opt
+import utils/twtstr
+
+type
+  EarlyHintState = enum
+    NO_EARLY_HINT, EARLY_HINT_STARTED, EARLY_HINT_DONE
+
+  HttpHandle = ref object
+    curl: CURL
+    statusline: bool
+    connectreport: bool
+    earlyhint: EarlyHintState
+    slist: curl_slist
+
+proc curlWriteHeader(p: cstring, size, nitems: csize_t, userdata: pointer):
+    csize_t {.cdecl.} =
+  var line = newString(nitems)
+  if nitems > 0:
+    prepareMutation(line)
+    copyMem(addr line[0], p, nitems)
+
+  let op = cast[HttpHandle](userdata)
+  if not op.statusline:
+    op.statusline = true
+    var status: clong
+    op.curl.getinfo(CURLINFO_RESPONSE_CODE, addr status)
+    if status == 103 and op.earlyhint == NO_EARLY_HINT:
+      op.earlyhint = EARLY_HINT_STARTED
+    else:
+      op.connectreport = true
+      stdout.write("Status: " & $status & "\n")
+      stdout.write("Cha-Control: ControlDone\n")
+    return nitems
+
+  if line == "":
+    # empty line (last, before body)
+    if op.earlyhint == EARLY_HINT_STARTED:
+      # ignore; we do not have a way to stream headers yet.
+      op.earlyhint = EARLY_HINT_DONE
+      # reset statusline; we are awaiting the next line.
+      op.statusline = false
+      return nitems
+    return nitems
+
+  if op.earlyhint != EARLY_HINT_STARTED:
+    # Regrettably, we can only write early hint headers after the status
+    # code is already known.
+    # For now, it seems easiest to just ignore them all.
+    stdout.write(line)
+  return nitems
+
+# From the documentation: size is always 1.
+proc curlWriteBody(p: cstring, size, nmemb: csize_t, userdata: pointer):
+    csize_t {.cdecl.} =
+  return csize_t(stdout.writeBuffer(p, int(nmemb)))
+
+# From the documentation: size is always 1.
+proc readFromStdin(buffer: cstring, size, nitems: csize_t, userdata: pointer):
+    csize_t {.cdecl.} =
+  return csize_t(stdin.readBuffer(buffer, nitems))
+
+proc curlPreRequest(clientp: pointer, conn_primary_ip, conn_local_ip: cstring,
+    conn_primary_port, conn_local_port: cint): cint {.cdecl.} =
+  let op = cast[HttpHandle](clientp)
+  op.connectreport = true
+  stdout.write("Cha-Control: Connected\n")
+  return 0 # ok
+
+proc main() =
+  let curl = curl_easy_init()
+  doAssert curl != nil
+  let surl = getEnv("QUERY_STRING")
+  curl.setopt(CURLOPT_URL, surl)
+  let op = HttpHandle(curl: curl)
+  curl.setopt(CURLOPT_WRITEFUNCTION, curlWriteBody)
+  curl.setopt(CURLOPT_HEADERDATA, op)
+  curl.setopt(CURLOPT_HEADERFUNCTION, curlWriteHeader)
+  curl.setopt(CURLOPT_PREREQDATA, op)
+  curl.setopt(CURLOPT_PREREQFUNCTION, curlPreRequest)
+  let proxy = getEnv("ALL_PROXY")
+  if proxy != "":
+    curl.setopt(CURLOPT_PROXY, proxy)
+  case getEnv("REQUEST_METHOD")
+  of "GET":
+    curl.setopt(CURLOPT_HTTPGET, 1)
+  of "POST":
+    curl.setopt(CURLOPT_POST, 1)
+    let len = parseInt64(getEnv("CONTENT_LENGTH")).get
+    # > For any given platform/compiler curl_off_t must be typedef'ed to
+    # a 64-bit
+    # > wide signed integral data type. The width of this data type must remain
+    # > constant and independent of any possible large file support settings.
+    # >
+    # > As an exception to the above, curl_off_t shall be typedef'ed to
+    # a 32-bit
+    # > wide signed integral data type if there is no 64-bit type.
+    # It seems safe to assume that if the platform has no uint64 then Nim won't
+    # compile either. In return, we are allowed to post >2G of data.
+    curl.setopt(CURLOPT_POSTFIELDSIZE_LARGE, uint64(len))
+    curl.setopt(CURLOPT_READFUNCTION, readFromStdin)
+  else: discard #TODO
+  let headers = getEnv("REQUEST_HEADERS")
+  for line in headers.split("\r\n"):
+    if line.startsWithNoCase("Accept-Encoding: "):
+      let s = line.after(' ')
+      # From the CURLOPT_ACCEPT_ENCODING manpage:
+      # > The application does not have to keep the string around after
+      # > setting this option.
+      curl.setopt(CURLOPT_ACCEPT_ENCODING, cstring(s))
+    # This is OK, because curl_slist_append strdup's line.
+    op.slist = curl_slist_append(op.slist, cstring(line))
+  if op.slist != nil:
+    curl.setopt(CURLOPT_HTTPHEADER, op.slist)
+  let res = curl_easy_perform(curl)
+  if res != CURLE_OK and not op.connectreport:
+    stdout.write(getCurlConnectionError(res))
+    op.connectreport = true
+  curl_easy_cleanup(curl)
+
+main()