loader: rework process model

Originally we had several loader processes so that the loader did not
need asynchronity for loading several buffers at once. Since then, the
scope of what loader does has been reduced significantly, and with
that loader has become mostly asynchronous.

This patch finishes the above work as follows:

* We only fork a single loader process for the browser. It is a waste of
  resources to do otherwise, and would have made future work on a
  download manager very difficult.

* loader becomes (almost) fully async. Now the only sync part is a)
  processing commands and b) waiting for clients to consume responses.
  b) is a bit more problematic than a), but should not cause problems
  unless some other horrible bug exists in a client. (TODO: make it
  fully async.)
  This gives us a noticable improvement in CSS loading speed, since all
  resources can now be queried at once (even before the previous ones
  are connected).

* Buffers now only get processes when the *connection* is finished. So
  headers, status code, etc. are handled by the client, and the buffer
  is forked when the loader starts streaming the response body.
  As a result, mailcap entries can simply dup2 the first UNIX domain
  socket connection as their stdin. This allows us to remove the ugly
  (and slow) `canredir' hack, which required us to send file handles on
  a tour accross the entire codebase.

* The "cache" has been reworked somewhat:
	- Since canredir is gone, buffer-level requests usually start
	  in a suspended state, and are explicitly resumed only after
	  the client could decide whether it wants to cache the response.
	- Instead of a flag on Request and the URL as the cache key,
	  we now use a global counter and the special `cache:' scheme.

* misc fixes: referer_from is now actually respected by buffers (not
  just the pager), load info display should work slightly better, etc.

3 files changed, 16 insertions, 18 deletions

diff --git a/src/config/config.nim b/src/config/config.nim
index f975e759..d08cf958 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -18,7 +18,7 @@ import types/cell
 import types/color
 import types/cookie
 import types/opt
-import types/referer
+import types/referrer
 import types/urimethodmap
 import types/url
 import utils/mimeguess
@@ -156,7 +156,7 @@ type
     scripting*: bool
     charsets*: seq[Charset]
     images*: bool
-    loaderConfig*: LoaderConfig
+    loaderConfig*: LoaderClientConfig
     mimeTypes*: MimeTypes
     isdump*: bool
     cgiDir*: seq[string]
@@ -236,14 +236,14 @@ func getProxy*(config: Config): URL =
 func getDefaultHeaders*(config: Config): Headers =
   return newHeaders(config.network.default_headers)
 
-proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar,
-    headers: Headers, referer_from, scripting: bool, charsets: seq[Charset],
-    images: bool, userstyle: string, proxy: URL, mimeTypes: MimeTypes,
-    urimethodmap: URIMethodMap, cgiDir: seq[string], tmpdir: string):
-    BufferConfig =
+proc getBufferConfig*(config: Config; location: URL; cookiejar: CookieJar,
+    headers: Headers; referer_from, scripting: bool; charsets: seq[Charset];
+    images: bool; userstyle: string; proxy: URL; mimeTypes: MimeTypes;
+    urimethodmap: URIMethodMap; cgiDir: seq[string]; tmpdir: string;
+    charsetOverride: Charset): BufferConfig =
   let filter = newURLFilter(
     scheme = some(location.scheme),
-    allowschemes = @["data", "stream"],
+    allowschemes = @["data", "cache"],
     default = true
   )
   return BufferConfig(
@@ -254,16 +254,12 @@ proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar,
     images: images,
     mimeTypes: mimeTypes,
     isdump: config.start.headless,
-    loaderConfig: LoaderConfig(
+    charsetOverride: charsetOverride,
+    loaderConfig: LoaderClientConfig(
       defaultHeaders: headers,
-      filter: filter,
       cookiejar: cookiejar,
       proxy: proxy,
-      cgiDir: cgiDir,
-      urimethodmap: urimethodmap,
-      w3mCGICompat: config.external.w3m_cgi_compat,
-      libexecPath: ChaPath("${%CHA_LIBEXEC_DIR}").unquote().get,
-      tmpdir: tmpdir
+      filter: filter
     )
   )
 
diff --git a/src/config/mailcap.nim b/src/config/mailcap.nim
index 89d268db..725c2c22 100644
--- a/src/config/mailcap.nim
+++ b/src/config/mailcap.nim
@@ -326,8 +326,8 @@ proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL,
   var canpipe: bool
   return unquoteCommand(ecmd, contentType, outpath, url, charset, canpipe)
 
-proc getMailcapEntry*(mailcap: Mailcap, mimeType, outpath: string,
-    url: URL, charset: Charset): ptr MailcapEntry =
+proc getMailcapEntry*(mailcap: Mailcap; mimeType, outpath: string;
+    url: URL; charset: Charset): ptr MailcapEntry =
   let mt = mimeType.until('/')
   if mt.len + 1 >= mimeType.len:
     return nil
@@ -343,7 +343,8 @@ proc getMailcapEntry*(mailcap: Mailcap, mimeType, outpath: string,
       var canpipe = true
       let cmd = unquoteCommand(entry.test, mimeType, outpath, url, charset,
         canpipe)
-      #TODO TODO TODO if not canpipe ...
+      if not canpipe:
+        continue
       if execCmd(cmd) != 0:
         continue
     return unsafeAddr entry
diff --git a/src/config/toml.nim b/src/config/toml.nim
index 6542b5a8..e46491c5 100644
--- a/src/config/toml.nim
+++ b/src/config/toml.nim
@@ -1,3 +1,4 @@
+import std/options
 import std/streams
 import std/strutils
 import std/tables