loader: rework process model

Originally we had several loader processes so that the loader did not
need asynchronity for loading several buffers at once. Since then, the
scope of what loader does has been reduced significantly, and with
that loader has become mostly asynchronous.

This patch finishes the above work as follows:

* We only fork a single loader process for the browser. It is a waste of
  resources to do otherwise, and would have made future work on a
  download manager very difficult.

* loader becomes (almost) fully async. Now the only sync part is a)
  processing commands and b) waiting for clients to consume responses.
  b) is a bit more problematic than a), but should not cause problems
  unless some other horrible bug exists in a client. (TODO: make it
  fully async.)
  This gives us a noticable improvement in CSS loading speed, since all
  resources can now be queried at once (even before the previous ones
  are connected).

* Buffers now only get processes when the *connection* is finished. So
  headers, status code, etc. are handled by the client, and the buffer
  is forked when the loader starts streaming the response body.
  As a result, mailcap entries can simply dup2 the first UNIX domain
  socket connection as their stdin. This allows us to remove the ugly
  (and slow) `canredir' hack, which required us to send file handles on
  a tour accross the entire codebase.

* The "cache" has been reworked somewhat:
	- Since canredir is gone, buffer-level requests usually start
	  in a suspended state, and are explicitly resumed only after
	  the client could decide whether it wants to cache the response.
	- Instead of a flag on Request and the URL as the cache key,
	  we now use a global counter and the special `cache:' scheme.

* misc fixes: referer_from is now actually respected by buffers (not
  just the pager), load info display should work slightly better, etc.

3 files changed, 720 insertions, 607 deletions

diff --git a/src/local/client.nim b/src/local/client.nim
index f8e1433d..b707fe84 100644
--- a/src/local/client.nim
+++ b/src/local/client.nim
@@ -24,6 +24,7 @@ import html/event
 import io/bufstream
 import io/posixstream
 import io/promise
+import io/serialize
 import io/socketstream
 import js/base64
 import js/console
@@ -61,7 +62,6 @@ type
     consoleWrapper: ConsoleWrapper
     fdmap: Table[int, Container]
     feednext: bool
-    forkserver: ForkServer
     ibuf: string
     jsctx: JSContext
     jsrt: JSRuntime
@@ -78,6 +78,9 @@ type
 
 jsDestructor(Client)
 
+func forkserver(client: Client): ForkServer {.inline.} =
+  client.pager.forkserver
+
 func console(client: Client): Console {.jsfget.} =
   return client.consoleWrapper.console
 
@@ -454,31 +457,51 @@ proc consoleBuffer(client: Client): Container {.jsfget.} =
   return client.consoleWrapper.container
 
 proc acceptBuffers(client: Client) =
-  while client.pager.unreg.len > 0:
-    let (pid, stream) = client.pager.unreg.pop()
+  let pager = client.pager
+  while pager.unreg.len > 0:
+    let (pid, stream) = pager.unreg.pop()
     let fd = int(stream.fd)
     if fd in client.fdmap:
       client.selector.unregister(fd)
       client.fdmap.del(fd)
     else:
-      client.pager.procmap.del(pid)
+      pager.procmap.del(pid)
     stream.close()
-  var accepted: seq[Pid]
   let registerFun = proc(fd: int) =
     client.selector.unregister(fd)
     client.selector.registerHandle(fd, {Read, Write}, 0)
-  for pid, container in client.pager.procmap:
-    let stream = connectSocketStream(pid, buffered = false, blocking = true)
+  for item in pager.procmap:
+    let container = item.container
+    let stream = connectSocketStream(container.process, buffered = false)
     if stream == nil:
-      client.pager.alert("Error: failed to set up buffer")
+      pager.alert("Error: failed to set up buffer")
       continue
-    container.setStream(stream, registerFun)
+    let key = pager.addLoaderClient(container.process,
+      container.config.loaderConfig)
+    stream.swrite(key)
+    let loader = pager.loader
+    if item.fdin != -1:
+      let outputId = item.istreamOutputId
+      if container.cacheId == -1:
+        container.cacheId = loader.addCacheFile(outputId, loader.clientPid)
+      var outCacheId = container.cacheId
+      let pid = container.process
+      if item.fdout == item.fdin:
+        loader.shareCachedItem(container.cacheId, pid)
+        loader.resume(@[item.istreamOutputId])
+      else:
+        outCacheId = loader.addCacheFile(item.ostreamOutputId, pid)
+        loader.resume(@[item.istreamOutputId, item.ostreamOutputId])
+      # pass down fdout
+      container.setStream(stream, registerFun, item.fdout, outCacheId)
+    else:
+      # buffer is cloned, no need to cache anything
+      container.setCloneStream(stream, registerFun)
     let fd = int(stream.fd)
     client.fdmap[fd] = container
     client.selector.registerHandle(fd, {Read}, 0)
-    client.pager.handleEvents(container)
-    accepted.add(pid)
-  client.pager.procmap.clear()
+    pager.handleEvents(container)
+  pager.procmap.setLen(0)
 
 proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize_t): cint {.
   importc: "setvbuf", header: "<stdio.h>", tags: [].}
@@ -488,6 +511,19 @@ proc handleRead(client: Client, fd: int) =
     client.input().then(proc() =
       client.handlePagerEvents()
     )
+  elif (let i = client.pager.findConnectingBuffer(fd); i != -1):
+    client.selector.unregister(fd)
+    client.loader.unregistered.add(fd)
+    let (container, stream) = client.pager.connectingBuffers[i]
+    let response = stream.readResponse(container.request)
+    if response.body == nil:
+      client.pager.fail(container, response.getErrorMessage())
+    elif response.redirect != nil:
+      client.pager.redirect(container, response)
+      response.body.close()
+    else:
+      client.pager.connected(container, response)
+    client.pager.connectingBuffers.del(i)
   elif fd == client.forkserver.estream.fd:
     const BufferSize = 4096
     const prefix = "STDERR: "
@@ -560,11 +596,18 @@ proc handleError(client: Client, fd: int) =
     client.loader.onError(fd)
   elif fd in client.loader.unregistered:
     discard # already unregistered...
+  elif (let i = client.pager.findConnectingBuffer(fd); i != -1):
+    # bleh
+    let (container, stream) = client.pager.connectingBuffers[i]
+    client.pager.fail(container, "loader died while loading")
+    client.selector.unregister(fd)
+    stream.close()
+    client.pager.connectingBuffers.del(i)
   else:
     if fd in client.fdmap:
       let container = client.fdmap[fd]
       if container != client.consoleWrapper.container:
-        client.console.log("Error in buffer", $container.location)
+        client.console.log("Error in buffer", $container.url)
       else:
         client.consoleWrapper.container = nil
       client.selector.unregister(fd)
@@ -675,7 +718,7 @@ proc writeFile(client: Client, path: string, content: string) {.jsfunc.} =
 
 const ConsoleTitle = "Browser Console"
 
-proc addConsole(pager: Pager, interactive: bool, clearFun, showFun, hideFun:
+proc addConsole(pager: Pager; interactive: bool; clearFun, showFun, hideFun:
     proc()): ConsoleWrapper =
   if interactive:
     var pipefd: array[0..1, cint]
@@ -683,25 +726,14 @@ proc addConsole(pager: Pager, interactive: bool, clearFun, showFun, hideFun:
       raise newException(Defect, "Failed to open console pipe.")
     let url = newURL("stream:console").get
     let container = pager.readPipe0("text/plain", CHARSET_UNKNOWN, pipefd[0],
-      some(url), ConsoleTitle, canreinterpret = false)
-    pager.registerContainer(container)
+      url, ConsoleTitle, canreinterpret = false)
     let err = newPosixStream(pipefd[1])
     err.writeLine("Type (M-c) console.hide() to return to buffer mode.")
-    let console = newConsole(
-      err,
-      clearFun = clearFun,
-      showFun = showFun,
-      hideFun = hideFun
-    )
-    return ConsoleWrapper(
-      console: console,
-      container: container
-    )
+    let console = newConsole(err, clearFun, showFun, hideFun)
+    return ConsoleWrapper(console: console, container: container)
   else:
-    let err = newFileStream(stderr)
-    return ConsoleWrapper(
-      console: newConsole(err)
-    )
+    let err = newPosixStream(stderr.getFileHandle())
+    return ConsoleWrapper(console: newConsole(err))
 
 proc clearConsole(client: Client) =
   var pipefd: array[0..1, cint]
@@ -710,9 +742,8 @@ proc clearConsole(client: Client) =
   let url = newURL("stream:console").get
   let pager = client.pager
   let replacement = pager.readPipe0("text/plain", CHARSET_UNKNOWN, pipefd[0],
-    some(url), ConsoleTitle, canreinterpret = false)
+    url, ConsoleTitle, canreinterpret = false)
   replacement.replace = client.consoleWrapper.container
-  pager.registerContainer(replacement)
   client.consoleWrapper.container = replacement
   let console = client.consoleWrapper.console
   console.err.close()
@@ -726,7 +757,7 @@ proc dumpBuffers(client: Client) =
       client.pager.drawBuffer(container, ostream)
       client.pager.handleEvents(container)
     except IOError:
-      client.console.log("Error in buffer", $container.location)
+      client.console.log("Error in buffer", $container.url)
       # check for errors
       client.handleRead(client.forkserver.estream.fd)
       quit(1)
@@ -846,17 +877,16 @@ proc newClient*(config: Config, forkserver: ForkServer): Client =
   JS_SetModuleLoaderFunc(jsrt, normalizeModuleName, clientLoadJSModule, nil)
   let jsctx = jsrt.newJSContext()
   let pager = newPager(config, forkserver, jsctx)
+  let loader = forkserver.newFileLoader(LoaderConfig(
+    urimethodmap: config.getURIMethodMap(),
+    w3mCGICompat: config.external.w3m_cgi_compat,
+    cgiDir: pager.cgiDir,
+    tmpdir: pager.tmpdir
+  ))
+  pager.setLoader(loader)
   let client = Client(
     config: config,
-    forkserver: forkserver,
-    loader: forkserver.newFileLoader(
-      defaultHeaders = config.getDefaultHeaders(),
-      proxy = config.getProxy(),
-      urimethodmap = config.getURIMethodMap(),
-      cgiDir = pager.cgiDir,
-      acceptProxy = true,
-      w3mCGICompat = config.external.w3m_cgi_compat
-    ),
+    loader: loader,
     jsrt: jsrt,
     jsctx: jsctx,
     pager: pager
diff --git a/src/local/container.nim b/src/local/container.nim
index 88848ac7..d7ee5c64 100644
--- a/src/local/container.nim
+++ b/src/local/container.nim
@@ -7,22 +7,21 @@ when defined(posix):
 
 import config/config
 import display/term
-import extern/stdio
 import io/promise
 import io/serialize
 import io/socketstream
 import js/javascript
 import js/jstypes
 import js/regex
-import loader/connecterror
+import loader/headers
 import loader/loader
 import loader/request
 import local/select
 import server/buffer
-import server/forkserver
 import types/cell
 import types/color
 import types/cookie
+import types/referrer
 import types/url
 import utils/luwrap
 import utils/mimeguess
@@ -43,25 +42,24 @@ type
     setxsave: bool
 
   ContainerEventType* = enum
-    FAIL, SUCCESS, NEEDS_AUTH, REDIRECT, ANCHOR, NO_ANCHOR, UPDATE, READ_LINE,
-    READ_AREA, OPEN, INVALID_COMMAND, STATUS, ALERT, LOADED, TITLE,
-    CHECK_MAILCAP, QUIT
+    cetAnchor, cetNoAnchor, cetUpdate, cetReadLine, cetReadArea, cetOpen,
+    cetSetLoadInfo, cetStatus, cetAlert, cetLoaded, cetTitle
 
   ContainerEvent* = object
     case t*: ContainerEventType
-    of READ_LINE:
+    of cetReadLine:
       prompt*: string
       value*: string
       password*: bool
-    of READ_AREA:
+    of cetReadArea:
       tvalue*: string
-    of OPEN, REDIRECT:
+    of cetOpen:
       request*: Request
-    of ANCHOR, NO_ANCHOR:
+    of cetAnchor, cetNoAnchor:
       anchor*: string
-    of ALERT:
+    of cetAlert:
       msg*: string
-    of UPDATE:
+    of cetUpdate:
       force*: bool
     else: discard
 
@@ -94,9 +92,15 @@ type
   Container* = ref object
     # note: this is not the same as source.request.url (but should be synced
     # with buffer.url)
-    url: URL
-    #TODO this is inaccurate, because only the network charset is passed through
+    url* {.jsget.}: URL
+    #TODO this is inaccurate, because charsetStack can desync
     charset*: Charset
+    charsetStack*: seq[Charset]
+    # note: this is *not* the same as Buffer.cacheId. buffer has the cache ID of
+    # the output, while container holds that of the input. Thus pager can
+    # re-interpret the original input, and buffer can rewind the (potentially
+    # mailcap) output.
+    cacheId* {.jsget.}: int
     parent* {.jsget.}: Container
     children* {.jsget.}: seq[Container]
     config*: BufferConfig
@@ -113,8 +117,7 @@ type
     pos: CursorPosition
     bpos: seq[CursorPosition]
     highlights: seq[Highlight]
-    process* {.jsget.}: Pid
-    loaderPid* {.jsget.}: Pid
+    process* {.jsget.}: int
     loadinfo*: string
     lines: SimpleFlexibleGrid
     lineshift: int
@@ -146,22 +149,12 @@ type
 jsDestructor(Highlight)
 jsDestructor(Container)
 
-proc newBuffer*(forkserver: ForkServer, config: BufferConfig,
-    request: Request, attrs: WindowAttributes, title: string,
-    redirectdepth: int, canreinterpret: bool, fd: FileHandle,
-    contentType: Option[string]): Container =
-  let (process, loaderPid) = forkserver.forkBuffer(request, config, attrs)
-  if fd != -1:
-    loaderPid.passFd(request.url.host, fd)
-    if fd == 0:
-      # We are passing stdin.
-      closeStdin()
-    else:
-      discard close(fd)
+proc newContainer*(config: BufferConfig; url: URL; request: Request;
+    attrs: WindowAttributes; title: string; redirectdepth: int;
+    canreinterpret: bool; contentType: Option[string];
+    charsetStack: seq[Charset]; cacheId: int): Container =
   return Container(
-    url: request.url,
-    process: process,
-    loaderPid: loaderPid,
+    url: url,
     request: request,
     contentType: contentType,
     width: attrs.width,
@@ -172,76 +165,31 @@ proc newBuffer*(forkserver: ForkServer, config: BufferConfig,
     pos: CursorPosition(
       setx: -1
     ),
-    canreinterpret: canreinterpret
-  )
-
-proc newBufferFrom*(forkserver: ForkServer, attrs: WindowAttributes,
-    container: Container, contentTypeOverride: string): Container =
-  let request = newRequest(container.request.url, fromcache = true)
-  let config = container.config
-  let loaderPid = container.loaderPid
-  let bufferPid = forkserver.forkBufferWithLoader(request, config, attrs,
-    loaderPid)
-  return Container(
-    url: request.url,
-    request: request,
-    width: container.width,
-    height: container.height,
-    title: container.title,
-    config: config,
-    process: bufferPid,
-    loaderPid: loaderPid,
-    pos: CursorPosition(
-      setx: -1
-    ),
-    canreinterpret: true,
-    contentType: some(contentTypeOverride)
+    canreinterpret: canreinterpret,
+    loadinfo: "Connecting to " & request.url.host & "...",
+    cacheId: cacheId
   )
 
-func location*(container: Container): URL {.jsfget.} =
+func location(container: Container): URL {.jsfget.} =
   return container.url
 
-proc clone*(container: Container, newurl: URL): Promise[Container] =
+proc clone*(container: Container; newurl: URL): Promise[Container] =
   let url = if newurl != nil:
     newurl
   else:
-    container.location
-  return container.iface.clone(url).then(proc(pid: Pid): Container =
+    container.url
+  return container.iface.clone(url).then(proc(pid: int): Container =
     if pid == -1:
       return nil
-    return Container(
-      url: url,
-      config: container.config,
-      iface: container.iface, # changed later in setStream
-      width: container.width,
-      height: container.height,
-      title: container.title,
-      hoverText: container.hoverText,
-      lastPeek: container.lastPeek,
-      request: container.request,
-      pos: container.pos,
-      bpos: container.bpos,
-      highlights: container.highlights,
-      process: pid,
-      loaderPid: container.loaderPid,
-      loadinfo: container.loadinfo,
-      lines: container.lines,
-      lineshift: container.lineshift,
-      numLines: container.numLines,
-      code: container.code,
-      retry: container.retry,
-      hlon: container.hlon,
-      #needslines: container.needslines,
-      loadState: container.loadState,
-      events: container.events,
-      startpos: container.startpos,
-      hasstart: container.hasstart,
-      redirectdepth: container.redirectdepth,
-      select: container.select,
-      canreinterpret: container.canreinterpret,
-      ishtml: container.ishtml,
-      cloned: true
-    )
+    let nc = Container()
+    nc[] = container[]
+    nc.url = url
+    nc.process = pid
+    nc.cloned = true
+    nc.retry = @[]
+    nc.parent = nil
+    nc.children = @[]
+    return nc
   )
 
 func lineLoaded(container: Container, y: int): bool =
@@ -355,7 +303,7 @@ func maxScreenWidth(container: Container): int =
 func getTitle*(container: Container): string {.jsfunc.} =
   if container.title != "":
     return container.title
-  return container.location.serialize(excludepassword = true)
+  return container.url.serialize(excludepassword = true)
 
 func currentLineWidth(container: Container): int =
   if container.numLines == 0: return 0
@@ -472,12 +420,9 @@ proc setNumLines(container: Container, lines: int, finish = false) =
 
 proc cursorLastLine*(container: Container)
 
-proc requestLines(container: Container): EmptyPromise
-    {.discardable.} =
+proc requestLines(container: Container): EmptyPromise {.discardable.} =
   if container.iface == nil:
-    let res = EmptyPromise()
-    res.resolve()
-    return res
+    return newResolvedPromise()
   let w = container.lineWindow
   return container.iface.getLines(w).then(proc(res: GetLinesResult) =
     container.lines.setLen(w.len)
@@ -491,7 +436,7 @@ proc requestLines(container: Container): EmptyPromise
     if res.numLines != container.numLines:
       container.setNumLines(res.numLines, true)
       if container.loadState != lsLoading:
-        container.triggerEvent(STATUS)
+        container.triggerEvent(cetStatus)
     if res.numLines > 0:
       container.updateCursor()
       if container.tailOnLoad:
@@ -499,20 +444,22 @@ proc requestLines(container: Container): EmptyPromise
         container.cursorLastLine()
     let cw = container.fromy ..< container.fromy + container.height
     if w.a in cw or w.b in cw or cw.a in w or cw.b in w or isBgNew:
-      container.triggerEvent(UPDATE)
+      container.triggerEvent(cetUpdate)
   )
 
 proc redraw(container: Container) {.jsfunc.} =
-  container.triggerEvent(ContainerEvent(t: UPDATE, force: true))
+  container.triggerEvent(ContainerEvent(t: cetUpdate, force: true))
 
 proc sendCursorPosition*(container: Container) =
+  if container.iface == nil:
+    return
   container.iface.updateHover(container.cursorx, container.cursory)
       .then(proc(res: UpdateHoverResult) =
     if res.hover.len > 0:
       assert res.hover.high <= int(HoverType.high)
       for (ht, s) in res.hover:
         container.hoverText[ht] = s
-      container.triggerEvent(STATUS)
+      container.triggerEvent(cetStatus)
     if res.repaint:
       container.needslines = true
   )
@@ -521,7 +468,7 @@ proc setFromY(container: Container, y: int) {.jsfunc.} =
   if container.pos.fromy != y:
     container.pos.fromy = max(min(y, container.maxfromy), 0)
     container.needslines = true
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
 
 proc setFromX(container: Container, x: int, refresh = true) {.jsfunc.} =
   if container.pos.fromx != x:
@@ -530,7 +477,7 @@ proc setFromX(container: Container, x: int, refresh = true) {.jsfunc.} =
       container.pos.cursorx = min(container.pos.fromx, container.currentLineWidth())
       if refresh:
         container.sendCursorPosition()
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
 
 proc setFromXY(container: Container, x, y: int) {.jsfunc.} =
   container.setFromY(y)
@@ -580,7 +527,7 @@ proc setCursorX(container: Container, x: int, refresh = true, save = true)
   if container.cursorx == x and container.currentSelection != nil and
       container.currentSelection.x2 != x:
     container.currentSelection.x2 = x
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
   if refresh:
     container.sendCursorPosition()
   if save:
@@ -602,7 +549,7 @@ proc setCursorY(container: Container, y: int, refresh = true) {.jsfunc.} =
       container.setFromY(y)
     container.pos.cursory = y
   if container.currentSelection != nil and container.currentSelection.y2 != y:
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
     container.currentSelection.y2 = y
   container.restoreCursorX()
   if refresh:
@@ -1053,7 +1000,7 @@ proc scrollLeft*(container: Container, n = 1) {.jsfunc.} =
     container.setFromX(x)
 
 proc alert(container: Container, msg: string) =
-  container.triggerEvent(ContainerEvent(t: ALERT, msg: msg))
+  container.triggerEvent(ContainerEvent(t: cetAlert, msg: msg))
 
 proc lineInfo(container: Container) {.jsfunc.} =
   container.alert("line " & $(container.cursory + 1) & "/" &
@@ -1080,7 +1027,7 @@ proc gotoLine*[T: string|int](container: Container, s: T) =
     elif s[0] == '$':
       container.cursorLastLine()
     else:
-      let i = parseUInt32(s)
+      let i = parseUInt32(s, allowSign = true)
       if i.isSome and i.get > 0:
         container.markPos0()
         container.setCursorY(int(i.get - 1))
@@ -1113,6 +1060,8 @@ proc copyCursorPos*(container, c2: Container) =
   container.hasstart = true
 
 proc cursorNextLink*(container: Container, n = 1) {.jsfunc.} =
+  if container.iface == nil:
+    return
   container.markPos0()
   container.iface
     .findNextLink(container.cursorx, container.cursory, n)
@@ -1123,6 +1072,8 @@ proc cursorNextLink*(container: Container, n = 1) {.jsfunc.} =
     )
 
 proc cursorPrevLink*(container: Container, n = 1) {.jsfunc.} =
+  if container.iface == nil:
+    return
   container.markPos0()
   container.iface
     .findPrevLink(container.cursorx, container.cursory, n)
@@ -1133,6 +1084,8 @@ proc cursorPrevLink*(container: Container, n = 1) {.jsfunc.} =
     )
 
 proc cursorNextParagraph*(container: Container, n = 1) {.jsfunc.} =
+  if container.iface == nil:
+    return
   container.markPos0()
   container.iface
     .findNextParagraph(container.cursory, n)
@@ -1142,6 +1095,8 @@ proc cursorNextParagraph*(container: Container, n = 1) {.jsfunc.} =
     )
 
 proc cursorPrevParagraph*(container: Container, n = 1) {.jsfunc.} =
+  if container.iface == nil:
+    return
   container.markPos0()
   container.iface
     .findPrevParagraph(container.cursory, n)
@@ -1156,17 +1111,17 @@ proc setMark*(container: Container, id: string, x = none(int),
   let y = y.get(container.cursory)
   container.marks.withValue(id, p):
     p[] = (x, y)
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
     return false
   do:
     container.marks[id] = (x, y)
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
     return true
 
 proc clearMark*(container: Container, id: string): bool {.jsfunc.} =
   result = id in container.marks
   container.marks.del(id)
-  container.triggerEvent(UPDATE)
+  container.triggerEvent(cetUpdate)
 
 proc getMarkPos(container: Container, id: string): Opt[PagePos] {.jsfunc.} =
   if id == "`" or id == "'":
@@ -1226,6 +1181,8 @@ proc findPrevMark*(container: Container, x = none(int), y = none(int)):
   return bestid
 
 proc cursorNthLink*(container: Container, n = 1) {.jsfunc.} =
+  if container.iface == nil:
+    return
   container.iface
     .findNthLink(n)
     .then(proc(res: tuple[x, y: int]) =
@@ -1233,6 +1190,8 @@ proc cursorNthLink*(container: Container, n = 1) {.jsfunc.} =
         container.setCursorXYCenter(res.x, res.y))
 
 proc cursorRevNthLink*(container: Container, n = 1) {.jsfunc.} =
+  if container.iface == nil:
+    return
   container.iface
     .findRevNthLink(n)
     .then(proc(res: tuple[x, y: int]) =
@@ -1258,12 +1217,12 @@ proc onMatch(container: Container, res: BufferMatch, refresh: bool) =
         y2: res.y
       )
       container.highlights.add(hl)
-      container.triggerEvent(UPDATE)
+      container.triggerEvent(cetUpdate)
       container.hlon = false
       container.needslines = true
   elif container.hlon:
     container.clearSearchHighlights()
-    container.triggerEvent(UPDATE)
+    container.triggerEvent(cetUpdate)
     container.needslines = true
     container.hlon = false
 
@@ -1275,6 +1234,8 @@ proc cursorNextMatch*(container: Container, regex: Regex, wrap, refresh: bool,
       container.select.cursorNextMatch(regex, wrap)
     return newResolvedPromise()
   else:
+    if container.iface == nil:
+      return
     return container.iface
       .findNextMatch(regex, container.cursorx, container.cursory, wrap, n)
       .then(proc(res: BufferMatch) =
@@ -1288,6 +1249,8 @@ proc cursorPrevMatch*(container: Container, regex: Regex, wrap, refresh: bool,
       container.select.cursorPrevMatch(regex, wrap)
     return newResolvedPromise()
   else:
+    if container.iface == nil:
+      return
     container.markPos0()
     return container.iface
       .findPrevMatch(regex, container.cursorx, container.cursory, wrap, n)
@@ -1321,13 +1284,15 @@ proc cursorToggleSelection(container: Container, n = 1,
     )
     container.highlights.add(hl)
     container.currentSelection = hl
-  container.triggerEvent(UPDATE)
+  container.triggerEvent(cetUpdate)
   return container.currentSelection
 
 #TODO I don't like this API
 # maybe make selection a subclass of highlight?
 proc getSelectionText(container: Container, hl: Highlight = nil):
     Promise[string] {.jsfunc.} =
+  if container.iface == nil:
+    return
   let hl = if hl == nil: container.currentSelection else: hl
   if hl.t != HL_SELECT:
     let p = newPromise[string]()
@@ -1370,7 +1335,7 @@ proc getSelectionText(container: Container, hl: Highlight = nil):
 
 proc setLoadInfo(container: Container, msg: string) =
   container.loadinfo = msg
-  container.triggerEvent(STATUS)
+  container.triggerEvent(cetSetLoadInfo)
 
 #TODO this should be called with a timeout.
 proc onload*(container: Container, res: int) =
@@ -1382,15 +1347,15 @@ proc onload*(container: Container, res: int) =
   elif res == -1:
     container.loadState = lsLoaded
     container.setLoadInfo("")
-    container.triggerEvent(STATUS)
+    container.triggerEvent(cetStatus)
     container.needslines = true
-    container.triggerEvent(LOADED)
+    container.triggerEvent(cetLoaded)
     container.iface.getTitle().then(proc(title: string) =
       if title != "":
         container.title = title
-        container.triggerEvent(TITLE)
+        container.triggerEvent(cetTitle)
     )
-    if not container.hasstart and container.location.anchor != "":
+    if not container.hasstart and container.url.anchor != "":
       container.iface.gotoAnchor().then(proc(res: Opt[tuple[x, y: int]]) =
         if res.isSome:
           let res = res.get
@@ -1403,72 +1368,62 @@ proc onload*(container: Container, res: int) =
       container.onload(res)
     )
 
-proc load(container: Container) =
-  container.setLoadInfo("Connecting to " & container.location.host & "...")
-  container.iface.connect().then(proc(res: ConnectResult) =
-    let info = container.loadinfo
-    if not res.invalid:
-      container.code = res.code
-      if res.code == 0:
-        container.triggerEvent(SUCCESS)
-        # accept cookies
-        let cookiejar = container.config.loaderConfig.cookiejar
-        if res.cookies.len > 0 and cookiejar != nil:
-          cookiejar.add(res.cookies)
-        # set referrer policy, if any
-        if res.referrerPolicy.isSome and container.config.referer_from:
-          container.config.referrerPolicy = res.referrerPolicy.get
-        container.setLoadInfo("Connected to " & $container.location &
-          ". Downloading...")
-        if res.needsAuth:
-          container.triggerEvent(NEEDS_AUTH)
-        if res.redirect != nil:
-          container.triggerEvent(ContainerEvent(t: REDIRECT, request: res.redirect))
-        container.charset = res.charset
-        if container.contentType.isNone:
-          if res.contentType == "application/octet-stream":
-            let contentType = guessContentType(container.location.pathname,
-              "application/octet-stream", container.config.mimeTypes)
-            if contentType != "application/octet-stream":
-              container.contentType = some(contentType)
-            else:
-              container.contentType = some(res.contentType)
-          else:
-            container.contentType = some(res.contentType)
-        container.ishtml = container.contentType.get == "text/html"
-        container.triggerEvent(CHECK_MAILCAP)
+proc extractCookies(response: Response): seq[Cookie] =
+  result = @[]
+  if "Set-Cookie" in response.headers.table:
+    for s in response.headers.table["Set-Cookie"]:
+      let cookie = newCookie(s, response.url)
+      if cookie.isOk:
+        result.add(cookie.get)
+
+proc extractReferrerPolicy(response: Response): Option[ReferrerPolicy] =
+  if "Referrer-Policy" in response.headers:
+    return getReferrerPolicy(response.headers["Referrer-Policy"])
+  return none(ReferrerPolicy)
+
+# Apply data received in response.
+# Note: pager must call this before checkMailcap.
+proc applyResponse*(container: Container; response: Response) =
+  container.code = response.res
+  # accept cookies
+  let cookieJar = container.config.loaderConfig.cookieJar
+  if cookieJar != nil:
+    cookieJar.add(response.extractCookies())
+  # set referrer policy, if any
+  let referrerPolicy = response.extractReferrerPolicy()
+  if container.config.referer_from:
+    if referrerPolicy.isSome:
+      container.config.referrerPolicy = referrerPolicy.get
+  else:
+    container.config.referrerPolicy = NO_REFERRER
+  container.setLoadInfo("Connected to " & $response.url & ". Downloading...")
+  # setup content type; note that isSome means an override so we skip it
+  if container.contentType.isNone:
+    if response.contentType == "application/octet-stream":
+      let contentType = guessContentType(container.url.pathname,
+        "application/octet-stream", container.config.mimeTypes)
+      if contentType != "application/octet-stream":
+        container.contentType = some(contentType)
       else:
-        if res.errorMessage != "":
-          container.errorMessage = res.errorMessage
-        else:
-          container.errorMessage = getLoaderErrorMessage(res.code)
-        container.setLoadInfo("")
-        container.triggerEvent(FAIL)
+        container.contentType = some(response.contentType)
     else:
-      container.setLoadInfo(info)
-  )
-
-proc startload*(container: Container) =
-  container.iface.load().then(proc(res: int) =
-    container.onload(res)
-  )
-
-proc connect2*(container: Container): EmptyPromise =
-  return container.iface.connect2(container.ishtml)
-
-proc redirectToFd*(container: Container, fdin: FileHandle, wait, cache: bool):
-    EmptyPromise =
-  return container.iface.redirectToFd(fdin, wait, cache)
-
-proc readFromFd*(container: Container, fdout: FileHandle, id: string,
-    ishtml: bool): EmptyPromise =
-  container.ishtml = ishtml
-  let url = newURL("stream:" & id).get
-  container.loaderPid.passFd(url.host, fdout)
-  return container.iface.readFromFd(url, ishtml)
-
-proc quit*(container: Container) =
-  container.triggerEvent(QUIT)
+      container.contentType = some(response.contentType)
+  # setup charsets:
+  # * override charset
+  # * network charset
+  # * default charset guesses
+  # HTML may override the last two (but not the override charset).
+  if container.config.charsetOverride != CHARSET_UNKNOWN:
+    container.charsetStack = @[container.config.charsetOverride]
+  elif response.charset != CHARSET_UNKNOWN:
+    container.charsetStack = @[response.charset]
+  else:
+    container.charsetStack = @[]
+    for i in countdown(container.config.charsets.high, 0):
+      container.charsetStack.add(container.config.charsets[i])
+    if container.charsetStack.len == 0:
+      container.charsetStack.add(DefaultCharset)
+  container.charset = container.charsetStack[^1]
 
 proc cancel*(container: Container) {.jsfunc.} =
   if container.select.open:
@@ -1477,26 +1432,30 @@ proc cancel*(container: Container) {.jsfunc.} =
     container.loadState = lsCanceled
     container.alert("Canceled loading")
 
-proc findAnchor*(container: Container, anchor: string) =
+proc findAnchor*(container: Container; anchor: string) =
   container.iface.findAnchor(anchor).then(proc(found: bool) =
     if found:
-      container.triggerEvent(ContainerEvent(t: ANCHOR, anchor: anchor))
+      container.triggerEvent(ContainerEvent(t: cetAnchor, anchor: anchor))
     else:
-      container.triggerEvent(NO_ANCHOR))
+      container.triggerEvent(ContainerEvent(t: cetNoAnchor, anchor: anchor))
+  )
 
 proc readCanceled*(container: Container) =
   container.iface.readCanceled().then(proc(repaint: bool) =
     if repaint:
       container.needslines = true)
 
-proc readSuccess*(container: Container, s: string) =
+proc readSuccess*(container: Container; s: string) =
   container.iface.readSuccess(s).then(proc(res: ReadSuccessResult) =
     if res.repaint:
       container.needslines = true
     if res.open.isSome:
-      container.triggerEvent(ContainerEvent(t: OPEN, request: res.open.get)))
+      container.triggerEvent(ContainerEvent(t: cetOpen, request: res.open.get))
+  )
 
 proc reshape(container: Container): EmptyPromise {.jsfunc.} =
+  if container.iface == nil:
+    return
   return container.iface.forceRender().then(proc(): EmptyPromise =
     return container.requestLines()
   )
@@ -1509,25 +1468,25 @@ proc displaySelect(container: Container, selectResult: SelectResult) =
       container.onclick(res))
   container.select.initSelect(selectResult, container.acursorx,
     container.acursory, container.height, submitSelect)
-  container.triggerEvent(UPDATE)
+  container.triggerEvent(cetUpdate)
 
-proc onclick(container: Container, res: ClickResult) =
+proc onclick(container: Container; res: ClickResult) =
   if res.repaint:
     container.needslines = true
   if res.open.isSome:
-    container.triggerEvent(ContainerEvent(t: OPEN, request: res.open.get))
+    container.triggerEvent(ContainerEvent(t: cetOpen, request: res.open.get))
   if res.select.isSome:
     container.displaySelect(res.select.get)
   if res.readline.isSome:
     let rl = res.readline.get
     let event = if rl.area:
       ContainerEvent(
-        t: READ_AREA,
+        t: cetReadArea,
         tvalue: rl.value
       )
     else:
       ContainerEvent(
-        t: READ_LINE,
+        t: cetReadLine,
         prompt: rl.prompt,
         value: rl.value,
         password: rl.hide
@@ -1538,6 +1497,8 @@ proc click*(container: Container) {.jsfunc.} =
   if container.select.open:
     container.select.click()
   else:
+    if container.iface == nil:
+      return
     container.iface.click(container.cursorx, container.cursory)
       .then(proc(res: ClickResult) = container.onclick(res))
 
@@ -1545,12 +1506,13 @@ proc windowChange*(container: Container, attrs: WindowAttributes) =
   if attrs.width != container.width or attrs.height - 1 != container.height:
     container.width = attrs.width
     container.height = attrs.height - 1
-    container.iface.windowChange(attrs).then(proc() =
-      container.needslines = true
-    )
+    if container.iface != nil:
+      container.iface.windowChange(attrs).then(proc() =
+        container.needslines = true
+      )
 
 proc peek(container: Container) {.jsfunc.} =
-  container.alert($container.location)
+  container.alert($container.url)
 
 proc clearHover*(container: Container) =
   container.lastPeek = low(HoverType)
@@ -1582,17 +1544,24 @@ proc handleCommand(container: Container) =
   container.iface.stream.sread(packetid)
   container.iface.resolve(packetid, len - slen(packetid))
 
-proc setStream*(container: Container, stream: SocketStream,
+proc setStream*(container: Container; stream: SocketStream;
+    registerFun: proc(fd: int); fd: FileHandle; outCacheId: int) =
+  assert not container.cloned
+  container.iface = newBufferInterface(stream, registerFun)
+  container.iface.passFd(fd, outCacheId)
+  discard close(fd)
+  discard container.iface.load().then(proc(res: int) =
+    container.onload(res)
+  )
+
+proc setCloneStream*(container: Container; stream: SocketStream;
     registerFun: proc(fd: int)) =
-  if not container.cloned:
-    container.iface = newBufferInterface(stream, registerFun)
-    container.load()
-  else:
-    container.iface = cloneInterface(stream, registerFun)
-    # Maybe we have to resume loading. Let's try.
-    discard container.iface.load().then(proc(res: int) =
-      container.onload(res)
-    )
+  assert container.cloned
+  container.iface = cloneInterface(stream, registerFun)
+  # Maybe we have to resume loading. Let's try.
+  discard container.iface.load().then(proc(res: int) =
+    container.onload(res)
+  )
 
 proc onreadline(container: Container, w: Slice[int],
     handle: (proc(line: SimpleFlexibleLine)), res: GetLinesResult) =
diff --git a/src/local/pager.nim b/src/local/pager.nim
index a9d06567..51c2f89f 100644
--- a/src/local/pager.nim
+++ b/src/local/pager.nim
@@ -23,11 +23,14 @@ import extern/stdio
 import extern/tempfile
 import io/posixstream
 import io/promise
+import io/socketstream
+import io/urlfilter
 import js/error
 import js/javascript
 import js/jstypes
 import js/regex
 import js/tojs
+import loader/headers
 import loader/loader
 import loader/request
 import local/container
@@ -38,6 +41,7 @@ import types/cell
 import types/color
 import types/cookie
 import types/opt
+import types/referrer
 import types/urimethodmap
 import types/url
 import utils/strwidth
@@ -50,8 +54,20 @@ type
     NO_LINEMODE, LOCATION, USERNAME, PASSWORD, COMMAND, BUFFER, SEARCH_F,
     SEARCH_B, ISEARCH_F, ISEARCH_B, GOTO_LINE
 
+  # fdin is the original fd; fdout may be the same, or different if mailcap
+  # is used.
+  ProcMapItem = object
+    container*: Container
+    fdin*: FileHandle
+    fdout*: FileHandle
+    istreamOutputId*: int
+    ostreamOutputId*: int
+
+  PagerAlertState = enum
+    pasNormal, pasAlertOn, pasLoadInfo
+
   Pager* = ref object
-    alerton: bool
+    alertState: PagerAlertState
     alerts: seq[string]
     askcharpromise*: Promise[string]
     askcursor: int
@@ -60,23 +76,26 @@ type
     cgiDir*: seq[string]
     commandMode {.jsget.}: bool
     config: Config
+    connectingBuffers*: seq[tuple[container: Container; stream: SocketStream]]
     container*: Container
     cookiejars: Table[string, CookieJar]
+    devRandom: PosixStream
     display: FixedGrid
-    forkserver: ForkServer
+    forkserver*: ForkServer
     inputBuffer*: string # currently uninterpreted characters
     iregex: Result[Regex, string]
     isearchpromise: EmptyPromise
     lineedit*: Option[LineEdit]
     linehist: array[LineMode, LineHistory]
     linemode: LineMode
+    loader*: FileLoader
     mailcap: Mailcap
     mimeTypes: MimeTypes
     notnum*: bool # has a non-numeric character been input already?
     numload*: int
     omnirules: seq[OmniRule]
     precnum*: int32 # current number prefix (when vi-numeric-prefix is true)
-    procmap*: Table[Pid, Container]
+    procmap*: seq[ProcMapItem]
     proxy: URL
     redraw*: bool
     regex: Opt[Regex]
@@ -85,8 +104,8 @@ type
     siteconf: seq[SiteConfig]
     statusgrid*: FixedGrid
     term*: Terminal
-    tmpdir: string
-    unreg*: seq[(Pid, PosixStream)]
+    tmpdir*: string
+    unreg*: seq[tuple[pid: int; stream: PosixStream]]
     urimethodmap: URIMethodMap
     username: string
 
@@ -94,6 +113,9 @@ jsDestructor(Pager)
 
 func attrs(pager: Pager): WindowAttributes = pager.term.attrs
 
+func loaderPid(pager: Pager): int64 {.jsfget.} =
+  int64(pager.loader.process)
+
 func getRoot(container: Container): Container =
   var c = container
   while c.parent != nil: c = c.parent
@@ -237,7 +259,7 @@ proc setPaths(pager: Pager): Err[string] =
   pager.cgiDir = cgiDir
   return ok()
 
-proc newPager*(config: Config, forkserver: ForkServer, ctx: JSContext): Pager =
+proc newPager*(config: Config; forkserver: ForkServer; ctx: JSContext): Pager =
   let (mailcap, errs) = config.getMailcap()
   let pager = Pager(
     config: config,
@@ -261,6 +283,29 @@ proc newPager*(config: Config, forkserver: ForkServer, ctx: JSContext): Pager =
     pager.alert("Error reading mailcap: " & err)
   return pager
 
+proc genClientKey(pager: Pager): ClientKey =
+  var key: ClientKey
+  let n = pager.devRandom.recvData(addr key[0], key.len)
+  doAssert n == key.len
+  return key
+
+proc addLoaderClient*(pager: Pager, pid: int, config: LoaderClientConfig):
+    ClientKey =
+  var key = pager.genClientKey()
+  while unlikely(not pager.loader.addClient(key, pid, config)):
+    key = pager.genClientKey()
+  return key
+
+proc setLoader*(pager: Pager, loader: FileLoader) =
+  pager.devRandom = newPosixStream("/dev/urandom", O_RDONLY, 0)
+  pager.loader = loader
+  let config = LoaderClientConfig(
+    defaultHeaders: pager.config.getDefaultHeaders(),
+    proxy: pager.config.getProxy(),
+    filter: newURLFilter(default = true),
+  )
+  loader.key = pager.addLoaderClient(pager.loader.clientPid, config)
+
 proc launchPager*(pager: Pager, infile: File) =
   case pager.term.start(infile)
   of tsrSuccess: discard
@@ -325,17 +370,13 @@ proc refreshStatusMsg*(pager: Pager) =
     pager.writeStatusMessage($pager.precnum & pager.inputBuffer)
   elif pager.inputBuffer != "":
     pager.writeStatusMessage(pager.inputBuffer)
-  elif container.loadinfo != "":
-    pager.alerton = true
-    pager.writeStatusMessage(container.loadinfo)
-    container.loadinfo = ""
   elif pager.alerts.len > 0:
-    pager.alerton = true
+    pager.alertState = pasAlertOn
     pager.writeStatusMessage(pager.alerts[0])
     pager.alerts.delete(0)
   else:
     var format = Format(flags: {FLAG_REVERSE})
-    pager.alerton = false
+    pager.alertState = pasNormal
     container.clearHover()
     var msg = $(container.cursory + 1) & "/" & $container.numLines & " (" &
               $container.atPercentOf() & "%)"
@@ -351,8 +392,12 @@ proc refreshStatusMsg*(pager: Pager) =
       pager.writeStatusMessage(hover2, format, tw)
 
 # Call refreshStatusMsg if no alert is being displayed on the screen.
+# Alerts take precedence over load info, but load info is preserved when no
+# pending alerts exist.
 proc showAlerts*(pager: Pager) =
-  if not pager.alerton and pager.inputBuffer == "" and pager.precnum == 0:
+  if (pager.alertState == pasNormal or
+      pager.alertState == pasLoadInfo and pager.alerts.len > 0) and
+      pager.inputBuffer == "" and pager.precnum == 0:
     pager.refreshStatusMsg()
 
 proc drawBuffer*(pager: Pager, container: Container, ostream: Stream) =
@@ -444,41 +489,90 @@ proc fulfillCharAsk*(pager: Pager, s: string) =
   pager.askcharpromise = nil
   pager.askprompt = ""
 
-proc registerContainer*(pager: Pager, container: Container) =
-  pager.procmap[container.process] = container
-
 proc addContainer*(pager: Pager, container: Container) =
   container.parent = pager.container
   if pager.container != nil:
     pager.container.children.insert(container, 0)
-  pager.registerContainer(container)
   pager.setContainer(container)
 
-proc newBuffer(pager: Pager, bufferConfig: BufferConfig, request: Request,
-    title = "", redirectdepth = 0, canreinterpret = true, fd = FileHandle(-1),
-    contentType = none(string)): Container =
-  return newBuffer(
-    pager.forkserver,
+proc onSetLoadInfo(pager: Pager; container: Container) =
+  if pager.alertState != pasAlertOn:
+    if container.loadinfo == "":
+      pager.alertState = pasNormal
+    else:
+      pager.writeStatusMessage(container.loadinfo)
+      pager.alertState = pasLoadInfo
+
+proc newContainer(pager: Pager; bufferConfig: BufferConfig; request: Request;
+    title = ""; redirectdepth = 0; canreinterpret = true;
+    contentType = none(string); charsetStack: seq[Charset] = @[];
+    url: URL = request.url; cacheId = -1): Container =
+  request.suspended = true
+  if bufferConfig.loaderConfig.cookieJar != nil:
+    # loader stores cookie jars per client, but we have no client yet.
+    # therefore we must set cookie here
+    let cookie = bufferConfig.loaderConfig.cookieJar.serialize(request.url)
+    if cookie != "":
+      request.headers["Cookie"] = cookie
+  if request.referrer != nil:
+    # same with referrer
+    let r = request.referrer.getReferrer(request.url,
+      bufferConfig.referrerPolicy)
+    if r != "":
+      request.headers["Referer"] = r
+  let stream = pager.loader.startRequest(request)
+  pager.loader.registerFun(stream.fd)
+  let container = newContainer(
     bufferConfig,
+    url,
     request,
     pager.term.attrs,
     title,
     redirectdepth,
     canreinterpret,
-    fd,
-    contentType
+    contentType,
+    charsetStack,
+    cacheId
   )
+  pager.connectingBuffers.add((container, stream))
+  pager.onSetLoadInfo(container)
+  return container
+
+proc newContainerFrom(pager: Pager; container: Container; contentType: string):
+    Container =
+  let url = newURL("cache:" & $container.cacheId).get
+  return pager.newContainer(
+    container.config,
+    newRequest(url),
+    contentType = some(contentType),
+    charsetStack = container.charsetStack,
+    url = container.url,
+    cacheId = container.cacheId
+  )
+
+func findConnectingBuffer*(pager: Pager; fd: int): int =
+  for i, (_, stream) in pager.connectingBuffers:
+    if stream.fd == fd:
+      return i
+  -1
 
-proc dupeBuffer(pager: Pager, container: Container, location: URL) =
-  container.clone(location).then(proc(container: Container) =
+proc dupeBuffer(pager: Pager, container: Container, url: URL) =
+  container.clone(url).then(proc(container: Container) =
     if container == nil:
       pager.alert("Failed to duplicate buffer.")
     else:
       pager.addContainer(container)
+      pager.procmap.add(ProcMapItem(
+        container: container,
+        fdin: -1,
+        fdout: -1,
+        istreamOutputId: -1,
+        ostreamOutputId: -1
+      ))
   )
 
 proc dupeBuffer(pager: Pager) {.jsfunc.} =
-  pager.dupeBuffer(pager.container, pager.container.location)
+  pager.dupeBuffer(pager.container, pager.container.url)
 
 # The prevBuffer and nextBuffer procedures emulate w3m's PREV and NEXT
 # commands by traversing the container tree in a depth-first order.
@@ -583,8 +677,10 @@ proc deleteContainer(pager: Pager, container: Container) =
       pager.setContainer(nil)
   container.parent = nil
   container.children.setLen(0)
-  pager.unreg.add((container.process, container.iface.stream))
-  pager.forkserver.removeChild(container.process)
+  if container.iface != nil:
+    pager.unreg.add((container.process, container.iface.stream))
+    pager.forkserver.removeChild(container.process)
+    pager.loader.removeClient(container.process)
 
 proc discardBuffer*(pager: Pager, container = none(Container)) {.jsfunc.} =
   let c = container.get(pager.container)
@@ -607,19 +703,17 @@ proc toggleSource(pager: Pager) {.jsfunc.} =
   if pager.container.sourcepair != nil:
     pager.setContainer(pager.container.sourcepair)
   else:
-    let contentType = if pager.container.ishtml:
-      "text/plain"
-    else:
+    let ishtml = not pager.container.ishtml
+    #TODO I wish I could set the contentType to whatever I wanted, not just HTML
+    let contentType = if ishtml:
       "text/html"
-    let container = newBufferFrom(
-      pager.forkserver,
-      pager.attrs,
-      pager.container,
-      contentType
-    )
-    container.sourcepair = pager.container
-    pager.container.sourcepair = container
-    pager.addContainer(container)
+    else:
+      "text/plain"
+    let container = pager.newContainerFrom(pager.container, contentType)
+    if container != nil:
+      container.sourcepair = pager.container
+      pager.container.sourcepair = container
+      pager.addContainer(container)
 
 proc windowChange*(pager: Pager) =
   let oldAttrs = pager.attrs
@@ -639,13 +733,13 @@ proc windowChange*(pager: Pager) =
 
 # Apply siteconf settings to a request.
 # Note that this may modify the URL passed.
-proc applySiteconf(pager: Pager, url: var URL): BufferConfig =
+proc applySiteconf(pager: Pager; url: var URL; cs: Charset): BufferConfig =
   let host = url.host
-  var referer_from: bool
-  var cookiejar: CookieJar
+  var referer_from = false
+  var cookieJar: CookieJar = nil
   var headers = pager.config.getDefaultHeaders()
-  var scripting: bool
-  var images: bool
+  var scripting = false
+  var images = false
   var charsets = pager.config.encoding.document_charset
   var userstyle = pager.config.css.stylesheet
   var proxy = pager.proxy
@@ -667,9 +761,9 @@ proc applySiteconf(pager: Pager, url: var URL): BufferConfig =
         if jarid notin pager.cookiejars:
           pager.cookiejars[jarid] = newCookieJar(url,
             sc.third_party_cookie)
-        cookiejar = pager.cookiejars[jarid]
+        cookieJar = pager.cookiejars[jarid]
       else:
-        cookiejar = nil # override
+        cookieJar = nil # override
     if sc.scripting.isSome:
       scripting = sc.scripting.get
     if sc.referer_from.isSome:
@@ -683,18 +777,17 @@ proc applySiteconf(pager: Pager, url: var URL): BufferConfig =
       userstyle &= sc.stylesheet.get
     if sc.proxy.isSome:
       proxy = sc.proxy.get
-  return pager.config.getBufferConfig(url, cookiejar, headers, referer_from,
+  return pager.config.getBufferConfig(url, cookieJar, headers, referer_from,
     scripting, charsets, images, userstyle, proxy, mimeTypes, urimethodmap,
-    pager.cgiDir, pager.tmpdir)
+    pager.cgiDir, pager.tmpdir, cs)
 
 # Load request in a new buffer.
 proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
     contentType = none(string), cs = CHARSET_UNKNOWN, replace: Container = nil,
     redirectdepth = 0, referrer: Container = nil) =
   if referrer != nil and referrer.config.referer_from:
-    request.referer = referrer.location
-  var bufferConfig = pager.applySiteconf(request.url)
-  bufferConfig.charsetOverride = cs
+    request.referrer = referrer.url
+  var bufferConfig = pager.applySiteconf(request.url, cs)
   if prevurl.isNone or not prevurl.get.equals(request.url, true) or
       request.url.hash == "" or request.httpMethod != HTTP_GET:
     # Basically, we want to reload the page *only* when
@@ -705,7 +798,7 @@ proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
     # feedback on what is actually going to happen when typing a URL; TODO.
     if referrer != nil:
       bufferConfig.referrerPolicy = referrer.config.referrerPolicy
-    let container = pager.newBuffer(
+    let container = pager.newContainer(
       bufferConfig,
       request,
       redirectdepth = redirectdepth,
@@ -714,7 +807,6 @@ proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
     if replace != nil:
       container.replace = replace
       container.copyCursorPos(container.replace)
-      pager.registerContainer(container)
     else:
       pager.addContainer(container)
     inc pager.numload
@@ -744,7 +836,7 @@ proc loadURL*(pager: Pager, url: string, ctype = none(string),
   let firstparse = parseURL(url)
   if firstparse.isSome:
     let prev = if pager.container != nil:
-      some(pager.container.location)
+      some(pager.container.url)
     else:
       none(URL)
     pager.gotoURL(newRequest(firstparse.get), prev, ctype, cs)
@@ -769,23 +861,23 @@ proc loadURL*(pager: Pager, url: string, ctype = none(string),
       pager.container.retry = urls
 
 proc readPipe0*(pager: Pager, contentType: string, cs: Charset,
-    fd: FileHandle, location: Option[URL], title: string,
-    canreinterpret: bool): Container =
-  var location = location.get(newURL("stream:-").get)
-  var bufferConfig = pager.applySiteconf(location)
-  bufferConfig.charsetOverride = cs
-  return pager.newBuffer(
+    fd: FileHandle, url: URL, title: string, canreinterpret: bool): Container =
+  var url = url
+  pager.loader.passFd(url.pathname, fd)
+  safeClose(fd)
+  let bufferConfig = pager.applySiteconf(url, cs)
+  return pager.newContainer(
     bufferConfig,
-    newRequest(location),
+    newRequest(url),
     title = title,
     canreinterpret = canreinterpret,
-    fd = fd,
     contentType = some(contentType)
   )
 
 proc readPipe*(pager: Pager, contentType: string, cs: Charset, fd: FileHandle,
     title: string) =
-  let container = pager.readPipe0(contentType, cs, fd, none(URL), title, true)
+  let url = newURL("stream:-").get
+  let container = pager.readPipe0(contentType, cs, fd, url, title, true)
   inc pager.numload
   pager.addContainer(container)
 
@@ -855,12 +947,12 @@ proc updateReadLine*(pager: Pager) =
         pager.username = lineedit.news
         pager.setLineEdit("Password: ", PASSWORD, hide = true)
       of PASSWORD:
-        let url = newURL(pager.container.location)
+        let url = newURL(pager.container.url)
         url.username = pager.username
         url.password = lineedit.news
         pager.username = ""
         pager.gotoURL(
-          newRequest(url), some(pager.container.location),
+          newRequest(url), some(pager.container.url),
           replace = pager.container,
           referrer = pager.container
         )
@@ -901,7 +993,7 @@ proc load(pager: Pager, s = "") {.jsfunc.} =
     if s.len > 1:
       pager.loadURL(s[0..^2])
   elif s == "":
-    pager.setLineEdit("URL: ", LOCATION, $pager.container.location)
+    pager.setLineEdit("URL: ", LOCATION, $pager.container.url)
   else:
     pager.setLineEdit("URL: ", LOCATION, s)
 
@@ -912,12 +1004,12 @@ proc jsGotoURL(pager: Pager, s: string): JSResult[void] {.jsfunc: "gotoURL".} =
 
 # Reload the page in a new buffer, then kill the previous buffer.
 proc reload(pager: Pager) {.jsfunc.} =
-  pager.gotoURL(newRequest(pager.container.location), none(URL),
+  pager.gotoURL(newRequest(pager.container.url), none(URL),
     pager.container.contentType, replace = pager.container)
 
 proc setEnvVars(pager: Pager) {.jsfunc.} =
   try:
-    putEnv("CHA_URL", $pager.container.location)
+    putEnv("CHA_URL", $pager.container.url)
     putEnv("CHA_CHARSET", $pager.container.charset)
   except OSError:
     pager.alert("Warning: failed to set some environment variables")
@@ -948,238 +1040,209 @@ proc externInto(pager: Pager, cmd, ins: string): bool {.jsfunc.} =
   pager.setEnvVars()
   return runProcessInto(cmd, ins)
 
-proc externFilterSource(pager: Pager, cmd: string, c: Container = nil,
+proc externFilterSource(pager: Pager; cmd: string; c: Container = nil;
     contentType = opt(string)) {.jsfunc.} =
-  let container = newBufferFrom(
-    pager.forkserver,
-    pager.attrs,
-    if c != nil: c else: pager.container,
-    contentType.get(pager.container.contentType.get(""))
-  )
+  let fromc = if c != nil: c else: pager.container
+  let contentType = contentType.get(pager.container.contentType.get(""))
+  let container = pager.newContainerFrom(fromc, contentType)
+  container.ishtml = contentType == "text/html"
   pager.addContainer(container)
   container.filter = BufferFilter(cmd: cmd)
 
 proc authorize(pager: Pager) =
   pager.setLineEdit("Username: ", USERNAME)
 
-type CheckMailcapResult = tuple[promise: EmptyPromise, connect: bool]
-
-proc checkMailcap(pager: Pager, container: Container,
-  contentTypeOverride = none(string)): CheckMailcapResult
+type CheckMailcapResult = object
+  fdout: int
+  ostreamOutputId: int
+  connect: bool
+  ishtml: bool
 
 # Pipe output of an x-ansioutput mailcap command to the text/x-ansi handler.
-proc ansiDecode(pager: Pager, container: Container, fdin: cint,
-    ishtml: var bool, fdout: var cint) =
-  let cs = container.charset
-  let url = container.location
-  let entry = pager.mailcap.getMailcapEntry("text/x-ansi", "", url, cs)
+proc ansiDecode(pager: Pager; url: URL; charset: Charset; ishtml: var bool;
+    fdin: cint): cint =
+  let entry = pager.mailcap.getMailcapEntry("text/x-ansi", "", url, charset)
   var canpipe = true
-  let cmd = unquoteCommand(entry.cmd, "text/x-ansi", "", url, cs, canpipe)
+  let cmd = unquoteCommand(entry.cmd, "text/x-ansi", "", url, charset, canpipe)
   if not canpipe:
     pager.alert("Error: could not pipe to text/x-ansi, decoding as text/plain")
+    return -1
+  var pipefdOutAnsi: array[2, cint]
+  if pipe(pipefdOutAnsi) == -1:
+    pager.alert("Error: failed to open pipe")
+    return
+  case fork()
+  of -1:
+    pager.alert("Error: failed to fork ANSI decoder process")
+    discard close(pipefdOutAnsi[0])
+    discard close(pipefdOutAnsi[1])
+    return -1
+  of 0: # child process
+    discard close(pipefdOutAnsi[0])
+    discard dup2(fdin, stdin.getFileHandle())
+    discard close(fdin)
+    discard dup2(pipefdOutAnsi[1], stdout.getFileHandle())
+    discard close(pipefdOutAnsi[1])
+    closeStderr()
+    myExec(cmd)
+    assert false
   else:
-    var pipefdOutAnsi: array[2, cint]
-    if pipe(pipefdOutAnsi) == -1:
-      raise newException(Defect, "Failed to open pipe.")
-    case fork()
-    of -1:
-      pager.alert("Error: failed to fork ANSI decoder process")
-      discard close(pipefdOutAnsi[0])
-      discard close(pipefdOutAnsi[1])
-    of 0: # child process
-      if fdin != -1:
-        discard close(fdin)
-      discard close(pipefdOutAnsi[0])
-      discard dup2(fdout, stdin.getFileHandle())
-      discard close(fdout)
-      discard dup2(pipefdOutAnsi[1], stdout.getFileHandle())
-      discard close(pipefdOutAnsi[1])
-      closeStderr()
-      myExec(cmd)
-      assert false
-    else:
-      discard close(pipefdOutAnsi[1])
-      discard close(fdout)
-      fdout = pipefdOutAnsi[0]
-      ishtml = HTMLOUTPUT in entry.flags
+    discard close(pipefdOutAnsi[1])
+    discard close(fdin)
+    ishtml = HTMLOUTPUT in entry.flags
+    return pipefdOutAnsi[0]
 
 # Pipe input into the mailcap command, then read its output into a buffer.
 # needsterminal is ignored.
-proc runMailcapReadPipe(pager: Pager, container: Container,
-    entry: MailcapEntry, cmd: string): CheckMailcapResult =
-  var pipefdIn: array[2, cint]
-  var pipefdOut: array[2, cint]
-  if pipe(pipefdIn) == -1 or pipe(pipefdOut) == -1:
-    raise newException(Defect, "Failed to open pipe.")
+proc runMailcapReadPipe(pager: Pager; stream: SocketStream; cmd: string;
+    pipefdOut: array[2, cint]): int =
   let pid = fork()
   if pid == -1:
-    pager.alert("Failed to fork process!")
-    return (nil, false)
-  elif pid == 0: # child process
-    discard close(pipefdIn[1])
+    pager.alert("Error: failed to fork mailcap read process")
+    return -1
+  elif pid == 0:
+    # child process
     discard close(pipefdOut[0])
-    discard dup2(pipefdIn[0], stdin.getFileHandle())
+    discard dup2(stream.fd, stdin.getFileHandle())
+    stream.close()
     discard dup2(pipefdOut[1], stdout.getFileHandle())
     closeStderr()
-    discard close(pipefdIn[0])
     discard close(pipefdOut[1])
     myExec(cmd)
-    assert false
-  else:
-    # parent
-    discard close(pipefdIn[0])
-    discard close(pipefdOut[1])
-    let fdin = pipefdIn[1]
-    var fdout = pipefdOut[0]
-    var ishtml = HTMLOUTPUT in entry.flags
-    if not ishtml and ANSIOUTPUT in entry.flags:
-      # decode ANSI sequence
-      pager.ansiDecode(container, fdin, ishtml, fdout)
-    let p = container.redirectToFd(fdin, wait = false, cache = true)
-    discard close(fdin)
-    let p2 = p.then(proc(): auto =
-      let p = container.readFromFd(fdout, $pid, ishtml)
-      discard close(fdout)
-      return p
-    )
-    return (p2, true)
+    doAssert false
+  # parent
+  pid
 
 # Pipe input into the mailcap command, and discard its output.
 # If needsterminal, leave stderr and stdout open and wait for the process.
-proc runMailcapWritePipe(pager: Pager, container: Container,
-    entry: MailcapEntry, cmd: string): CheckMailcapResult =
-  let needsterminal = NEEDSTERMINAL in entry.flags
-  var pipefd: array[2, cint]
-  if pipe(pipefd) == -1:
-    raise newException(Defect, "Failed to open pipe.")
+proc runMailcapWritePipe(pager: Pager; stream: SocketStream;
+    needsterminal: bool; cmd: string) =
   if needsterminal:
     pager.term.quit()
   let pid = fork()
   if pid == -1:
-    return (nil, false)
+    pager.alert("Error: failed to fork mailcap write process")
   elif pid == 0:
     # child process
-    discard close(pipefd[1])
-    discard dup2(pipefd[0], stdin.getFileHandle())
+    discard dup2(stream.fd, stdin.getFileHandle())
+    stream.close()
     if not needsterminal:
       closeStdout()
       closeStderr()
-    discard close(pipefd[0])
     myExec(cmd)
-    assert false
+    doAssert false
   else:
     # parent
-    discard close(pipefd[0])
-    let fd = pipefd[1]
-    let p = container.redirectToFd(fd, wait = true, cache = false)
-    discard close(fd)
+    stream.close()
     if needsterminal:
       var x: cint
       discard waitpid(pid, x, 0)
       pager.term.restart()
-    return (p, false)
+
+proc writeToFile(istream: SocketStream; outpath: string): bool =
+  let ps = newPosixStream(outpath, O_WRONLY or O_CREAT, 0o600)
+  if ps == nil:
+    return false
+  var buffer: array[4096, uint8]
+  while true:
+    let n = istream.recvData(buffer)
+    if n == 0:
+      break
+    if ps.sendData(buffer.toOpenArray(0, n - 1)) < n:
+      ps.close()
+      return false
+    if n < buffer.len:
+      break
+  ps.close()
+  true
 
 # Save input in a file, run the command, and redirect its output to a
 # new buffer.
 # needsterminal is ignored.
-proc runMailcapReadFile(pager: Pager, container: Container,
-    entry: MailcapEntry, cmd, outpath: string): CheckMailcapResult =
-  let fd = open(outpath, O_WRONLY or O_CREAT, 0o600)
-  if fd == -1:
-    return (nil, false)
-  let p = container.redirectToFd(fd, wait = true, cache = true).then(proc():
-      auto =
-    var pipefd: array[2, cint] # redirect stdout here
-    if pipe(pipefd) == -1:
-      raise newException(Defect, "Failed to open pipe.")
+proc runMailcapReadFile(pager: Pager; stream: SocketStream;
+    cmd, outpath: string; pipefdOut: array[2, cint]): int =
+  let pid = fork()
+  if pid == 0:
+    # child process
+    discard close(pipefdOut[0])
+    discard dup2(pipefdOut[1], stdout.getFileHandle())
+    discard close(pipefdOut[1])
+    closeStderr()
+    if not stream.writeToFile(outpath):
+      #TODO print error message
+      quit(1)
+    stream.close()
+    let ret = execCmd(cmd)
+    discard tryRemoveFile(outpath)
+    quit(ret)
+  # parent
+  pid
+
+# Save input in a file, run the command, and discard its output.
+# If needsterminal, leave stderr and stdout open and wait for the process.
+proc runMailcapWriteFile(pager: Pager; stream: SocketStream;
+    needsterminal: bool; cmd, outpath: string) =
+  if needsterminal:
+    pager.term.quit()
+    if not stream.writeToFile(outpath):
+      pager.term.restart()
+      pager.alert("Error: failed to write file for mailcap process")
+    else:
+      discard execCmd(cmd)
+      discard tryRemoveFile(outpath)
+      pager.term.restart()
+  else:
+    # don't block
     let pid = fork()
     if pid == 0:
       # child process
-      discard close(pipefd[0])
-      discard dup2(pipefd[1], stdout.getFileHandle())
-      discard close(pipefd[1])
+      closeStdin()
+      closeStdout()
       closeStderr()
+      if not stream.writeToFile(outpath):
+        #TODO print error message (maybe in parent?)
+        quit(1)
+      stream.close()
       let ret = execCmd(cmd)
       discard tryRemoveFile(outpath)
       quit(ret)
     # parent
-    discard close(pipefd[1])
-    var fdout = pipefd[0]
-    var ishtml = HTMLOUTPUT in entry.flags
-    if not ishtml and ANSIOUTPUT in entry.flags:
-      pager.ansiDecode(container, -1, ishtml, fdout)
-    let p = container.readFromFd(fdout, $pid, ishtml)
-    discard close(fdout)
-    return p
-  )
-  return (p, true)
+    stream.close()
 
-# Save input in a file, run the command, and discard its output.
-# If needsterminal, leave stderr and stdout open and wait for the process.
-proc runMailcapWriteFile(pager: Pager, container: Container,
-    entry: MailcapEntry, cmd, outpath: string): CheckMailcapResult =
-  let needsterminal = NEEDSTERMINAL in entry.flags
-  let fd = open(outpath, O_WRONLY or O_CREAT, 0o600)
-  if fd == -1:
-    return (nil, false)
-  let p = container.redirectToFd(fd, wait = true, cache = false).then(proc() =
-    if needsterminal:
-      pager.term.quit()
-      discard execCmd(cmd)
-      discard tryRemoveFile(outpath)
-      pager.term.restart()
-    else:
-      # don't block
-      let pid = fork()
-      if pid == 0:
-        # child process
-        closeStdin()
-        closeStdout()
-        closeStderr()
-        let ret = execCmd(cmd)
-        discard tryRemoveFile(outpath)
-        quit(ret)
-  )
-  return (p, false)
-
-proc filterBuffer(pager: Pager, container: Container): CheckMailcapResult =
+proc filterBuffer(pager: Pager; stream: SocketStream; cmd: string;
+    ishtml: bool): CheckMailcapResult =
   pager.setEnvVars()
-  let cmd = container.filter.cmd
-  var pipefd_in: array[2, cint]
-  if pipe(pipefd_in) == -1:
-    raise newException(Defect, "Failed to open pipe.")
   var pipefd_out: array[2, cint]
   if pipe(pipefd_out) == -1:
-    raise newException(Defect, "Failed to open pipe.")
+    pager.alert("Error: failed to open pipe")
+    return CheckMailcapResult(connect: false, fdout: -1)
   let pid = fork()
   if pid == -1:
-    return (nil, true)
+    pager.alert("Error: failed to fork buffer filter process")
+    return CheckMailcapResult(connect: false, fdout: -1)
   elif pid == 0:
     # child
-    discard close(pipefd_in[1])
     discard close(pipefd_out[0])
-    stdout.flushFile()
-    discard dup2(pipefd_in[0], stdin.getFileHandle())
+    discard dup2(stream.fd, stdin.getFileHandle())
+    stream.close()
     discard dup2(pipefd_out[1], stdout.getFileHandle())
     closeStderr()
-    discard close(pipefd_in[0])
     discard close(pipefd_out[1])
     myExec(cmd)
-    assert false
-  else:
-    # parent
-    discard close(pipefd_in[0])
-    discard close(pipefd_out[1])
-    let fdin = pipefd_in[1]
-    let fdout = pipefd_out[0]
-    let p = container.redirectToFd(fdin, wait = false, cache = false)
-    let p2 = p.then(proc(): auto =
-      discard close(fdin)
-      return container.readFromFd(fdout, $pid, container.ishtml)
-    ).then(proc() =
-      discard close(fdout)
-    )
-    return (p2, true)
+    doAssert false
+  # parent
+  discard close(pipefd_out[1])
+  let fdout = pipefd_out[0]
+  let url = parseURL("stream:" & $pid).get
+  pager.loader.passFd(url.pathname, FileHandle(fdout))
+  safeClose(fdout)
+  let response = pager.loader.doRequest(newRequest(url, suspended = true))
+  return CheckMailcapResult(
+    connect: true,
+    fdout: response.body.fd,
+    ostreamOutputId: response.outputId,
+    ishtml: ishtml
+  )
 
 # Search for a mailcap entry, and if found, execute the specified command
 # and pipeline the input and output appropriately.
@@ -1190,129 +1253,194 @@ proc filterBuffer(pager: Pager, container: Container): CheckMailcapResult =
 # * write to file, run, read stdout
 # If needsterminal is specified, and stdout is not being read, then the
 # pager is suspended until the command exits.
-#TODO add support for edit/compose, better error handling (use Promise[bool]
-# instead of tuple[EmptyPromise, bool])
-proc checkMailcap(pager: Pager, container: Container,
-    contentTypeOverride = none(string)): CheckMailcapResult =
+#TODO add support for edit/compose, better error handling
+proc checkMailcap(pager: Pager; container: Container; stream: SocketStream;
+    istreamOutputId: int): CheckMailcapResult =
   if container.filter != nil:
-    return pager.filterBuffer(container)
-  if container.contentType.isNone:
-    return (nil, true)
-  let contentType = contentTypeOverride.get(container.contentType.get)
+    return pager.filterBuffer(stream, container.filter.cmd, container.ishtml)
+  # contentType must exist, because we set it in applyResponse
+  let contentType = container.contentType.get
   if contentType == "text/html":
-    # We support HTML natively, so it would make little sense to execute
+    # We support text/html natively, so it would make little sense to execute
     # mailcap filters for it.
-    return (nil, true)
-  elif contentType == "text/plain":
-    # This could potentially be useful. Unfortunately, many mailcaps include
-    # a text/plain entry with less by default, so it's probably better to
-    # ignore this.
-    return (nil, true)
+    return CheckMailcapResult(connect: true, fdout: stream.fd, ishtml: true)
+  if contentType == "text/plain":
+    # text/plain could potentially be useful. Unfortunately, many mailcaps
+    # include a text/plain entry with less by default, so it's probably better
+    # to ignore this.
+    return CheckMailcapResult(connect: true, fdout: stream.fd)
   #TODO callback for outpath or something
-  let url = container.location
+  let url = container.url
   let cs = container.charset
   let entry = pager.mailcap.getMailcapEntry(contentType, "", url, cs)
-  if entry != nil:
-    let tmpdir = pager.tmpdir
-    let ext = container.location.pathname.afterLast('.')
-    let tempfile = getTempFile(tmpdir, ext)
-    let outpath = if entry.nametemplate != "":
-      unquoteCommand(entry.nametemplate, contentType, tempfile, url, cs)
-    else:
-      tempfile
-    var canpipe = true
-    let cmd = unquoteCommand(entry.cmd, contentType, outpath, url, cs, canpipe)
-    putEnv("MAILCAP_URL", $url) #TODO delEnv this after command is finished?
-    if {COPIOUSOUTPUT, HTMLOUTPUT, ANSIOUTPUT} * entry.flags == {}:
-      # no output.
+  if entry == nil:
+    return CheckMailcapResult(connect: true, fdout: stream.fd)
+  let tmpdir = pager.tmpdir
+  let ext = url.pathname.afterLast('.')
+  let tempfile = getTempFile(tmpdir, ext)
+  let outpath = if entry.nametemplate != "":
+    unquoteCommand(entry.nametemplate, contentType, tempfile, url, cs)
+  else:
+    tempfile
+  var canpipe = true
+  let cmd = unquoteCommand(entry.cmd, contentType, outpath, url, cs, canpipe)
+  var ishtml = HTMLOUTPUT in entry.flags
+  let needsterminal = NEEDSTERMINAL in entry.flags
+  putEnv("MAILCAP_URL", $url)
+  block needsConnect:
+    if entry.flags * {COPIOUSOUTPUT, HTMLOUTPUT, ANSIOUTPUT} == {}:
+      # No output. Resume here, so that blocking needsterminal filters work.
+      pager.loader.resume(@[istreamOutputId])
       if canpipe:
-        return pager.runMailcapWritePipe(container, entry[], cmd)
+        pager.runMailcapWritePipe(stream, needsterminal, cmd)
       else:
-        return pager.runMailcapWriteFile(container, entry[], cmd, outpath)
+        pager.runMailcapWriteFile(stream, needsterminal, cmd, outpath)
+      # stream is already closed
+      break needsConnect # never connect here, since there's no output
+    var pipefdOut: array[2, cint]
+    if pipe(pipefdOut) == -1:
+      pager.alert("Error: failed to open pipe")
+      stream.close() # connect: false implies that we consumed the stream
+      break needsConnect
+    let pid = if canpipe:
+      pager.runMailcapReadPipe(stream, cmd, pipefdOut)
     else:
-      if canpipe:
-        return pager.runMailcapReadPipe(container, entry[], cmd)
-      else:
-        return pager.runMailcapReadFile(container, entry[], cmd, outpath)
-  return (nil, true)
+      pager.runMailcapReadFile(stream, cmd, outpath, pipefdOut)
+    discard close(pipefdOut[1]) # close write
+    let fdout = if not ishtml and ANSIOUTPUT in entry.flags:
+      pager.ansiDecode(url, cs, ishtml, pipefdOut[0])
+    else:
+      pipefdOut[0]
+    delEnv("MAILCAP_URL")
+    let url = parseURL("stream:" & $pid).get
+    pager.loader.passFd(url.pathname, FileHandle(fdout))
+    safeClose(cint(fdout))
+    let response = pager.loader.doRequest(newRequest(url, suspended = true))
+    return CheckMailcapResult(
+      connect: true,
+      fdout: response.body.fd,
+      ostreamOutputId: response.outputId,
+      ishtml: ishtml
+    )
+  delEnv("MAILCAP_URL")
+  return CheckMailcapResult(connect: false, fdout: -1)
 
-proc redirectTo(pager: Pager, container: Container, request: Request) =
+proc redirectTo(pager: Pager; container: Container; request: Request) =
   pager.alert("Redirecting to " & $request.url)
-  pager.gotoURL(request, some(container.location), replace = container,
+  pager.gotoURL(request, some(container.url), replace = container,
     redirectdepth = container.redirectdepth + 1, referrer = container)
 
-proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bool =
-  case event.t
-  of FAIL:
-    dec pager.numload
-    pager.deleteContainer(container)
-    if container.retry.len > 0:
-      pager.gotoURL(newRequest(container.retry.pop()),
-        contentType = container.contentType)
+proc fail*(pager: Pager; container: Container; errorMessage: string) =
+  dec pager.numload
+  pager.deleteContainer(container)
+  if container.retry.len > 0:
+    pager.gotoURL(newRequest(container.retry.pop()),
+      contentType = container.contentType)
+  else:
+    pager.alert("Can't load " & $container.url & " (" & errorMessage & ")")
+
+proc redirect*(pager: Pager; container: Container; response: Response) =
+  # still need to apply response, or we lose cookie jars.
+  container.applyResponse(response)
+  let request = response.redirect
+  if container.redirectdepth < pager.config.network.max_redirect:
+    if container.url.scheme == request.url.scheme or
+        container.url.scheme == "cgi-bin" or
+        container.url.scheme == "http" and request.url.scheme == "https" or
+        container.url.scheme == "https" and request.url.scheme == "http":
+      pager.redirectTo(container, request)
     else:
-      pager.alert("Can't load " & $container.location & " (" &
-        container.errorMessage & ")")
-    return false
-  of SUCCESS:
+      let url = request.url
+      pager.ask("Warning: switch protocols? " & $url).then(proc(x: bool) =
+        if x:
+          pager.redirectTo(container, request)
+      )
+  else:
+    pager.alert("Error: maximum redirection depth reached")
+    pager.deleteContainer(container)
+
+proc replace(pager: Pager; container: Container) =
+  let n = container.replace.children.find(container)
+  if n != -1:
+    container.replace.children.delete(n)
+    container.parent = nil
+  let n2 = container.children.find(container.replace)
+  if n2 != -1:
+    container.children.delete(n2)
+    container.replace.parent = nil
+  container.children.add(container.replace.children)
+  for child in container.children:
+    child.parent = container
+  container.replace.children.setLen(0)
+  if container.replace.parent != nil:
+    container.parent = container.replace.parent
+    let n = container.replace.parent.children.find(container.replace)
+    assert n != -1, "Container not a child of its parent"
+    container.parent.children[n] = container
+    container.replace.parent = nil
+  if pager.container == container.replace:
+    pager.setContainer(container)
+  pager.deleteContainer(container.replace)
+  container.replace = nil
+
+proc connected*(pager: Pager; container: Container; response: Response) =
+  let istream = response.body
+  container.applyResponse(response)
+  if response.status == 401: # unauthorized
+    pager.authorize()
+    istream.close()
+    return
+  let mailcapRes = pager.checkMailcap(container, istream, response.outputId)
+  if mailcapRes.connect:
+    container.ishtml = mailcapRes.ishtml
+    container.applyResponse(response)
+    # buffer now actually exists; create a process for it
+    container.process = pager.forkserver.forkBuffer(
+      container.config,
+      container.url,
+      container.request,
+      pager.attrs,
+      container.ishtml,
+      container.charsetStack
+    )
+    if mailcapRes.fdout != istream.fd:
+      # istream has been redirected into a filter
+      istream.close()
+    pager.procmap.add(ProcMapItem(
+      container: container, 
+      fdout: FileHandle(mailcapRes.fdout),
+      fdin: FileHandle(istream.fd),
+      ostreamOutputId: mailcapRes.ostreamOutputId,
+      istreamOutputId: response.outputId
+    ))
     if container.replace != nil:
-      let n = container.replace.children.find(container)
-      if n != -1:
-        container.replace.children.delete(n)
-        container.parent = nil
-      let n2 = container.children.find(container.replace)
-      if n2 != -1:
-        container.children.delete(n2)
-        container.replace.parent = nil
-      container.children.add(container.replace.children)
-      for child in container.children:
-        child.parent = container
-      container.replace.children.setLen(0)
-      if container.replace.parent != nil:
-        container.parent = container.replace.parent
-        let n = container.replace.parent.children.find(container.replace)
-        assert n != -1, "Container not a child of its parent"
-        container.parent.children[n] = container
-        container.replace.parent = nil
-      if pager.container == container.replace:
-        pager.setContainer(container)
-      pager.deleteContainer(container.replace)
-      container.replace = nil
-  of LOADED:
+      pager.replace(container)
+  else:
     dec pager.numload
-  of NEEDS_AUTH:
-    if pager.container == container:
-      pager.authorize()
-  of REDIRECT:
-    if container.redirectdepth < pager.config.network.max_redirect:
-      let url = event.request.url
-      if container.location.scheme == url.scheme or
-          container.location.scheme == "cgi-bin" or
-          container.location.scheme == "http" and url.scheme == "https" or
-          container.location.scheme == "https" and url.scheme == "http":
-        pager.redirectTo(container, event.request)
-      else:
-        pager.ask("Warning: switch protocols? " & $url).then(proc(x: bool) =
-          if x:
-            pager.redirectTo(container, event.request)
-        )
-    else:
-      pager.alert("Error: maximum redirection depth reached")
-      pager.deleteContainer(container)
-      return false
-  of ANCHOR:
-    let url2 = newURL(container.location)
+    pager.deleteContainer(container)
+    pager.redraw = true
+    pager.refreshStatusMsg()
+
+proc handleEvent0(pager: Pager; container: Container; event: ContainerEvent):
+    bool =
+  case event.t
+  of cetLoaded:
+    dec pager.numload
+  of cetAnchor:
+    let url2 = newURL(container.url)
     url2.setHash(event.anchor)
     pager.dupeBuffer(container, url2)
-  of NO_ANCHOR:
+  of cetNoAnchor:
     pager.alert("Couldn't find anchor " & event.anchor)
-  of UPDATE:
+  of cetUpdate:
     if container == pager.container:
       pager.redraw = true
-      if event.force: pager.term.clearCanvas()
-  of READ_LINE:
+      if event.force:
+        pager.term.clearCanvas()
+  of cetReadLine:
     if container == pager.container:
       pager.setLineEdit("(BUFFER) " & event.prompt, BUFFER, event.value, hide = event.password)
-  of READ_AREA:
+  of cetReadArea:
     if container == pager.container:
       var s = event.tvalue
       if openInEditor(pager.term, pager.config, pager.tmpdir, s):
@@ -1320,44 +1448,30 @@ proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bo
       else:
         pager.container.readCanceled()
       pager.redraw = true
-  of OPEN:
+  of cetOpen:
     let url = event.request.url
     if pager.container == nil or not pager.container.isHoverURL(url):
       pager.ask("Open pop-up? " & $url).then(proc(x: bool) =
         if x:
-          pager.gotoURL(event.request, some(container.location),
+          pager.gotoURL(event.request, some(container.url),
             referrer = pager.container)
       )
     else:
-      pager.gotoURL(event.request, some(container.location),
+      pager.gotoURL(event.request, some(container.url),
         referrer = pager.container)
-  of INVALID_COMMAND: discard
-  of STATUS:
+  of cetStatus:
     if pager.container == container:
       pager.refreshStatusMsg()
-  of TITLE:
+  of cetSetLoadInfo:
+    if pager.container == container:
+      pager.onSetLoadInfo(container)
+  of cetTitle:
     if pager.container == container:
       pager.showAlerts()
       pager.term.setTitle(container.getTitle())
-  of ALERT:
+  of cetAlert:
     if pager.container == container:
       pager.alert(event.msg)
-  of CHECK_MAILCAP:
-    var (cm, connect) = pager.checkMailcap(container)
-    if cm == nil:
-      cm = container.connect2()
-    if connect:
-      cm.then(proc() =
-        container.startload())
-    else:
-      # remove "connecting..." message left by connect2
-      pager.refreshStatusMsg()
-      cm.then(proc(): auto =
-        container.quit())
-  of QUIT:
-    dec pager.numload
-    pager.deleteContainer(container)
-    return false
   return true
 
 proc handleEvents*(pager: Pager, container: Container) =