about summary refs log tree commit diff stats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/config/config.nim6
-rw-r--r--src/extern/tempfile.nim2
-rw-r--r--src/html/chadombuilder.nim64
-rw-r--r--src/io/posixstream.nim6
-rw-r--r--src/io/serialize.nim37
-rw-r--r--src/loader/connecterror.nim2
-rw-r--r--src/loader/loader.nim238
-rw-r--r--src/loader/loaderhandle.nim21
-rw-r--r--src/loader/request.nim11
-rw-r--r--src/loader/response.nim7
-rw-r--r--src/loader/streamid.nim7
-rw-r--r--src/local/client.nim3
-rw-r--r--src/local/container.nim37
-rw-r--r--src/local/pager.nim46
-rw-r--r--src/render/rendertext.nim11
-rw-r--r--src/server/buffer.nim211
-rw-r--r--src/server/forkserver.nim60
-rw-r--r--src/types/buffersource.nim12
18 files changed, 443 insertions, 338 deletions
diff --git a/src/config/config.nim b/src/config/config.nim
index 0305d7c7..c66e7883 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -226,7 +226,8 @@ func getDefaultHeaders*(config: Config): Headers =
 proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar,
     headers: Headers, referer_from, scripting: bool, charsets: seq[Charset],
     images: bool, userstyle: string, proxy: URL, mimeTypes: MimeTypes,
-    urimethodmap: URIMethodMap, cgiDir: seq[string]): BufferConfig =
+    urimethodmap: URIMethodMap, cgiDir: seq[string], tmpdir: string):
+    BufferConfig =
   let filter = newURLFilter(
     scheme = some(location.scheme),
     allowschemes = @["data", "stream"],
@@ -247,7 +248,8 @@ proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar,
       cgiDir: cgiDir,
       urimethodmap: urimethodmap,
       w3mCGICompat: config.external.w3m_cgi_compat,
-      libexecPath: ChaPath("${%CHA_LIBEXEC_DIR}").unquote().get
+      libexecPath: ChaPath("${%CHA_LIBEXEC_DIR}").unquote().get,
+      tmpdir: tmpdir
     )
   )
 
diff --git a/src/extern/tempfile.nim b/src/extern/tempfile.nim
index 75c09835..5968270b 100644
--- a/src/extern/tempfile.nim
+++ b/src/extern/tempfile.nim
@@ -4,7 +4,7 @@ var tmpf_seq: int
 proc getTempFile*(tmpdir: string, ext = ""): string =
   if not dirExists(tmpdir):
     createDir(tmpdir)
-  var tmpf = tmpdir / "chatmp" & $tmpf_seq
+  var tmpf = tmpdir / "chatmp" & $getCurrentProcessId() & "-" & $tmpf_seq
   if ext != "":
     tmpf &= "."
     tmpf &= ext
diff --git a/src/html/chadombuilder.nim b/src/html/chadombuilder.nim
index 044d8643..d604f455 100644
--- a/src/html/chadombuilder.nim
+++ b/src/html/chadombuilder.nim
@@ -29,12 +29,15 @@ type
     seekable: bool
     builder*: ChaDOMBuilder
     opts: HTML5ParserOpts[Node, CAtom]
-    inputStream: Stream
+    stream: StringStream
     encoder: EncoderStream
     decoder: DecoderStream
+    rewindImpl: proc()
     # hack so we don't have to worry about leaks or the GC deallocating parser
     refs: seq[Document]
     stoppedFromScript: bool
+    needsBOMSniff: bool
+    wasICE: bool # inhibitCheckEnd
 
   ChaDOMBuilder = ref object of DOMBuilder[Node, CAtom]
     charset: Charset
@@ -260,17 +263,21 @@ proc parseHTMLFragment*(element: Element, s: string): seq[Node] =
   builder.finish()
   return root.childList
 
-#TODO this should be handled by decoderstream
-proc bomSniff(inputStream: Stream): Charset =
-  let bom = inputStream.readStr(2)
+#TODO this should be handled by decoderstream or buffer
+proc bomSniff(wrapper: HTML5ParserWrapper): Charset =
+  let stream = wrapper.stream
+  let op = stream.getPosition()
+  if op + 2 >= stream.data.len:
+    return CHARSET_UNKNOWN
+  let bom = stream.readStr(2)
   if bom == "\xFE\xFF":
     return CHARSET_UTF_16_BE
   if bom == "\xFF\xFE":
     return CHARSET_UTF_16_LE
   if bom == "\xEF\xBB":
-    if inputStream.readChar() == '\xBF':
+    if op + 3 < stream.data.len and stream.readChar() == '\xBF':
       return CHARSET_UTF_8
-  inputStream.setPosition(0)
+  wrapper.stream.setPosition(op)
   return CHARSET_UNKNOWN
 
 proc switchCharset(wrapper: HTML5ParserWrapper) =
@@ -284,16 +291,18 @@ proc switchCharset(wrapper: HTML5ParserWrapper) =
     DECODER_ERROR_MODE_REPLACEMENT
   else:
     DECODER_ERROR_MODE_FATAL
+  let ice = wrapper.decoder == nil or wrapper.wasICE
   wrapper.parser = initHTML5Parser(builder, wrapper.opts)
-  wrapper.decoder = newDecoderStream(wrapper.inputStream, builder.charset,
+  wrapper.decoder = newDecoderStream(wrapper.stream, builder.charset,
     errormode = em)
-  wrapper.decoder.setInhibitCheckEnd(true)
+  wrapper.decoder.setInhibitCheckEnd(ice)
+  wrapper.wasICE = ice
   wrapper.encoder = newEncoderStream(wrapper.decoder, CHARSET_UTF_8,
     errormode = ENCODER_ERROR_MODE_FATAL)
 
-proc newHTML5ParserWrapper*(inputStream: Stream, window: Window, url: URL,
-    factory: CAtomFactory, charsets: seq[Charset] = @[], seekable = true):
-    HTML5ParserWrapper =
+proc newHTML5ParserWrapper*(stream: StringStream, window: Window, url: URL,
+    factory: CAtomFactory, rewindImpl: proc(), charsets: seq[Charset],
+    seekable: bool): HTML5ParserWrapper =
   let opts = HTML5ParserOpts[Node, CAtom](
     isIframeSrcdoc: false, #TODO?
     scripting: window != nil and window.settings.scripting
@@ -303,14 +312,12 @@ proc newHTML5ParserWrapper*(inputStream: Stream, window: Window, url: URL,
     seekable: seekable,
     builder: builder,
     opts: opts,
-    inputStream: inputStream
+    stream: stream,
+    rewindImpl: rewindImpl,
+    needsBOMSniff: seekable
   )
   builder.document.setActiveParser(wrapper)
-  if seekable and (let scs = inputStream.bomSniff(); scs != CHARSET_UNKNOWN):
-    builder.confidence = ccCertain
-    wrapper.charsetStack = @[scs]
-    wrapper.seekable = false
-  elif charsets.len == 0:
+  if charsets.len == 0:
     wrapper.charsetStack = @[DefaultCharset] # UTF-8
   else:
     for i in countdown(charsets.high, 0):
@@ -385,13 +392,23 @@ proc CDB_parseDocumentWriteChunk(wrapper: pointer) {.exportc.} =
 
 proc parseAll*(wrapper: HTML5ParserWrapper) =
   let builder = wrapper.builder
+  if wrapper.needsBOMSniff:
+    if wrapper.stream.getPosition() + 3 >= wrapper.stream.data.len:
+      return
+    let scs = wrapper.bomSniff()
+    if scs != CHARSET_UNKNOWN:
+      builder.confidence = ccCertain
+      wrapper.charsetStack = @[scs]
+      wrapper.seekable = false
+      wrapper.switchCharset()
+    wrapper.needsBOMSniff = false
   while true:
     let buffer = wrapper.encoder.readAll()
     if wrapper.decoder.failed:
       assert wrapper.seekable
       # Retry with another charset.
       builder.restart(wrapper)
-      wrapper.inputStream.setPosition(0)
+      wrapper.rewindImpl()
       wrapper.switchCharset()
       continue
     if buffer.len == 0:
@@ -402,13 +419,22 @@ proc parseAll*(wrapper: HTML5ParserWrapper) =
     # res == PRES_STOP: A meta tag describing the charset has been found; force
     # use of this charset.
     builder.restart(wrapper)
-    wrapper.inputStream.setPosition(0)
+    wrapper.rewindImpl()
     wrapper.charsetStack.add(builder.charset)
     wrapper.seekable = false
     wrapper.switchCharset()
 
 proc finish*(wrapper: HTML5ParserWrapper) =
+  if wrapper.needsBOMSniff:
+    let scs = wrapper.bomSniff()
+    if scs != CHARSET_UNKNOWN:
+      wrapper.builder.confidence = ccCertain
+      wrapper.charsetStack = @[scs]
+      wrapper.seekable = false
+      wrapper.switchCharset()
+    wrapper.needsBOMSniff = false
   wrapper.decoder.setInhibitCheckEnd(false)
+  wrapper.wasICE = false
   wrapper.parseAll()
   wrapper.parser.finish()
   wrapper.builder.finish()
diff --git a/src/io/posixstream.nim b/src/io/posixstream.nim
index 0cc6ff73..1b615dce 100644
--- a/src/io/posixstream.nim
+++ b/src/io/posixstream.nim
@@ -104,3 +104,9 @@ proc newPosixStream*(fd: FileHandle): PosixStream =
     writeDataImpl: psWriteData,
     atEndImpl: psAtEnd
   )
+
+proc newPosixStream*(path: string, flags, mode: cint): PosixStream =
+  let fd = open(cstring(path), flags, mode)
+  if fd == -1:
+    return nil
+  return newPosixStream(fd)
diff --git a/src/io/serialize.nim b/src/io/serialize.nim
index f27fb40d..2e2480e6 100644
--- a/src/io/serialize.nim
+++ b/src/io/serialize.nim
@@ -5,9 +5,7 @@ import std/sets
 import std/streams
 import std/tables
 
-import loader/request
 import types/blob
-import types/buffersource
 import types/formdata
 import types/url
 import types/opt
@@ -72,10 +70,6 @@ proc swrite*[T, E](stream: Stream, o: Result[T, E])
 proc sread*[T, E](stream: Stream, o: var Result[T, E])
 func slen*[T, E](o: Result[T, E]): int
 
-proc swrite*(stream: Stream, source: BufferSource)
-proc sread*(stream: Stream, source: var BufferSource)
-func slen*(source: BufferSource): int
-
 proc swrite*(stream: Stream, n: SomeNumber) =
   stream.write(n)
 
@@ -383,34 +377,3 @@ func slen*[T, E](o: Result[T, E]): int =
   else:
     when not (E is void):
       result += slen(o.error)
-
-proc swrite*(stream: Stream, source: BufferSource) =
-  stream.swrite(source.t)
-  case source.t
-  of CLONE: stream.swrite(source.clonepid)
-  of LOAD_REQUEST: stream.swrite(source.request)
-  stream.swrite(source.location)
-  stream.swrite(source.contentType)
-  stream.swrite(source.charset)
-
-proc sread*(stream: Stream, source: var BufferSource) =
-  var t: BufferSourceType
-  stream.sread(t)
-  case t
-  of CLONE:
-    source = BufferSource(t: CLONE)
-    stream.sread(source.clonepid)
-  of LOAD_REQUEST:
-    source = BufferSource(t: LOAD_REQUEST)
-    stream.sread(source.request)
-  stream.sread(source.location)
-  stream.sread(source.contentType)
-  stream.sread(source.charset)
-
-func slen*(source: BufferSource): int =
-  result += slen(source.t)
-  case source.t
-  of CLONE: result += slen(source.clonepid)
-  of LOAD_REQUEST: result += slen(source.request)
-  result += slen(source.location)
-  result += slen(source.contentType)
diff --git a/src/loader/connecterror.nim b/src/loader/connecterror.nim
index 1b529128..08f7b436 100644
--- a/src/loader/connecterror.nim
+++ b/src/loader/connecterror.nim
@@ -1,4 +1,6 @@
 type ConnectErrorCode* = enum
+  ERROR_URL_NOT_IN_CACHE = (-16, "URL was not found in the cache")
+  ERROR_FILE_NOT_IN_CACHE = (-15, "file was not found in the cache")
   ERROR_FAILED_TO_EXECUTE_CGI_SCRIPT = (-14, "failed to execute CGI script")
   ERROR_CGI_NO_DATA = (-13, "CGI script returned no data")
   ERROR_CGI_MALFORMED_HEADER = (-12, "CGI script returned a malformed header")
diff --git a/src/loader/loader.nim b/src/loader/loader.nim
index 95d119e2..0755d427 100644
--- a/src/loader/loader.nim
+++ b/src/loader/loader.nim
@@ -28,6 +28,7 @@ import std/streams
 import std/strutils
 import std/tables
 
+import extern/tempfile
 import io/posixstream
 import io/promise
 import io/serialize
@@ -42,6 +43,7 @@ import loader/headers
 import loader/loaderhandle
 import loader/request
 import loader/response
+import loader/streamid
 import types/cookie
 import types/referer
 import types/urimethodmap
@@ -57,6 +59,7 @@ export response
 type
   FileLoader* = ref object
     process*: Pid
+    clientPid*: int
     connecting*: Table[int, ConnectData]
     ongoing*: Table[int, OngoingData]
     unregistered*: seq[int]
@@ -65,7 +68,7 @@ type
 
   ConnectData = object
     promise: Promise[JSResult[Response]]
-    stream: Stream
+    stream: SocketStream
     request: Request
 
   OngoingData = object
@@ -78,13 +81,12 @@ type
     TEE
     SUSPEND
     RESUME
+    REWIND
     ADDREF
     UNREF
     SET_REFERRER_POLICY
     PASS_FD
 
-  ClientFdMap = seq[tuple[pid, fd: int, output: OutputHandle]]
-
   LoaderContext = ref object
     refcount: int
     ssock: ServerSocket
@@ -92,10 +94,11 @@ type
     config: LoaderConfig
     handleMap: Table[int, LoaderHandle]
     outputMap: Table[int, OutputHandle]
-    clientFdMap: ClientFdMap
     referrerpolicy: ReferrerPolicy
     selector: Selector[int]
     fd: int
+    # List of cached files. Note that fds from passFd are never cached.
+    cacheMap: Table[string, string] # URL -> path
     # List of file descriptors passed by the pager.
     passedFdMap: Table[string, FileHandle]
 
@@ -111,6 +114,7 @@ type
     uriMethodMap*: URIMethodMap
     w3mCGICompat*: bool
     libexecPath*: string
+    tmpdir*: string
 
   FetchPromise* = Promise[JSResult[Response]]
 
@@ -129,24 +133,27 @@ proc rejectHandle(handle: LoaderHandle, code: ConnectErrorCode, msg = "") =
   handle.sendResult(code, msg)
   handle.close()
 
-func findOutputIdx(clientFdMap: ClientFdMap, pid, fd: int): int =
-  for i, (itpid, itfd, _) in clientFdMap:
-    if pid == itpid and fd == itfd:
-      return i
-  return -1
-
-proc delOutput(clientFdMap: var ClientFdMap, pid, fd: int) =
-  let i = clientFdMap.findOutputIdx(pid, fd)
-  if i != -1:
-    clientFdMap.del(i)
-
-func findOutput(clientFdMap: ClientFdMap, pid, fd: int): OutputHandle =
-  let i = clientFdMap.findOutputIdx(pid, fd)
-  if i != -1:
-    return clientFdMap[i].output
+func findOutput(ctx: LoaderContext, id: StreamId): OutputHandle =
+  assert id.pid != -1 and id.fd != -1
+  for it in ctx.outputMap.values:
+    if it.clientId == id:
+      return it
+  return nil
+
+#TODO linear search over strings :(
+func findCachedHandle(ctx: LoaderContext, cachepath: string): LoaderHandle =
+  assert cachepath != ""
+  for it in ctx.handleMap.values:
+    if it.cached and it.cachepath == cachepath:
+      return it
   return nil
 
-proc addFd(ctx: LoaderContext, handle: LoaderHandle) =
+proc delOutput(ctx: LoaderContext, id: StreamId) =
+  let output = ctx.findOutput(id)
+  if output != nil:
+    ctx.outputMap.del(output.ostream.fd)
+
+proc addFd(ctx: LoaderContext, handle: LoaderHandle, originalUrl: URL) =
   let output = handle.output
   output.ostream.setBlocking(false)
   ctx.selector.registerHandle(handle.istream.fd, {Read}, 0)
@@ -159,10 +166,17 @@ proc addFd(ctx: LoaderContext, handle: LoaderHandle) =
     # (kind of a hack, but should always work)
     ctx.outputMap[output.ostream.fd] = output
     ctx.outputMap.del(output.sostream.fd)
-    if output.clientPid != -1:
-      ctx.clientFdMap.delOutput(output.clientPid, output.clientFd)
-      output.clientFd = -1
-      output.clientPid = -1
+    if output.clientId != NullStreamId:
+      ctx.delOutput(output.clientId)
+      output.clientId = NullStreamId
+  if originalUrl != nil:
+    let tmpf = getTempFile(ctx.config.tmpdir)
+    let ps = newPosixStream(tmpf, O_CREAT or O_WRONLY, 0o600)
+    if ps != nil:
+      output.tee(ps, NullStreamId)
+      let path = $originalUrl
+      ctx.cacheMap[path] = tmpf
+      handle.cachepath = path
 
 proc loadStream(ctx: LoaderContext, handle: LoaderHandle, request: Request) =
   ctx.passedFdMap.withValue(request.url.host, fdp):
@@ -172,12 +186,13 @@ proc loadStream(ctx: LoaderContext, handle: LoaderHandle, request: Request) =
     handle.istream = newPosixStream(fdp[])
     ctx.passedFdMap.del(request.url.host)
   do:
-    handle.rejectHandle(ERROR_FILE_NOT_FOUND, "stream not found")
+    handle.sendResult(ERROR_FILE_NOT_FOUND, "stream not found")
 
 proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
   var redo = true
   var tries = 0
   var prevurl: URL = nil
+  let originalUrl = request.url
   while redo and tries < MaxRewrites:
     redo = false
     if ctx.config.w3mCGICompat and request.url.scheme == "file":
@@ -190,15 +205,20 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
           redo = true
           continue
     if request.url.scheme == "cgi-bin":
-      handle.loadCGI(request, ctx.config.cgiDir, ctx.config.libexecPath, prevurl)
+      handle.loadCGI(request, ctx.config.cgiDir, ctx.config.libexecPath,
+        prevurl)
       if handle.istream != nil:
-        ctx.addFd(handle)
+        let originalUrl = if handle.cached: originalUrl else: nil
+        ctx.addFd(handle, originalUrl)
       else:
         handle.close()
     elif request.url.scheme == "stream":
       ctx.loadStream(handle, request)
       if handle.istream != nil:
-        ctx.addFd(handle)
+        let originalUrl = if handle.cached: originalUrl else: nil
+        ctx.addFd(handle, originalUrl)
+      else:
+        handle.close()
     else:
       prevurl = request.url
       case ctx.config.uriMethodMap.findAndRewrite(request.url)
@@ -212,20 +232,61 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
   if tries >= MaxRewrites:
     handle.rejectHandle(ERROR_TOO_MANY_REWRITES)
 
+proc loadFromCache(ctx: LoaderContext, stream: SocketStream, request: Request) =
+  let handle = newLoaderHandle(stream, false, request.clientId)
+  let surl = $request.url
+  let cachedHandle = ctx.findCachedHandle(surl)
+  ctx.cacheMap.withValue(surl, p):
+    let ps = newPosixStream(p[], O_RDONLY, 0)
+    if ps == nil:
+      handle.rejectHandle(ERROR_FILE_NOT_IN_CACHE)
+      ctx.cacheMap.del(surl)
+      return
+    handle.sendResult(0)
+    handle.sendStatus(200)
+    handle.sendHeaders(newHeaders())
+    var buffer {.noinit.}: array[BufferSize, uint8]
+    try:
+      while true:
+        let n = ps.recvData(addr buffer[0], buffer.len)
+        if buffer.len == 0:
+          break
+        if handle.output.sendData(addr buffer[0], n) < n:
+          break
+        if n < buffer.len:
+          break
+    except ErrorBrokenPipe:
+      handle.close()
+      raise
+    ps.close()
+  do:
+    if cachedHandle == nil:
+      handle.sendResult(ERROR_URL_NOT_IN_CACHE)
+  if cachedHandle != nil:
+    # download is still ongoing; move output to the original handle
+    let output = handle.output
+    output.ostream.setBlocking(false)
+    handle.outputs.setLen(0)
+    output.parent = cachedHandle
+    cachedHandle.outputs.add(output)
+    ctx.outputMap[output.ostream.fd] = output
+  handle.close()
+
 proc onLoad(ctx: LoaderContext, stream: SocketStream) =
   var request: Request
   stream.sread(request)
   let handle = newLoaderHandle(
     stream,
     request.canredir,
-    request.clientPid,
-    request.clientFd
+    request.clientId
   )
-  assert request.clientPid != 0
+  assert request.clientId.pid != 0
   when defined(debug):
     handle.url = request.url
   if not ctx.config.filter.match(request.url):
     handle.rejectHandle(ERROR_DISALLOWED_URL)
+  elif request.fromcache:
+    ctx.loadFromCache(stream, request)
   else:
     for k, v in ctx.config.defaultheaders.table:
       if k notin request.headers.table:
@@ -243,9 +304,35 @@ proc onLoad(ctx: LoaderContext, stream: SocketStream) =
       request.proxy = ctx.config.proxy
     let fd = int(stream.source.getFd())
     ctx.outputMap[fd] = handle.output
-    ctx.clientFdMap.add((request.clientPid, request.clientFd, handle.output))
     ctx.loadResource(request, handle)
 
+proc rewind(ctx: LoaderContext, stream: PosixStream, clientId: StreamId) =
+  let output = ctx.findOutput(clientId)
+  if output == nil or output.ostream == nil:
+    stream.swrite(false)
+    return
+  let handle = output.parent
+  if not handle.cached:
+    stream.swrite(false)
+    return
+  assert handle.cachepath != ""
+  let ps = newPosixStream(handle.cachepath, O_RDONLY, 0)
+  if ps == nil:
+    stream.swrite(false)
+    return
+  stream.swrite(true)
+  output.ostream.setBlocking(true) #TODO
+  var buffer {.noinit.}: array[BufferSize, uint8]
+  while true:
+    let n = ps.recvData(addr buffer[0], BufferSize)
+    if n == 0:
+      break
+    if output.sendData(addr buffer[0], n) < n:
+      break
+    if n < BufferSize:
+      break
+  ps.close()
+
 proc acceptConnection(ctx: LoaderContext) =
   let stream = ctx.ssock.acceptSocketStream()
   try:
@@ -255,25 +342,22 @@ proc acceptConnection(ctx: LoaderContext) =
     of LOAD:
       ctx.onLoad(stream)
     of TEE:
-      var clientPid: int
-      var clientFd: int
-      var pid: int
-      var fd: int
-      stream.sread(pid)
-      stream.sread(fd)
-      stream.sread(clientPid)
-      stream.sread(clientFd)
-      let output = ctx.clientFdMap.findOutput(pid, fd)
+      var targetId: StreamId
+      var clientId: StreamId
+      stream.sread(targetId)
+      stream.sread(clientId)
+      let output = ctx.findOutput(targetId)
       if output != nil:
-        output.tee(stream, clientPid, clientFd)
+        output.tee(stream, clientId)
       stream.swrite(output != nil)
+      stream.setBlocking(false)
     of SUSPEND:
       var pid: int
       var fds: seq[int]
       stream.sread(pid)
       stream.sread(fds)
       for fd in fds:
-        let output = ctx.clientFdMap.findOutput(pid, fd)
+        let output = ctx.findOutput((pid, fd))
         if output != nil:
           # remove from the selector, so any new reads will be just placed
           # in the handle's buffer
@@ -284,11 +368,15 @@ proc acceptConnection(ctx: LoaderContext) =
       stream.sread(pid)
       stream.sread(fds)
       for fd in fds:
-        let output = ctx.clientFdMap.findOutput(pid, fd)
+        let output = ctx.findOutput((pid, fd))
         if output != nil:
           # place the stream back into the selector, so we can write to it
           # again
           ctx.selector.registerHandle(output.ostream.fd, {Write}, 0)
+    of REWIND:
+      var targetId: StreamId
+      stream.sread(targetId)
+      ctx.rewind(stream, targetId)
     of ADDREF:
       inc ctx.refcount
     of UNREF:
@@ -313,6 +401,8 @@ proc acceptConnection(ctx: LoaderContext) =
 
 proc exitLoader(ctx: LoaderContext) =
   ctx.ssock.close()
+  for path in ctx.cacheMap.values:
+    discard unlink(cstring(path))
   quit(0)
 
 var gctx: LoaderContext
@@ -434,8 +524,8 @@ proc finishCycle(ctx: LoaderContext, unregRead: var seq[LoaderHandle],
       if output.registered:
         ctx.selector.unregister(output.ostream.fd)
       ctx.outputMap.del(output.ostream.fd)
-      if output.clientFd != -1:
-        ctx.clientFdMap.delOutput(output.clientPid, output.clientFd)
+      if output.clientId != NullStreamId:
+        ctx.delOutput(output.clientId)
       output.ostream.close()
       output.ostream = nil
       let handle = output.parent
@@ -548,8 +638,7 @@ proc applyHeaders(loader: FileLoader, request: Request, response: Response) =
 #TODO: add init
 proc fetch*(loader: FileLoader, input: Request): FetchPromise =
   let stream = connectSocketStream(loader.process, false, blocking = true)
-  input.clientPid = getpid()
-  input.clientFd = int(stream.fd)
+  input.clientId = (loader.clientPid, int(stream.fd))
   stream.swrite(LOAD)
   stream.swrite(input)
   stream.flush()
@@ -565,8 +654,7 @@ proc fetch*(loader: FileLoader, input: Request): FetchPromise =
 
 proc reconnect*(loader: FileLoader, data: ConnectData) =
   let stream = connectSocketStream(loader.process, false, blocking = true)
-  data.request.clientPid = getpid()
-  data.request.clientFd = int(stream.fd)
+  data.request.clientId = (loader.clientPid, int(stream.fd))
   stream.swrite(LOAD)
   stream.swrite(data.request)
   stream.flush()
@@ -578,7 +666,7 @@ proc reconnect*(loader: FileLoader, data: ConnectData) =
     stream: stream
   )
 
-proc switchStream*(data: var ConnectData, stream: Stream) =
+proc switchStream*(data: var ConnectData, stream: SocketStream) =
   data.stream = stream
 
 proc switchStream*(loader: FileLoader, data: var OngoingData,
@@ -593,33 +681,41 @@ proc switchStream*(loader: FileLoader, data: var OngoingData,
     loader.unregisterFun(fd)
     realCloseImpl(stream)
 
-proc suspend*(loader: FileLoader, pid: int, fds: seq[int]) =
+proc suspend*(loader: FileLoader, fds: seq[int]) =
   let stream = connectSocketStream(loader.process, false, blocking = true)
   stream.swrite(SUSPEND)
-  stream.swrite(pid)
+  stream.swrite(loader.clientPid)
   stream.swrite(fds)
   stream.close()
 
-proc resume*(loader: FileLoader, pid: int, fds: seq[int]) =
+proc resume*(loader: FileLoader, fds: seq[int]) =
   let stream = connectSocketStream(loader.process, false, blocking = true)
   stream.swrite(RESUME)
-  stream.swrite(pid)
+  stream.swrite(loader.clientPid)
   stream.swrite(fds)
   stream.close()
 
-proc tee*(loader: FileLoader, pid, fd: int): Stream =
+proc tee*(loader: FileLoader, targetId: StreamId): SocketStream =
   let stream = connectSocketStream(loader.process, false, blocking = true)
   stream.swrite(TEE)
-  stream.swrite(pid)
-  stream.swrite(fd)
-  stream.swrite(int(getpid()))
-  stream.swrite(int(stream.fd))
+  stream.swrite(targetId)
+  let clientId: StreamId = (loader.clientPid, int(stream.fd))
+  stream.swrite(clientId)
   return stream
 
+proc rewind*(loader: FileLoader, fd: int): bool =
+  let stream = connectSocketStream(loader.process, false, blocking = true)
+  stream.swrite(REWIND)
+  let id: StreamId = (loader.clientPid, fd)
+  stream.swrite(id)
+  var res: bool
+  stream.sread(res)
+  return res
+
 const BufferSize = 4096
 
 proc handleHeaders(loader: FileLoader, request: Request, response: Response,
-    stream: Stream): bool =
+    stream: SocketStream) =
   var status: int
   stream.sread(status)
   response.status = cast[uint16](status)
@@ -628,7 +724,6 @@ proc handleHeaders(loader: FileLoader, request: Request, response: Response,
   loader.applyHeaders(request, response)
   # Only a stream of the response body may arrive after this point.
   response.body = stream
-  return true # success
 
 proc onConnected*(loader: FileLoader, fd: int) =
   let connectData = loader.connecting[fd]
@@ -638,7 +733,8 @@ proc onConnected*(loader: FileLoader, fd: int) =
   var res: int
   stream.sread(res)
   let response = newResponse(res, request, fd, stream)
-  if res == 0 and loader.handleHeaders(request, response, stream):
+  if res == 0:
+    loader.handleHeaders(request, response, stream)
     assert loader.unregisterFun != nil
     let realCloseImpl = stream.closeImpl
     stream.closeImpl = nil
@@ -651,7 +747,7 @@ proc onConnected*(loader: FileLoader, fd: int) =
       response: response,
       bodyRead: response.bodyRead
     )
-    SocketStream(stream).source.getFd().setBlocking(false)
+    stream.source.getFd().setBlocking(false)
     promise.resolve(JSResult[Response].ok(response))
   else:
     var msg: string
@@ -698,22 +794,19 @@ proc onError*(loader: FileLoader, fd: int) =
     buffer[].buf = ""
     response.unregisterFun()
 
-proc doRequest*(loader: FileLoader, request: Request, blocking = true,
-    canredir = false): Response =
+proc doRequest*(loader: FileLoader, request: Request, canredir = false):
+    Response =
   let response = Response(url: request.url)
   let stream = connectSocketStream(loader.process, false, blocking = true)
   if canredir:
     request.canredir = true #TODO set this somewhere else?
-  request.clientPid = getpid()
-  request.clientFd = int(stream.fd)
+  request.clientId = (loader.clientPid, int(stream.fd))
   stream.swrite(LOAD)
   stream.swrite(request)
   stream.flush()
   stream.sread(response.res)
   if response.res == 0:
-    if loader.handleHeaders(request, response, stream):
-      if not blocking:
-        stream.source.getFd().setBlocking(blocking)
+    loader.handleHeaders(request, response, stream)
   else:
     var msg: string
     stream.sread(msg)
@@ -725,12 +818,13 @@ proc addref*(loader: FileLoader) =
   let stream = connectSocketStream(loader.process)
   if stream != nil:
     stream.swrite(ADDREF)
-  stream.close()
+    stream.close()
 
 proc unref*(loader: FileLoader) =
   let stream = connectSocketStream(loader.process)
   if stream != nil:
     stream.swrite(UNREF)
+    stream.close()
 
 proc setReferrerPolicy*(loader: FileLoader, referrerpolicy: ReferrerPolicy) =
   let stream = connectSocketStream(loader.process)
diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim
index 7a0e893a..54054ed1 100644
--- a/src/loader/loaderhandle.nim
+++ b/src/loader/loaderhandle.nim
@@ -1,11 +1,13 @@
 import std/deques
 import std/net
 import std/streams
+import std/tables
 
 import io/posixstream
 import io/serialize
 import io/socketstream
 import loader/headers
+import loader/streamid
 
 when defined(debug):
   import types/url
@@ -27,8 +29,7 @@ type
     ostream*: PosixStream
     istreamAtEnd*: bool
     sostream*: PosixStream # saved ostream when redirected
-    clientFd*: int
-    clientPid*: int
+    clientId*: StreamId
     registered*: bool
 
   LoaderHandle* = ref object
@@ -39,6 +40,8 @@ type
     # conditions that would be difficult to untangle.
     canredir: bool
     outputs*: seq[OutputHandle]
+    cached*: bool
+    cachepath*: string
     when defined(debug):
       url*: URL
 
@@ -49,16 +52,15 @@ type
       buffer.page = nil
 
 # Create a new loader handle, with the output stream ostream.
-proc newLoaderHandle*(ostream: PosixStream, canredir: bool,
-    clientPid, clientFd: int): LoaderHandle =
+proc newLoaderHandle*(ostream: PosixStream, canredir: bool, clientId: StreamId):
+    LoaderHandle =
   let handle = LoaderHandle(
     canredir: canredir
   )
   handle.outputs.add(OutputHandle(
     ostream: ostream,
     parent: handle,
-    clientPid: clientPid,
-    clientFd: clientFd
+    clientId: clientId
   ))
   return handle
 
@@ -100,8 +102,7 @@ proc bufferCleared*(output: OutputHandle) =
   else:
     output.currentBuffer = nil
 
-proc tee*(outputIn: OutputHandle, ostream: PosixStream,
-    clientFd, clientPid: int) =
+proc tee*(outputIn: OutputHandle, ostream: PosixStream, clientId: StreamId) =
   outputIn.parent.outputs.add(OutputHandle(
     parent: outputIn.parent,
     ostream: ostream,
@@ -109,8 +110,7 @@ proc tee*(outputIn: OutputHandle, ostream: PosixStream,
     currentBufferIdx: outputIn.currentBufferIdx,
     buffers: outputIn.buffers,
     istreamAtEnd: outputIn.istreamAtEnd,
-    clientFd: clientFd,
-    clientPid: clientPid
+    clientId: clientId
   ))
 
 template output*(handle: LoaderHandle): OutputHandle =
@@ -132,6 +132,7 @@ proc sendHeaders*(handle: LoaderHandle, headers: Headers) =
   if handle.canredir:
     var redir: bool
     output.ostream.sread(redir)
+    output.ostream.sread(handle.cached)
     if redir:
       let fd = SocketStream(output.ostream).recvFileHandle()
       output.sostream = output.ostream
diff --git a/src/loader/request.nim b/src/loader/request.nim
index 805345f3..fcad1681 100644
--- a/src/loader/request.nim
+++ b/src/loader/request.nim
@@ -9,6 +9,7 @@ import js/fromjs
 import js/javascript
 import js/jstypes
 import loader/headers
+import loader/streamid
 import types/blob
 import types/formdata
 import types/referer
@@ -80,8 +81,8 @@ type
     credentialsMode* {.jsget.}: CredentialsMode
     proxy*: URL #TODO do something with this
     canredir*: bool
-    clientFd*: int
-    clientPid*: int
+    fromcache*: bool
+    clientId*: StreamId
 
   ReadableStream* = ref object of Stream
     isource*: Stream
@@ -164,7 +165,7 @@ func newRequest*(url: URL, httpMethod = HTTP_GET, headers = newHeaders(),
     body = opt(string), multipart = opt(FormData), mode = RequestMode.NO_CORS,
     credentialsMode = CredentialsMode.SAME_ORIGIN,
     destination = RequestDestination.NO_DESTINATION, proxy: URL = nil,
-    referrer: URL = nil, canredir = false): Request =
+    referrer: URL = nil, canredir = false, fromcache = false): Request =
   return Request(
     url: url,
     httpMethod: httpMethod,
@@ -175,7 +176,9 @@ func newRequest*(url: URL, httpMethod = HTTP_GET, headers = newHeaders(),
     credentialsMode: credentialsMode,
     destination: destination,
     referer: referrer,
-    proxy: proxy
+    proxy: proxy,
+    canredir: canredir,
+    fromcache: fromcache
   )
 
 func newRequest*(url: URL, httpMethod = HTTP_GET,
diff --git a/src/loader/response.nim b/src/loader/response.nim
index a8bc31e4..0869a73e 100644
--- a/src/loader/response.nim
+++ b/src/loader/response.nim
@@ -3,6 +3,7 @@ import std/unicode
 
 import bindings/quickjs
 import io/promise
+import io/socketstream
 import js/error
 import js/javascript
 import loader/headers
@@ -35,7 +36,7 @@ type
     responseType* {.jsget: "type".}: ResponseType
     res*: int
     fd*: int
-    body*: Stream
+    body*: SocketStream
     bodyUsed* {.jsget.}: bool
     contentType*: string
     status* {.jsget.}: uint16
@@ -50,8 +51,8 @@ type
 
 jsDestructor(Response)
 
-proc newResponse*(res: int, request: Request, fd = -1, stream: Stream = nil):
-    Response =
+proc newResponse*(res: int, request: Request, fd = -1,
+    stream: SocketStream = nil): Response =
   return Response(
     res: res,
     url: request.url,
diff --git a/src/loader/streamid.nim b/src/loader/streamid.nim
new file mode 100644
index 00000000..a1ce3455
--- /dev/null
+++ b/src/loader/streamid.nim
@@ -0,0 +1,7 @@
+# Identifier for remote streams; it is a tuple of the client's process ID and
+# file descriptor.
+
+type
+  StreamId* = tuple[pid, fd: int]
+
+const NullStreamId* = StreamId((-1, -1))
diff --git a/src/local/client.nim b/src/local/client.nim
index 76cbd4e7..20e1a6ef 100644
--- a/src/local/client.nim
+++ b/src/local/client.nim
@@ -95,9 +95,6 @@ proc finalize(client: Client) {.jsfin.} =
   if client.jsrt != nil:
     free(client.jsrt)
 
-proc doRequest(client: Client, req: Request): Response {.jsfunc.} =
-  return client.loader.doRequest(req)
-
 proc fetch[T: Request|string](client: Client, req: T,
     init = none(RequestInit)): JSResult[FetchPromise] {.jsfunc.} =
   let req = ?newRequest(client.jsctx, req, init)
diff --git a/src/local/container.nim b/src/local/container.nim
index d4d44b0f..585c8a3f 100644
--- a/src/local/container.nim
+++ b/src/local/container.nim
@@ -156,6 +156,29 @@ proc newBuffer*(forkserver: ForkServer, config: BufferConfig,
     canreinterpret: canreinterpret
   )
 
+proc newBufferFrom*(forkserver: ForkServer, attrs: WindowAttributes,
+    container: Container, contentTypeOverride: string): Container =
+  var source = container.source
+  source.contentType = some(contentTypeOverride)
+  source.request = newRequest(source.request.url, fromcache = true)
+  let config = container.config
+  let loaderPid = container.loaderPid
+  let bufferPid = forkserver.forkBufferWithLoader(source, config, attrs,
+    loaderPid)
+  return Container(
+    source: source,
+    width: container.width,
+    height: container.height,
+    title: container.title,
+    config: config,
+    process: bufferPid,
+    loaderPid: loaderPid,
+    pos: CursorPosition(
+      setx: -1
+    ),
+    canreinterpret: true
+  )
+
 func location*(container: Container): URL {.jsfget.} =
   return container.source.location
 
@@ -1385,9 +1408,9 @@ proc startload*(container: Container) =
 proc connect2*(container: Container): EmptyPromise =
   return container.iface.connect2()
 
-proc redirectToFd*(container: Container, fdin: FileHandle, wait: bool):
+proc redirectToFd*(container: Container, fdin: FileHandle, wait, cache: bool):
     EmptyPromise =
-  return container.iface.redirectToFd(fdin, wait)
+  return container.iface.redirectToFd(fdin, wait, cache)
 
 proc readFromFd*(container: Container, fdout: FileHandle, id: string,
     ishtml: bool): EmptyPromise =
@@ -1430,11 +1453,6 @@ proc reshape(container: Container): EmptyPromise {.discardable, jsfunc.} =
     container.setNumLines(lines)
     return container.requestLines())
 
-proc pipeBuffer*(container, pipeTo: Container) =
-  container.iface.getSource().then(proc() =
-    pipeTo.load() #TODO do not load if pipeTo is killed first?
-  )
-
 proc onclick(container: Container, res: ClickResult)
 
 proc displaySelect(container: Container, selectResult: SelectResult) =
@@ -1531,7 +1549,8 @@ proc setStream*(container: Container, stream: Stream) =
     discard container.iface.load().then(proc(res: LoadResult) =
       container.onload(res))
 
-proc onreadline(container: Container, w: Slice[int], handle: (proc(line: SimpleFlexibleLine)), res: GetLinesResult) =
+proc onreadline(container: Container, w: Slice[int],
+    handle: (proc(line: SimpleFlexibleLine)), res: GetLinesResult) =
   for line in res.lines:
     handle(line)
   if res.numLines > w.b + 1:
@@ -1544,7 +1563,7 @@ proc onreadline(container: Container, w: Slice[int], handle: (proc(line: SimpleF
     container.setNumLines(res.numLines, true)
 
 # Synchronously read all lines in the buffer.
-proc readLines*(container: Container, handle: (proc(line: SimpleFlexibleLine))) =
+proc readLines*(container: Container, handle: proc(line: SimpleFlexibleLine)) =
   if container.code == 0:
     # load succeded
     let w = 0 .. 23
diff --git a/src/local/pager.nim b/src/local/pager.nim
index cc33d06b..3233b012 100644
--- a/src/local/pager.nim
+++ b/src/local/pager.nim
@@ -458,26 +458,6 @@ proc newBuffer(pager: Pager, bufferConfig: BufferConfig, source: BufferSource,
     fd
   )
 
-proc dupeBuffer2(pager: Pager, container: Container, location: URL,
-    contentType = ""): Container =
-  let contentType = if contentType != "":
-    some(contentType)
-  else:
-    container.contentType
-  let location = if location != nil:
-    location
-  else:
-    container.source.location
-  let source = BufferSource(
-    t: CLONE,
-    location: location,
-    contentType: contentType,
-    clonepid: container.process,
-  )
-  let pipeTo = pager.newBuffer(container.config, source, container.title)
-  container.pipeBuffer(pipeTo)
-  return pipeTo
-
 proc dupeBuffer(pager: Pager, container: Container, location: URL) =
   container.clone(location).then(proc(container: Container) =
     if container == nil:
@@ -620,7 +600,12 @@ proc toggleSource(pager: Pager) {.jsfunc.} =
       "text/plain"
     else:
       "text/html"
-    let container = pager.dupeBuffer2(pager.container, nil, contentType)
+    let container = newBufferFrom(
+      pager.forkserver,
+      pager.attrs,
+      pager.container,
+      contentType
+    )
     container.sourcepair = pager.container
     pager.container.sourcepair = container
     pager.addContainer(container)
@@ -687,7 +672,7 @@ proc applySiteconf(pager: Pager, url: var URL): BufferConfig =
       proxy = sc.proxy.get
   return pager.config.getBufferConfig(url, cookiejar, headers, referer_from,
     scripting, charsets, images, userstyle, proxy, mimeTypes, urimethodmap,
-    pager.cgiDir)
+    pager.cgiDir, pager.tmpdir)
 
 # Load request in a new buffer.
 proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
@@ -705,7 +690,6 @@ proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
     # what other browsers do. Still, it would be nice if we got some visual
     # feedback on what is actually going to happen when typing a URL; TODO.
     let source = BufferSource(
-      t: LOAD_REQUEST,
       request: request,
       contentType: ctype,
       charset: cs,
@@ -779,7 +763,6 @@ proc readPipe0*(pager: Pager, ctype: Option[string], cs: Charset,
   var location = location.get(newURL("stream:-").get)
   let bufferconfig = pager.applySiteconf(location)
   let source = BufferSource(
-    t: LOAD_REQUEST,
     request: newRequest(location),
     contentType: some(ctype.get("text/plain")),
     charset: cs,
@@ -973,7 +956,7 @@ proc runMailcapReadPipe(pager: Pager, container: Container,
   discard close(pipefd_out[1])
   let fdin = pipefd_in[1]
   let fdout = pipefd_out[0]
-  let p = container.redirectToFd(fdin, wait = false)
+  let p = container.redirectToFd(fdin, wait = false, cache = true)
   let p2 = p.then(proc(): auto =
     discard close(fdin)
     let ishtml = HTMLOUTPUT in entry.flags
@@ -1013,7 +996,7 @@ proc runMailcapWritePipe(pager: Pager, container: Container,
     # parent
     discard close(pipefd[0])
     let fd = pipefd[1]
-    let p = container.redirectToFd(fd, wait = false)
+    let p = container.redirectToFd(fd, wait = false, cache = false)
     discard close(fd)
     if needsterminal:
       var x: cint
@@ -1026,10 +1009,11 @@ proc runMailcapWritePipe(pager: Pager, container: Container,
 # needsterminal is ignored.
 proc runMailcapReadFile(pager: Pager, container: Container,
     entry: MailcapEntry, cmd, outpath: string): (EmptyPromise, bool) =
-  let fd = open(outpath, O_WRONLY or O_CREAT, 0o644)
+  let fd = open(outpath, O_WRONLY or O_CREAT, 0o600)
   if fd == -1:
     return (nil, false)
-  let p = container.redirectToFd(fd, wait = true).then(proc(): auto =
+  let p = container.redirectToFd(fd, wait = true, cache = true).then(proc():
+      auto =
     var pipefd: array[2, cint] # redirect stdout here
     if pipe(pipefd) == -1:
       raise newException(Defect, "Failed to open pipe.")
@@ -1060,10 +1044,10 @@ proc runMailcapReadFile(pager: Pager, container: Container,
 proc runMailcapWriteFile(pager: Pager, container: Container,
     entry: MailcapEntry, cmd, outpath: string): (EmptyPromise, bool) =
   let needsterminal = NEEDSTERMINAL in entry.flags
-  let fd = open(outpath, O_WRONLY or O_CREAT, 0o644)
+  let fd = open(outpath, O_WRONLY or O_CREAT, 0o600)
   if fd == -1:
     return (nil, false)
-  let p = container.redirectToFd(fd, wait = true).then(proc() =
+  let p = container.redirectToFd(fd, wait = true, cache = false).then(proc() =
     if needsterminal:
       pager.term.quit()
       discard execCmd(cmd)
@@ -1099,8 +1083,6 @@ proc runMailcapWriteFile(pager: Pager, container: Container,
 proc checkMailcap(pager: Pager, container: Container): (EmptyPromise, bool) =
   if container.contentType.isNone:
     return (nil, true)
-  if container.source.t == CLONE:
-    return (nil, true) # clone cannot use mailcap
   let contentType = container.contentType.get
   if contentType == "text/html":
     # We support HTML natively, so it would make little sense to execute
diff --git a/src/render/rendertext.nim b/src/render/rendertext.nim
index fc258710..bbf2ec49 100644
--- a/src/render/rendertext.nim
+++ b/src/render/rendertext.nim
@@ -20,9 +20,11 @@ type StreamRenderer* = object
   newline: bool
   w: int
   j: int # byte in line
+  rewindImpl: proc()
 
-proc newStreamRenderer*(stream: Stream, charsets0: openArray[Charset]):
-    StreamRenderer =
+#TODO pass bool for whether we can rewind
+proc newStreamRenderer*(stream: Stream, charsets0: openArray[Charset],
+    rewindImpl: proc()): StreamRenderer =
   var charsets = newSeq[Charset](charsets0.len)
   for i in 0 ..< charsets.len:
     charsets[i] = charsets0[charsets.high - i]
@@ -44,11 +46,12 @@ proc newStreamRenderer*(stream: Stream, charsets0: openArray[Charset]):
     charsets: charsets,
     ansiparser: AnsiCodeParser(
       state: PARSE_DONE
-    )
+    ),
+    rewindImpl: rewindImpl
   )
 
 proc rewind(renderer: var StreamRenderer) =
-  renderer.stream.setPosition(0)
+  renderer.rewindImpl()
   let cs = renderer.charsets.pop()
   let em = if renderer.charsets.len > 0:
     DECODER_ERROR_MODE_FATAL
diff --git a/src/server/buffer.nim b/src/server/buffer.nim
index 817dae82..af857928 100644
--- a/src/server/buffer.nim
+++ b/src/server/buffer.nim
@@ -29,13 +29,11 @@ import io/promise
 import io/serialize
 import io/serversocket
 import io/socketstream
-import io/teestream
 import js/fromjs
 import js/javascript
 import js/regex
 import js/timeout
 import js/tojs
-import loader/connecterror
 import loader/headers
 import loader/loader
 import render/renderdocument
@@ -63,8 +61,8 @@ type
   BufferCommand* = enum
     LOAD, RENDER, WINDOW_CHANGE, FIND_ANCHOR, READ_SUCCESS, READ_CANCELED,
     CLICK, FIND_NEXT_LINK, FIND_PREV_LINK, FIND_NTH_LINK, FIND_REV_NTH_LINK,
-    FIND_NEXT_MATCH, FIND_PREV_MATCH, GET_SOURCE, GET_LINES, UPDATE_HOVER,
-    CONNECT, CONNECT2, GOTO_ANCHOR, CANCEL, GET_TITLE, SELECT, REDIRECT_TO_FD,
+    FIND_NEXT_MATCH, FIND_PREV_MATCH, GET_LINES, UPDATE_HOVER, CONNECT,
+    CONNECT2, GOTO_ANCHOR, CANCEL, GET_TITLE, SELECT, REDIRECT_TO_FD,
     READ_FROM_FD, SET_CONTENT_TYPE, CLONE, FIND_PREV_PARAGRAPH,
     FIND_NEXT_PARAGRAPH
 
@@ -98,8 +96,8 @@ type
     document: Document
     prevstyled: StyledNode
     selector: Selector[int]
-    istream: Stream
-    sstream: Stream
+    istream: SocketStream
+    sstream: StringStream
     available: int
     pstream: SocketStream # pipe stream
     srenderer: StreamRenderer
@@ -711,8 +709,25 @@ type ConnectResult* = object
   referrerpolicy*: Option[ReferrerPolicy]
   charset*: Charset
 
+proc rewind(buffer: Buffer): bool =
+  if buffer.loader.rewind(buffer.fd):
+    return true
+  let request = newRequest(buffer.url, fromcache = true)
+  let response = buffer.loader.doRequest(request, canredir = false)
+  if response.body != nil:
+    buffer.selector.unregister(buffer.fd)
+    buffer.loader.unregistered.add(buffer.fd)
+    buffer.istream.close()
+    buffer.istream = response.body
+    buffer.fd = response.body.fd
+    buffer.selector.registerHandle(buffer.fd, {Read}, 0)
+    return true
+  return false
+
 proc setHTML(buffer: Buffer, ishtml: bool) =
   buffer.ishtml = ishtml
+  let rewindImpl = proc() =
+    doAssert buffer.rewind()
   if ishtml:
     let factory = newCAtomFactory()
     buffer.factory = factory
@@ -741,6 +756,7 @@ proc setHTML(buffer: Buffer, ishtml: bool) =
       buffer.window,
       buffer.url,
       buffer.factory,
+      rewindImpl = rewindImpl,
       buffer.charsets,
       seekable = true
     )
@@ -749,6 +765,9 @@ proc setHTML(buffer: Buffer, ishtml: bool) =
     buffer.uastyle = css.parseStylesheet(factory)
     buffer.quirkstyle = quirk.parseStylesheet(factory)
     buffer.userstyle = parseStylesheet(buffer.config.userstyle, factory)
+  else:
+    buffer.srenderer = newStreamRenderer(buffer.sstream, buffer.charsets,
+      rewindImpl)
 
 proc connect*(buffer: Buffer): ConnectResult {.proxy.} =
   if buffer.connected:
@@ -761,45 +780,31 @@ proc connect*(buffer: Buffer): ConnectResult {.proxy.} =
   var redirect: Request
   var cookies: seq[Cookie]
   var referrerpolicy: Option[ReferrerPolicy]
-  case source.t
-  of CLONE:
-    #TODO there is only one function for CLONE left: to get the source for
-    # the "view buffer" operation.
-    # This does not belong in buffers at all, and should be requested from
-    # the networking module instead.
-    let s = connectSocketStream(source.clonepid, blocking = false)
-    buffer.istream = s
-    buffer.fd = int(s.source.getFd())
-    if buffer.istream == nil:
-      return ConnectResult(code: ERROR_SOURCE_NOT_FOUND)
-    if buffer.source.contentType.isNone:
-      buffer.source.contentType = some("text/plain")
-  of LOAD_REQUEST:
-    let request = source.request
-    let response = buffer.loader.doRequest(request, blocking = true, canredir = true)
-    if response.body == nil:
-      return ConnectResult(
-        code: response.res,
-        errorMessage: response.internalMessage
-      )
-    if response.charset != CHARSET_UNKNOWN:
-      charset = charset
-    if buffer.source.contentType.isNone:
-      buffer.source.contentType = some(response.contentType)
-    buffer.istream = response.body
-    let fd = SocketStream(response.body).source.getFd()
-    buffer.fd = int(fd)
-    needsAuth = response.status == 401 # Unauthorized
-    redirect = response.redirect
-    if "Set-Cookie" in response.headers.table:
-      for s in response.headers.table["Set-Cookie"]:
-        let cookie = newCookie(s, response.url)
-        if cookie.isOk:
-          cookies.add(cookie.get)
-    if "Referrer-Policy" in response.headers:
-      referrerpolicy = getReferrerPolicy(response.headers["Referrer-Policy"])
-      if referrerpolicy.isSome:
-        buffer.loader.setReferrerPolicy(referrerpolicy.get)
+  let request = source.request
+  let response = buffer.loader.doRequest(request, canredir = true)
+  if response.body == nil:
+    return ConnectResult(
+      code: response.res,
+      errorMessage: response.internalMessage
+    )
+  if response.charset != CHARSET_UNKNOWN:
+    charset = charset
+  if buffer.source.contentType.isNone:
+    buffer.source.contentType = some(response.contentType)
+  buffer.istream = response.body
+  let fd = response.body.source.getFd()
+  buffer.fd = int(fd)
+  needsAuth = response.status == 401 # Unauthorized
+  redirect = response.redirect
+  if "Set-Cookie" in response.headers.table:
+    for s in response.headers.table["Set-Cookie"]:
+      let cookie = newCookie(s, response.url)
+      if cookie.isOk:
+        cookies.add(cookie.get)
+  if "Referrer-Policy" in response.headers:
+    referrerpolicy = getReferrerPolicy(response.headers["Referrer-Policy"])
+    if referrerpolicy.isSome:
+      buffer.loader.setReferrerPolicy(referrerpolicy.get)
   buffer.connected = true
   let contentType = buffer.source.contentType.get("")
   buffer.setHTML(contentType == "text/html")
@@ -815,31 +820,28 @@ proc connect*(buffer: Buffer): ConnectResult {.proxy.} =
 # * connect2, telling loader to load at last (we block loader until then)
 # * redirectToFd, telling loader to load into the passed fd
 proc connect2*(buffer: Buffer) {.proxy.} =
-  if buffer.source.t == LOAD_REQUEST and buffer.istream of SocketStream:
+  if not buffer.source.request.fromcache:
     # Notify loader that we can proceed with loading the input stream.
-    let ss = SocketStream(buffer.istream)
-    ss.swrite(false)
-    ss.setBlocking(false)
+    buffer.istream.swrite(false)
+    buffer.istream.swrite(true)
+    buffer.istream.setBlocking(false)
   buffer.selector.registerHandle(buffer.fd, {Read}, 0)
 
-proc redirectToFd*(buffer: Buffer, fd: FileHandle, wait: bool) {.proxy.} =
-  case buffer.source.t
-  of LOAD_REQUEST:
-    let ss = SocketStream(buffer.istream)
-    ss.swrite(true)
-    ss.sendFileHandle(fd)
-    if wait:
-      #TODO this is kind of dumb
-      # Basically, after redirect the network process keeps the socket open,
-      # and writes a boolean after transfer has been finished. This way,
-      # we can block this promise so it only returns after e.g. the whole
-      # file has been saved.
-      var dummy: bool
-      ss.sread(dummy)
-    discard close(fd)
-    ss.close()
-  of CLONE:
-    discard
+proc redirectToFd*(buffer: Buffer, fd: FileHandle, wait, cache: bool)
+    {.proxy.} =
+  buffer.istream.swrite(true)
+  buffer.istream.swrite(cache)
+  buffer.istream.sendFileHandle(fd)
+  if wait:
+    #TODO this is kind of dumb
+    # Basically, after redirect the network process keeps the socket open,
+    # and writes a boolean after transfer has been finished. This way,
+    # we can block this promise so it only returns after e.g. the whole
+    # file has been saved.
+    var dummy: bool
+    buffer.istream.sread(dummy)
+  discard close(fd)
+  buffer.istream.close()
 
 proc readFromFd*(buffer: Buffer, url: URL, ishtml: bool) {.proxy.} =
   let contentType = if ishtml:
@@ -848,17 +850,15 @@ proc readFromFd*(buffer: Buffer, url: URL, ishtml: bool) {.proxy.} =
     "text/plain"
   let request = newRequest(url)
   buffer.source = BufferSource(
-    t: LOAD_REQUEST,
     request: request,
     location: buffer.source.location,
     contentType: some(contentType),
     charset: buffer.source.charset
   )
   buffer.setHTML(ishtml)
-  let response = buffer.loader.doRequest(request, blocking = true,
-    canredir = false)
+  let response = buffer.loader.doRequest(request, canredir = false)
   buffer.istream = response.body
-  buffer.fd = int(SocketStream(response.body).source.getFd())
+  buffer.fd = int(response.body.source.getFd())
   buffer.selector.registerHandle(buffer.fd, {Read}, 0)
 
 proc setContentType*(buffer: Buffer, contentType: string) {.proxy.} =
@@ -878,24 +878,10 @@ proc clone*(buffer: Buffer, newurl: URL): Pid {.proxy.} =
   if pipe(pipefd) == -1:
     buffer.estream.write("Failed to open pipe.\n")
     return -1
-  # Naturally, we have to solve the problem of splitting up input streams here.
-  # The "cleanest" way is to get the source to duplicate the stream, and
-  # also send the new buffer the data over a separate stream. We do this
-  # for resources we retrieve with fetch().
-  # This is unfortunately not possible for the main source input stream,
-  # because it may come from a pipe that we receive from the client.
-  # So for istream, we just use a TeeStream from the original buffer and
-  # pray that no interruptions happen along the way.
-  # TODO: this is fundamentally broken and should be changed once the istream
-  # mess is untangled. A good first step would be to remove sstream from
-  # buffer.
+  # We have to solve the problem of splitting up open input streams here.
+  # To "split up" all open streams, we request a new handle to all open streams
+  # (possibly including buffer.istream) from the FileLoader process.
   let needsPipe = not buffer.istream.atEnd
-  var pipefd_write: array[2, cint]
-  if needsPipe:
-    assert buffer.fd != -1
-    if pipe(pipefd_write) == -1:
-      buffer.estream.write("Failed to open pipe.\n")
-      return -1
   var fds: seq[int]
   for fd in buffer.loader.connecting.keys:
     fds.add(fd)
@@ -903,8 +889,9 @@ proc clone*(buffer: Buffer, newurl: URL): Pid {.proxy.} =
     fds.add(fd)
   #TODO maybe we still have some data in sockets... we should probably split
   # this up to be executed after the main loop is finished...
-  let parentPid = getpid()
-  buffer.loader.suspend(parentPid, fds)
+  buffer.loader.suspend(fds)
+  if needsPipe:
+    buffer.loader.suspend(@[buffer.fd])
   buffer.loader.addref()
   let pid = fork()
   if pid == -1:
@@ -919,12 +906,15 @@ proc clone*(buffer: Buffer, newurl: URL): Pid {.proxy.} =
     when not bsdPlatform:
       buffer.selector.close()
     buffer.selector = newSelector[int]()
+    let parentPid = buffer.loader.clientPid
+    # We have a new process ID.
+    buffer.loader.clientPid = getCurrentProcessId()
     #TODO set buffer.window.timeouts.selector
     var cfds: seq[int]
     for fd in buffer.loader.connecting.keys:
       cfds.add(fd)
     for fd in cfds:
-      let stream = SocketStream(buffer.loader.tee(parentPid, fd))
+      let stream = buffer.loader.tee((parentPid, fd))
       var success: bool
       stream.sread(success)
       let sfd = int(stream.source.getFd())
@@ -942,7 +932,7 @@ proc clone*(buffer: Buffer, newurl: URL): Pid {.proxy.} =
     for fd in buffer.loader.ongoing.keys:
       ofds.add(fd)
     for fd in ofds:
-      let stream = SocketStream(buffer.loader.tee(parentPid, fd))
+      let stream = buffer.loader.tee((parentPid, fd))
       var success: bool
       stream.sread(success)
       let sfd = int(stream.source.getFd())
@@ -954,10 +944,10 @@ proc clone*(buffer: Buffer, newurl: URL): Pid {.proxy.} =
         #TODO what to do?
         discard
     if needsPipe:
-      discard close(pipefd_write[1]) # close write
-      buffer.fd = pipefd_write[0]
+      let ofd = int(buffer.istream.fd)
+      buffer.istream = buffer.loader.tee((parentPid, ofd))
+      buffer.fd = buffer.istream.fd
       buffer.selector.registerHandle(buffer.fd, {Read}, 0)
-      buffer.istream = newPosixStream(pipefd_write[0])
     buffer.pstream.close()
     let ssock = initServerSocket(buffered = false)
     buffer.ssock = ssock
@@ -972,17 +962,12 @@ proc clone*(buffer: Buffer, newurl: URL): Pid {.proxy.} =
     return 0
   else: # parent
     discard close(pipefd[1]) # close write
-    if needsPipe:
-      discard close(pipefd_write[0]) # close read
     # We must wait for child to tee its ongoing streams.
     let ps = newPosixStream(pipefd[0])
     let c = ps.readChar()
     assert c == char(0)
     ps.close()
-    if needsPipe:
-      let istrmp = newPosixStream(pipefd_write[1])
-      buffer.istream = newTeeStream(buffer.istream, istrmp)
-    buffer.loader.resume(parentPid, fds)
+    buffer.loader.resume(fds)
     return pid
 
 proc dispatchDOMContentLoadedEvent(buffer: Buffer) =
@@ -1113,18 +1098,20 @@ proc onload(buffer: Buffer) =
   of LOADING_PAGE:
     discard
   while true:
-    let op = buffer.sstream.getPosition()
-    var s {.noinit.}: array[BufferSize, uint8]
+    buffer.sstream.setPosition(0)
+    buffer.sstream.data.setLen(BufferSize)
     try:
-      let n = buffer.istream.readData(addr s[0], s.len)
+      buffer.sstream.data.prepareMutation()
+      let n = buffer.istream.readData(addr buffer.sstream.data[0], BufferSize)
+      if n != buffer.sstream.data.len:
+        buffer.sstream.data.setLen(n)
       if n != 0:
-        buffer.sstream.writeData(addr s[0], n)
-        buffer.sstream.setPosition(op)
         buffer.available += n
         buffer.processData()
         res.bytes = buffer.available
       res.lines = buffer.lines.len
       if buffer.istream.atEnd():
+        buffer.sstream = nil
         # EOF
         res.atend = true
         buffer.finishLoad().then(proc() =
@@ -1619,17 +1606,6 @@ proc getLines*(buffer: Buffer, w: Slice[int]): GetLinesResult {.proxy.} =
     result.lines.add(line)
   result.numLines = buffer.lines.len
 
-#TODO this is mostly broken
-proc getSource*(buffer: Buffer) {.proxy.} =
-  let ssock = initServerSocket()
-  let stream = ssock.acceptSocketStream()
-  let op = buffer.sstream.getPosition()
-  buffer.sstream.setPosition(0)
-  stream.write(buffer.sstream.readAll())
-  buffer.sstream.setPosition(op)
-  stream.close()
-  ssock.close()
-
 macro bufferDispatcher(funs: static ProxyMap, buffer: Buffer,
     cmd: BufferCommand, packetid: int) =
   let switch = newNimNode(nnkCaseStmt)
@@ -1781,7 +1757,6 @@ proc launchBuffer*(config: BufferConfig, source: BufferSource,
   onSignal SIGTERM:
     discard sig
     gbuffer.cleanup()
-  buffer.srenderer = newStreamRenderer(buffer.sstream, buffer.charsets)
   loader.registerFun = proc(fd: int) =
     buffer.selector.registerHandle(fd, {Read}, 0)
   loader.unregisterFun = proc(fd: int) =
diff --git a/src/server/forkserver.nim b/src/server/forkserver.nim
index ced32341..f788aa24 100644
--- a/src/server/forkserver.nim
+++ b/src/server/forkserver.nim
@@ -1,4 +1,5 @@
 import std/options
+import std/os
 import std/posix
 import std/streams
 import std/tables
@@ -20,10 +21,9 @@ import utils/strwidth
 
 type
   ForkCommand* = enum
-    FORK_BUFFER, FORK_LOADER, REMOVE_CHILD, LOAD_CONFIG
+    FORK_BUFFER, FORK_LOADER, REMOVE_CHILD, LOAD_CONFIG, FORK_BUFFER_WITH_LOADER
 
   ForkServer* = ref object
-    process*: Pid
     istream: Stream
     ostream: Stream
     estream*: PosixStream
@@ -50,7 +50,7 @@ proc newFileLoader*(forkserver: ForkServer, defaultHeaders: Headers,
   forkserver.ostream.flush()
   var process: Pid
   forkserver.istream.sread(process)
-  return FileLoader(process: process)
+  return FileLoader(process: process, clientPid: getCurrentProcessId())
 
 proc loadForkServerConfig*(forkserver: ForkServer, config: Config) =
   forkserver.ostream.swrite(LOAD_CONFIG)
@@ -76,6 +76,18 @@ proc forkBuffer*(forkserver: ForkServer, source: BufferSource,
   forkserver.istream.sread(loaderPid)
   return (process, loaderPid)
 
+proc forkBufferWithLoader*(forkserver: ForkServer, source: BufferSource,
+    config: BufferConfig, attrs: WindowAttributes, loaderPid: Pid): Pid =
+  forkserver.ostream.swrite(FORK_BUFFER_WITH_LOADER)
+  forkserver.ostream.swrite(source)
+  forkserver.ostream.swrite(config)
+  forkserver.ostream.swrite(attrs)
+  forkserver.ostream.swrite(loaderPid)
+  forkserver.ostream.flush()
+  var bufferPid: Pid
+  forkserver.istream.sread(bufferPid)
+  return bufferPid
+
 proc trapSIGINT() =
   # trap SIGINT, so e.g. an external editor receiving an interrupt in the
   # same process group can't just kill the process
@@ -117,14 +129,8 @@ proc forkLoader(ctx: var ForkServerContext, config: LoaderConfig): Pid =
   return pid
 
 var gssock: ServerSocket
-proc forkBuffer(ctx: var ForkServerContext): tuple[process, loaderPid: Pid] =
-  var source: BufferSource
-  var config: BufferConfig
-  var attrs: WindowAttributes
-  ctx.istream.sread(source)
-  ctx.istream.sread(config)
-  ctx.istream.sread(attrs)
-  let loaderPid = ctx.forkLoader(config.loaderConfig)
+proc forkBuffer0(ctx: var ForkServerContext, source: BufferSource,
+    config: BufferConfig, attrs: WindowAttributes, loaderPid: Pid): Pid =
   var pipefd: array[2, cint]
   if pipe(pipefd) == -1:
     raise newException(Defect, "Failed to open pipe.")
@@ -150,7 +156,10 @@ proc forkBuffer(ctx: var ForkServerContext): tuple[process, loaderPid: Pid] =
     ps.close()
     discard close(stdin.getFileHandle())
     discard close(stdout.getFileHandle())
-    let loader = FileLoader(process: loaderPid)
+    let loader = FileLoader(
+      process: loaderPid,
+      clientPid: getCurrentProcessId()
+    )
     try:
       launchBuffer(config, source, attrs, loader, ssock)
     except CatchableError:
@@ -167,7 +176,30 @@ proc forkBuffer(ctx: var ForkServerContext): tuple[process, loaderPid: Pid] =
   assert c == char(0)
   ps.close()
   ctx.children.add((pid, loaderPid))
-  return (pid, loaderPid)
+  return pid
+
+proc forkBuffer(ctx: var ForkServerContext): tuple[process, loaderPid: Pid] =
+  var source: BufferSource
+  var config: BufferConfig
+  var attrs: WindowAttributes
+  ctx.istream.sread(source)
+  ctx.istream.sread(config)
+  ctx.istream.sread(attrs)
+  let loaderPid = ctx.forkLoader(config.loaderConfig)
+  let process = ctx.forkBuffer0(source, config, attrs, loaderPid)
+  return (process, loaderPid)
+
+proc forkBufferWithLoader(ctx: var ForkServerContext): Pid =
+  var source: BufferSource
+  var config: BufferConfig
+  var attrs: WindowAttributes
+  var loaderPid: Pid
+  ctx.istream.sread(source)
+  ctx.istream.sread(config)
+  ctx.istream.sread(attrs)
+  ctx.istream.sread(loaderPid)
+  FileLoader(process: loaderPid).addref()
+  return ctx.forkBuffer0(source, config, attrs, loaderPid)
 
 proc runForkServer() =
   var ctx = ForkServerContext(
@@ -188,6 +220,8 @@ proc runForkServer() =
             break
       of FORK_BUFFER:
         ctx.ostream.swrite(ctx.forkBuffer())
+      of FORK_BUFFER_WITH_LOADER:
+        ctx.ostream.swrite(ctx.forkBufferWithLoader())
       of FORK_LOADER:
         var config: LoaderConfig
         ctx.istream.sread(config)
diff --git a/src/types/buffersource.nim b/src/types/buffersource.nim
index 6377464e..675d2a2a 100644
--- a/src/types/buffersource.nim
+++ b/src/types/buffersource.nim
@@ -1,23 +1,13 @@
 import std/options
 
-when defined(posix):
-  import std/posix
-
 import loader/request
 import types/url
 
 import chakasu/charset
 
 type
-  BufferSourceType* = enum
-    CLONE, LOAD_REQUEST
-
   BufferSource* = object
     location*: URL
     contentType*: Option[string] # override
     charset*: Charset # fallback
-    case t*: BufferSourceType
-    of CLONE:
-      clonepid*: Pid
-    of LOAD_REQUEST:
-      request*: Request
+    request*: Request
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-01-08 06:13:39 +0000