Remove CLONE BufferSource; cache document sources in tmpdir

At last all BufferSources are unified.

To achieve the same effect as the previous CLONE source type, we now
use the "fromcache" flag in Request. This *forces* the document to be
streamed from the disk; if the file no longer exists for some reason,
an error is returned (i.e. the document is not re-downloaded).

For a document to be cached, it has to be the main document of the
buffer (i.e. no additional resources requested with fetch()), and
also not an x-htmloutput HTML file (for those, the original source is
saved). The result is that toggleSource now always returns the actual
source for e.g. markdown files, not the HTML-transformed version.

Also, it is now possible to view the source of a document that is
still being downloaded.

buffer.sstream has almost been eliminated; it still exists, but only as
a pseudo-buffer to interface with EncoderStream and DecoderStream. It no
longer holds the entire source of a buffer at any point, and is cleared
as soon as the buffer is completely loaded.

3 files changed, 42 insertions, 44 deletions

diff --git a/src/local/client.nim b/src/local/client.nim
index 76cbd4e7..20e1a6ef 100644
--- a/src/local/client.nim
+++ b/src/local/client.nim
@@ -95,9 +95,6 @@ proc finalize(client: Client) {.jsfin.} =
   if client.jsrt != nil:
     free(client.jsrt)
 
-proc doRequest(client: Client, req: Request): Response {.jsfunc.} =
-  return client.loader.doRequest(req)
-
 proc fetch[T: Request|string](client: Client, req: T,
     init = none(RequestInit)): JSResult[FetchPromise] {.jsfunc.} =
   let req = ?newRequest(client.jsctx, req, init)
diff --git a/src/local/container.nim b/src/local/container.nim
index d4d44b0f..585c8a3f 100644
--- a/src/local/container.nim
+++ b/src/local/container.nim
@@ -156,6 +156,29 @@ proc newBuffer*(forkserver: ForkServer, config: BufferConfig,
     canreinterpret: canreinterpret
   )
 
+proc newBufferFrom*(forkserver: ForkServer, attrs: WindowAttributes,
+    container: Container, contentTypeOverride: string): Container =
+  var source = container.source
+  source.contentType = some(contentTypeOverride)
+  source.request = newRequest(source.request.url, fromcache = true)
+  let config = container.config
+  let loaderPid = container.loaderPid
+  let bufferPid = forkserver.forkBufferWithLoader(source, config, attrs,
+    loaderPid)
+  return Container(
+    source: source,
+    width: container.width,
+    height: container.height,
+    title: container.title,
+    config: config,
+    process: bufferPid,
+    loaderPid: loaderPid,
+    pos: CursorPosition(
+      setx: -1
+    ),
+    canreinterpret: true
+  )
+
 func location*(container: Container): URL {.jsfget.} =
   return container.source.location
 
@@ -1385,9 +1408,9 @@ proc startload*(container: Container) =
 proc connect2*(container: Container): EmptyPromise =
   return container.iface.connect2()
 
-proc redirectToFd*(container: Container, fdin: FileHandle, wait: bool):
+proc redirectToFd*(container: Container, fdin: FileHandle, wait, cache: bool):
     EmptyPromise =
-  return container.iface.redirectToFd(fdin, wait)
+  return container.iface.redirectToFd(fdin, wait, cache)
 
 proc readFromFd*(container: Container, fdout: FileHandle, id: string,
     ishtml: bool): EmptyPromise =
@@ -1430,11 +1453,6 @@ proc reshape(container: Container): EmptyPromise {.discardable, jsfunc.} =
     container.setNumLines(lines)
     return container.requestLines())
 
-proc pipeBuffer*(container, pipeTo: Container) =
-  container.iface.getSource().then(proc() =
-    pipeTo.load() #TODO do not load if pipeTo is killed first?
-  )
-
 proc onclick(container: Container, res: ClickResult)
 
 proc displaySelect(container: Container, selectResult: SelectResult) =
@@ -1531,7 +1549,8 @@ proc setStream*(container: Container, stream: Stream) =
     discard container.iface.load().then(proc(res: LoadResult) =
       container.onload(res))
 
-proc onreadline(container: Container, w: Slice[int], handle: (proc(line: SimpleFlexibleLine)), res: GetLinesResult) =
+proc onreadline(container: Container, w: Slice[int],
+    handle: (proc(line: SimpleFlexibleLine)), res: GetLinesResult) =
   for line in res.lines:
     handle(line)
   if res.numLines > w.b + 1:
@@ -1544,7 +1563,7 @@ proc onreadline(container: Container, w: Slice[int], handle: (proc(line: SimpleF
     container.setNumLines(res.numLines, true)
 
 # Synchronously read all lines in the buffer.
-proc readLines*(container: Container, handle: (proc(line: SimpleFlexibleLine))) =
+proc readLines*(container: Container, handle: proc(line: SimpleFlexibleLine)) =
   if container.code == 0:
     # load succeded
     let w = 0 .. 23
diff --git a/src/local/pager.nim b/src/local/pager.nim
index cc33d06b..3233b012 100644
--- a/src/local/pager.nim
+++ b/src/local/pager.nim
@@ -458,26 +458,6 @@ proc newBuffer(pager: Pager, bufferConfig: BufferConfig, source: BufferSource,
     fd
   )
 
-proc dupeBuffer2(pager: Pager, container: Container, location: URL,
-    contentType = ""): Container =
-  let contentType = if contentType != "":
-    some(contentType)
-  else:
-    container.contentType
-  let location = if location != nil:
-    location
-  else:
-    container.source.location
-  let source = BufferSource(
-    t: CLONE,
-    location: location,
-    contentType: contentType,
-    clonepid: container.process,
-  )
-  let pipeTo = pager.newBuffer(container.config, source, container.title)
-  container.pipeBuffer(pipeTo)
-  return pipeTo
-
 proc dupeBuffer(pager: Pager, container: Container, location: URL) =
   container.clone(location).then(proc(container: Container) =
     if container == nil:
@@ -620,7 +600,12 @@ proc toggleSource(pager: Pager) {.jsfunc.} =
       "text/plain"
     else:
       "text/html"
-    let container = pager.dupeBuffer2(pager.container, nil, contentType)
+    let container = newBufferFrom(
+      pager.forkserver,
+      pager.attrs,
+      pager.container,
+      contentType
+    )
     container.sourcepair = pager.container
     pager.container.sourcepair = container
     pager.addContainer(container)
@@ -687,7 +672,7 @@ proc applySiteconf(pager: Pager, url: var URL): BufferConfig =
       proxy = sc.proxy.get
   return pager.config.getBufferConfig(url, cookiejar, headers, referer_from,
     scripting, charsets, images, userstyle, proxy, mimeTypes, urimethodmap,
-    pager.cgiDir)
+    pager.cgiDir, pager.tmpdir)
 
 # Load request in a new buffer.
 proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
@@ -705,7 +690,6 @@ proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
     # what other browsers do. Still, it would be nice if we got some visual
     # feedback on what is actually going to happen when typing a URL; TODO.
     let source = BufferSource(
-      t: LOAD_REQUEST,
       request: request,
       contentType: ctype,
       charset: cs,
@@ -779,7 +763,6 @@ proc readPipe0*(pager: Pager, ctype: Option[string], cs: Charset,
   var location = location.get(newURL("stream:-").get)
   let bufferconfig = pager.applySiteconf(location)
   let source = BufferSource(
-    t: LOAD_REQUEST,
     request: newRequest(location),
     contentType: some(ctype.get("text/plain")),
     charset: cs,
@@ -973,7 +956,7 @@ proc runMailcapReadPipe(pager: Pager, container: Container,
   discard close(pipefd_out[1])
   let fdin = pipefd_in[1]
   let fdout = pipefd_out[0]
-  let p = container.redirectToFd(fdin, wait = false)
+  let p = container.redirectToFd(fdin, wait = false, cache = true)
   let p2 = p.then(proc(): auto =
     discard close(fdin)
     let ishtml = HTMLOUTPUT in entry.flags
@@ -1013,7 +996,7 @@ proc runMailcapWritePipe(pager: Pager, container: Container,
     # parent
     discard close(pipefd[0])
     let fd = pipefd[1]
-    let p = container.redirectToFd(fd, wait = false)
+    let p = container.redirectToFd(fd, wait = false, cache = false)
     discard close(fd)
     if needsterminal:
       var x: cint
@@ -1026,10 +1009,11 @@ proc runMailcapWritePipe(pager: Pager, container: Container,
 # needsterminal is ignored.
 proc runMailcapReadFile(pager: Pager, container: Container,
     entry: MailcapEntry, cmd, outpath: string): (EmptyPromise, bool) =
-  let fd = open(outpath, O_WRONLY or O_CREAT, 0o644)
+  let fd = open(outpath, O_WRONLY or O_CREAT, 0o600)
   if fd == -1:
     return (nil, false)
-  let p = container.redirectToFd(fd, wait = true).then(proc(): auto =
+  let p = container.redirectToFd(fd, wait = true, cache = true).then(proc():
+      auto =
     var pipefd: array[2, cint] # redirect stdout here
     if pipe(pipefd) == -1:
       raise newException(Defect, "Failed to open pipe.")
@@ -1060,10 +1044,10 @@ proc runMailcapReadFile(pager: Pager, container: Container,
 proc runMailcapWriteFile(pager: Pager, container: Container,
     entry: MailcapEntry, cmd, outpath: string): (EmptyPromise, bool) =
   let needsterminal = NEEDSTERMINAL in entry.flags
-  let fd = open(outpath, O_WRONLY or O_CREAT, 0o644)
+  let fd = open(outpath, O_WRONLY or O_CREAT, 0o600)
   if fd == -1:
     return (nil, false)
-  let p = container.redirectToFd(fd, wait = true).then(proc() =
+  let p = container.redirectToFd(fd, wait = true, cache = false).then(proc() =
     if needsterminal:
       pager.term.quit()
       discard execCmd(cmd)
@@ -1099,8 +1083,6 @@ proc runMailcapWriteFile(pager: Pager, container: Container,
 proc checkMailcap(pager: Pager, container: Container): (EmptyPromise, bool) =
   if container.contentType.isNone:
     return (nil, true)
-  if container.source.t == CLONE:
-    return (nil, true) # clone cannot use mailcap
   let contentType = container.contentType.get
   if contentType == "text/html":
     # We support HTML natively, so it would make little sense to execute