Remove CLONE BufferSource; cache document sources in tmpdir

At last all BufferSources are unified.

To achieve the same effect as the previous CLONE source type, we now
use the "fromcache" flag in Request. This *forces* the document to be
streamed from the disk; if the file no longer exists for some reason,
an error is returned (i.e. the document is not re-downloaded).

For a document to be cached, it has to be the main document of the
buffer (i.e. no additional resources requested with fetch()), and
also not an x-htmloutput HTML file (for those, the original source is
saved). The result is that toggleSource now always returns the actual
source for e.g. markdown files, not the HTML-transformed version.

Also, it is now possible to view the source of a document that is
still being downloaded.

buffer.sstream has almost been eliminated; it still exists, but only as
a pseudo-buffer to interface with EncoderStream and DecoderStream. It no
longer holds the entire source of a buffer at any point, and is cleared
as soon as the buffer is completely loaded.

1 files changed, 11 insertions, 10 deletions

diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim
index 7a0e893a..54054ed1 100644
--- a/src/loader/loaderhandle.nim
+++ b/src/loader/loaderhandle.nim
@@ -1,11 +1,13 @@
 import std/deques
 import std/net
 import std/streams
+import std/tables
 
 import io/posixstream
 import io/serialize
 import io/socketstream
 import loader/headers
+import loader/streamid
 
 when defined(debug):
   import types/url
@@ -27,8 +29,7 @@ type
     ostream*: PosixStream
     istreamAtEnd*: bool
     sostream*: PosixStream # saved ostream when redirected
-    clientFd*: int
-    clientPid*: int
+    clientId*: StreamId
     registered*: bool
 
   LoaderHandle* = ref object
@@ -39,6 +40,8 @@ type
     # conditions that would be difficult to untangle.
     canredir: bool
     outputs*: seq[OutputHandle]
+    cached*: bool
+    cachepath*: string
     when defined(debug):
       url*: URL
 
@@ -49,16 +52,15 @@ type
       buffer.page = nil
 
 # Create a new loader handle, with the output stream ostream.
-proc newLoaderHandle*(ostream: PosixStream, canredir: bool,
-    clientPid, clientFd: int): LoaderHandle =
+proc newLoaderHandle*(ostream: PosixStream, canredir: bool, clientId: StreamId):
+    LoaderHandle =
   let handle = LoaderHandle(
     canredir: canredir
   )
   handle.outputs.add(OutputHandle(
     ostream: ostream,
     parent: handle,
-    clientPid: clientPid,
-    clientFd: clientFd
+    clientId: clientId
   ))
   return handle
 
@@ -100,8 +102,7 @@ proc bufferCleared*(output: OutputHandle) =
   else:
     output.currentBuffer = nil
 
-proc tee*(outputIn: OutputHandle, ostream: PosixStream,
-    clientFd, clientPid: int) =
+proc tee*(outputIn: OutputHandle, ostream: PosixStream, clientId: StreamId) =
   outputIn.parent.outputs.add(OutputHandle(
     parent: outputIn.parent,
     ostream: ostream,
@@ -109,8 +110,7 @@ proc tee*(outputIn: OutputHandle, ostream: PosixStream,
     currentBufferIdx: outputIn.currentBufferIdx,
     buffers: outputIn.buffers,
     istreamAtEnd: outputIn.istreamAtEnd,
-    clientFd: clientFd,
-    clientPid: clientPid
+    clientId: clientId
   ))
 
 template output*(handle: LoaderHandle): OutputHandle =
@@ -132,6 +132,7 @@ proc sendHeaders*(handle: LoaderHandle, headers: Headers) =
   if handle.canredir:
     var redir: bool
     output.ostream.sread(redir)
+    output.ostream.sread(handle.cached)
     if redir:
       let fd = SocketStream(output.ostream).recvFileHandle()
       output.sostream = output.ostream