Incremental rendering

Yay!

Admittedly, it is not very useful in its current form, except maybe on
very slow networks.

The problem is that renderDocument is *slow*, so we only run it when
onload fails to consume all bytes from the network in a single pass.
Even then, we are guaranteed to get a FOUC, since CSS is only downloaded
in finishLoad(). Well, I think it's cool, anyway.

8 files changed, 389 insertions, 224 deletions

diff --git a/lib/chame b/lib/chame
-Subproject 946af631dc066b4cd291fc5a96ee0db4b075866
+Subproject f6018b45b8d3d41e31d5ab003813504333c42eb
diff --git a/src/html/chadombuilder.nim b/src/html/chadombuilder.nim
index 21e94901..a2e80f3b 100644
--- a/src/html/chadombuilder.nim
+++ b/src/html/chadombuilder.nim
@@ -20,14 +20,23 @@ import chame/tags
 # DOMBuilder implementation for Chawan.
 
 type CharsetConfidence = enum
-  CONFIDENCE_TENTATIVE, CONFIDENCE_CERTAIN, CONFIDENCE_IRRELEVANT
+  ccTentative, ccCertain, ccIrrelevant
 
 type
+  HTML5ParserWrapper* = ref object
+    parser: HTML5Parser[Node, CAtom]
+    charsetStack: seq[Charset]
+    seekable: bool
+    builder*: ChaDOMBuilder
+    opts: HTML5ParserOpts[Node, CAtom]
+    inputStream: Stream
+    encoder: EncoderStream
+    decoder: DecoderStream
+
   ChaDOMBuilder = ref object of DOMBuilder[Node, CAtom]
     charset: Charset
     confidence: CharsetConfidence
-    document: Document
-    isFragment: bool
+    document*: Document
     factory: CAtomFactory
     poppedScript: HTMLScriptElement
 
@@ -79,19 +88,21 @@ proc setQuirksModeImpl(builder: ChaDOMBuilder, quirksMode: QuirksMode) =
 
 proc setEncodingImpl(builder: ChaDOMBuilder, encoding: string):
     SetEncodingResult =
-  let charset = getCharset(encoding)
-  if charset == CHARSET_UNKNOWN:
+  if builder.confidence != ccTentative:
     return SET_ENCODING_CONTINUE
   if builder.charset in {CHARSET_UTF_16_LE, CHARSET_UTF_16_BE}:
-    builder.confidence = CONFIDENCE_CERTAIN
+    builder.confidence = ccCertain
     return SET_ENCODING_CONTINUE
-  builder.confidence = CONFIDENCE_CERTAIN
+  let charset = getCharset(encoding)
+  if charset == CHARSET_UNKNOWN:
+    return SET_ENCODING_CONTINUE
+  builder.confidence = ccCertain
   if charset == builder.charset:
     return SET_ENCODING_CONTINUE
-  if charset == CHARSET_X_USER_DEFINED:
-    builder.charset = CHARSET_WINDOWS_1252
+  builder.charset = if charset == CHARSET_X_USER_DEFINED:
+    CHARSET_WINDOWS_1252
   else:
-    builder.charset = charset
+    charset
   return SET_ENCODING_STOP
 
 proc getTemplateContentImpl(builder: ChaDOMBuilder, handle: Node): Node =
@@ -189,7 +200,7 @@ proc elementPoppedImpl(builder: ChaDOMBuilder, element: Node) =
     builder.poppedScript = HTMLScriptElement(element)
 
 proc newChaDOMBuilder(url: URL, window: Window, factory: CAtomFactory,
-    isFragment = false): ChaDOMBuilder =
+    confidence: CharsetConfidence): ChaDOMBuilder =
   let document = newDocument(factory)
   document.contentType = "text/html"
   document.url = url
@@ -198,17 +209,15 @@ proc newChaDOMBuilder(url: URL, window: Window, factory: CAtomFactory,
     window.document = document
   return ChaDOMBuilder(
     document: document,
-    isFragment: isFragment,
-    factory: factory
+    factory: factory,
+    confidence: confidence
   )
 
 # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments
 proc parseHTMLFragment*(element: Element, s: string): seq[Node] =
   let url = parseURL("about:blank").get
   let factory = element.document.factory
-  let builder = newChaDOMBuilder(url, nil, factory)
-  let inputStream = newStringStream(s)
-  builder.isFragment = true
+  let builder = newChaDOMBuilder(url, nil, factory, ccIrrelevant)
   let document = builder.document
   document.mode = element.document.mode
   let state = case element.tagType
@@ -234,12 +243,9 @@ proc parseHTMLFragment*(element: Element, s: string): seq[Node] =
     pushInTemplate: element.tagType == TAG_TEMPLATE
   )
   var parser = initHTML5Parser(builder, opts)
-  var buffer: array[4096, char]
-  while true:
-    let n = inputStream.readData(addr buffer[0], buffer.len)
-    if n == 0: break
-    let res = parser.parseChunk(buffer.toOpenArray(0, n - 1))
-    assert res == PRES_CONTINUE # scripting is false, so this must be continue
+  let res = parser.parseChunk(s.toOpenArray(0, s.high))
+  # scripting is false and confidence is certain -> this must be continue
+  assert res == PRES_CONTINUE
   parser.finish()
   builder.finish()
   return root.childList
@@ -257,107 +263,126 @@ proc bomSniff(inputStream: Stream): Charset =
   inputStream.setPosition(0)
   return CHARSET_UNKNOWN
 
-proc parseHTML*(inputStream: Stream, window: Window, url: URL,
-    factory: CAtomFactory, charsets: seq[Charset] = @[],
-    seekable = true): Document =
+proc switchCharset(wrapper: HTML5ParserWrapper) =
+  let builder = wrapper.builder
+  builder.charset = wrapper.charsetStack.pop()
+  if wrapper.seekable:
+    builder.confidence = ccTentative # used in the next iteration
+  else:
+    builder.confidence = ccCertain
+  let em = if wrapper.charsetStack.len == 0 or not wrapper.seekable:
+    DECODER_ERROR_MODE_REPLACEMENT
+  else:
+    DECODER_ERROR_MODE_FATAL
+  wrapper.parser = initHTML5Parser(builder, wrapper.opts)
+  wrapper.decoder = newDecoderStream(wrapper.inputStream, builder.charset,
+    errormode = em)
+  wrapper.decoder.setInhibitCheckEnd(true)
+  wrapper.encoder = newEncoderStream(wrapper.decoder, CHARSET_UTF_8,
+    errormode = ENCODER_ERROR_MODE_FATAL)
+
+proc newHTML5ParserWrapper*(inputStream: Stream, window: Window, url: URL,
+    factory: CAtomFactory, charsets: seq[Charset] = @[], seekable = true):
+    HTML5ParserWrapper =
   let opts = HTML5ParserOpts[Node, CAtom](
     isIframeSrcdoc: false, #TODO?
     scripting: window != nil and window.settings.scripting
   )
-  let builder = newChaDOMBuilder(url, window, factory)
-  var charsetStack: seq[Charset]
-  for i in countdown(charsets.high, 0):
-    charsetStack.add(charsets[i])
-  var seekable = seekable
-  var inputStream = inputStream
-  if seekable:
-    let scs = inputStream.bomSniff()
-    if scs != CHARSET_UNKNOWN:
-      charsetStack.add(scs)
-      builder.confidence = CONFIDENCE_CERTAIN
-      seekable = false
-  if charsetStack.len == 0:
-    charsetStack.add(DefaultCharset) # UTF-8
-  while true:
-    builder.charset = charsetStack.pop()
-    if seekable:
-      builder.confidence = CONFIDENCE_TENTATIVE # used in the next iteration
-    else:
-      builder.confidence = CONFIDENCE_CERTAIN
-    let em = if charsetStack.len == 0 or not seekable:
-      DECODER_ERROR_MODE_REPLACEMENT
+  let builder = newChaDOMBuilder(url, window, factory, ccTentative)
+  let wrapper = HTML5ParserWrapper(
+    seekable: seekable,
+    builder: builder,
+    opts: opts,
+    inputStream: inputStream
+  )
+  if seekable and (let scs = inputStream.bomSniff(); scs != CHARSET_UNKNOWN):
+    builder.confidence = ccCertain
+    wrapper.charsetStack = @[scs]
+    wrapper.seekable = false
+  elif charsets.len == 0:
+    wrapper.charsetStack = @[DefaultCharset] # UTF-8
+  else:
+    for i in countdown(charsets.high, 0):
+      wrapper.charsetStack.add(charsets[i])
+  wrapper.switchCharset()
+  return wrapper
+
+proc parseBuffer(wrapper: HTML5ParserWrapper, buffer: openArray[char]):
+    ParseResult =
+  let builder = wrapper.builder
+  let document = builder.document
+  var res = wrapper.parser.parseChunk(buffer)
+  # set insertion point for when it's needed
+  var ip = wrapper.parser.getInsertionPoint()
+  while res == PRES_SCRIPT:
+    if builder.poppedScript != nil:
+      #TODO microtask
+      document.writeBuffers.add(DocumentWriteBuffer())
+      builder.poppedScript.prepare()
+    while document.parserBlockingScript != nil:
+      let script = document.parserBlockingScript
+      document.parserBlockingScript = nil
+      #TODO style sheet
+      script.execute()
+      assert document.parserBlockingScript != script
+    builder.poppedScript = nil
+    if document.writeBuffers.len == 0:
+      if ip == buffer.len:
+        # nothing left to re-parse.
+        break
+      # parse rest of input buffer
+      res = wrapper.parser.parseChunk(buffer.toOpenArray(ip, buffer.high))
+      ip += wrapper.parser.getInsertionPoint() # move insertion point
     else:
-      DECODER_ERROR_MODE_FATAL
-    let decoder = newDecoderStream(inputStream, builder.charset, errormode = em)
-    let encoder = newEncoderStream(decoder, CHARSET_UTF_8,
-      errormode = ENCODER_ERROR_MODE_FATAL)
-    var parser = initHTML5Parser(builder, opts)
-    let document = builder.document
-    var buffer: array[4096, char]
-    while true:
-      let n = encoder.readData(addr buffer[0], buffer.len)
-      if n == 0: break
-      var res = parser.parseChunk(buffer.toOpenArray(0, n - 1))
-      # set insertion point for when it's needed
-      var ip = parser.getInsertionPoint()
-      while res == PRES_SCRIPT:
-        if builder.poppedScript != nil:
-          #TODO microtask
-          document.writeBuffers.add(DocumentWriteBuffer())
-          builder.poppedScript.prepare()
-        while document.parserBlockingScript != nil:
-          let script = document.parserBlockingScript
-          document.parserBlockingScript = nil
-          #TODO style sheet
-          script.execute()
-          assert document.parserBlockingScript != script
-        builder.poppedScript = nil
-        if document.writeBuffers.len == 0:
-          if ip == n:
-            # nothing left to re-parse.
-            break
-          # parse rest of input buffer
-          res = parser.parseChunk(buffer.toOpenArray(ip, n - 1))
-          ip += parser.getInsertionPoint() # move insertion point
+      let writeBuffer = document.writeBuffers[^1]
+      let p = writeBuffer.i
+      let H = writeBuffer.data.high
+      res = wrapper.parser.parseChunk(writeBuffer.data.toOpenArray(p, H))
+      case res
+      of PRES_CONTINUE:
+        discard document.writeBuffers.pop()
+        res = PRES_SCRIPT
+      of PRES_SCRIPT:
+        let pp = p + wrapper.parser.getInsertionPoint()
+        if pp == writeBuffer.data.len:
+          discard document.writeBuffers.pop()
         else:
-          let writeBuffer = document.writeBuffers[^1]
-          let p = writeBuffer.i
-          let n = writeBuffer.data.len
-          res = parser.parseChunk(writeBuffer.data.toOpenArray(p, n - 1))
-          case res
-          of PRES_CONTINUE:
-            discard document.writeBuffers.pop()
-            res = PRES_SCRIPT
-          of PRES_SCRIPT:
-            let pp = p + parser.getInsertionPoint()
-            if pp == writeBuffer.data.len:
-              discard document.writeBuffers.pop()
-            else:
-              writeBuffer.i = pp
-          of PRES_STOP:
-            break
-            {.linearScanEnd.}
-      # PRES_STOP is returned when we return SET_ENCODING_STOP from
-      # setEncodingImpl. We immediately stop parsing in this case.
-      if res == PRES_STOP:
+          writeBuffer.i = pp
+      of PRES_STOP:
         break
-    parser.finish()
-    if builder.confidence == CONFIDENCE_CERTAIN and seekable:
-      # A meta tag describing the charset has been found; force use of this
-      # charset.
+        {.linearScanEnd.}
+  return res
+
+proc parseAll*(wrapper: HTML5ParserWrapper) =
+  let builder = wrapper.builder
+  while true:
+    let buffer = wrapper.encoder.readAll()
+    if wrapper.decoder.failed:
+      assert wrapper.seekable
+      # Retry with another charset.
       builder.restart()
-      inputStream.setPosition(0)
-      charsetStack.add(builder.charset)
-      seekable = false
+      wrapper.inputStream.setPosition(0)
+      wrapper.switchCharset()
       continue
-    if decoder.failed and seekable:
-      # Retry with another charset.
+    if buffer.len == 0:
+      break
+    let res = wrapper.parseBuffer(buffer)
+    if res == PRES_STOP:
+      # A meta tag describing the charset has been found; force use of this
+      # charset.
       builder.restart()
-      inputStream.setPosition(0)
+      wrapper.inputStream.setPosition(0)
+      wrapper.charsetStack.add(builder.charset)
+      wrapper.seekable = false
+      wrapper.switchCharset()
       continue
     break
-  builder.finish()
-  return builder.document
+
+proc finish*(wrapper: HTML5ParserWrapper) =
+  wrapper.decoder.setInhibitCheckEnd(false)
+  wrapper.parseAll()
+  wrapper.parser.finish()
+  wrapper.builder.finish()
 
 proc newDOMParser(): DOMParser {.jsctor.} =
   return DOMParser()
@@ -378,8 +403,13 @@ proc parseFromString(ctx: JSContext, parser: DOMParser, str, t: string):
       newURL("about:blank").get
     #TODO this is probably broken in client (or at least sub-optimal)
     let factory = if window != nil: window.factory else: newCAtomFactory()
-    let res = parseHTML(newStringStream(str), Window(nil), url, factory)
-    return ok(res)
+    let builder = newChaDOMBuilder(url, window, factory, ccIrrelevant)
+    var parser = initHTML5Parser(builder, HTML5ParserOpts[Node, CAtom]())
+    let res = parser.parseChunk(str)
+    assert res == PRES_CONTINUE
+    parser.finish()
+    builder.finish()
+    return ok(builder.document)
   of "text/xml", "application/xml", "application/xhtml+xml", "image/svg+xml":
     return err(newInternalError("XML parsing is not supported yet"))
   else:
diff --git a/src/html/dom.nim b/src/html/dom.nim
index 114634e6..cea593cb 100644
--- a/src/html/dom.nim
+++ b/src/html/dom.nim
@@ -268,7 +268,7 @@ type
     value* {.jsget.}: Option[int32]
 
   HTMLStyleElement* = ref object of HTMLElement
-    sheet*: CSSStylesheet
+    sheet: CSSStylesheet
 
   HTMLLinkElement* = ref object of HTMLElement
     sheet*: CSSStylesheet
@@ -2315,6 +2315,11 @@ func form(label: HTMLLabelElement): HTMLFormElement {.jsfget.} =
 proc setRelList(link: HTMLLinkElement, s: string) {.jsfset: "relList".} =
   link.attr("rel", s)
 
+proc setSheet*(link: HTMLLinkElement, sheet: CSSStylesheet) =
+  link.sheet = sheet
+  if link.document != nil:
+    link.document.cachedSheetsInvalid = true
+
 # <form>
 proc setRelList(form: HTMLFormElement, s: string) {.jsfset: "relList".} =
   form.attr("rel", s)
@@ -3276,7 +3281,7 @@ proc fetchClassicScript(element: HTMLScriptElement, url: URL,
     return
   let loader = window.loader.get
   let request = createPotentialCORSRequest(url, RequestDestination.SCRIPT, cors)
-  let response = loader.doRequest(request)
+  let response = loader.doRequest(request, canredir = false)
   if response.res != 0:
     element.onComplete(ScriptResult(t: RESULT_NULL))
     return
diff --git a/src/io/posixstream.nim b/src/io/posixstream.nim
index 73a957f8..04fe0e5c 100644
--- a/src/io/posixstream.nim
+++ b/src/io/posixstream.nim
@@ -62,11 +62,19 @@ proc psReadData(s: Stream, buffer: pointer, len: int): int =
   if result == -1:
     raisePosixIOError()
 
+method sendData*(s: PosixStream, buffer: pointer, len: int): int {.base.} =
+  #TODO use sendData instead
+  let n = write(s.fd, buffer, len)
+  if n < 0:
+    raisePosixIOError()
+  return n
+
 proc psWriteData(s: Stream, buffer: pointer, len: int) =
+  #TODO use sendData instead
   let s = cast[PosixStream](s)
   let res = write(s.fd, buffer, len)
   if res == -1:
-    raise newException(IOError, $strerror(errno))
+    raisePosixIOError()
 
 proc psAtEnd(s: Stream): bool =
   return cast[PosixStream](s).isend
diff --git a/src/io/socketstream.nim b/src/io/socketstream.nim
index 9426f7a7..fb378083 100644
--- a/src/io/socketstream.nim
+++ b/src/io/socketstream.nim
@@ -9,10 +9,9 @@ when defined(posix):
 import io/posixstream
 import io/serversocket
 
-type SocketStream* = ref object of Stream
+type SocketStream* = ref object of PosixStream
   source*: Socket
   blk*: bool
-  isend: bool
 
 proc sockReadData(s: Stream, buffer: pointer, len: int): int =
   assert len != 0
@@ -50,6 +49,12 @@ proc sockWriteData(s: Stream, buffer: pointer, len: int) =
       raisePosixIOError()
     i += n
 
+method sendData*(s: SocketStream, buffer: pointer, len: int): int =
+  let n = s.source.send(buffer, len)
+  if n < 0:
+    raisePosixIOError()
+  return n
+
 proc sockAtEnd(s: Stream): bool =
   SocketStream(s).isend
 
@@ -124,3 +129,5 @@ proc acceptSocketStream*(ssock: ServerSocket, blocking = true): SocketStream =
   var sock: Socket
   ssock.sock.accept(sock, inheritable = true)
   result.source = sock
+  if not blocking:
+    sock.getFd().setBlocking(false)
diff --git a/src/loader/loader.nim b/src/loader/loader.nim
index 02585630..749f7d29 100644
--- a/src/loader/loader.nim
+++ b/src/loader/loader.nim
@@ -139,7 +139,9 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
         handle.close()
       else:
         let fd = handle.istream.fd
+        handle.setBlocking(false)
         ctx.selector.registerHandle(fd, {Read}, 0)
+        ctx.selector.registerHandle(handle.fd, {Write}, 0)
         let ofl = fcntl(fd, F_GETFL, 0)
         discard fcntl(fd, F_SETFL, ofl or O_NONBLOCK)
         # yes, this puts the istream fd in addition to the ostream fd in
@@ -164,7 +166,7 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
 proc onLoad(ctx: LoaderContext, stream: SocketStream) =
   var request: Request
   stream.sread(request)
-  let handle = newLoaderHandle(stream, request.canredir)
+  let handle = newLoaderHandle(stream, request.canredir, request.url)
   if not ctx.config.filter.match(request.url):
     handle.sendResult(ERROR_DISALLOWED_URL)
     handle.close()
@@ -188,9 +190,6 @@ proc onLoad(ctx: LoaderContext, stream: SocketStream) =
     ctx.loadResource(request, handle)
 
 proc acceptConnection(ctx: LoaderContext) =
-  #TODO TODO TODO acceptSocketStream should be non-blocking here,
-  # otherwise the client disconnecting between poll and accept could
-  # block this indefinitely.
   let stream = ctx.ssock.acceptSocketStream()
   try:
     var cmd: LoaderCommand
@@ -250,7 +249,7 @@ proc initLoaderContext(fd: cint, config: LoaderConfig): LoaderContext =
   gctx = ctx
   #TODO ideally, buffered would be true. Unfortunately this conflicts with
   # sendFileHandle/recvFileHandle.
-  ctx.ssock = initServerSocket(buffered = false)
+  ctx.ssock = initServerSocket(buffered = false, blocking = false)
   ctx.fd = int(ctx.ssock.sock.getFd())
   ctx.selector.registerHandle(ctx.fd, {Read}, 0)
   # The server has been initialized, so the main process can resume execution.
@@ -271,41 +270,81 @@ proc initLoaderContext(fd: cint, config: LoaderConfig): LoaderContext =
 
 proc runFileLoader*(fd: cint, config: LoaderConfig) =
   var ctx = initLoaderContext(fd, config)
-  var buffer {.noinit.}: array[16384, uint8]
   while ctx.alive:
     let events = ctx.selector.select(-1)
-    var unreg: seq[int]
+    var unregRead: seq[LoaderHandle]
+    var unregWrite: seq[LoaderHandle]
     for event in events:
       if Read in event.events:
         if event.fd == ctx.fd: # incoming connection
           ctx.acceptConnection()
         else:
           let handle = ctx.handleMap[event.fd]
-          while not handle.istream.atEnd:
+          assert event.fd != handle.fd
+          while true:
             try:
-              let n = handle.istream.readData(addr buffer[0], buffer.len)
-              handle.sendData(addr buffer[0], n)
+              let buffer = newLoaderBuffer()
+              buffer.len = handle.istream.readData(addr buffer[0], buffer.cap)
+              if buffer.len == 0:
+                dealloc(buffer)
+                break
+              handle.addBuffer(buffer)
+              if buffer.len < buffer.cap:
+                break
             except ErrorAgain, ErrorWouldBlock: # retry later
               break
-            except ErrorBrokenPipe: # receiver died; stop streaming
-              unreg.add(event.fd)
+            except ErrorBrokenPipe: # sender died; stop streaming
+              unregRead.add(handle)
               break
+      if Write in event.events:
+        let handle = ctx.handleMap[event.fd]
+        assert event.fd == handle.fd
+        while handle.currentBuffer != nil:
+          let buffer = handle.currentBuffer
+          try:
+            let i = handle.currentBufferIdx
+            assert buffer.len - i > 0
+            let n = handle.sendData(addr buffer[i], buffer.len - i)
+            handle.currentBufferIdx += n
+            if handle.currentBufferIdx < buffer.len:
+              break
+            handle.bufferCleared() # swap out buffer
+          except ErrorAgain, ErrorWouldBlock: # never mind
+            break
+          except ErrorBrokenPipe: # receiver died; stop streaming
+            unregWrite.add(handle)
+            break
+        if handle.istream == nil and handle.currentBuffer == nil and
+            (unregWrite.len == 0 or unregWrite[^1] != handle):
+          # after EOF, but not appended in this send cycle
+          unregWrite.add(handle)
       if Error in event.events:
         assert event.fd != ctx.fd
-        when defined(debug):
-          # sanity check
-          let handle = ctx.handleMap[event.fd]
-          if not handle.istream.atEnd():
-            let n = handle.istream.readData(addr buffer[0], buffer.len)
-            assert n == 0
-            assert handle.istream.atEnd()
-        unreg.add(event.fd)
-    for fd in unreg:
-      ctx.selector.unregister(fd)
-      let handle = ctx.handleMap[fd]
-      ctx.handleMap.del(fd)
-      ctx.handleMap.del(handle.getFd())
-      handle.close()
+        let handle = ctx.handleMap[event.fd]
+        if handle.fd == event.fd: # ostream died
+          unregWrite.add(handle)
+        else: # istream died
+          unregRead.add(handle)
+    for handle in unregRead:
+      ctx.selector.unregister(handle.istream.fd)
+      ctx.handleMap.del(handle.istream.fd)
+      handle.istream.close()
+      handle.istream = nil
+      if handle.currentBuffer == nil:
+        unregWrite.add(handle)
+      #TODO TODO TODO what to do about sostream
+    for handle in unregWrite:
+      ctx.selector.unregister(handle.fd)
+      ctx.handleMap.del(handle.fd)
+      handle.ostream.close()
+      handle.ostream = nil
+      if handle.istream != nil:
+        handle.istream.close()
+        ctx.handleMap.del(handle.istream.fd)
+        ctx.selector.unregister(handle.istream.fd)
+        handle.istream.close()
+        handle.istream = nil
+      #TODO TODO TODO what to do about sostream
   ctx.exitLoader()
 
 proc getAttribute(contentType, attrname: string): string =
@@ -478,6 +517,8 @@ proc onRead*(loader: FileLoader, fd: int) =
         buffer[].buf.setLen(olen + BufferSize)
         let n = response.body.readData(addr buffer[].buf[olen], BufferSize)
         buffer[].buf.setLen(olen + n)
+        if n == 0:
+          break
       except ErrorAgain, ErrorWouldBlock:
         break
     if response.body.atEnd():
diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim
index 7a9b3434..5d2dee4e 100644
--- a/src/loader/loaderhandle.nim
+++ b/src/loader/loaderhandle.nim
@@ -1,3 +1,4 @@
+import std/deques
 import std/net
 import std/streams
 
@@ -7,28 +8,72 @@ import io/serialize
 import io/socketstream
 import loader/headers
 
-type LoaderHandle* = ref object
-  ostream: Stream
-  # Stream for taking input
-  istream*: PosixStream
-  # Only the first handle can be redirected, because a) mailcap can only
-  # redirect the first handle and b) async redirects would result in race
-  # conditions that would be difficult to untangle.
-  canredir: bool
-  sostream: Stream # saved ostream when redirected
-  sostream_suspend: Stream # saved ostream when suspended
-  fd: int
+import types/url
+type
+  LoaderBufferPage = array[4056, uint8] # 4096 - 8 - 32
+
+  LoaderBufferObj = object
+    page*: LoaderBufferPage
+    len: int
+
+  LoaderBuffer* = ptr LoaderBufferObj
+
+  LoaderHandle* = ref object
+    ostream*: PosixStream #TODO un-extern
+    # Stream for taking input
+    istream*: PosixStream
+    # Only the first handle can be redirected, because a) mailcap can only
+    # redirect the first handle and b) async redirects would result in race
+    # conditions that would be difficult to untangle.
+    canredir: bool
+    sostream: Stream # saved ostream when redirected
+    sostream_suspend: Stream # saved ostream when suspended
+    fd*: int # ostream fd
+    currentBuffer*: LoaderBuffer
+    currentBufferIdx*: int
+    buffers: Deque[LoaderBuffer]
+    url*: URL #TODO TODO TODO debug
 
 # Create a new loader handle, with the output stream ostream.
-proc newLoaderHandle*(ostream: Stream, canredir: bool): LoaderHandle =
+proc newLoaderHandle*(ostream: PosixStream, canredir: bool, url: URL): LoaderHandle =
   return LoaderHandle(
     ostream: ostream,
     canredir: canredir,
-    fd: int(SocketStream(ostream).source.getFd())
+    fd: int(SocketStream(ostream).source.getFd()),
+    url: url
   )
 
-proc getFd*(handle: LoaderHandle): int =
-  return handle.fd
+func `[]`*(buffer: LoaderBuffer, i: int): var uint8 {.inline.} =
+  return buffer[].page[i]
+
+func cap*(buffer: LoaderBuffer): int {.inline.} =
+  return buffer[].page.len
+
+func len*(buffer: LoaderBuffer): var int {.inline.} =
+  return buffer[].len
+
+proc `len=`*(buffer: LoaderBuffer, i: int) {.inline.} =
+  buffer[].len = i
+
+proc newLoaderBuffer*(): LoaderBuffer =
+  let buffer = cast[LoaderBuffer](alloc(sizeof(LoaderBufferObj)))
+  buffer.len = 0
+  return buffer
+
+proc addBuffer*(handle: LoaderHandle, buffer: LoaderBuffer) =
+  if handle.currentBuffer == nil:
+    handle.currentBuffer = buffer
+  else:
+    handle.buffers.addLast(buffer)
+
+proc bufferCleared*(handle: LoaderHandle) =
+  assert handle.currentBuffer != nil
+  handle.currentBufferIdx = 0
+  dealloc(handle.currentBuffer)
+  if handle.buffers.len > 0:
+    handle.currentBuffer = handle.buffers.popFirst()
+  else:
+    handle.currentBuffer = nil
 
 proc addOutputStream*(handle: LoaderHandle, stream: Stream) =
   if likely(handle.sostream_suspend != nil):
@@ -43,8 +88,18 @@ proc addOutputStream*(handle: LoaderHandle, stream: Stream) =
     # sostream_suspend is never nil when the function is called.
     # (Feel free to remove this assertion if this changes.)
     doAssert false
-    let ms = newMultiStream(handle.ostream, stream)
-    handle.ostream = ms
+    #TODO TODO TODO fix this
+    #let ms = newMultiStream(handle.ostream, stream)
+    #handle.ostream = ms
+
+proc setBlocking*(handle: LoaderHandle, blocking: bool) =
+  #TODO this is stupid
+  if handle.sostream_suspend != nil and handle.sostream_suspend of SocketStream:
+    SocketStream(handle.sostream_suspend).setBlocking(blocking)
+  elif handle.sostream != nil and handle.sostream of SocketStream:
+    SocketStream(handle.sostream).setBlocking(blocking)
+  else:
+    SocketStream(handle.ostream).setBlocking(blocking)
 
 proc sendResult*(handle: LoaderHandle, res: int, msg = "") =
   handle.ostream.swrite(res)
@@ -67,23 +122,25 @@ proc sendHeaders*(handle: LoaderHandle, headers: Headers) =
       let stream = newPosixStream(fd)
       handle.ostream = stream
 
-proc sendData*(handle: LoaderHandle, p: pointer, nmemb: int) =
-  handle.ostream.writeData(p, nmemb)
-
-proc sendData*(handle: LoaderHandle, s: string) =
-  if s.len > 0:
-    handle.sendData(unsafeAddr s[0], s.len)
+proc sendData*(handle: LoaderHandle, p: pointer, nmemb: int): int =
+  return handle.ostream.sendData(p, nmemb)
 
 proc suspend*(handle: LoaderHandle) =
+  #TODO TODO TODO fix suspend
+  doAssert false
   handle.sostream_suspend = handle.ostream
-  handle.ostream = newStringStream()
+  #handle.ostream = newStringStream()
 
 proc resume*(handle: LoaderHandle) =
+  #TODO TODO TODO fix resume
+  doAssert false
+  #[
   let ss = handle.ostream
   handle.ostream = handle.sostream_suspend
   handle.sostream_suspend = nil
   handle.sendData(ss.readAll())
   ss.close()
+  ]#
 
 proc close*(handle: LoaderHandle) =
   if handle.sostream != nil:
@@ -93,6 +150,9 @@ proc close*(handle: LoaderHandle) =
       # ignore error, that just means the buffer has already closed the stream
       discard
     handle.sostream.close()
-  handle.ostream.close()
+  if handle.ostream != nil:
+    handle.ostream.close()
+    handle.ostream = nil
   if handle.istream != nil:
     handle.istream.close()
+    handle.istream = nil
diff --git a/src/server/buffer.nim b/src/server/buffer.nim
index abb116c6..b3b7f4a6 100644
--- a/src/server/buffer.nim
+++ b/src/server/buffer.nim
@@ -93,7 +93,6 @@ type
     rfd: int # file descriptor of command pipe
     fd: int # file descriptor of buffer source
     alive: bool
-    readbufsize: int
     lines: FlexibleGrid
     rendered: bool
     source: BufferSource
@@ -124,6 +123,7 @@ type
     factory: CAtomFactory
     uastyle: CSSStylesheet
     quirkstyle: CSSStylesheet
+    htmlParser: HTML5ParserWrapper
 
   InterfaceOpaque = ref object
     stream: Stream
@@ -644,7 +644,13 @@ proc do_reshape(buffer: Buffer) =
     buffer.prevstyled = styledRoot
   else:
     buffer.lines.renderStream(buffer.srenderer)
-    buffer.available = 0
+
+proc processData(buffer: Buffer) =
+  if buffer.ishtml:
+    buffer.htmlParser.parseAll()
+    buffer.document = buffer.htmlParser.builder.document
+  else:
+    buffer.lines.renderStream(buffer.srenderer)
 
 proc windowChange*(buffer: Buffer, attrs: WindowAttributes) {.proxy.} =
   buffer.attrs = attrs
@@ -695,14 +701,14 @@ proc updateHover*(buffer: Buffer, cursorx, cursory: int): UpdateHoverResult {.pr
 
   buffer.prevnode = thisnode
 
-proc loadResource(buffer: Buffer, elem: HTMLLinkElement): EmptyPromise =
+proc loadResource(buffer: Buffer, link: HTMLLinkElement): EmptyPromise =
   let document = buffer.document
-  let href = elem.attr("href")
+  let href = link.attr("href")
   if href == "": return
   let url = parseURL(href, document.url.some)
   if url.isSome:
     let url = url.get
-    let media = elem.media
+    let media = link.media
     if media != "":
       let cvals = parseListOfComponentValues(newStringStream(media))
       let media = parseMediaQueryList(cvals)
@@ -724,7 +730,8 @@ proc loadResource(buffer: Buffer, elem: HTMLLinkElement): EmptyPromise =
           #TODO non-utf-8 css
           let ds = newDecoderStream(ss, cs = CHARSET_UTF_8)
           let source = newEncoderStream(ds, cs = CHARSET_UTF_8)
-          elem.sheet = parseStylesheet(source, buffer.factory))
+          link.setSheet(parseStylesheet(source, buffer.factory))
+      )
 
 proc loadResource(buffer: Buffer, elem: HTMLImageElement): EmptyPromise =
   let document = buffer.document
@@ -803,6 +810,14 @@ proc setHTML(buffer: Buffer, ishtml: bool) =
         buffer.attrs,
         buffer.factory
       )
+    buffer.htmlParser = newHTML5ParserWrapper(
+      buffer.sstream,
+      buffer.window,
+      buffer.url,
+      buffer.factory,
+      buffer.charsets,
+      seekable = true
+    )
     const css = staticRead"res/ua.css"
     const quirk = css & staticRead"res/quirk.css"
     buffer.uastyle = css.parseStylesheet(factory)
@@ -1138,17 +1153,9 @@ proc finishLoad(buffer: Buffer): EmptyPromise =
     return p
   var p: EmptyPromise
   if buffer.ishtml:
-    buffer.sstream.setPosition(0)
-    buffer.available = 0
-    let document = parseHTML(
-      buffer.sstream,
-      charsets = buffer.charsets,
-      window = buffer.window,
-      url = buffer.url,
-      factory = buffer.factory
-    )
-    buffer.document = document
-    document.readyState = READY_STATE_INTERACTIVE
+    buffer.htmlParser.finish()
+    buffer.document = buffer.htmlParser.builder.document
+    buffer.document.readyState = READY_STATE_INTERACTIVE
     buffer.state = LOADING_RESOURCES
     buffer.dispatchDOMContentLoadedEvent()
     p = buffer.loadResources()
@@ -1194,34 +1201,47 @@ proc onload(buffer: Buffer) =
     return
   of LOADING_PAGE:
     discard
-  let op = buffer.sstream.getPosition()
-  var s {.noinit.}: array[BufferSize, uint8]
-  try:
-    buffer.sstream.setPosition(op + buffer.available)
-    let n = buffer.istream.readData(addr s[0], buffer.readbufsize)
-    if n != 0: # n can be 0 if we get EOF. (in which case we shouldn't reshape unnecessarily.)
-      buffer.sstream.writeData(addr s[0], n)
-      buffer.sstream.setPosition(op)
-      if buffer.readbufsize < BufferSize:
-        buffer.readbufsize = min(BufferSize, buffer.readbufsize * 2)
-      buffer.available += n
-      if buffer.ishtml:
+  while true:
+    let op = buffer.sstream.getPosition()
+    var s {.noinit.}: array[BufferSize, uint8]
+    try:
+      let n = buffer.istream.readData(addr s[0], s.len)
+      if n != 0:
+        buffer.sstream.writeData(addr s[0], n)
+        buffer.sstream.setPosition(op)
+        buffer.available += n
+        buffer.processData()
         res.bytes = buffer.available
-      else:
-        buffer.do_reshape()
-    if buffer.istream.atEnd():
-      res.atend = true
-      buffer.finishLoad().then(proc() =
-        buffer.state = LOADED
-        if buffer.document != nil: # may be nil if not buffer.ishtml
-          buffer.document.readyState = READY_STATE_COMPLETE
-        buffer.dispatchLoadEvent()
-        buffer.resolveTask(LOAD, res))
-      return
-    buffer.resolveTask(LOAD, res)
-  except ErrorAgain, ErrorWouldBlock:
-    if buffer.readbufsize > 1:
-      buffer.readbufsize = buffer.readbufsize div 2
+      res.lines = buffer.lines.len
+      if buffer.istream.atEnd():
+        # EOF
+        res.atend = true
+        buffer.finishLoad().then(proc() =
+          buffer.prevstyled = nil # for incremental rendering
+          buffer.do_reshape()
+          res.lines = buffer.lines.len
+          buffer.state = LOADED
+          if buffer.document != nil: # may be nil if not buffer.ishtml
+            buffer.document.readyState = READY_STATE_COMPLETE
+          buffer.dispatchLoadEvent()
+          buffer.resolveTask(LOAD, res)
+        )
+        return # skip incr render
+      buffer.resolveTask(LOAD, res)
+    except ErrorAgain, ErrorWouldBlock:
+      break
+  if buffer.document != nil:
+    # incremental rendering: only if we cannot read the entire stream in one
+    # pass
+    #TODO this is too simplistic to be really useful
+    let uastyle = if buffer.document.mode != QUIRKS:
+      buffer.uastyle
+    else:
+      buffer.quirkstyle
+    let styledRoot = buffer.document.applyStylesheets(uastyle,
+      buffer.userstyle, buffer.prevstyled)
+    buffer.lines = renderDocument(styledRoot, buffer.attrs)
+    buffer.prevstyled = styledRoot
 
 proc getTitle*(buffer: Buffer): string {.proxy.} =
   if buffer.document != nil:
@@ -1237,16 +1257,10 @@ proc cancel*(buffer: Buffer): int {.proxy.} =
   buffer.istream.close()
   buffer.state = LOADED
   if buffer.ishtml:
-    buffer.sstream.setPosition(0)
-    buffer.available = 0
-    buffer.document = parseHTML(
-      buffer.sstream,
-      charsets = buffer.charsets,
-      window = buffer.window,
-      url = buffer.url,
-      factory = buffer.factory,
-      seekable = false
-    )
+    buffer.htmlParser.finish()
+    buffer.document = buffer.htmlParser.builder.document
+    buffer.document.readyState = READY_STATE_INTERACTIVE
+    buffer.state = LOADING_RESOURCES
     buffer.do_reshape()
   return buffer.lines.len
 
@@ -1784,6 +1798,7 @@ proc handleRead(buffer: Buffer, fd: int) =
     buffer.onload()
   elif fd in buffer.loader.connecting:
     buffer.loader.onConnected(fd)
+    buffer.loader.onRead(fd)
     if buffer.config.scripting:
       buffer.window.runJSJobs()
   elif fd in buffer.loader.ongoing:
@@ -1848,7 +1863,6 @@ proc launchBuffer*(config: BufferConfig, source: BufferSource,
     sstream: newStringStream(),
     width: attrs.width,
     height: attrs.height - 1,
-    readbufsize: BufferSize,
     selector: newSelector[int](),
     estream: newFileStream(stderr),
     pstream: socks,