client: fix blocking reads on container connection

Sometimes, headers take a while to reach us even after the result has
been sent. e.g.

echo 'Cha-Control: Connected'
sleep 5
echo 'Cha-Control: ControlDone'

^ this froze the UI for 5 seconds, that's certainly not what we want.

Since we don't have a proper buffered reader yet, and I don't want to
write another disgusting hack like BufStream, we just use a state
machine to figure out how much we can read. Sounds bad, but in practice
it works just fine since loader's response patterns are very simple.

3 files changed, 104 insertions, 43 deletions

diff --git a/src/local/client.nim b/src/local/client.nim
index 6b605311..07f493e6 100644
--- a/src/local/client.nim
+++ b/src/local/client.nim
@@ -1,3 +1,4 @@
+import std/exitprocs
 import std/nativesockets
 import std/net
 import std/options
@@ -11,8 +12,6 @@ import std/unicode
 when defined(posix):
   import std/posix
 
-import std/exitprocs
-
 import bindings/constcharp
 import bindings/quickjs
 import config/config
@@ -65,9 +64,7 @@ type
     ibuf: string
     jsctx: JSContext
     jsrt: JSRuntime
-    loader: FileLoader
     pager {.jsget.}: Pager
-    selector: Selector[int]
     timeouts: TimeoutState
     pressed: tuple[col: int, row: int]
 
@@ -78,12 +75,18 @@ type
 
 jsDestructor(Client)
 
-func forkserver(client: Client): ForkServer {.inline.} =
-  client.pager.forkserver
-
 func console(client: Client): Console {.jsfget.} =
   return client.consoleWrapper.console
 
+template selector(client: Client): Selector[int] =
+  client.pager.selector
+
+template loader(client: Client): FileLoader =
+  client.pager.loader
+
+template forkserver(client: Client): ForkServer =
+  client.pager.forkserver
+
 proc readChar(client: Client): char =
   if client.ibuf == "":
     try:
@@ -506,25 +509,13 @@ proc acceptBuffers(client: Client) =
 proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize_t): cint {.
   importc: "setvbuf", header: "<stdio.h>", tags: [].}
 
-proc handleRead(client: Client, fd: int) =
+proc handleRead(client: Client; fd: int) =
   if client.pager.infile != nil and fd == client.pager.infile.getFileHandle():
     client.input().then(proc() =
       client.handlePagerEvents()
     )
-  elif (let i = client.pager.findConnectingBuffer(fd); i != -1):
-    client.selector.unregister(fd)
-    client.loader.unregistered.add(fd)
-    let (container, stream) = client.pager.connectingBuffers[i]
-    let response = stream.readResponse(container.request)
-    if response.body == nil:
-      client.pager.fail(container, response.getErrorMessage())
-    elif (let redirect = response.getRedirect(container.request);
-        redirect != nil):
-      client.pager.redirect(container, response, redirect)
-      response.body.close()
-    else:
-      client.pager.connected(container, response)
-    client.pager.connectingBuffers.del(i)
+  elif (let i = client.pager.findConnectingContainer(fd); i != -1):
+    client.pager.handleConnectingContainer(i)
   elif fd == client.forkserver.estream.fd:
     const BufferSize = 4096
     const prefix = "STDERR: "
@@ -597,13 +588,8 @@ proc handleError(client: Client, fd: int) =
     client.loader.onError(fd)
   elif fd in client.loader.unregistered:
     discard # already unregistered...
-  elif (let i = client.pager.findConnectingBuffer(fd); i != -1):
-    # bleh
-    let (container, stream) = client.pager.connectingBuffers[i]
-    client.pager.fail(container, "loader died while loading")
-    client.selector.unregister(fd)
-    stream.close()
-    client.pager.connectingBuffers.del(i)
+  elif (let i = client.pager.findConnectingContainer(fd); i != -1):
+    client.pager.handleConnectingContainerError(i)
   else:
     if fd in client.fdmap:
       let container = client.fdmap[fd]
@@ -784,8 +770,7 @@ proc launchClient*(client: Client, pages: seq[string],
     selector.registerHandle(fd, {Read}, 0)
   client.loader.unregisterFun = proc(fd: int) =
     selector.unregister(fd)
-  client.selector = selector
-  client.pager.launchPager(infile)
+  client.pager.launchPager(infile, selector)
   let clearFun = proc() =
     client.clearConsole()
   let showFun = proc() =
@@ -888,7 +873,6 @@ proc newClient*(config: Config, forkserver: ForkServer): Client =
   pager.setLoader(loader)
   let client = Client(
     config: config,
-    loader: loader,
     jsrt: jsrt,
     jsctx: jsctx,
     pager: pager
diff --git a/src/local/container.nim b/src/local/container.nim
index fc3384d9..a2b78aa5 100644
--- a/src/local/container.nim
+++ b/src/local/container.nim
@@ -1399,7 +1399,6 @@ proc applyResponse*(container: Container; response: Response) =
       container.config.referrerPolicy = referrerPolicy.get
   else:
     container.config.referrerPolicy = NO_REFERRER
-  container.setLoadInfo("Connected to " & $response.url & ". Downloading...")
   # setup content type; note that isSome means an override so we skip it
   if container.contentType.isNone:
     var contentType = response.getContentType()
diff --git a/src/local/pager.nim b/src/local/pager.nim
index a0f63aca..984b3ec5 100644
--- a/src/local/pager.nim
+++ b/src/local/pager.nim
@@ -3,6 +3,7 @@ import std/net
 import std/options
 import std/os
 import std/osproc
+import std/selectors
 import std/streams
 import std/tables
 import std/unicode
@@ -24,6 +25,7 @@ import extern/tempfile
 import io/bufstream
 import io/posixstream
 import io/promise
+import io/serialize
 import io/socketstream
 import io/urlfilter
 import js/error
@@ -31,6 +33,7 @@ import js/javascript
 import js/jstypes
 import js/regex
 import js/tojs
+import loader/connecterror
 import loader/headers
 import loader/loader
 import loader/request
@@ -67,6 +70,17 @@ type
   PagerAlertState = enum
     pasNormal, pasAlertOn, pasLoadInfo
 
+  ContainerConnectionState = enum
+    ccsBeforeResult, ccsBeforeStatus, ccsBeforeHeaders
+
+  ConnectingContainerItem = ref object
+    state: ContainerConnectionState
+    container: Container
+    stream: SocketStream
+    res: int
+    outputId: int
+    status: uint16
+
   Pager* = ref object
     alertState: PagerAlertState
     alerts: seq[string]
@@ -77,7 +91,7 @@ type
     cgiDir*: seq[string]
     commandMode {.jsget.}: bool
     config: Config
-    connectingBuffers*: seq[tuple[container: Container; stream: SocketStream]]
+    connectingContainers: seq[ConnectingContainerItem]
     container*: Container
     cookiejars: Table[string, CookieJar]
     devRandom: PosixStream
@@ -98,10 +112,11 @@ type
     precnum*: int32 # current number prefix (when vi-numeric-prefix is true)
     procmap*: seq[ProcMapItem]
     proxy: URL
-    redraw*: bool
+    redraw: bool
     regex: Opt[Regex]
     reverseSearch: bool
     scommand*: string
+    selector*: Selector[int]
     siteconf: seq[SiteConfig]
     statusgrid*: FixedGrid
     term*: Terminal
@@ -307,7 +322,8 @@ proc setLoader*(pager: Pager, loader: FileLoader) =
   )
   loader.key = pager.addLoaderClient(pager.loader.clientPid, config)
 
-proc launchPager*(pager: Pager, infile: File) =
+proc launchPager*(pager: Pager; infile: File; selector: Selector[int]) =
+  pager.selector = selector
   case pager.term.start(infile)
   of tsrSuccess: discard
   of tsrDA1Fail:
@@ -535,7 +551,11 @@ proc newContainer(pager: Pager; bufferConfig: BufferConfig; request: Request;
     charsetStack,
     cacheId
   )
-  pager.connectingBuffers.add((container, stream))
+  pager.connectingContainers.add(ConnectingContainerItem(
+    state: ccsBeforeResult,
+    container: container,
+    stream: stream
+  ))
   pager.onSetLoadInfo(container)
   return container
 
@@ -551,9 +571,9 @@ proc newContainerFrom(pager: Pager; container: Container; contentType: string):
     cacheId = container.cacheId
   )
 
-func findConnectingBuffer*(pager: Pager; fd: int): int =
-  for i, (_, stream) in pager.connectingBuffers:
-    if stream.fd == fd:
+func findConnectingContainer*(pager: Pager; fd: int): int =
+  for i, item in pager.connectingContainers:
+    if item.stream.fd == fd:
       return i
   -1
 
@@ -1357,7 +1377,7 @@ proc redirectTo(pager: Pager; container: Container; request: Request) =
     redirectdepth = container.redirectdepth + 1, referrer = container)
   dec pager.numload
 
-proc fail*(pager: Pager; container: Container; errorMessage: string) =
+proc fail(pager: Pager; container: Container; errorMessage: string) =
   dec pager.numload
   pager.deleteContainer(container)
   if container.retry.len > 0:
@@ -1366,7 +1386,7 @@ proc fail*(pager: Pager; container: Container; errorMessage: string) =
   else:
     pager.alert("Can't load " & $container.url & " (" & errorMessage & ")")
 
-proc redirect*(pager: Pager; container: Container; response: Response;
+proc redirect(pager: Pager; container: Container; response: Response;
     request: Request) =
   # still need to apply response, or we lose cookie jars.
   container.applyResponse(response)
@@ -1386,7 +1406,7 @@ proc redirect*(pager: Pager; container: Container; response: Response;
     pager.alert("Error: maximum redirection depth reached")
     pager.deleteContainer(container)
 
-proc connected*(pager: Pager; container: Container; response: Response) =
+proc connected(pager: Pager; container: Container; response: Response) =
   let istream = response.body
   container.applyResponse(response)
   if response.status == 401: # unauthorized
@@ -1430,6 +1450,64 @@ proc connected*(pager: Pager; container: Container; response: Response) =
     pager.redraw = true
     pager.refreshStatusMsg()
 
+# true if done, false if keep
+proc handleConnectingContainer*(pager: Pager; i: int) =
+  let item = pager.connectingContainers[i]
+  let container = item.container
+  let stream = item.stream
+  case item.state
+  of ccsBeforeResult:
+    var res: int
+    stream.sread(res)
+    if res == 0:
+      stream.sread(item.outputId)
+      inc item.state
+      container.loadinfo = "Connected to " & $container.url & ". Downloading..."
+      pager.onSetLoadInfo(container)
+      # continue
+    else:
+      var msg: string
+      stream.sread(msg)
+      if msg == "":
+        msg = getLoaderErrorMessage(res)
+      pager.fail(container, msg)
+      # done
+      pager.connectingContainers.del(i)
+      pager.selector.unregister(item.stream.fd)
+      pager.loader.unregistered.add(item.stream.fd)
+      stream.close()
+  of ccsBeforeStatus:
+    stream.sread(item.status)
+    inc item.state
+    # continue
+  of ccsBeforeHeaders:
+    let response = Response(
+      res: item.res,
+      outputId: item.outputId,
+      status: item.status,
+      url: container.request.url,
+      body: stream
+    )
+    stream.sread(response.headers)
+    # done
+    pager.connectingContainers.del(i)
+    pager.selector.unregister(item.stream.fd)
+    pager.loader.unregistered.add(item.stream.fd)
+    let redirect = response.getRedirect(container.request)
+    if redirect != nil:
+      stream.close()
+      pager.redirect(container, response, redirect)
+    else:
+      pager.connected(container, response)
+
+proc handleConnectingContainerError*(pager: Pager; i: int) =
+  let item = pager.connectingContainers[i]
+  pager.fail(item.container, "loader died while loading")
+  pager.selector.unregister(item.stream.fd)
+  pager.loader.unregistered.add(item.stream.fd)
+  item.stream.close()
+  pager.connectingContainers.del(i)
+
 proc handleEvent0(pager: Pager; container: Container; event: ContainerEvent):
     bool =
   case event.t