client, forkserver, dynstream: misc refactorings, fixes

* fix broken int conversion in dynstream
* fix EPIPE handling in forkserver
* merge fdmap and connectingContainers into loader map

7 files changed, 107 insertions, 101 deletions

diff --git a/src/io/dynstream.nim b/src/io/dynstream.nim
index 129ecf05..66a2a932 100644
--- a/src/io/dynstream.nim
+++ b/src/io/dynstream.nim
@@ -186,7 +186,7 @@ proc recvDataLoopOrMmap*(ps: PosixStream; len = -1): MaybeMappedMemory =
     let srcOff = lseek(ps.fd, 0, SEEK_CUR) # skip headers
     if len != -1:
       doAssert len == stats.st_size - srcOff
-    let len = int(stats.st_size) - srcOff
+    let len = int(stats.st_size - srcOff)
     let p0 = mmap(nil, len, PROT_READ, MAP_SHARED, ps.fd, 0)
     if p0 == MAP_FAILED:
       return nil
diff --git a/src/loader/loader.nim b/src/loader/loader.nim
index b5a6b3ba..85490b84 100644
--- a/src/loader/loader.nim
+++ b/src/loader/loader.nim
@@ -694,6 +694,8 @@ proc loadCGI(ctx: LoaderContext; client: ClientData; handle: InputHandle;
     # expects SIGCHLD to be untouched. (e.g. git dies a horrible death with
     # SIGCHLD as SIG_IGN)
     signal(SIGCHLD, SIG_DFL)
+    # let's also reset SIGPIPE, which we ignored in forkserver
+    signal(SIGPIPE, SIG_DFL)
     # close the parent handles
     for i in 0 ..< ctx.handleMap.len:
       if ctx.handleMap[i] != nil:
diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim
index 23ddff30..b43149fb 100644
--- a/src/loader/loaderhandle.nim
+++ b/src/loader/loaderhandle.nim
@@ -42,9 +42,6 @@ type
     suspended*: bool
     dead*: bool
 
-  SigchldHandle* = ref object of LoaderHandle
-    notifyMap*: seq[tuple[pid: int; input: InputHandle]]
-
   HandleParserState* = enum
     hpsBeforeLines, hpsAfterFirstLine, hpsControlDone
 
diff --git a/src/loader/loaderiface.nim b/src/loader/loaderiface.nim
index 8cc5ff42..51f02480 100644
--- a/src/loader/loaderiface.nim
+++ b/src/loader/loaderiface.nim
@@ -25,7 +25,7 @@ type
     key*: ClientKey
     process*: int
     clientPid*: int
-    map: seq[LoaderData]
+    map: seq[MapData]
     mapFds*: int # number of fds in map
     unregistered*: seq[int]
     registerFun*: proc(fd: int)
@@ -38,9 +38,11 @@ type
   ConnectDataState = enum
     cdsBeforeResult, cdsBeforeStatus, cdsBeforeHeaders
 
-  LoaderData = ref object of RootObj
+  MapData* = ref object of RootObj
     stream*: SocketStream
 
+  LoaderData = ref object of MapData
+
   ConnectData* = ref object of LoaderData
     state: ConnectDataState
     status: uint16
@@ -123,7 +125,7 @@ proc startRequest*(loader: FileLoader; request: Request;
     w.swrite(config)
   return stream
 
-iterator data*(loader: FileLoader): LoaderData {.inline.} =
+iterator data*(loader: FileLoader): MapData {.inline.} =
   for it in loader.map:
     if it != nil:
       yield it
@@ -133,27 +135,32 @@ iterator ongoing*(loader: FileLoader): OngoingData {.inline.} =
     if it of OngoingData:
       yield OngoingData(it)
 
-func fd*(data: LoaderData): int =
+func fd*(data: MapData): int =
   return int(data.stream.fd)
 
-proc put*(loader: FileLoader; data: LoaderData) =
+proc put*(loader: FileLoader; data: MapData) =
   let fd = int(data.stream.fd)
   if loader.map.len <= fd:
     loader.map.setLen(fd + 1)
   assert loader.map[fd] == nil
   loader.map[fd] = data
-  inc loader.mapFds
+  if data of LoaderData:
+    inc loader.mapFds
 
-proc get*(loader: FileLoader; fd: int): LoaderData =
+proc get*(loader: FileLoader; fd: int): MapData =
   if fd < loader.map.len:
     return loader.map[fd]
   return nil
 
-proc unset*(loader: FileLoader; data: LoaderData) =
+proc unset*(loader: FileLoader; fd: int) =
+  if loader.map[fd] != nil and loader.map[fd] of LoaderData:
+    dec loader.mapFds
+  loader.map[fd] = nil
+
+proc unset*(loader: FileLoader; data: MapData) =
   let fd = int(data.stream.fd)
   if loader.get(fd) != nil:
-    dec loader.mapFds
-    loader.map[fd] = nil
+    loader.unset(fd)
 
 proc fetch0(loader: FileLoader; input: Request; promise: FetchPromise;
     redirectNum: int) =
diff --git a/src/local/client.nim b/src/local/client.nim
index c20a460b..6c1d505c 100644
--- a/src/local/client.nim
+++ b/src/local/client.nim
@@ -57,13 +57,15 @@ type
     alive: bool
     config {.jsget.}: Config
     consoleWrapper: ConsoleWrapper
-    fdmap: seq[Container]
     feednext: bool
     pager {.jsget.}: Pager
-    pressed: tuple[col: int; row: int]
+    pressed: tuple[col, row: int]
     exitCode: int
     inEval: bool
 
+  ContainerData = ref object of MapData
+    container: Container
+
   ConsoleWrapper = object
     console: Console
     container: Container
@@ -380,17 +382,17 @@ proc acceptBuffers(client: Client) =
       let stream = container.iface.stream
       let fd = int(stream.source.fd)
       client.selector.unregister(fd)
-      client.fdmap[fd] = nil
+      client.loader.unset(fd)
       stream.sclose()
     elif container.process != -1: # connecting to buffer process
       let i = pager.findProcMapItem(container.process)
       pager.procmap.del(i)
-    elif (let i = pager.findConnectingContainer(container); i != -1):
+    elif (let item = pager.findConnectingContainer(container); item != nil):
       # connecting to URL
-      let stream = pager.connectingContainers[i].stream
+      let stream = item.stream
       client.selector.unregister(int(stream.fd))
       stream.sclose()
-      pager.connectingContainers.del(i)
+      client.loader.unset(item)
   let registerFun = proc(fd: int) =
     client.selector.unregister(fd)
     client.selector.registerHandle(fd, {Read, Write}, 0)
@@ -441,63 +443,67 @@ proc acceptBuffers(client: Client) =
       loader.shareCachedItem(container.cacheId, loader.clientPid)
       container.setCloneStream(stream, registerFun)
     let fd = int(stream.fd)
-    if client.fdmap.len <= fd:
-      client.fdmap.setLen(fd + 1)
-    client.fdmap[fd] = container
+    client.loader.put(ContainerData(stream: stream, container: container))
     client.selector.registerHandle(fd, {Read}, 0)
     pager.handleEvents(container)
   pager.procmap.setLen(0)
 
+proc handleStderr(client: Client) =
+  const BufferSize = 4096
+  const prefix = "STDERR: "
+  var buffer {.noinit.}: array[BufferSize, char]
+  let estream = client.forkserver.estream
+  var hadlf = true
+  while true:
+    try:
+      let n = estream.recvData(buffer)
+      if n == 0:
+        break
+      var i = 0
+      while i < n:
+        var j = n
+        var found = false
+        for k in i ..< n:
+          if buffer[k] == '\n':
+            j = k + 1
+            found = true
+            break
+        if hadlf:
+          client.console.err.write(prefix)
+        if j - i > 0:
+          client.console.err.write(buffer.toOpenArray(i, j - 1))
+        i = j
+        hadlf = found
+    except ErrorAgain:
+      break
+  if not hadlf:
+    client.console.err.write('\n')
+  client.console.err.sflush()
+
 proc handleRead(client: Client; fd: int) =
   if client.pager.term.istream != nil and fd == client.pager.term.istream.fd:
     client.input().then(proc() =
       client.pager.handleEvents()
     )
-  elif (let i = client.pager.findConnectingContainer(fd); i != -1):
-    client.pager.handleConnectingContainer(i)
   elif fd == client.forkserver.estream.fd:
-    const BufferSize = 4096
-    const prefix = "STDERR: "
-    var buffer {.noinit.}: array[BufferSize, char]
-    let estream = client.forkserver.estream
-    var hadlf = true
-    while true:
-      try:
-        let n = estream.recvData(buffer)
-        if n == 0:
-          break
-        var i = 0
-        while i < n:
-          var j = n
-          var found = false
-          for k in i ..< n:
-            if buffer[k] == '\n':
-              j = k + 1
-              found = true
-              break
-          if hadlf:
-            client.console.err.write(prefix)
-          if j - i > 0:
-            client.console.err.write(buffer.toOpenArray(i, j - 1))
-          i = j
-          hadlf = found
-      except ErrorAgain:
-        break
-    if not hadlf:
-      client.console.err.write('\n')
-    client.console.err.sflush()
+    client.handleStderr()
   elif (let data = client.loader.get(fd); data != nil):
-    client.loader.onRead(fd)
-    if data of ConnectData:
-      client.runJSJobs()
+    if data of ConnectingContainer:
+      client.pager.handleRead(ConnectingContainer(data))
+    elif data of ContainerData:
+      let container = ContainerData(data).container
+      client.pager.handleEvent(container)
+    else:
+      client.loader.onRead(fd)
+      if data of ConnectData:
+        client.runJSJobs()
   elif fd in client.loader.unregistered:
     discard # ignore
   else:
-    let container = client.fdmap[fd]
-    client.pager.handleEvent(container)
+    assert false
 
 proc handleWrite(client: Client; fd: int) =
-  let container = client.fdmap[fd]
+  let container = ContainerData(client.loader.get(fd)).container
   if container.iface.stream.flushWrite():
     client.selector.unregister(fd)
     client.selector.registerHandle(fd, {Read}, 0)
@@ -519,25 +525,26 @@ proc handleError(client: Client; fd: int) =
     #TODO do something here...
     stderr.write("Fork server crashed :(\n")
     client.quit(1)
-  elif client.loader.get(fd) != nil:
-    discard client.loader.onError(fd) #TODO handle connection error?
-  elif fd in client.loader.unregistered:
-    discard # already unregistered...
-  elif (let i = client.pager.findConnectingContainer(fd); i != -1):
-    client.pager.handleConnectingContainerError(i)
-  else:
-    if fd < client.fdmap.len and client.fdmap[fd] != nil:
-      let container = client.fdmap[fd]
+  elif (let data = client.loader.get(fd); data != nil):
+    if data of ConnectingContainer:
+      client.pager.handleError(ConnectingContainer(data))
+    elif data of ContainerData:
+      let container = ContainerData(data).container
       if container != client.consoleWrapper.container:
         client.console.error("Error in buffer", $container.url)
       else:
         client.consoleWrapper.container = nil
       client.selector.unregister(fd)
-      client.fdmap[fd] = nil
-    if client.consoleWrapper.container != nil:
+      client.loader.unset(fd)
+      doAssert client.consoleWrapper.container != nil
       client.showConsole()
     else:
-      doAssert false
+      discard client.loader.onError(fd) #TODO handle connection error?
+  elif fd in client.loader.unregistered:
+    discard # already unregistered...
+  else:
+    doAssert client.consoleWrapper.container != nil
+    client.showConsole()
 
 proc inputLoop(client: Client) =
   let selector = client.selector
diff --git a/src/local/pager.nim b/src/local/pager.nim
index edcf614b..35f6ab4c 100644
--- a/src/local/pager.nim
+++ b/src/local/pager.nim
@@ -85,10 +85,9 @@ type
   ContainerConnectionState = enum
     ccsBeforeResult, ccsBeforeStatus, ccsBeforeHeaders
 
-  ConnectingContainerItem = ref object
+  ConnectingContainer* = ref object of MapData
     state: ContainerConnectionState
     container: Container
-    stream*: SocketStream
     res: int
     outputId: int
     status: uint16
@@ -124,7 +123,6 @@ type
     askprompt: string
     commandMode {.jsget.}: bool
     config*: Config
-    connectingContainers*: seq[ConnectingContainerItem]
     container*: Container
     cookiejars: Table[string, CookieJar]
     devRandom: PosixStream
@@ -788,7 +786,7 @@ proc newContainer(pager: Pager; bufferConfig: BufferConfig;
     cacheId,
     pager.config
   )
-  pager.connectingContainers.add(ConnectingContainerItem(
+  pager.loader.put(ConnectingContainer(
     state: ccsBeforeResult,
     container: container,
     stream: stream
@@ -808,17 +806,14 @@ proc newContainerFrom(pager: Pager; container: Container; contentType: string):
     url = container.url
   )
 
-func findConnectingContainer*(pager: Pager; fd: int): int =
-  for i, item in pager.connectingContainers:
-    if item.stream.fd == fd:
-      return i
-  -1
-
-func findConnectingContainer*(pager: Pager; container: Container): int =
-  for i, item in pager.connectingContainers:
-    if item.container == container:
-      return i
-  -1
+func findConnectingContainer*(pager: Pager; container: Container):
+    ConnectingContainer =
+  for item in pager.loader.data:
+    if item of ConnectingContainer:
+      let item = ConnectingContainer(item)
+      if item.container == container:
+        return item
+  return nil
 
 func findProcMapItem*(pager: Pager; pid: int): int =
   for i, item in pager.procmap:
@@ -2053,9 +2048,7 @@ proc unregisterFd(pager: Pager; fd: int) =
   pager.selector.unregister(fd)
   pager.loader.unregistered.add(fd)
 
-# true if done, false if keep
-proc handleConnectingContainer*(pager: Pager; i: int) =
-  let item = pager.connectingContainers[i]
+proc handleRead*(pager: Pager; item: ConnectingContainer) =
   let container = item.container
   let stream = item.stream
   case item.state
@@ -2076,7 +2069,7 @@ proc handleConnectingContainer*(pager: Pager; i: int) =
         msg = getLoaderErrorMessage(res)
       pager.fail(container, msg)
       # done
-      pager.connectingContainers.del(i)
+      pager.loader.unset(item)
       pager.unregisterFd(int(item.stream.fd))
       stream.sclose()
   of ccsBeforeStatus:
@@ -2090,7 +2083,7 @@ proc handleConnectingContainer*(pager: Pager; i: int) =
     var r = stream.initPacketReader()
     r.sread(response.headers)
     # done
-    pager.connectingContainers.del(i)
+    pager.loader.unset(item)
     pager.unregisterFd(int(item.stream.fd))
     let redirect = response.getRedirect(container.request)
     if redirect != nil:
@@ -2099,12 +2092,11 @@ proc handleConnectingContainer*(pager: Pager; i: int) =
     else:
       pager.connected(container, response)
 
-proc handleConnectingContainerError*(pager: Pager; i: int) =
-  let item = pager.connectingContainers[i]
+proc handleError*(pager: Pager; item: ConnectingContainer) =
   pager.fail(item.container, "loader died while loading")
   pager.unregisterFd(int(item.stream.fd))
   item.stream.sclose()
-  pager.connectingContainers.del(i)
+  pager.loader.unset(item)
 
 proc metaRefresh(pager: Pager; container: Container; n: int; url: URL) =
   let ctx = pager.jsctx
@@ -2174,16 +2166,15 @@ proc handleEvent0(pager: Pager; container: Container; event: ContainerEvent):
     if pager.container == container:
       pager.alert(event.msg)
   of cetCancel:
-    let i = pager.findConnectingContainer(container)
-    if i == -1:
+    let item = pager.findConnectingContainer(container)
+    if item == nil:
       # whoops. we tried to cancel, but the event loop did not favor us...
       # at least cancel it in the buffer
       container.remoteCancel()
     else:
-      let item = pager.connectingContainers[i]
       dec pager.numload
       pager.deleteContainer(container, container.find(ndAny))
-      pager.connectingContainers.del(i)
+      pager.loader.unset(item)
       pager.unregisterFd(int(item.stream.fd))
       item.stream.sclose()
   of cetMetaRefresh:
diff --git a/src/server/forkserver.nim b/src/server/forkserver.nim
index aa66a42b..eaea5075 100644
--- a/src/server/forkserver.nim
+++ b/src/server/forkserver.nim
@@ -168,6 +168,7 @@ proc forkBuffer(ctx: var ForkServerContext; r: var BufferedReader): int =
       gpstream.sclose()
       gssock.close(unlink = false)
       exitnow(1)
+    signal(SIGPIPE, SIG_DFL)
     enterBufferSandbox(sockDir)
     let loader = FileLoader(
       process: loaderPid,
@@ -202,6 +203,7 @@ proc runForkServer() =
     sockDirFd: -1
   )
   signal(SIGCHLD, SIG_IGN)
+  signal(SIGPIPE, SIG_IGN)
   while true:
     try:
       ctx.istream.withPacketReader r:
@@ -231,7 +233,7 @@ proc runForkServer() =
           let r = ctx.forkBuffer(r)
           ctx.ostream.withPacketWriter w:
             w.swrite(r)
-    except EOFError:
+    except EOFError, ErrorBrokenPipe:
       # EOF
       break
   ctx.istream.sclose()