loader: mmap intermediate image files, misc refactoring

* refactor parseHeader
* optimize response blob()
* add direct "to cache" mode for loader requests which sets stdout to a
  file, and use it for image processing
* move image resizing into a separate process
* mmap cache files in between processing steps when possible

At last, resize is no longer a part of image decoding. Also, it feels
much nicer to keep encoded image data in the same cache as everything
else.

The mmap operations *should* be more efficient than copying the whole
RGBA data through a pipe. In practice, it only makes a difference for
loading (well, now just mmapping) the encoded image into the pager,
where it singlehandedly speeds up image display by 10x on my test image.

For the other steps, the unfortunate fact that "tocache" must delay the
next fork/exec in the pipeline until the entire image is processed seems
to equal out any wins we might have gotten from skipping a single raw
RGBA copy.

I have tried moving the delay before the exec (it's possible with yet
another pipe), but it didn't help much and made the code much
uglier. (Not that tocache didn't, but I can live with this...)

7 files changed, 88 insertions, 74 deletions

diff --git a/adapter/img/jebp.c b/adapter/img/jebp.c
index d6dfb904..e48ed43f 100644
--- a/adapter/img/jebp.c
+++ b/adapter/img/jebp.c
@@ -2,6 +2,3 @@
 /* #define JEBP_NO_STDIO */
 #define JEBP_IMPLEMENTATION
 #include "jebp.h"
-#define STB_IMAGE_RESIZE_IMPLEMENTATION
-#include "stb_image_resize.h"
-/**/
diff --git a/adapter/img/jebp.nim b/adapter/img/jebp.nim
index afeb283e..7a10a27e 100644
--- a/adapter/img/jebp.nim
+++ b/adapter/img/jebp.nim
@@ -58,13 +58,6 @@ proc myRead(data: pointer; size: csize_t; user: pointer): csize_t {.cdecl.} =
     n += csize_t(i)
   return n
 
-{.push header: "stb_image_resize.h".}
-proc stbir_resize_uint8(input_pixels: ptr uint8;
-  input_w, input_h, input_stride_in_bytes: cint; output_pixels: ptr uint8;
-  output_w, output_h, output_stride_in_bytes, num_channels: cint): cint
-  {.importc.}
-{.pop.}
-
 proc writeAll(data: pointer; size: int) =
   var n = 0
   while n < size:
@@ -123,17 +116,6 @@ proc main() =
     if res != 0:
       die("Cha-Control: ConnectionError 1 jebp error " &
         $jebp_error_string(res))
-    elif targetWidth != -1 and targetHeight != -1:
-      let hdr = "Cha-Image-Dimensions: " & $targetWidth & "x" & $targetHeight &
-        "\n\n"
-      let p2 = cast[ptr UncheckedArray[uint8]](alloc(hdr.len +
-        targetWidth * targetHeight * 4))
-      copyMem(addr p2[0], unsafeAddr hdr[0], hdr.len)
-      doAssert stbir_resize_uint8(cast[ptr uint8](image.pixels), image.width,
-        image.height, 0, addr p2[hdr.len], targetWidth, targetHeight, 0, 4) == 1
-      writeAll(p2, hdr.len + targetWidth * targetHeight * 4)
-      dealloc(p2)
-      jebp_free_image(addr image)
     else:
       puts("Cha-Image-Dimensions: " & $image.width & "x" & $image.height &
         "\n\n")
diff --git a/adapter/img/resize.nim b/adapter/img/resize.nim
new file mode 100644
index 00000000..5bd29b63
--- /dev/null
+++ b/adapter/img/resize.nim
@@ -0,0 +1,60 @@
+import std/options
+import std/os
+import std/posix
+import std/strutils
+
+import io/dynstream
+import utils/sandbox
+import utils/twtstr
+
+{.compile("stb_image_resize.c", "-O3").}
+
+{.push header: "stb_image_resize.h".}
+proc stbir_resize_uint8(input_pixels: ptr uint8;
+  input_w, input_h, input_stride_in_bytes: cint; output_pixels: ptr uint8;
+  output_w, output_h, output_stride_in_bytes, num_channels: cint): cint
+  {.importc.}
+{.pop.}
+
+proc die(s: string) {.noreturn.} =
+  let os = newPosixStream(STDOUT_FILENO)
+  os.sendDataLoop(s)
+  quit(1)
+
+proc main() =
+  var srcWidth = cint(-1)
+  var srcHeight = cint(-1)
+  var dstWidth = cint(-1)
+  var dstHeight = cint(-1)
+  for hdr in getEnv("REQUEST_HEADERS").split('\n'):
+    let k = hdr.until(':')
+    if k == "Cha-Image-Target-Dimensions" or k == "Cha-Image-Dimensions":
+      let v = hdr.after(':').strip()
+      let s = v.split('x')
+      if s.len != 2:
+        die("Cha-Control: ConnectionError 1 wrong dimensions\n")
+      let w = parseUInt32(s[0], allowSign = false)
+      let h = parseUInt32(s[1], allowSign = false)
+      if w.isNone or w.isNone:
+        die("Cha-Control: ConnectionError 1 wrong dimensions\n")
+      if k == "Cha-Image-Target-Dimensions":
+        dstWidth = cint(w.get)
+        dstHeight = cint(h.get)
+      else:
+        srcWidth = cint(w.get)
+        srcHeight = cint(h.get)
+  let ps = newPosixStream(STDIN_FILENO)
+  let os = newPosixStream(STDOUT_FILENO)
+  let src = ps.recvDataLoopOrMmap(int(srcWidth * srcHeight * 4))
+  let dst = os.maybeMmapForSend(int(dstWidth * dstHeight * 4 + 1))
+  if src == nil or dst == nil:
+    die("Cha-Control: ConnectionError 1 failed to open i/o\n")
+  dst.p[0] = uint8('\n') # for CGI
+  enterNetworkSandbox()
+  doAssert stbir_resize_uint8(addr src.p[0], srcWidth, srcHeight, 0,
+    addr dst.p[1], dstWidth, dstHeight, 0, 4) == 1
+  os.sendDataLoop(dst)
+  dealloc(src)
+  dealloc(dst)
+
+main()
diff --git a/adapter/img/sixel.nim b/adapter/img/sixel.nim
index e503dbbd..19c3782d 100644
--- a/adapter/img/sixel.nim
+++ b/adapter/img/sixel.nim
@@ -157,7 +157,7 @@ proc trim(trimMap: var TrimMap; K: var uint) =
   )
   K = k
 
-proc getPixel(img: seq[RGBAColorBE]; m: int; bgcolor: ARGBColor): RGBColor
+proc getPixel(img: openArray[RGBAColorBE]; m: int; bgcolor: ARGBColor): RGBColor
     {.inline.} =
   let c0 = img[m].toARGBColor()
   if c0.a != 255:
@@ -165,7 +165,7 @@ proc getPixel(img: seq[RGBAColorBE]; m: int; bgcolor: ARGBColor): RGBColor
     return RGBColor(uint32(c1).fastmul(100))
   return RGBColor(uint32(c0).fastmul(100))
 
-proc quantize(img: seq[RGBAColorBE]; bgcolor: ARGBColor; outk: var uint):
+proc quantize(img: openArray[RGBAColorBE]; bgcolor: ARGBColor; outk: var uint):
     NodeChildren =
   var root = default(NodeChildren)
   if outk <= 2: # monochrome; not much we can do with an octree...
@@ -366,7 +366,7 @@ proc createBands(bands: var seq[SixelBand]; activeChunks: seq[ptr SixelChunk]) =
     if not found:
       bands.add(SixelBand(head: chunk, tail: chunk))
 
-proc encode(img: seq[RGBAColorBE]; width, height, offx, offy, cropw: int;
+proc encode(img: openArray[RGBAColorBE]; width, height, offx, offy, cropw: int;
     halfdump: bool; bgcolor: ARGBColor; palette: int) =
   var palette = uint(palette)
   var root = img.quantize(bgcolor, palette)
@@ -478,7 +478,6 @@ proc parseDimensions(s: string): (int, int) =
   return (int(w.get), int(h.get))
 
 proc main() =
-  enterNetworkSandbox()
   let scheme = getEnv("MAPPED_URI_SCHEME")
   let f = scheme.after('+')
   if f != "x-sixel":
@@ -487,7 +486,6 @@ proc main() =
   of "decode":
     die("Cha-Control: ConnectionError 1 not implemented\n")
   of "encode":
-    let headers = getEnv("REQUEST_HEADERS")
     var width = 0
     var height = 0
     var offx = 0
@@ -497,7 +495,7 @@ proc main() =
     var bgcolor = rgb(0, 0, 0)
     var cropw = -1
     var quality = -1
-    for hdr in headers.split('\n'):
+    for hdr in getEnv("REQUEST_HEADERS").split('\n'):
       let s = hdr.after(':').strip()
       case hdr.until(':')
       of "Cha-Image-Dimensions":
@@ -537,9 +535,15 @@ proc main() =
       os.sendDataLoop("Cha-Image-Dimensions: 0x0\n")
       quit(0) # done...
     let n = width * height
-    var img = cast[seq[RGBAColorBE]](newSeqUninitialized[uint32](n))
+    let L = n * 4
     let ps = newPosixStream(STDIN_FILENO)
-    ps.recvDataLoop(addr img[0], n * 4)
-    img.encode(width, height, offx, offy, cropw, halfdump, bgcolor, palette)
+    let src = ps.recvDataLoopOrMmap(L)
+    if src == nil:
+      die("Cha-Control: ConnectionError 1 failed to read input\n")
+    enterNetworkSandbox() # don't swallow stat
+    let p = cast[ptr UncheckedArray[RGBAColorBE]](src.p)
+    p.toOpenArray(0, n - 1).encode(width, height, offx, offy, cropw, halfdump,
+      bgcolor, palette)
+    dealloc(src)
 
 main()
diff --git a/adapter/img/stb_image.c b/adapter/img/stb_image.c
index 2c2bac34..b5afab43 100644
--- a/adapter/img/stb_image.c
+++ b/adapter/img/stb_image.c
@@ -15,5 +15,3 @@
 #define STB_IMAGE_WRITE_IMPLEMENTATION
 #define STIBW_NO_STDIO
 #include "stb_image_write.h"
-#define STB_IMAGE_RESIZE_IMPLEMENTATION
-#include "stb_image_resize.h"
diff --git a/adapter/img/stb_image_resize.c b/adapter/img/stb_image_resize.c
new file mode 100644
index 00000000..b2d47755
--- /dev/null
+++ b/adapter/img/stb_image_resize.c
@@ -0,0 +1,2 @@
+#define STB_IMAGE_RESIZE_IMPLEMENTATION
+#include "stb_image_resize.h"
diff --git a/adapter/img/stbi.nim b/adapter/img/stbi.nim
index f0cfe2bb..dec221f1 100644
--- a/adapter/img/stbi.nim
+++ b/adapter/img/stbi.nim
@@ -84,13 +84,6 @@ proc writeAll(data: pointer; size: int) =
 proc myWriteFunc(context, data: pointer; size: cint) {.cdecl.} =
   writeAll(data, int(size))
 
-{.push header: "stb_image_resize.h".}
-proc stbir_resize_uint8(input_pixels: ptr uint8;
-  input_w, input_h, input_stride_in_bytes: cint; output_pixels: ptr uint8;
-  output_w, output_h, output_stride_in_bytes, num_channels: cint): cint
-  {.importc.}
-{.pop.}
-
 proc puts(s: string) =
   if s.len > 0:
     writeAll(unsafeAddr s[0], s.len)
@@ -100,13 +93,12 @@ proc die(s: string) {.noreturn.} =
   quit(1)
 
 proc main() =
-  enterNetworkSandbox()
-  let scheme = getEnv("MAPPED_URI_SCHEME")
-  let f = scheme.after('+')
+  let f = getEnv("MAPPED_URI_SCHEME").after('+')
   case getEnv("MAPPED_URI_PATH")
   of "decode":
     if f notin ["jpeg", "gif", "bmp", "png", "x-unknown"]:
       die("Cha-Control: ConnectionError 1 unknown format " & f)
+    enterNetworkSandbox()
     var user = StbiUser()
     var x: cint
     var y: cint
@@ -116,25 +108,12 @@ proc main() =
       skip: mySkip,
       eof: myEof
     )
-    let headers = getEnv("REQUEST_HEADERS")
-    var targetWidth = cint(-1)
-    var targetHeight = cint(-1)
     var infoOnly = false
-    for hdr in headers.split('\n'):
+    for hdr in getEnv("REQUEST_HEADERS").split('\n'):
       let v = hdr.after(':').strip()
-      case hdr.until(':')
-      of "Cha-Image-Info-Only":
+      if hdr.until(':') == "Cha-Image-Info-Only":
         infoOnly = v == "1"
-      of "Cha-Image-Target-Dimensions":
-        let s = v.split('x')
-        if s.len != 2:
-          die("Cha-Control: ConnectionError 1 wrong dimensions\n")
-        let w = parseUInt32(s[0], allowSign = false)
-        let h = parseUInt32(s[1], allowSign = false)
-        if w.isNone or w.isNone:
-          die("Cha-Control: ConnectionError 1 wrong dimensions\n")
-        targetWidth = cint(w.get)
-        targetHeight = cint(h.get)
+        break
     if infoOnly:
       if stbi_info_from_callbacks(addr clbk, addr user, x, y,
           channels_in_file) == 1:
@@ -148,22 +127,13 @@ proc main() =
     if p == nil:
       die("Cha-Control: ConnectionError 1 stbi error " &
         $stbi_failure_reason())
-    elif targetWidth != -1 and targetHeight != -1:
-      let hdr = "Cha-Image-Dimensions: " & $targetWidth & "x" &
-        $targetHeight & "\n\n"
-      let p2 = cast[ptr UncheckedArray[uint8]](alloc(hdr.len +
-        targetWidth * targetHeight * 4))
-      copyMem(addr p2[0], unsafeAddr hdr[0], hdr.len)
-      doAssert stbir_resize_uint8(p, x, y, 0, addr p2[hdr.len], targetWidth,
-        targetHeight, 0, 4) == 1
-      writeAll(p2, hdr.len + targetWidth * targetHeight * 4)
-      dealloc(p2)
-      stbi_image_free(p)
     else:
       puts("Cha-Image-Dimensions: " & $x & "x" & $y & "\n\n")
       writeAll(p, x * y * 4)
       stbi_image_free(p)
   of "encode":
+    if f notin ["png", "bmp", "jpeg"]:
+      die("Cha-Control: ConnectionError 1 unknown format " & f)
     let headers = getEnv("REQUEST_HEADERS")
     var quality = cint(50)
     var width = cint(0)
@@ -185,10 +155,12 @@ proc main() =
           die("Cha-Control: ConnectionError 1 wrong quality")
         quality = cint(q)
     let ps = newPosixStream(STDIN_FILENO)
-    var s = newSeqUninitialized[uint8](width * height * 4)
-    ps.recvDataLoop(s)
+    let src = ps.recvDataLoopOrMmap(width * height * 4)
+    if src == nil:
+      die("Cha-Control: ConnectionError 1 failed to read input\n")
+    enterNetworkSandbox() # don't swallow stat
     puts("Cha-Image-Dimensions: " & $width & 'x' & $height & "\n\n")
-    let p = unsafeAddr s[0]
+    let p = src.p
     case f
     of "png":
       stbi_write_png_to_func(myWriteFunc, nil, cint(width), cint(height), 4, p,
@@ -198,7 +170,6 @@ proc main() =
     of "jpeg":
       stbi_write_jpg_to_func(myWriteFunc, nil, cint(width), cint(height), 4, p,
         quality)
-    else:
-      die("Cha-Control: ConnectionError 1 unknown format " & f)
+    dealloc(src)
 
 main()