dom, pager: cache images from network

With many limitations:

* slightly randomized expiry, so it's harder to fingerprint
* only images. so e.g. CSS is still left uncached
* it's per-buffer and non-persistent, so images are still redownloaded
  for every new page load

so it's more of an image sharing between placements than true caching.

7 files changed, 86 insertions, 50 deletions

diff --git a/src/html/dom.nim b/src/html/dom.nim
index bb5128dc..0fbdda7d 100644
--- a/src/html/dom.nim
+++ b/src/html/dom.nim
@@ -5,6 +5,7 @@ import std/options
 import std/sets
 import std/strutils
 import std/tables
+import std/times
 
 import chagashi/charset
 import chagashi/decoder
@@ -70,6 +71,12 @@ type
   Location = ref object
     window: Window
 
+  CachedURLImage = ref object
+    expiry: int64
+    loading: bool
+    shared: seq[HTMLImageElement]
+    bmp: NetworkBitmap
+
   Window* = ref object of EventTarget
     attrs*: WindowAttributes
     internalConsole*: Console
@@ -86,6 +93,7 @@ type
     importMapsAllowed*: bool
     factory*: CAtomFactory
     loadingResourcePromises*: seq[EmptyPromise]
+    imageURLCache: Table[string, CachedURLImage]
     images*: bool
     styling*: bool
     # ID of the next image
@@ -2912,6 +2920,16 @@ proc loadResource(window: Window; image: HTMLImageElement) =
       # mixed content :/
       #TODO maybe do this in loader?
       url.scheme = "https"
+    let surl = $url
+    window.imageURLCache.withValue(surl, p):
+      if p[].expiry > getTime().utc().toTime().toUnix():
+        image.bitmap = p[].bmp
+        return
+      elif p[].loading:
+        p[].shared.add(image)
+        return
+    let cachedURL = CachedURLImage(expiry: -1, loading: true)
+    window.imageURLCache[surl] = cachedURL
     let p = window.loader.fetch(newRequest(url)).then(
       proc(res: JSResult[Response]): EmptyPromise =
         if res.isNone:
@@ -2932,9 +2950,21 @@ proc loadResource(window: Window; image: HTMLImageElement) =
         response.resume()
         response.unregisterFun()
         response.body.sclose()
+        var expiry = -1i64
+        if "Cache-Control" in response.headers:
+          for hdr in response.headers.table["Cache-Control"]:
+            var i = hdr.find("max-age=")
+            if i != -1:
+              i = hdr.skipBlanks(i + "max-age=".len)
+              let s = hdr.until(AllChars - AsciiDigit, i)
+              let pi = parseInt64(s)
+              if pi.isSome:
+                expiry = getTime().utc().toTime().toUnix() + pi.get
+              break
+        cachedURL.loading = false
+        cachedURL.expiry = expiry
         return r.then(proc(res: JSResult[Response]): EmptyPromise =
           if res.isNone:
-            window.console.error("Failed to decode", $response.url)
             return
           let response = res.get
           # close immediately; all data we're interested in is in the headers.
@@ -2951,13 +2981,17 @@ proc loadResource(window: Window; image: HTMLImageElement) =
           if width.isNone or height.isNone:
             window.console.error("wrong Cha-Image-Dimensions in", $response.url)
             return
-          image.bitmap = NetworkBitmap(
+          let bmp = NetworkBitmap(
             width: width.get,
             height: height.get,
             cacheId: cacheId,
             imageId: window.getImageId(),
             contentType: contentType
           )
+          image.bitmap = bmp
+          cachedURL.bmp = bmp
+          for share in cachedURL.shared:
+            share.bitmap = bmp
         )
       )
     window.loadingResourcePromises.add(p)
diff --git a/src/local/container.nim b/src/local/container.nim
index a7702101..d78f9120 100644
--- a/src/local/container.nim
+++ b/src/local/container.nim
@@ -99,6 +99,8 @@ type
 
   CachedImage* = ref object
     loaded*: bool
+    width*: int
+    height*: int
     bmp*: NetworkBitmap
 
   Container* = ref object
@@ -2038,10 +2040,12 @@ proc highlightMarks*(container: Container; display: var FixedGrid;
       hlformat.bgcolor = hlcolor
       display[y * display.width + x].format = hlformat
 
-func findCachedImage*(container: Container; id: int): CachedImage =
-  for image in container.cachedImages:
-    if image.bmp.imageId == id:
-      return image
+func findCachedImage*(container: Container; image: PosBitmap): CachedImage =
+  let imageId = NetworkBitmap(image.bmp).imageId
+  for it in container.cachedImages:
+    if it.bmp.imageId == imageId and it.width == image.width and
+        it.height == image.height:
+      return it
   return nil
 
 proc handleEvent*(container: Container) =
diff --git a/src/local/pager.nim b/src/local/pager.nim
index cdc08422..9f5eb593 100644
--- a/src/local/pager.nim
+++ b/src/local/pager.nim
@@ -474,17 +474,14 @@ proc redraw(pager: Pager) {.jsfunc.} =
       pager.container.select.redraw = true
 
 proc loadCachedImage(pager: Pager; container: Container; image: PosBitmap) =
-  #TODO this is kinda dumb, because we cannot unload cached images.
-  # ideally the filesystem cache should serve as the only cache, but right
-  # now it's just sort of a temporary place before the image is dumped to
-  # memory.
-  # maybe allow the buffer to add a cache file? or receive a separate "image
-  # load start" event in container, and then add one in the pager?
-  # the first option seems better; it's simpler, and buffers can add arbitrary
-  # cache files if they just tell the pager it's an image anyway.
+  #TODO we should only cache the final output in memory, not the full bitmap.
   let bmp = NetworkBitmap(image.bmp)
   let request = newRequest(newURL("cache:" & $bmp.cacheId).get)
-  let cachedImage = CachedImage(bmp: bmp)
+  let cachedImage = CachedImage(
+    bmp: bmp,
+    width: image.width,
+    height: image.height
+  )
   pager.loader.shareCachedItem(bmp.cacheId, pager.loader.clientPid,
     container.process)
   pager.loader.fetch(request).then(proc(res: JSResult[Response]):
@@ -508,15 +505,18 @@ proc loadCachedImage(pager: Pager; container: Container; image: PosBitmap) =
     response.body.sclose()
     return r
   ).then(proc(res: JSResult[Response]): EmptyPromise =
+    if res.isNone:
+      pager.loader.removeCachedItem(bmp.cacheId)
+      return newResolvedPromise()
     let response = res.get
     # take target sizes
     bmp.width = uint64(image.width)
     bmp.height = uint64(image.height)
-    return response.saveToBitmap(bmp)
-  ).then(proc() =
-    container.redraw = true
-    cachedImage.loaded = true
-    pager.loader.removeCachedItem(bmp.cacheId)
+    return response.saveToBitmap(bmp).then(proc() =
+      container.redraw = true
+      cachedImage.loaded = true
+      pager.loader.removeCachedItem(bmp.cacheId)
+    )
   )
   container.cachedImages.add(cachedImage)
 
@@ -525,9 +525,8 @@ proc initImages(pager: Pager; container: Container) =
   for image in container.images:
     var imageId = -1
     if image.bmp of NetworkBitmap:
-      # add cache file to pager, but source it from the container.
       let bmp = NetworkBitmap(image.bmp)
-      let cached = container.findCachedImage(bmp.imageId)
+      let cached = container.findCachedImage(image)
       imageId = bmp.imageId
       if cached == nil:
         pager.loadCachedImage(container, image)
@@ -538,7 +537,7 @@ proc initImages(pager: Pager; container: Container) =
     else:
       imageId = pager.imageId
       inc pager.imageId
-    let canvasImage = pager.term.loadImage(image.bmp, container.process, imageId,
+    let canvasImage = pager.term.loadImage(image, container.process, imageId,
       image.x - container.fromx, image.y - container.fromy, pager.bufWidth,
       pager.bufHeight)
     if canvasImage != nil:
@@ -548,6 +547,7 @@ proc initImages(pager: Pager; container: Container) =
 
 proc draw*(pager: Pager) =
   var redraw = false
+  var imageRedraw = false
   let container = pager.container
   if container != nil:
     if container.redraw:
@@ -558,6 +558,7 @@ proc draw*(pager: Pager) =
         container.highlightMarks(pager.display.grid, hlcolor)
       container.redraw = false
       pager.display.redraw = true
+      imageRedraw = true
     if (let select = container.select; select != nil and select.redraw):
       select.drawSelect(pager.display.grid)
       select.redraw = false
@@ -580,7 +581,7 @@ proc draw*(pager: Pager) =
     pager.term.writeGrid(pager.status.grid, 0, pager.attrs.height - 1)
     pager.status.redraw = false
     redraw = true
-  if container != nil and pager.term.imageMode != imNone:
+  if imageRedraw and pager.term.imageMode != imNone:
     # init images only after term canvas has been finalized
     pager.initImages(container)
   if redraw:
diff --git a/src/local/term.nim b/src/local/term.nim
index 546b16fa..8dcff9ba 100644
--- a/src/local/term.nim
+++ b/src/local/term.nim
@@ -7,10 +7,14 @@ import std/termios
 import std/unicode
 
 import bindings/termcap
+import chagashi/charset
+import chagashi/decoder
+import chagashi/encoder
 import config/config
 import img/bitmap
 import io/posixstream
 import js/base64
+import layout/renderdocument
 import types/cell
 import types/color
 import types/opt
@@ -18,10 +22,6 @@ import types/winattrs
 import utils/strwidth
 import utils/twtstr
 
-import chagashi/charset
-import chagashi/decoder
-import chagashi/encoder
-
 #TODO switch away from termcap...
 
 type
@@ -66,7 +66,7 @@ type
     damaged: bool
     marked*: bool
     kittyId: int
-    bmp: Bitmap
+    pbmp: PosBitmap
 
   Terminal* = ref object
     cs*: Charset
@@ -222,6 +222,9 @@ const ANSIColorMap = [
   rgb(255, 255, 255)
 ]
 
+template bmp(image: CanvasImage): Bitmap =
+  image.pbmp.bmp
+
 proc flush*(term: Terminal) =
   term.outfile.flushFile()
 
@@ -616,9 +619,12 @@ proc outputGrid*(term: Terminal) =
   term.cursorx = -1
   term.cursory = -1
 
-func findImage(term: Terminal; pid, imageId: int): CanvasImage =
+func findImage(term: Terminal; pid, imageId: int; pbmp: PosBitmap):
+    CanvasImage =
   for it in term.canvasImages:
-    if it.pid == pid and it.imageId == imageId:
+    if it.pid == pid and it.imageId == imageId and
+        it.pbmp.width == pbmp.width and it.pbmp.height == pbmp.height and
+        it.pbmp.x == pbmp.x and it.pbmp.y == pbmp.y:
       return it
   return nil
 
@@ -660,9 +666,9 @@ proc clearImages*(term: Terminal; maxh: int) =
       term.clearImage(image, maxh)
     image.marked = false
 
-proc loadImage*(term: Terminal; bmp: Bitmap; pid, imageId, x, y, maxw,
+proc loadImage*(term: Terminal; pbmp: PosBitmap; pid, imageId, x, y, maxw,
     maxh: int): CanvasImage =
-  if (let image = term.findImage(pid, imageId); image != nil):
+  if (let image = term.findImage(pid, imageId, pbmp); image != nil):
     # reuse image on screen
     if image.x != x or image.y != y:
       # only clear sixels; with kitty we just move the existing image
@@ -672,7 +678,7 @@ proc loadImage*(term: Terminal; bmp: Bitmap; pid, imageId, x, y, maxw,
         # no longer on screen
         return nil
     elif term.imageMode == imSixel:
-      # check if any line our image is on is damaged
+      # check if any line of our image is damaged
       let ey = min(image.y + int(image.bmp.height), maxh)
       let mx = (image.offx + image.dispw) div term.attrs.ppc
       for y in max(image.y, 0) ..< ey:
@@ -684,7 +690,7 @@ proc loadImage*(term: Terminal; bmp: Bitmap; pid, imageId, x, y, maxw,
     image.marked = true
     return image
   # new image
-  let image = CanvasImage(bmp: bmp, pid: pid, imageId: imageId)
+  let image = CanvasImage(pbmp: pbmp, pid: pid, imageId: imageId)
   if term.positionImage(image, x, y, maxw, maxh):
     return image
   # no longer on screen
diff --git a/src/server/buffer.nim b/src/server/buffer.nim
index d84bc29c..af14634a 100644
--- a/src/server/buffer.nim
+++ b/src/server/buffer.nim
@@ -27,7 +27,6 @@ import html/enums
 import html/env
 import html/event
 import html/formdata as formdata_impl
-import img/bitmap
 import io/bufreader
 import io/bufstream
 import io/bufwriter
@@ -1699,7 +1698,7 @@ proc getLines*(buffer: Buffer; w: Slice[int]): GetLinesResult {.proxy.} =
   result.bgcolor = buffer.bgcolor
   if buffer.config.images:
     for image in buffer.images:
-      if image.y <= w.b and image.y + int(image.bmp.height) >= w.a:
+      if image.y <= w.b and image.y + image.height >= w.a:
         result.images.add(image)
 
 proc markURL*(buffer: Buffer; schemes: seq[string]) {.proxy.} =
diff --git a/src/server/forkserver.nim b/src/server/forkserver.nim
index de81bace..dfe3febb 100644
--- a/src/server/forkserver.nim
+++ b/src/server/forkserver.nim
@@ -4,6 +4,7 @@ import std/posix
 import std/selectors
 import std/tables
 
+import chagashi/charset
 import config/config
 import html/formdata
 import io/bufreader
@@ -22,8 +23,6 @@ import utils/proctitle
 import utils/sandbox
 import utils/strwidth
 
-import chagashi/charset
-
 type
   ForkCommand = enum
     fcForkBuffer, fcForkLoader, fcRemoveChild, fcLoadConfig
diff --git a/src/types/cookie.nim b/src/types/cookie.nim
index 1b1c82cd..4f7b3ef4 100644
--- a/src/types/cookie.nim
+++ b/src/types/cookie.nim
@@ -188,7 +188,7 @@ proc add*(cookiejar: CookieJar; cookies: seq[Cookie]) =
 proc serialize*(cookiejar: CookieJar; url: URL): string =
   if not cookiejar.filter.match(url):
     return "" # fail
-  let t = now().toTime().toUnix()
+  let t = getTime().utc().toTime().toUnix()
   #TODO sort
   for i in countdown(cookiejar.cookies.high, 0):
     let cookie = cookiejar.cookies[i]
@@ -208,10 +208,7 @@ proc serialize*(cookiejar: CookieJar; url: URL): string =
     result &= cookie.value
 
 proc newCookie*(str: string; url: URL = nil): JSResult[Cookie] {.jsctor.} =
-  let cookie = Cookie(
-    expires: -1,
-    created: now().toTime().toUnix()
-  )
+  let cookie = Cookie(expires: -1, created: getTime().utc().toTime().toUnix())
   var first = true
   var haspath = false
   var hasdomain = false
@@ -222,12 +219,8 @@ proc newCookie*(str: string; url: URL = nil): JSResult[Cookie] {.jsctor.} =
       first = false
       continue
     let part = part.strip(leading = true, trailing = false, AsciiWhitespace)
-    var n = 0
-    for i in 0..part.high:
-      if part[i] == '=':
-        n = i
-        break
-    if n == 0:
+    let n = part.find('=')
+    if n <= 0:
       continue
     let key = part.substr(0, n - 1)
     let val = part.substr(n + 1)