diff options
| author | bptato <nincsnevem662@gmail.com> | 2024-09-19 17:46:27 +0200 |
|---|---|---|
| committer | bptato <nincsnevem662@gmail.com> | 2024-09-22 22:44:53 +0200 |
| commit | 080493c058f52a5c20638f1b975d032af45f4d3f (patch) | |
| tree | 60e6ba6b3cb967d29d349018b3f315e7637b4b9e /src | |
| parent | e23fa780cf2fff7146efcd64b2806ce428858b80 (diff) | |
| download | chawan-080493c058f52a5c20638f1b975d032af45f4d3f.tar.gz | |
loader: mmap intermediate image files, misc refactoring
* refactor parseHeader
* optimize response blob()
* add direct "to cache" mode for loader requests which sets stdout to a
file, and use it for image processing
* move image resizing into a separate process
* mmap cache files in between processing steps when possible
At last, resize is no longer a part of image decoding. Also, it feels
much nicer to keep encoded image data in the same cache as everything
else.
The mmap operations *should* be more efficient than copying the whole
RGBA data through a pipe. In practice, it only makes a difference for
loading (well, now just mmapping) the encoded image into the pager,
where it singlehandedly speeds up image display by 10x on my test image.
For the other steps, the unfortunate fact that "tocache" must delay the
next fork/exec in the pipeline until the entire image is processed seems
to equal out any wins we might have gotten from skipping a single raw
RGBA copy.
I have tried moving the delay before the exec (it's possible with yet
another pipe), but it didn't help much and made the code much
uglier. (Not that tocache didn't, but I can live with this...)
Diffstat (limited to 'src')
| -rw-r--r-- | src/io/dynstream.nim | 82 | ||||
| -rw-r--r-- | src/io/tempfile.nim | 2 | ||||
| -rw-r--r-- | src/loader/loader.nim | 136 | ||||
| -rw-r--r-- | src/loader/loaderhandle.nim | 3 | ||||
| -rw-r--r-- | src/loader/loaderiface.nim | 20 | ||||
| -rw-r--r-- | src/loader/request.nim | 6 | ||||
| -rw-r--r-- | src/loader/response.nim | 35 | ||||
| -rw-r--r-- | src/local/client.nim | 6 | ||||
| -rw-r--r-- | src/local/container.nim | 8 | ||||
| -rw-r--r-- | src/local/pager.nim | 101 | ||||
| -rw-r--r-- | src/types/url.nim | 10 |
11 files changed, 320 insertions, 89 deletions
diff --git a/src/io/dynstream.nim b/src/io/dynstream.nim index 2bfa02f5..129ecf05 100644 --- a/src/io/dynstream.nim +++ b/src/io/dynstream.nim @@ -169,6 +169,88 @@ proc newPosixStream*(path: string; flags, mode: cint): PosixStream = return nil return newPosixStream(fd) +type + MaybeMappedMemory* = ptr MaybeMappedMemoryObj + + MaybeMappedMemoryObj = object + p0: pointer + p0len: int + fromMmap: bool + p*: ptr UncheckedArray[uint8] + len*: int + +# Read data of size "len", or mmap it if the stream is a file. +proc recvDataLoopOrMmap*(ps: PosixStream; len = -1): MaybeMappedMemory = + var stats: Stat + if fstat(ps.fd, stats) != -1 and S_ISREG(stats.st_mode): + let srcOff = lseek(ps.fd, 0, SEEK_CUR) # skip headers + if len != -1: + doAssert len == stats.st_size - srcOff + let len = int(stats.st_size) - srcOff + let p0 = mmap(nil, len, PROT_READ, MAP_SHARED, ps.fd, 0) + if p0 == MAP_FAILED: + return nil + let p1 = addr cast[ptr UncheckedArray[uint8]](p0)[srcOff] + let res = create(MaybeMappedMemoryObj) + res[] = MaybeMappedMemoryObj( + p0: p0, + p0len: int(stats.st_size), + p: cast[ptr UncheckedArray[uint8]](p1), + len: len, + fromMmap: true + ) + return res + let p = cast[ptr UncheckedArray[uint8]](alloc(len)) + ps.recvDataLoop(p, len) + let res = create(MaybeMappedMemoryObj) + res[] = MaybeMappedMemoryObj( + p0: p, + p0len: len, + p: p, + len: len, + fromMmap: false + ) + return res + +proc maybeMmapForSend*(ps: PosixStream; len: int): MaybeMappedMemory = + var stats: Stat + if fstat(0, stats) != -1 and S_ISREG(stats.st_mode): + let p0 = mmap(nil, len, PROT_WRITE, MAP_SHARED, ps.fd, 0) + if p0 == MAP_FAILED: + return nil + ps.seek(len - 1) + ps.sendDataLoop([char(0)]) + let res = create(MaybeMappedMemoryObj) + res[] = MaybeMappedMemoryObj( + p0: p0, + p0len: int(stats.st_size), + p: cast[ptr UncheckedArray[uint8]](p0), + len: len, + fromMmap: true + ) + return res + let p = cast[ptr UncheckedArray[uint8]](alloc(len)) + let res = create(MaybeMappedMemoryObj) + res[] = MaybeMappedMemoryObj( + p0: p, + p0len: len, + p: p, + len: len, + fromMmap: false + ) + return res + +proc sendDataLoop*(ps: PosixStream; mem: MaybeMappedMemory) = + # only send if not mmapped; otherwise everything is already where it should be + if not mem.fromMmap: + ps.sendDataLoop(mem.p, mem.len) + +proc dealloc*(mem: MaybeMappedMemory) = + if mem.fromMmap: + discard munmap(mem.p0, mem.p0len) + else: + dealloc(mem.p0) + type SocketStream* = ref object of PosixStream source*: Socket diff --git a/src/io/tempfile.nim b/src/io/tempfile.nim index 44c63297..132a12b1 100644 --- a/src/io/tempfile.nim +++ b/src/io/tempfile.nim @@ -1,6 +1,6 @@ import std/os -var tmpf_seq: int +var tmpf_seq = 0 proc getTempFile*(tmpdir: string; ext = ""): string = if not dirExists(tmpdir): createDir(tmpdir) diff --git a/src/loader/loader.nim b/src/loader/loader.nim index e7a45833..b5a6b3ba 100644 --- a/src/loader/loader.nim +++ b/src/loader/loader.nim @@ -58,8 +58,9 @@ export response type CachedItem = ref object id: int - path: string refc: int + offset: int + path: string ClientData = ref object pid: int @@ -263,10 +264,41 @@ proc addCacheFile(ctx: LoaderContext; client: ClientData; output: OutputHandle): return cacheId return -1 +proc findOffset(ps: PosixStream): int = + try: + var buffer = default(array[512, char]) + var off = 0 + var lf = 1u # we start at EOL + while true: + let n = ps.recvData(buffer) + if n == 0: + return off + for i in 0 ..< n: + let c = buffer[i] + if c == '\n': + inc lf + if lf == 2: + return off + i + 1 + elif c != '\r': + lf = 0 + off += n + except IOError: + discard + return -1 + proc openCachedItem(client: ClientData; id: int): (PosixStream, int) = let n = client.cacheMap.find(id) if n != -1: - return (newPosixStream(client.cacheMap[n].path, O_RDONLY, 0), n) + let item = client.cacheMap[n] + let ps = newPosixStream(client.cacheMap[n].path, O_RDONLY, 0) + if item.offset == -1: + let offset = ps.findOffset() + if offset == -1: + client.cacheMap.del(n) + return (nil, -1) + item.offset = offset + ps.seek(item.offset) + return (ps, n) return (nil, -1) proc put(ctx: LoaderContext; handle: LoaderHandle) = @@ -357,18 +389,13 @@ proc handleLine(handle: InputHandle; line: string; headers: Headers) = let v = line.substr(k.len + 1).strip() headers.add(k, v) -proc parseHeaders0(handle: InputHandle; buffer: LoaderBuffer): int = +proc parseHeaders0(handle: InputHandle; data: openArray[char]): int = let parser = handle.parser var s = parser.lineBuffer - let L = if buffer == nil: 1 else: buffer.len - for i in 0 ..< L: + for i, c in data: template die = handle.parser = nil return -1 - let c = if buffer != nil: - char(buffer.page[i]) - else: - '\n' if parser.crSeen and c != '\n': die parser.crSeen = false @@ -402,11 +429,15 @@ proc parseHeaders0(handle: InputHandle; buffer: LoaderBuffer): int = s &= c if s != "": parser.lineBuffer = s - return L + return data.len proc parseHeaders(handle: InputHandle; buffer: LoaderBuffer): int = try: - return handle.parseHeaders0(buffer) + if buffer == nil: + return handle.parseHeaders0(['\n']) + assert buffer.page != nil + let p = cast[ptr UncheckedArray[char]](buffer.page) + return handle.parseHeaders0(p.toOpenArray(0, buffer.len - 1)) except ErrorBrokenPipe: handle.parser = nil return -1 @@ -457,9 +488,10 @@ proc handleRead(ctx: LoaderContext; handle: InputHandle; hrrDone # stream is a regular file, so we can't select on it. -# cachedHandle is used for attaching the output handle to a different -# InputHandle when loadFromCache is called while a download is still ongoing -# (and thus some parts of the document are not cached yet). +# +# cachedHandle is used for attaching the output handle to another +# InputHandle when loadFromCache is called while a download is still +# ongoing (and thus some parts of the document are not cached yet). proc loadStreamRegular(ctx: LoaderContext; handle, cachedHandle: InputHandle) = assert handle.parser == nil # parser is only used with CGI var unregWrite: seq[OutputHandle] = @[] @@ -482,7 +514,7 @@ proc loadStreamRegular(ctx: LoaderContext; handle, cachedHandle: InputHandle) = output.istreamAtEnd = true ctx.put(output) else: - assert output.stream.fd < ctx.handleMap.len or + assert output.stream.fd >= ctx.handleMap.len or ctx.handleMap[output.stream.fd] == nil output.oclose() handle.outputs.setLen(0) @@ -506,7 +538,7 @@ type CGIPath = object myDir: string proc setupEnv(cpath: CGIPath; request: Request; contentLen: int; prevURL: URL; - insecureSSLNoVerify: bool) = + config: LoaderClientConfig) = let url = request.url putEnv("SCRIPT_NAME", cpath.scriptName) putEnv("SCRIPT_FILENAME", cpath.cmd) @@ -532,9 +564,9 @@ proc setupEnv(cpath: CGIPath; request: Request; contentLen: int; prevURL: URL; putEnv("HTTP_COOKIE", request.headers["Cookie"]) if request.referrer != nil: putEnv("HTTP_REFERER", $request.referrer) - if request.proxy != nil: - putEnv("ALL_PROXY", $request.proxy) - if insecureSSLNoVerify: + if config.proxy != nil: + putEnv("ALL_PROXY", $config.proxy) + if config.insecureSSLNoVerify: putEnv("CHA_INSECURE_SSL_NO_VERIFY", "1") setCurrentDir(cpath.myDir) @@ -572,7 +604,7 @@ proc parseCGIPath(ctx: LoaderContext; request: Request): CGIPath = return cpath proc loadCGI(ctx: LoaderContext; client: ClientData; handle: InputHandle; - request: Request; prevURL: URL; insecureSSLNoVerify: bool) = + request: Request; prevURL: URL; config: LoaderClientConfig) = if ctx.config.cgiDir.len == 0: handle.sendResult(ERROR_NO_CGI_DIR) return @@ -584,10 +616,31 @@ proc loadCGI(ctx: LoaderContext; client: ClientData; handle: InputHandle; if not fileExists(cpath.cmd): handle.sendResult(ERROR_CGI_FILE_NOT_FOUND) return - var pipefd: array[0..1, cint] # child -> parent + # Pipe the response body as stdout. + var pipefd: array[2, cint] # child -> parent if pipe(pipefd) == -1: handle.sendResult(ERROR_FAIL_SETUP_CGI) return + let istreamOut = newPosixStream(pipefd[0]) # read by loader + var ostreamOut = newPosixStream(pipefd[1]) # written by child + var ostreamOut2: PosixStream = nil + if request.tocache: + # Set stdout to a file, and repurpose the pipe as a dummy to detect when + # the process ends. outputId is the cache id. + let tmpf = getTempFile(ctx.config.tmpdir) + ostreamOut2 = ostreamOut + # RDWR, otherwise mmap won't work + ostreamOut = newPosixStream(tmpf, O_CREAT or O_RDWR, 0o600) + if ostreamOut == nil: + handle.sendResult(ERROR_FAIL_SETUP_CGI) + return + let cacheId = handle.output.outputId # welp + client.cacheMap.add(CachedItem( + id: cacheId, + path: tmpf, + refc: 1, + offset: -1 + )) # Pipe the request body as stdin for POST. var istream: PosixStream = nil # child end (read) var ostream: PosixStream = nil # parent end (write) @@ -617,15 +670,14 @@ proc loadCGI(ctx: LoaderContext; client: ClientData; handle: InputHandle; istream = newPosixStream(pipefdRead[0]) ostream = newPosixStream(pipefdRead[1]) let contentLen = request.body.contentLength() - stdout.flushFile() stderr.flushFile() let pid = fork() if pid == -1: handle.sendResult(ERROR_FAIL_SETUP_CGI) elif pid == 0: - discard close(pipefd[0]) # close read - discard dup2(pipefd[1], 1) # dup stdout - discard close(pipefd[1]) + istreamOut.sclose() # close read + discard dup2(ostreamOut.fd, 1) # dup stdout + ostreamOut.sclose() if ostream != nil: ostream.sclose() # close write if istream2 != nil: @@ -637,7 +689,7 @@ proc loadCGI(ctx: LoaderContext; client: ClientData; handle: InputHandle; else: closeStdin() # we leave stderr open, so it can be seen in the browser console - setupEnv(cpath, request, contentLen, prevURL, insecureSSLNoVerify) + setupEnv(cpath, request, contentLen, prevURL, config) # reset SIGCHLD to the default handler. this is useful if the child process # expects SIGCHLD to be untouched. (e.g. git dies a horrible death with # SIGCHLD as SIG_IGN) @@ -652,11 +704,13 @@ proc loadCGI(ctx: LoaderContext; client: ClientData; handle: InputHandle; ($strerror(errno)).deleteChars({'\n', '\r'})) quit(1) else: - discard close(pipefd[1]) # close write + ostreamOut.sclose() # close write + if ostreamOut2 != nil: + ostreamOut2.sclose() # close write if request.body.t != rbtNone: istream.sclose() # close read handle.parser = HeaderParser(headers: newHeaders()) - handle.stream = newPosixStream(pipefd[0]) + handle.stream = istreamOut case request.body.t of rbtString: ostream.write(request.body.s) @@ -709,10 +763,7 @@ proc loadStream(ctx: LoaderContext; client: ClientData; handle: InputHandle; proc loadFromCache(ctx: LoaderContext; client: ClientData; handle: InputHandle; request: Request) = let id = parseInt32(request.url.pathname).get(-1) - let startFrom = if request.url.query.isSome: - parseInt32(request.url.query.get).get(0) - else: - 0 + let startFrom = parseInt32(request.url.query.get("")).get(0) let (ps, n) = client.openCachedItem(id) if ps != nil: if startFrom != 0: @@ -798,7 +849,7 @@ proc loadResource(ctx: LoaderContext; client: ClientData; redo = true continue if request.url.scheme == "cgi-bin": - ctx.loadCGI(client, handle, request, prevurl, config.insecureSSLNoVerify) + ctx.loadCGI(client, handle, request, prevurl, config) if handle.stream != nil: ctx.addFd(handle) else: @@ -852,8 +903,6 @@ proc load(ctx: LoaderContext; stream: SocketStream; request: Request; handle.rejectHandle(ERROR_DISALLOWED_URL) else: request.setupRequestDefaults(config) - if request.proxy == nil or not ctx.isPrivileged(client): - request.proxy = config.proxy ctx.loadResource(client, config, request, handle) proc load(ctx: LoaderContext; stream: SocketStream; client: ClientData; @@ -969,6 +1018,18 @@ proc shareCachedItem(ctx: LoaderContext; stream: SocketStream; targetClient.cacheMap.add(item) stream.sclose() +proc openCachedItem(ctx: LoaderContext; stream: SocketStream; + client: ClientData; r: var BufferedReader) = + # open a cached item + var id: int + r.sread(id) + let (ps, _) = client.openCachedItem(id) + stream.withPacketWriter w: + w.swrite(ps != nil) + if ps != nil: + w.sendAux.add(FileHandle(ps.fd)) + stream.sclose() + proc passFd(ctx: LoaderContext; stream: SocketStream; client: ClientData; r: var BufferedReader) = var id: string @@ -1076,6 +1137,9 @@ proc acceptConnection(ctx: LoaderContext) = of lcShareCachedItem: privileged_command ctx.shareCachedItem(stream, r) + of lcOpenCachedItem: + privileged_command + ctx.openCachedItem(stream, client, r) of lcRedirectToFile: privileged_command ctx.redirectToFile(stream, r) @@ -1123,7 +1187,7 @@ proc initLoaderContext(fd: cint; config: LoaderConfig): LoaderContext = ctx.ssock = initServerSocket(config.sockdir, -1, myPid, blocking = true) let sfd = int(ctx.ssock.sock.getFd()) ctx.selector.registerHandle(sfd, {Read}, 0) - if ctx.handleMap.len <= sfd: + if sfd >= ctx.handleMap.len: ctx.handleMap.setLen(sfd + 1) ctx.handleMap[sfd] = LoaderHandle() # pseudo handle # The server has been initialized, so the main process can resume execution. diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim index b43149fb..23ddff30 100644 --- a/src/loader/loaderhandle.nim +++ b/src/loader/loaderhandle.nim @@ -42,6 +42,9 @@ type suspended*: bool dead*: bool + SigchldHandle* = ref object of LoaderHandle + notifyMap*: seq[tuple[pid: int; input: InputHandle]] + HandleParserState* = enum hpsBeforeLines, hpsAfterFirstLine, hpsControlDone diff --git a/src/loader/loaderiface.nim b/src/loader/loaderiface.nim index c8635c6f..8cc5ff42 100644 --- a/src/loader/loaderiface.nim +++ b/src/loader/loaderiface.nim @@ -67,6 +67,7 @@ type lcShareCachedItem lcSuspend lcTee + lcOpenCachedItem ClientKey* = array[32, uint8] @@ -78,8 +79,6 @@ type referrerPolicy*: ReferrerPolicy insecureSSLNoVerify*: bool - FetchPromise* = Promise[JSResult[Response]] - proc getRedirect*(response: Response; request: Request): Request = if "Location" in response.headers.table: if response.status in 301u16..303u16 or response.status in 307u16..308u16: @@ -378,6 +377,23 @@ proc shareCachedItem*(loader: FileLoader; id, targetPid: int; sourcePid = -1) = w.swrite(id) stream.sclose() +proc openCachedItem*(loader: FileLoader; cacheId: int): PosixStream = + let stream = loader.connect() + if stream != nil: + stream.withLoaderPacketWriter loader, w: + w.swrite(lcOpenCachedItem) + w.swrite(cacheId) + var fd = FileHandle(-1) + stream.withPacketReader r: + var success: bool + r.sread(success) + if success: + fd = r.recvAux.pop() + stream.sclose() + if fd != -1: + return newPosixStream(fd) + return nil + proc passFd*(loader: FileLoader; id: string; fd: FileHandle) = let stream = loader.connect() if stream != nil: diff --git a/src/loader/request.nim b/src/loader/request.nim index bd102093..69daaf79 100644 --- a/src/loader/request.nim +++ b/src/loader/request.nim @@ -81,7 +81,7 @@ type headers*: Headers body*: RequestBody referrer*: URL - proxy*: URL #TODO do something with this + tocache*: bool JSRequest* = ref object request*: Request @@ -143,14 +143,14 @@ iterator pairs*(headers: Headers): (string, string) = yield (k, v) func newRequest*(url: URL; httpMethod = hmGet; headers = newHeaders(); - body = RequestBody(); proxy: URL = nil; referrer: URL = nil): Request = + body = RequestBody(); referrer: URL = nil; tocache = false): Request = return Request( url: url, httpMethod: httpMethod, headers: headers, body: body, referrer: referrer, - proxy: proxy + tocache: tocache ) func createPotentialCORSRequest*(url: URL; destination: RequestDestination; diff --git a/src/loader/response.nim b/src/loader/response.nim index 1d7307cd..23c386ef 100644 --- a/src/loader/response.nim +++ b/src/loader/response.nim @@ -47,6 +47,8 @@ type opaque*: RootRef flags*: set[ResponseFlag] + FetchPromise* = Promise[JSResult[Response]] + jsDestructor(Response) proc newResponse*(res: int; request: Request; stream: SocketStream; @@ -59,6 +61,9 @@ proc newResponse*(res: int; request: Request; stream: SocketStream; status: status ) +proc newFetchPromise*(): FetchPromise = + return newPromise[JSResult[Response]]() + func makeNetworkError*(): Response {.jsstfunc: "Response.error".} = #TODO use "create" function return Response( @@ -115,6 +120,10 @@ func getReferrerPolicy*(this: Response): Option[ReferrerPolicy] = this.headers.table.withValue("Referrer-Policy", p): return strictParseEnum[ReferrerPolicy](p[][0]) +proc resume*(response: Response) = + response.resumeFun(response.outputId) + response.resumeFun = nil + type TextOpaque = ref object of RootObj buf: string bodyRead: Promise[JSResult[string]] @@ -145,15 +154,9 @@ proc onFinishText(response: Response; success: bool) = let err = newTypeError("NetworkError when attempting to fetch resource") bodyRead.resolve(JSResult[string].err(err)) -proc resume*(response: Response) = - response.resumeFun(response.outputId) - response.resumeFun = nil - proc text*(response: Response): Promise[JSResult[string]] {.jsfunc.} = if response.body == nil: - let p = newPromise[JSResult[string]]() - p.resolve(JSResult[string].ok("")) - return p + return newResolvedPromise(JSResult[string].ok("")) if response.bodyUsed: let p = newPromise[JSResult[string]]() let err = JSResult[string] @@ -179,12 +182,12 @@ proc onReadBlob(response: Response) = let opaque = BlobOpaque(response.opaque) while true: try: - let targetLen = opaque.len + BufferSize - if targetLen > opaque.size: - opaque.size = targetLen - opaque.p = realloc(opaque.p, targetLen) + if opaque.len + BufferSize > opaque.size: + opaque.size *= 2 + opaque.p = realloc(opaque.p, opaque.size) let p = cast[ptr UncheckedArray[uint8]](opaque.p) - let n = response.body.recvData(addr p[opaque.len], BufferSize) + let diff = opaque.size - opaque.len + let n = response.body.recvData(addr p[opaque.len], diff) opaque.len += n if n == 0: break @@ -211,13 +214,13 @@ proc onFinishBlob(response: Response; success: bool) = proc blob*(response: Response): Promise[JSResult[Blob]] {.jsfunc.} = if response.bodyUsed: - let p = newPromise[JSResult[Blob]]() let err = JSResult[Blob].err(newTypeError("Body has already been consumed")) - p.resolve(err) - return p + return newResolvedPromise(err) let opaque = BlobOpaque( bodyRead: newPromise[JSResult[Blob]](), - contentType: response.getContentType() + contentType: response.getContentType(), + p: alloc(BufferSize), + size: BufferSize ) response.opaque = opaque response.onRead = onReadBlob diff --git a/src/local/client.nim b/src/local/client.nim index c8bf9b24..c20a460b 100644 --- a/src/local/client.nim +++ b/src/local/client.nim @@ -356,7 +356,7 @@ proc input(client: Client): EmptyPromise = p.resolve() return p -when not defined(android): +when ioselSupportedPlatform: let SIGWINCH {.importc, header: "<signal.h>", nodecl.}: cint proc showConsole(client: Client) {.jsfunc.} = @@ -542,7 +542,7 @@ proc handleError(client: Client; fd: int) = proc inputLoop(client: Client) = let selector = client.selector selector.registerHandle(int(client.pager.term.istream.fd), {Read}, 0) - when not defined(android): + when ioselSupportedPlatform: let sigwinch = selector.registerSignal(int(SIGWINCH), 0) var keys: array[64, ReadyKey] while true: @@ -554,7 +554,7 @@ proc inputLoop(client: Client) = client.handleWrite(event.fd) if Error in event.events: client.handleError(event.fd) - when not defined(android): + when ioselSupportedPlatform: if Signal in event.events: assert event.fd == sigwinch client.pager.windowChange() diff --git a/src/local/container.nim b/src/local/container.nim index 8f295ad9..e22f0f4e 100644 --- a/src/local/container.nim +++ b/src/local/container.nim @@ -101,11 +101,15 @@ type ContainerFlag* = enum cfCloned, cfUserRequested, cfHasStart, cfCanReinterpret, cfSave, cfIsHTML + CachedImageState* = enum + cisLoading, cisCanceled, cisLoaded + CachedImage* = ref object - loaded*: bool + state*: CachedImageState width*: int height*: int - data*: Blob + data*: Blob # mmapped blob of image data + cacheId*: int # cache id of the file backing "data" bmp*: NetworkBitmap # Following variables are always 0 in kitty mode; they exist to support # sixel cropping. diff --git a/src/local/pager.nim b/src/local/pager.nim index fa205596..d6e31e2a 100644 --- a/src/local/pager.nim +++ b/src/local/pager.nim @@ -214,6 +214,10 @@ proc clearStatus(pager: Pager) = proc setContainer*(pager: Pager; c: Container) {.jsfunc.} = if pager.term.imageMode != imNone and pager.container != nil: + for cachedImage in pager.container.cachedImages: + if cachedImage.state == cisLoaded: + pager.loader.removeCachedItem(cachedImage.cacheId) + cachedImage.state = cisCanceled pager.container.cachedImages.setLen(0) pager.container = c if c != nil: @@ -503,6 +507,9 @@ proc redraw(pager: Pager) {.jsfunc.} = if pager.container.select != nil: pager.container.select.redraw = true +proc getTempFile(pager: Pager; ext = ""): string = + return getTempFile(pager.config.external.tmpdir, ext) + proc loadCachedImage(pager: Pager; container: Container; image: PosBitmap; offx, erry, dispw: int) = let bmp = image.bmp @@ -517,20 +524,53 @@ proc loadCachedImage(pager: Pager; container: Container; image: PosBitmap; pager.loader.shareCachedItem(bmp.cacheId, pager.loader.clientPid, container.process) let imageMode = pager.term.imageMode - let headers = newHeaders() - if image.width != bmp.width or image.height != bmp.height: - headers.add("Cha-Image-Target-Dimensions", $image.width & 'x' & - $image.height) pager.loader.fetch(newRequest( newURL("img-codec+" & bmp.contentType.after('/') & ":decode").get, httpMethod = hmPost, - headers = headers, body = RequestBody(t: rbtCache, cacheId: bmp.cacheId), - )).then(proc(res: JSResult[Response]) = + tocache = true + )).then(proc(res: JSResult[Response]): FetchPromise = + # remove previous step + pager.loader.removeCachedItem(bmp.cacheId) + if res.isNone: + return nil + let response = res.get + let cacheId = response.outputId # set by loader in tocache + if cachedImage.state == cisCanceled: # container is no longer visible + pager.loader.removeCachedItem(cacheId) + return nil + if image.width == bmp.width and image.height == bmp.height: + # skip resize + return newResolvedPromise(res) + # resize + # use a temp file, so that img-resize can mmap its output + let headers = newHeaders({ + "Cha-Image-Dimensions": $bmp.width & 'x' & $bmp.height, + "Cha-Image-Target-Dimensions": $image.width & 'x' & $image.height + }) + let p = pager.loader.fetch(newRequest( + newURL("cgi-bin:resize").get, + httpMethod = hmPost, + headers = headers, + body = RequestBody(t: rbtCache, cacheId: cacheId), + tocache = true + )).then(proc(res: JSResult[Response]): FetchPromise = + # ugh. I must remove the previous cached item, but only after + # resize is done... + pager.loader.removeCachedItem(cacheId) + return newResolvedPromise(res) + ) + response.resume() + response.close() + return p + ).then(proc(res: JSResult[Response]) = if res.isNone: - pager.loader.removeCachedItem(bmp.cacheId) return let response = res.get + let cacheId = response.outputId + if cachedImage.state == cisCanceled: + pager.loader.removeCachedItem(cacheId) + return let headers = newHeaders({ "Cha-Image-Dimensions": $image.width & 'x' & $image.height }) @@ -550,23 +590,40 @@ proc loadCachedImage(pager: Pager; container: Container; image: PosBitmap; url, httpMethod = hmPost, headers = headers, - body = RequestBody(t: rbtOutput, outputId: response.outputId), + body = RequestBody(t: rbtCache, cacheId: cacheId), + tocache = true ) let r = pager.loader.fetch(request) response.resume() response.close() - r.then(proc(res: JSResult[Response]): Promise[JSResult[Blob]] = + r.then(proc(res: JSResult[Response]) = + # remove previous step + pager.loader.removeCachedItem(cacheId) if res.isNone: - let p = newPromise[JSResult[Blob]]() - p.resolve(JSResult[Blob].err(res.error)) - return p - return res.get.blob() - ).then(proc(res: JSResult[Blob]) = - if res.isSome: - container.redraw = true - cachedImage.data = res.get - cachedImage.loaded = true - pager.loader.removeCachedItem(bmp.cacheId) + return + let cacheId = res.get.outputId + if cachedImage.state == cisCanceled: + pager.loader.removeCachedItem(cacheId) + return + let ps = pager.loader.openCachedItem(cacheId) + if ps == nil: + pager.loader.removeCachedItem(cacheId) + return + let mem = ps.recvDataLoopOrMmap() + ps.sclose() + if mem == nil: + pager.loader.removeCachedItem(cacheId) + return + let blob = newBlob(mem.p, mem.len, "image/x-sixel", + (proc(opaque, p: pointer) = + let mem = cast[MaybeMappedMemory](opaque) + dealloc(mem) + ), mem + ) + container.redraw = true + cachedImage.data = blob + cachedImage.state = cisLoaded + cachedImage.cacheId = cacheId ) ) container.cachedImages.add(cachedImage) @@ -592,7 +649,7 @@ proc initImages(pager: Pager; container: Container) = if cached == nil: pager.loadCachedImage(container, image, offx, erry, dispw) continue - if not cached.loaded: + if cached.state != cisLoaded: continue # loading let canvasImage = pager.term.loadImage(cached.data, container.process, imageId, image.x - container.fromx, image.y - container.fromy, @@ -1110,7 +1167,7 @@ proc getEditorCommand(pager: Pager; file: string; line = 1): string {.jsfunc.} = proc openInEditor(pager: Pager; input: var string): bool = try: - let tmpf = getTempFile(pager.config.external.tmpdir) + let tmpf = pager.getTempFile() if input != "": writeFile(tmpf, input) let cmd = pager.getEditorCommand(tmpf) @@ -1815,7 +1872,7 @@ proc checkMailcap(pager: Pager; container: Container; stream: SocketStream; if entry == nil: return CheckMailcapResult(connect: true, fdout: stream.fd, found: false) let ext = url.pathname.afterLast('.') - let tempfile = getTempFile(pager.config.external.tmpdir, ext) + let tempfile = pager.getTempFile(ext) let outpath = if entry.nametemplate != "": unquoteCommand(entry.nametemplate, contentType, tempfile, url) else: diff --git a/src/types/url.nim b/src/types/url.nim index ab2aa9c9..61f051ac 100644 --- a/src/types/url.nim +++ b/src/types/url.nim @@ -1169,11 +1169,11 @@ proc update(params: URLSearchParams) = else: params.url.query = some(serializedQuery) -proc append*(params: URLSearchParams; name, value: string) {.jsfunc.} = +proc append(params: URLSearchParams; name, value: string) {.jsfunc.} = params.list.add((name, value)) params.update() -proc delete*(params: URLSearchParams; name: string) {.jsfunc.} = +proc delete(params: URLSearchParams; name: string) {.jsfunc.} = for i in countdown(params.list.high, 0): if params.list[i][0] == name: params.list.delete(i) @@ -1182,13 +1182,15 @@ proc get*(params: URLSearchParams; name: string): Option[string] {.jsfunc.} = for it in params.list: if it[0] == name: return some(it[1]) + return none(string) -proc getAll*(params: URLSearchParams; name: string): seq[string] {.jsfunc.} = +proc getAll(params: URLSearchParams; name: string): seq[string] {.jsfunc.} = + result = @[] for it in params.list: if it[0] == name: result.add(it[1]) -proc set*(params: URLSearchParams; name, value: string) {.jsfunc.} = +proc set(params: URLSearchParams; name, value: string) {.jsfunc.} = var first = true for i in 0..params.list.high: if params.list[i][0] == name: |