diff options
30 files changed, 1431 insertions, 378 deletions
diff --git a/doc/mailcap.md b/doc/mailcap.md new file mode 100644 index 00000000..03fbebb1 --- /dev/null +++ b/doc/mailcap.md @@ -0,0 +1,82 @@ +# Mailcap + +Chawan's buffers can only handle HTML and plain text. To make Chawan recognize +other file formats, the mailcap file format can be used. + +For an exact description of the mailcap format, see +[RFC 1524](https://www.rfc-editor.org/rfc/rfc1524). + +## Search path + +The search path for mailcap files can be overridden using the configuration +variable `external.mailcap`. If no mailcap files were found, Chawan simply +uses the xdg-open command for all entries. NOTE: this will change once file +downloading is implemented. + +## Format + +Chawan tries to adhere to the format described in RFC 1524, with a few +extensions. + +### Templating + +%s, %t works as described in the standard. %{...} in general does not work, +only %{charset}. (TODO: fix this.) + +Also, the non-standard template %u may be specified to get the original URL +of the resource. + +If no quoting is applied, Chawan will quote the templates automatically. + +Note that $(subprocesses) are not quoted properly yet. We recommend using +something like: + +``` +x=%s; echo "$(cat "$x")" +``` + +### Fields + +The `test`, `nametemplate` and `copiousoutput` fields are +recognized. Additionally, the non-standard `x-htmloutput` extension field +is recognized too. + +* When the `test` named field is specified, the mailcap entry is only used + if the test command returns 0. + Warning: as of now, %s does not work with test. +* `copiousoutput` makes Chawan redirect the output of the external command + into a new buffer. +* The `x-htmloutput` extension field behaves the same as `copiousoutput`, + but makes Chawan interpret the command's output as HTML. +* For a description of nametemplate, see the RFC. + +## Notes + +Entries with a content type of text/html are ignored. + +## Examples + +``` +# Note: these examples require an entry in mime.types that sets e.g. md as +# the markdown content type. + +# Handle markdown files using pandoc. +text/markdown; pandoc - -f markdown -t html -o -; x-htmloutput + +# Show syntax highlighting for JavaScript source files using bat. +text/javascript; bat -f -l es6 --file-name %u -; copiousoutput + +# Play music using mpv, and hand over control of the terminal until mpv exits. +audio/*; mpv -; needsterminal + +# Play videos using mpv in the background, redirecting its standard output +# and standard error to /dev/null. +video/*; mpv - + +# Open OpenOffice files using LibreOffice Writer. +application/vnd.openxmlformats-officedocument.wordprocessingml.document;lowriter %s +# (Wow that was ugly.) + +# Following entry will be ignored, as text/html is supported natively by Chawan. +text/html; cha -T text/html -I %{charset}; copiousoutput +``` diff --git a/doc/mime.types.md b/doc/mime.types.md new file mode 100644 index 00000000..626ebf22 --- /dev/null +++ b/doc/mime.types.md @@ -0,0 +1,38 @@ +# mime.types + +Chawan uses the mime.types file to recognize certain file extensions for +matching mailcap entries. See the [mailcap](mailcap.md) documentation for +a description of mailcap. + +## Search path + +Chawan parses all mime.types files defined in `external.mime-types`. If no +mime.types file was found, the built-in mime type associations are used. + +## Format + +The mime.types file is a list of whitespace-separated columns. The first +column represents the mime type, all following columns are file extensions. + +Lines starting with a hash character (#) are recognized as comments, and +are ignored. + +Example: + +``` +# comment +application/x-example exmpl ex +``` + +This mime.types file would register the file extensions "exmpl" and "ex" +to be recognized as the mime type `application/x-example`. + +## Note + +Chawan only uses mime.types files for finding mailcap entries; buffers use an +internal mime.types file for content type detection instead. + +The default mime.types file only includes file formats that buffers can handle, +which is rather limited (at the time of writing, 5 file formats). Therefore it +is highly recommended to configure at least one external mime.types file if you +use mailcap. diff --git a/res/config.toml b/res/config.toml index c12c10ff..eac589c3 100644 --- a/res/config.toml +++ b/res/config.toml @@ -12,6 +12,18 @@ display-charset = "auto" #system-charset = "auto" #TODO [external] +mailcap = [ + "~/.mailcap", + "/etc/mailcap", + "/usr/etc/mailcap", + "/usr/local/etc/mailcap" +] +mime-types = [ + "~/.mime.types", + "/etc/mime.types", + "/usr/etc/mime.types", + "/usr/local/etc/mime.types" +] tmpdir = "/tmp/cha" editor = "vi %s +%d" diff --git a/res/mime.types b/res/mime.types new file mode 100644 index 00000000..cb3ad6cc --- /dev/null +++ b/res/mime.types @@ -0,0 +1,10 @@ +# This file only includes mime types recognized in some form by Chawan buffers. +# For all other purposes, we recommend using the mime.types file provided +# by your distribution. +# See extern.mime-types in res/config.toml for a list of paths sourced +# by default. +text/html html htm +application/xhtml+xml xhtml xhtm xht +text/plain txt +text/css css +image/png png diff --git a/src/buffer/buffer.nim b/src/buffer/buffer.nim index ca421e67..b793360a 100644 --- a/src/buffer/buffer.nim +++ b/src/buffer/buffer.nim @@ -24,6 +24,7 @@ import html/dom import html/env import html/tags import img/png +import io/connecterror import io/loader import io/posixstream import io/promise @@ -56,8 +57,9 @@ type BufferCommand* = enum LOAD, RENDER, WINDOW_CHANGE, FIND_ANCHOR, READ_SUCCESS, READ_CANCELED, CLICK, FIND_NEXT_LINK, FIND_PREV_LINK, FIND_NEXT_MATCH, FIND_PREV_MATCH, - GET_SOURCE, GET_LINES, UPDATE_HOVER, PASS_FD, CONNECT, GOTO_ANCHOR, CANCEL, - GET_TITLE, SELECT + GET_SOURCE, GET_LINES, UPDATE_HOVER, PASS_FD, CONNECT, CONNECT2, + GOTO_ANCHOR, CANCEL, GET_TITLE, SELECT, REDIRECT_TO_FD, READ_FROM_FD, + SET_CONTENT_TYPE # LOADING_PAGE: istream open # LOADING_RESOURCES: istream closed, resources open @@ -80,7 +82,7 @@ type fd: int # file descriptor of buffer source alive: bool readbufsize: int - contenttype: string + contenttype: string #TODO already stored in source lines: FlexibleGrid rendered: bool source: BufferSource @@ -91,7 +93,7 @@ type document: Document viewport: Viewport prevstyled: StyledNode - url: URL + url: URL #TODO already stored in source selector: Selector[int] istream: Stream sstream: Stream @@ -181,7 +183,10 @@ proc buildInterfaceProc(fun: NimNode, funid: string): tuple[fun, name: NimNode] for i in 2 ..< params2.len: let s = params2[i][0] # sym e.g. url body.add(quote do: - `thisval`.stream.swrite(`s`)) + when typeof(`s`) is FileHandle: + SocketStream(`thisval`.stream).sendFileHandle(`s`) + else: + `thisval`.stream.swrite(`s`)) body.add(quote do: `thisval`.stream.flush()) body.add(quote do: @@ -245,8 +250,8 @@ macro task(fun: typed) = fun func charsets(buffer: Buffer): seq[Charset] = - if buffer.source.charset.isSome: - return @[buffer.source.charset.get] + if buffer.source.charset != CHARSET_UNKNOWN: + return @[buffer.source.charset] return buffer.config.charsets func getTitleAttr(node: StyledNode): string = @@ -634,16 +639,23 @@ type ConnectResult* = object contentType*: string cookies*: seq[Cookie] referrerpolicy*: Option[ReferrerPolicy] + charset*: Charset -proc setupSource(buffer: Buffer): ConnectResult = +proc connect*(buffer: Buffer): ConnectResult {.proxy.} = if buffer.connected: - result.invalid = true - return + return ConnectResult(invalid: true) let source = buffer.source + # Warning: source content type overrides received content types, but source + # charset is just a fallback. let setct = source.contenttype.isNone if not setct: buffer.contenttype = source.contenttype.get buffer.url = source.location + var charset = source.charset + var needsAuth = false + var redirect: Request + var cookies: seq[Cookie] + var referrerpolicy: Option[ReferrerPolicy] case source.t of CLONE: #TODO clone should probably just fork() the buffer instead. @@ -651,8 +663,7 @@ proc setupSource(buffer: Buffer): ConnectResult = buffer.istream = s buffer.fd = int(s.source.getFd()) if buffer.istream == nil: - result.code = ERROR_SOURCE_NOT_FOUND - return + return ConnectResult(code: ERROR_SOURCE_NOT_FOUND) if setct: buffer.contenttype = "text/plain" of LOAD_PIPE: @@ -663,33 +674,86 @@ proc setupSource(buffer: Buffer): ConnectResult = buffer.contenttype = "text/plain" of LOAD_REQUEST: let request = source.request - let response = buffer.loader.doRequest(request, blocking = false) + let response = buffer.loader.doRequest(request, blocking = true, canredir = true) if response.body == nil: - result.code = response.res - return + return ConnectResult(code: response.res) + if response.charset != CHARSET_UNKNOWN: + charset = charset if setct: buffer.contenttype = response.contenttype buffer.istream = response.body let fd = SocketStream(response.body).source.getFd() buffer.fd = int(fd) - result.needsAuth = response.status == 401 # Unauthorized - result.redirect = response.redirect + needsAuth = response.status == 401 # Unauthorized + redirect = response.redirect if "Set-Cookie" in response.headers.table: for s in response.headers.table["Set-Cookie"]: let cookie = newCookie(s, response.url) if cookie.isOk: - result.cookies.add(cookie.get) + cookies.add(cookie.get) if "Referrer-Policy" in response.headers.table: - result.referrerpolicy = getReferrerPolicy(response.headers.table["Referrer-Policy"][0]) - buffer.istream = newTeeStream(buffer.istream, buffer.sstream, closedest = false) - buffer.selector.registerHandle(buffer.fd, {Read}, 0) - if setct: - result.contentType = buffer.contenttype + referrerpolicy = getReferrerPolicy(response.headers.table["Referrer-Policy"][0]) buffer.connected = true + return ConnectResult( + charset: charset, + needsAuth: needsAuth, + redirect: redirect, + cookies: cookies, + contentType: if setct: buffer.contenttype else: "" + ) -proc connect*(buffer: Buffer): ConnectResult {.proxy.} = - let code = buffer.setupSource() - return code +# After connect, pager will call one of the following: +# * connect2, telling loader to load at last (we block loader until then) +# * redirectToFd, telling loader to load into the passed fd +proc connect2*(buffer: Buffer) {.proxy.} = + if buffer.source.t == LOAD_REQUEST: + # Notify loader that we can proceed with loading the input stream. + let ss = SocketStream(buffer.istream) + ss.swrite(false) + ss.setBlocking(false) + buffer.istream = newTeeStream(buffer.istream, buffer.sstream, + closedest = false) + buffer.selector.registerHandle(buffer.fd, {Read}, 0) + +proc redirectToFd*(buffer: Buffer, fd: FileHandle, wait: bool) {.proxy.} = + #TODO also clone & fd + if buffer.source.t == LOAD_REQUEST: + let ss = SocketStream(buffer.istream) + ss.swrite(true) + ss.sendFileHandle(fd) + if wait: + #TODO this is kind of dumb + # Basically, after redirect the network process keeps the socket open, + # and writes a boolean after transfer has been finished. This way, + # we can block this promise so it only returns after e.g. the whole + # file has been saved. + var dummy: bool + ss.sread(dummy) + discard close(fd) + ss.close() + +proc readFromFd*(buffer: Buffer, fd: FileHandle, ishtml: bool) {.proxy.} = + let contentType = if ishtml: + "text/html" + else: + "text/plain" + buffer.source = BufferSource( + t: LOAD_PIPE, + fd: fd, + location: buffer.source.location, + contenttype: some(contentType), + charset: buffer.source.charset + ) + buffer.contenttype = contentType + discard fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) or O_NONBLOCK) + let ps = newPosixStream(fd) + buffer.istream = newTeeStream(ps, buffer.sstream, + closedest = false) + buffer.fd = fd + buffer.selector.registerHandle(buffer.fd, {Read}, 0) + +proc setContentType*(buffer: Buffer, contentType: string) {.proxy.} = + buffer.source.contenttype = some(contentType) const BufferSize = 4096 @@ -1158,10 +1222,10 @@ proc getLines*(buffer: Buffer, w: Slice[int]): GetLinesResult {.proxy.} = result.lines.add(line) result.numLines = buffer.lines.len -proc passFd*(buffer: Buffer) {.proxy.} = - let fd = SocketStream(buffer.pstream).recvFileHandle() +proc passFd*(buffer: Buffer, fd: FileHandle) {.proxy.} = buffer.source.fd = fd +#TODO this is mostly broken proc getSource*(buffer: Buffer) {.proxy.} = let ssock = initServerSocket() let stream = ssock.acceptSocketStream() @@ -1172,7 +1236,8 @@ proc getSource*(buffer: Buffer) {.proxy.} = stream.close() ssock.close() -macro bufferDispatcher(funs: static ProxyMap, buffer: Buffer, cmd: BufferCommand, packetid: int) = +macro bufferDispatcher(funs: static ProxyMap, buffer: Buffer, + cmd: BufferCommand, packetid: int) = let switch = newNimNode(nnkCaseStmt) switch.add(ident("cmd")) for k, v in funs: @@ -1186,8 +1251,11 @@ macro bufferDispatcher(funs: static ProxyMap, buffer: Buffer, cmd: BufferCommand let id = ident(param[i].strVal) let typ = param[^2] stmts.add(quote do: - var `id`: `typ` - `buffer`.pstream.sread(`id`)) + when `typ` is FileHandle: + let `id` = SocketStream(`buffer`.pstream).recvFileHandle() + else: + var `id`: `typ` + `buffer`.pstream.sread(`id`)) call.add(id) var rval: NimNode if v.params[0].kind == nnkEmpty: @@ -1287,8 +1355,7 @@ proc runBuffer(buffer: Buffer, rfd: int) = quit(0) proc launchBuffer*(config: BufferConfig, source: BufferSource, - attrs: WindowAttributes, loader: FileLoader, - mainproc: Pid) = + attrs: WindowAttributes, loader: FileLoader, mainproc: Pid) = let buffer = Buffer( alive: true, userstyle: parseStylesheet(config.userstyle), diff --git a/src/buffer/container.nim b/src/buffer/container.nim index 04900474..e5d2b700 100644 --- a/src/buffer/container.nim +++ b/src/buffer/container.nim @@ -17,14 +17,13 @@ import io/request import io/window import ips/forkserver import ips/serialize -import ips/socketstream import js/javascript import js/regex import types/buffersource import types/color import types/cookie -import types/dispatcher import types/url +import utils/mimeguess import utils/twtstr type @@ -39,7 +38,8 @@ type ContainerEventType* = enum NO_EVENT, FAIL, SUCCESS, NEEDS_AUTH, REDIRECT, ANCHOR, NO_ANCHOR, UPDATE, - READ_LINE, READ_AREA, OPEN, INVALID_COMMAND, STATUS, ALERT, LOADED, TITLE + READ_LINE, READ_AREA, OPEN, INVALID_COMMAND, STATUS, ALERT, LOADED, TITLE, + CHECK_MAILCAP, QUIT ContainerEvent* = object case t*: ContainerEventType @@ -73,8 +73,7 @@ type attrs: WindowAttributes width* {.jsget.}: int height* {.jsget.}: int - contenttype* {.jsget.}: Option[string] - title: string + title*: string # used in status msg hovertext: array[HoverType, string] lastpeek: HoverType source*: BufferSource @@ -89,9 +88,8 @@ type replace*: Container code*: int retry*: seq[URL] - hlon*: bool - sourcepair*: Container - pipeto: Container + hlon*: bool # highlight on? + sourcepair*: Container # pointer to buffer with a source view (may be nil) redraw*: bool needslines*: bool canceled: bool @@ -103,27 +101,38 @@ type jsDestructor(Container) -proc newBuffer*(dispatcher: Dispatcher, config: BufferConfig, - source: BufferSource, title = "", redirectdepth = 0): Container = +proc newBuffer*(forkserver: ForkServer, mainproc: Pid, config: BufferConfig, + source: BufferSource, title = "", redirectdepth = 0): Container = let attrs = getWindowAttributes(stdout) - let ostream = dispatcher.forkserver.ostream - let istream = dispatcher.forkserver.istream + let ostream = forkserver.ostream + let istream = forkserver.istream ostream.swrite(FORK_BUFFER) ostream.swrite(source) ostream.swrite(config) ostream.swrite(attrs) - ostream.swrite(dispatcher.mainproc) + ostream.swrite(mainproc) ostream.flush() - result = Container( - source: source, attrs: attrs, width: attrs.width, - height: attrs.height - 1, contenttype: source.contenttype, - title: title, config: config, redirectdepth: redirectdepth + var process: Pid + istream.sread(process) + return Container( + source: source, + attrs: attrs, + width: attrs.width, + height: attrs.height - 1, + title: title, + config: config, + redirectdepth: redirectdepth, + process: process, + pos: CursorPosition( + setx: -1 + ) ) - istream.sread(result.process) - result.pos.setx = -1 -func location*(container: Container): URL {.jsfunc.} = - container.source.location +func contentType*(container: Container): Option[string] {.jsfget.} = + return container.source.contenttype + +func location*(container: Container): URL {.jsfget.} = + return container.source.location func lineLoaded(container: Container, y: int): bool = return y - container.lineshift in 0..container.lines.high @@ -699,7 +708,7 @@ proc setLoadInfo(container: Container, msg: string) = container.triggerEvent(STATUS) #TODO TODO TODO this should be called with a timeout. -proc onload(container: Container, res: LoadResult) = +proc onload*(container: Container, res: LoadResult) = if container.canceled: container.setLoadInfo("") #TODO we wouldn't need the then part if we had incremental rendering of @@ -737,7 +746,7 @@ proc onload(container: Container, res: LoadResult) = proc load(container: Container) = container.setLoadInfo("Connecting to " & container.location.host & "...") - container.iface.connect().then(proc(res: ConnectResult): auto = + container.iface.connect().then(proc(res: ConnectResult) = let info = container.loadinfo if not res.invalid: container.code = res.code @@ -753,16 +762,41 @@ proc load(container: Container) = container.triggerEvent(NEEDS_AUTH) if res.redirect != nil: container.triggerEvent(ContainerEvent(t: REDIRECT, request: res.redirect)) - if res.contentType != "": - container.contenttype = some(res.contentType) - return container.iface.load() + container.source.charset = res.charset + if res.contentType == "application/octet-stream": + let contentType = guessContentType(container.location.pathname, + "application/octet-stream", container.config.mimeTypes) + if contentType != "application/octet-stream": + container.iface.setContentType(contentType) + container.source.contenttype = some(contentType) + elif res.contentType != "": + container.source.contenttype = some(res.contentType) + container.triggerEvent(CHECK_MAILCAP) else: container.setLoadInfo("") container.triggerEvent(FAIL) else: container.setLoadInfo(info) - ).then(proc(res: tuple[atend: bool, lines, bytes: int]) = - container.onload(res)) + ) + +proc startload*(container: Container) = + container.iface.load() + .then(proc(res: tuple[atend: bool, lines, bytes: int]) = + container.onload(res)) + +proc connect2*(container: Container): EmptyPromise = + return container.iface.connect2() + +proc redirectToFd*(container: Container, fdin: FileHandle, wait: bool): + EmptyPromise = + return container.iface.redirectToFd(fdin, wait) + +proc readFromFd*(container: Container, fdout: FileHandle, ishtml: bool): + EmptyPromise = + return container.iface.readFromFd(fdout, ishtml) + +proc quit*(container: Container) = + container.triggerEvent(QUIT) proc cancel*(container: Container) {.jsfunc.} = if container.select.open: @@ -795,19 +829,10 @@ proc reshape(container: Container): EmptyPromise {.discardable, jsfunc.} = container.setNumLines(lines) return container.requestLines()) -proc dupeBuffer*(dispatcher: Dispatcher, container: Container, config: Config, location: URL, contenttype: string): Container = - let source = BufferSource( - t: CLONE, - location: if location != nil: location else: container.source.location, - contenttype: if contenttype != "": some(contenttype) else: container.contenttype, - clonepid: container.process, - ) - container.pipeto = dispatcher.newBuffer(container.config, source, container.title) +proc pipeBuffer*(container, pipeTo: Container) = container.iface.getSource().then(proc() = - if container.pipeto != nil: - container.pipeto.load() - container.pipeto = nil) - return container.pipeto + pipeTo.load() #TODO do not load if pipeTo is killed first? + ) proc onclick(container: Container, res: ClickResult) @@ -893,10 +918,8 @@ proc handleCommand(container: Container) = proc setStream*(container: Container, stream: Stream) = container.iface = newBufferInterface(stream) if container.source.t == LOAD_PIPE: - container.iface.passFd() - let s = SocketStream(stream) - s.sendFileHandle(container.source.fd) - discard close(container.source.fd) + container.iface.passFd(container.source.fd).then(proc() = + discard close(container.source.fd)) stream.flush() container.load() diff --git a/src/config/config.nim b/src/config/config.nim index f379cb3b..6b9c7032 100644 --- a/src/config/config.nim +++ b/src/config/config.nim @@ -4,6 +4,8 @@ import os import streams import buffer/cell +import config/mailcap +import config/mimetypes import config/toml import data/charset import io/headers @@ -14,6 +16,7 @@ import types/color import types/cookie import types/referer import types/url +import utils/mimeguess import utils/opt import utils/twtstr @@ -79,6 +82,8 @@ type ExternalConfig = object tmpdir*: string editor*: string + mailcap*: seq[string] + mime_types*: seq[string] NetworkConfig = object max_redirect*: int32 @@ -102,6 +107,7 @@ type #TODO: add JS wrappers for objects Config* = ref ConfigObj ConfigObj* = object + configdir: string includes: seq[string] start*: StartConfig search*: SearchConfig @@ -126,6 +132,7 @@ type charsets*: seq[Charset] images*: bool proxy*: URL + mimeTypes*: MimeTypes ForkServerConfig* = object tmpdir*: string @@ -159,7 +166,8 @@ func getProxy*(config: Config): URL = proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar, headers: Headers, referer_from, scripting: bool, charsets: seq[Charset], - images: bool, userstyle: string, proxy: URL): BufferConfig = + images: bool, userstyle: string, proxy: URL, mimeTypes: MimeTypes): + BufferConfig = result = BufferConfig( userstyle: userstyle, filter: newURLFilter(scheme = some(location.scheme), default = true), @@ -169,7 +177,8 @@ proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar, scripting: scripting, charsets: charsets, images: images, - proxy: proxy + proxy: proxy, + mimeTypes: mimeTypes ) new(result.headers) result.headers[] = DefaultHeaders @@ -269,19 +278,58 @@ func constructActionTable*(origTable: Table[string, string]): Table[string, stri result[teststr] = "client.feedNext()" result[realk] = v -proc readUserStylesheet(dir, file: string): string = +proc openFileExpand(dir, file: string): FileStream = if file.len == 0: - return "" + return nil if file[0] == '~' or file[0] == '/': - var f: File - if f.open(expandPath(file)): - result = f.readAll() - f.close() + return newFileStream(expandPath(file)) else: - var f: File - if f.open(dir / file): - result = f.readAll() - f.close() + return newFileStream(expandPath(dir / file)) + +proc readUserStylesheet(dir, file: string): string = + let s = openFileExpand(dir, file) + if s != nil: + result = s.readAll() + s.close() + +# The overall configuration will be obtained through the virtual concatenation +# of several individual configuration files known as mailcap files. +proc getMailcap*(config: Config): tuple[mailcap: Mailcap, errs: seq[string]] = + let configDir = getConfigDir() / "chawan" #TODO store this in config? + var mailcap: Mailcap + var errs: seq[string] + var found = false + for p in config.external.mailcap: + let f = openFileExpand(configDir, p) + if f != nil: + let res = parseMailcap(f) + if res.isSome: + mailcap.add(res.get) + else: + errs.add(res.error) + found = true + if not found: + return (DefaultMailcap, errs) + return (mailcap, errs) + +# We try to source mime types declared in config. +# If none of these files can be found, fall back to DefaultGuess. +#TODO some error handling would be nice, to at least show a warning to +# the user. Not sure how this could be done, though. +proc getMimeTypes*(config: Config): MimeTypes = + if config.external.mime_types.len == 0: + return DefaultGuess + var mimeTypes: MimeTypes + let configDir = getConfigDir() / "chawan" #TODO store this in config? + var found = false + for p in config.external.mime_types: + let f = openFileExpand(configDir, p) + if f != nil: + mimeTypes.parseMimeTypes(f) + found = true + if not found: + return DefaultGuess + return mimeTypes proc parseConfig(config: Config, dir: string, stream: Stream, name = "<input>") proc parseConfig*(config: Config, dir: string, s: string, name = "<input>") @@ -489,6 +537,7 @@ proc parseConfig(config: Config, dir: string, t: TomlValue) = config.parseConfig(dir, staticRead(dir / s)) else: config.parseConfig(dir, newFileStream(dir / s)) + config.configdir = dir #TODO: for omnirule/siteconf, check if substitution rules are specified? proc parseConfig(config: Config, dir: string, stream: Stream, name = "<input>") = diff --git a/src/config/mailcap.nim b/src/config/mailcap.nim new file mode 100644 index 00000000..da9b8f69 --- /dev/null +++ b/src/config/mailcap.nim @@ -0,0 +1,342 @@ +# See https://www.rfc-editor.org/rfc/rfc1524 + +import osproc +import streams +import strutils + +import data/charset +import types/url +import utils/opt +import utils/twtstr + +type + MailcapParser = object + stream: Stream + hasbuf: bool + buf: char + + MailcapFlags* = enum + NEEDSTERMINAL = "needsterminal" + COPIOUSOUTPUT = "copiousoutput" + HTMLOUTPUT = "x-htmloutput" # from w3m + + MailcapEntry* = object + mt*: string + subt*: string + cmd*: string + flags*: set[MailcapFlags] + nametemplate*: string + edit*: string + test*: string + + Mailcap* = seq[MailcapEntry] + +const DefaultMailcap* = @[ + MailcapEntry( + mt: "*", + subt: "*", + cmd: "xdg-open '%s'" + ) +] + +proc has(state: MailcapParser): bool {.inline.} = + return not state.stream.atEnd + +proc consume(state: var MailcapParser): char = + if state.hasbuf: + state.hasbuf = false + return state.buf + return state.stream.readChar() + +proc reconsume(state: var MailcapParser, c: char) = + state.buf = c + state.hasbuf = true + +proc skipBlanks(state: var MailcapParser, c: var char): bool = + while state.has(): + c = state.consume() + if c notin AsciiWhitespace: + return true + +proc skipBlanks(state: var MailcapParser) = + var c: char + if state.skipBlanks(c): + state.reconsume(c) + +proc skipLine(state: var MailcapParser) = + while state.has(): + let c = state.consume() + if c == '\n': + break + +proc consumeTypeField(state: var MailcapParser): Result[string, string] = + var s = "" + # type + while state.has(): + let c = state.consume() + if c == '/': + s &= c + break + if c notin AsciiAlphaNumeric + {'-', '*'}: + return err("Invalid character encountered in type field") + s &= c.tolower() + if not state.has(): + return err("Missing subtype") + # subtype + while state.has(): + let c = state.consume() + if c == ';': + state.reconsume(c) + break + if c notin AsciiAlphaNumeric + {'-', '.', '*', '_', '+'}: + return err("Invalid character encountered in subtype field") + s &= c.tolower() + var c: char + if not state.skipBlanks(c) or c != ';': + return err("Semicolon not found") + return ok(s) + +proc consumeCommand(state: var MailcapParser): Result[string, string] = + state.skipBlanks() + var quoted = false + var s = "" + while state.has(): + let c = state.consume() + if not quoted: + if c == '\r': + continue + if c == ';' or c == '\n': + state.reconsume(c) + return ok(s) + if c == '\\': + quoted = true + continue + if c notin Ascii - Controls: + return err("Invalid character encountered in command") + else: + quoted = false + s &= c + return ok(s) + +type NamedField = enum + NO_NAMED_FIELD, NAMED_FIELD_TEST, NAMED_FIELD_NAMETEMPLATE, NAMED_FIELD_EDIT + +proc parseFieldKey(entry: var MailcapEntry, k: string): NamedField = + case k + of "needsterminal": + entry.flags.incl(NEEDSTERMINAL) + of "copiousoutput": + entry.flags.incl(COPIOUSOUTPUT) + of "x-htmloutput": + entry.flags.incl(HTMLOUTPUT) + of "test": + return NAMED_FIELD_TEST + of "nametemplate": + return NAMED_FIELD_NAMETEMPLATE + of "edit": + return NAMED_FIELD_EDIT + return NO_NAMED_FIELD + +proc consumeField(state: var MailcapParser, entry: var MailcapEntry): + Result[bool, string] = + state.skipBlanks() + if not state.has(): + return ok(false) + var buf = "" + while state.has(): + let c = state.consume() + case c + of ';', '\n': + if parseFieldKey(entry, buf) != NO_NAMED_FIELD: + return err("Expected command") + return ok(c == ';') + of '\r': + continue + of '=': + let f = parseFieldKey(entry, buf) + let cmd = ?state.consumeCommand() + case f + of NO_NAMED_FIELD: + discard + of NAMED_FIELD_TEST: + entry.test = cmd + of NAMED_FIELD_NAMETEMPLATE: + entry.nametemplate = cmd + of NAMED_FIELD_EDIT: + entry.edit = cmd + return ok(state.consume() == ';') + else: + if c in Controls: + return err("Invalid character encountered in field") + buf &= c + +proc parseMailcap*(stream: Stream): Result[Mailcap, string] = + var state = MailcapParser(stream: stream) + var mailcap: Mailcap + while not stream.atEnd(): + let c = state.consume() + if c == '#': + state.skipLine() + continue + state.reconsume(c) + state.skipBlanks() + let c2 = state.consume() + if c2 == '\n' or c2 == '\r': + continue + state.reconsume(c2) + let t = ?state.consumeTypeField() + let mt = t.until('/') #TODO this could be more efficient + let subt = t[mt.len + 1 .. ^1] + var entry = MailcapEntry( + mt: mt, + subt: subt, + cmd: ?state.consumeCommand() + ) + if state.consume() == ';': + while ?state.consumeField(entry): + discard + mailcap.add(entry) + return ok(mailcap) + +# Mostly based on w3m's mailcap quote/unquote +type UnquoteState = enum + STATE_NORMAL, STATE_QUOTED, STATE_PERC, STATE_ATTR, STATE_ATTR_QUOTED + +type UnquoteResult* = object + canpipe*: bool + cmd*: string + +type QuoteState = enum + QS_DQUOTED, QS_SQUOTED + +proc quoteFile(file: string, qs: set[QuoteState]): string = + var s = "" + for c in file: + case c + of '$', '`', '"', '\\': + if QS_SQUOTED notin qs: + s &= '\\' + of '\'': + s &= "'\\'" # then re-open the quote by appending c + of '_', '.', ':', '/': + discard # no need to quote + else: + if c notin AsciiAlpha and qs == {}: + s &= '\\' + s &= c + return s + +proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL, + charset: Charset, canpipe: var bool): string = + var cmd = "" + var attrname = "" + var state: UnquoteState + var filename = "" + var qs: set[QuoteState] + for c in ecmd: + case state + of STATE_QUOTED: + cmd &= c.tolower() + state = STATE_NORMAL + of STATE_ATTR_QUOTED: + attrname &= c.tolower() + state = STATE_ATTR + of STATE_NORMAL: + case c + of '%': + state = STATE_PERC + of '\\': + state = STATE_QUOTED + of '\'': + if QS_SQUOTED in qs: + qs.excl(QS_SQUOTED) + else: + qs.incl(QS_SQUOTED) + cmd &= c + of '"': + if QS_DQUOTED in qs: + qs.excl(QS_DQUOTED) + else: + qs.incl(QS_DQUOTED) + cmd &= c + else: + cmd &= c.tolower() + of STATE_PERC: + if c == '%': + cmd &= c.tolower() + elif c == 's': + filename = quoteFile(outpath, qs) + cmd &= filename + canpipe = false + elif c == 't': + cmd &= contentType.until(';') + elif c == 'u': # extension + cmd &= $url + elif c == '{': + state = STATE_ATTR + continue + state = STATE_NORMAL + of STATE_ATTR: + if c == '}': + if attrname == "charset": + cmd &= $charset + continue + #TODO this is broken, because content-type is stripped of ; fields + let kvs = contentType.after(';').toLowerAscii() + var i = kvs.find(attrname) + if i != -1 and kvs.len > i + attrname.len and + kvs[i + attrname.len] == '=': + i += attrname.len + 1 + while i < kvs.len and kvs[i] in AsciiWhitespace: + inc i + var q = false + for j in i ..< kvs.len: + if q: + cmd &= kvs[j] + else: + if kvs[j] == '\\': + q = true + elif kvs[j] == ';' or kvs[j] in AsciiWhitespace: + break + else: + cmd &= kvs[j] + attrname = "" + return cmd + +proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL, + charset: Charset): string = + var canpipe: bool + return unquoteCommand(ecmd, contentType, outpath, url, charset, canpipe) + +proc getMailcapEntry*(mailcap: Mailcap, mimeType, outpath: string, + url: URL, charset: Charset): ptr MailcapEntry = + let mt = mimeType.until('/') + if mt.len + 1 >= mimeType.len: + return nil + let st = mimeType[mt.len + 1 .. ^1] + for entry in mailcap: + block try_entry: + block check_mt: + if entry.mt.len == 1 and entry.mt[0] == '*': + break check_mt + if entry.mt.len != mt.len: + break try_entry + for i in 0 ..< mt.len: + if entry.mt[i] != mt[i]: + break try_entry + block check_subt: + if entry.subt.len == 1 and entry.subt[0] == '*': + break check_subt + if entry.subt.len != st.len: + break try_entry + for i in 0 ..< st.len: + if entry.subt[i] != st[i]: + break try_entry + if entry.test != "": + var canpipe = true + let cmd = unquoteCommand(entry.test, mimeType, outpath, url, charset, + canpipe) + #TODO TODO TODO if not canpipe ... + if execCmd(cmd) != 0: + break try_entry + return unsafeAddr entry diff --git a/src/config/mimetypes.nim b/src/config/mimetypes.nim new file mode 100644 index 00000000..2b0900d5 --- /dev/null +++ b/src/config/mimetypes.nim @@ -0,0 +1,38 @@ +import streams +import tables + +import utils/twtstr + +# extension -> type +type MimeTypes* = Table[string, string] + +# Add mime types found in stream to mimeTypes. +# No error handling for now. +proc parseMimeTypes*(mimeTypes: var MimeTypes, stream: Stream) = + while not stream.atEnd(): + let line = stream.readLine() + if line.len == 0: + continue + if line[0] == '#': + continue + var t = "" + var i = 0 + while i < line.len and line[i] notin AsciiWhitespace: + t &= line[i].tolower() + inc i + if t == "": continue + while i < line.len: + while i < line.len and line[i] in AsciiWhitespace: + inc i + var ext = "" + while i < line.len and line[i] notin AsciiWhitespace: + ext &= line[i].tolower() + inc i + if ext == "": continue + mimeTypes[ext] = t + stream.close() + +proc parseMimeTypes*(stream: Stream): MimeTypes = + var mimeTypes: MimeTypes + mimeTypes.parseMimeTypes(stream) + return mimeTypes diff --git a/src/display/client.nim b/src/display/client.nim index 5b961841..3d40fe63 100644 --- a/src/display/client.nim +++ b/src/display/client.nim @@ -42,7 +42,6 @@ import js/module import js/timeout import types/blob import types/cookie -import types/dispatcher import types/url import utils/opt import xhr/formdata @@ -55,15 +54,16 @@ type attrs: WindowAttributes config {.jsget.}: Config console {.jsget.}: Console - dispatcher: Dispatcher errormessage: string fd: int fdmap: Table[int, Container] feednext: bool + forkserver: ForkServer jsctx: JSContext jsrt: JSRuntime line {.jsget.}: LineEdit loader: FileLoader + mainproc: Pid pager {.jsget.}: Pager s: string selector: Selector[Container] @@ -156,7 +156,7 @@ proc command(client: Client, src: string) = proc suspend(client: Client) {.jsfunc.} = client.pager.term.quit() - discard kill(getpid(), cint(SIGSTOP)) + discard kill(client.mainproc, cint(SIGSTOP)) client.pager.term.restart() proc quit(client: Client, code = 0) {.jsfunc.} = @@ -305,13 +305,13 @@ proc handleRead(client: Client, fd: int) = if client.console.tty != nil and fd == client.console.tty.getFileHandle(): client.input() client.handlePagerEvents() - elif fd == client.dispatcher.forkserver.estream.fd: + elif fd == client.forkserver.estream.fd: var nl = false const prefix = "STDERR: " var s = prefix while true: try: - let c = client.dispatcher.forkserver.estream.readChar() + let c = client.forkserver.estream.readChar() if nl and s.len > prefix.len: client.console.err.write(s) s = prefix @@ -337,14 +337,14 @@ proc handleRead(client: Client, fd: int) = client.pager.handleEvent(container) proc flushConsole*(client: Client) {.jsfunc.} = - client.handleRead(client.dispatcher.forkserver.estream.fd) + client.handleRead(client.forkserver.estream.fd) proc handleError(client: Client, fd: int) = if client.console.tty != nil and fd == client.console.tty.getFileHandle(): #TODO do something here... stderr.write("Error in tty\n") quit(1) - elif fd == client.dispatcher.forkserver.estream.fd: + elif fd == client.forkserver.estream.fd: #TODO do something here... stderr.write("Fork server crashed :(\n") quit(1) @@ -467,7 +467,7 @@ proc newConsole(pager: Pager, tty: File): Console = if pipe(pipefd) == -1: raise newException(Defect, "Failed to open console pipe.") let url = newURL("javascript:console.show()") - result.container = pager.readPipe0(some("text/plain"), none(Charset), + result.container = pager.readPipe0(some("text/plain"), CHARSET_UNKNOWN, pipefd[0], option(url.get(nil)), "Browser console") var f: File if not open(f, pipefd[1], fmWrite): @@ -491,12 +491,12 @@ proc dumpBuffers(client: Client) = except IOError: client.console.log("Error in buffer", $container.location) # check for errors - client.handleRead(client.dispatcher.forkserver.estream.fd) + client.handleRead(client.forkserver.estream.fd) quit(1) stdout.close() -proc launchClient*(client: Client, pages: seq[string], ctype: Option[string], - cs: Option[Charset], dump: bool) = +proc launchClient*(client: Client, pages: seq[string], + contentType: Option[string], cs: Charset, dump: bool) = var tty: File var dump = dump if not dump: @@ -511,7 +511,7 @@ proc launchClient*(client: Client, pages: seq[string], ctype: Option[string], client.fd = int(client.ssock.sock.getFd()) let selector = newSelector[Container]() selector.registerHandle(client.fd, {Read}, nil) - let efd = int(client.dispatcher.forkserver.estream.fd) + let efd = int(client.forkserver.estream.fd) selector.registerHandle(efd, {Read}, nil) client.loader.registerFun = proc(fd: int) = selector.registerHandle(fd, {Read}, nil) @@ -537,11 +537,11 @@ proc launchClient*(client: Client, pages: seq[string], ctype: Option[string], client.userstyle = client.config.css.stylesheet.parseStylesheet() if not stdin.isatty(): - client.pager.readPipe(ctype, cs, stdin.getFileHandle()) + client.pager.readPipe(contentType, cs, stdin.getFileHandle()) for page in pages: - client.pager.loadURL(page, ctype = ctype, cs = cs) - client.pager.refreshStatusMsg() + client.pager.loadURL(page, ctype = contentType, cs = cs) + client.pager.showAlerts() if not dump: client.inputLoop() else: @@ -565,32 +565,7 @@ proc jsCollect(client: Client) {.jsfunc.} = proc sleep(client: Client, millis: int) {.jsfunc.} = sleep millis -proc newClient*(config: Config, dispatcher: Dispatcher): Client = - new(result) - setControlCHook(proc() {.noconv.} = quit(1)) - result.config = config - result.dispatcher = dispatcher - result.attrs = getWindowAttributes(stdout) - let forkserver = dispatcher.forkserver - result.loader = forkserver.newFileLoader( - proxy = config.getProxy(), - acceptProxy = true - ) - result.jsrt = newJSRuntime() - result.jsrt.setInterruptHandler(interruptHandler, cast[pointer](result)) - JS_SetModuleLoaderFunc(result.jsrt, normalizeModuleName, clientLoadJSModule, - nil) - let ctx = result.jsrt.newJSContext() - result.jsctx = ctx - result.pager = newPager(config, result.attrs, dispatcher, ctx) - var global = JS_GetGlobalObject(ctx) - ctx.registerType(Client, asglobal = true) - setGlobal(ctx, global, result) - ctx.setProperty(global, "client", global) - JS_FreeValue(ctx, global) - - ctx.registerType(Console) - +proc addJSModules(client: Client, ctx: JSContext) = ctx.addDOMExceptionModule() ctx.addCookieModule() ctx.addURLModule() @@ -608,3 +583,32 @@ proc newClient*(config: Config, dispatcher: Dispatcher): Client = ctx.addConfigModule() ctx.addPagerModule() ctx.addContainerModule() + +proc newClient*(config: Config, forkserver: ForkServer, mainproc: Pid): Client = + setControlCHook(proc() {.noconv.} = quit(1)) + let jsrt = newJSRuntime() + JS_SetModuleLoaderFunc(jsrt, normalizeModuleName, clientLoadJSModule, nil) + let jsctx = jsrt.newJSContext() + let attrs = getWindowAttributes(stdout) + let client = Client( + config: config, + forkserver: forkserver, + mainproc: mainproc, + attrs: attrs, + loader: forkserver.newFileLoader( + proxy = config.getProxy(), + acceptProxy = true + ), + jsrt: jsrt, + jsctx: jsctx, + pager: newPager(config, attrs, forkserver, mainproc, jsctx) + ) + jsrt.setInterruptHandler(interruptHandler, cast[pointer](client)) + var global = JS_GetGlobalObject(jsctx) + jsctx.registerType(Client, asglobal = true) + setGlobal(jsctx, global, client) + jsctx.setProperty(global, "client", global) + JS_FreeValue(jsctx, global) + jsctx.registerType(Console) + client.addJSModules(jsctx) + return client diff --git a/src/display/pager.nim b/src/display/pager.nim index 95057978..0b8197fc 100644 --- a/src/display/pager.nim +++ b/src/display/pager.nim @@ -2,6 +2,7 @@ import deques import net import options import os +import osproc import streams import tables import unicode @@ -13,13 +14,17 @@ import buffer/cell import buffer/container import buffer/select import config/config +import config/mailcap +import config/mimetypes import data/charset import display/term +import io/connecterror import io/headers import io/lineedit import io/loader import io/promise import io/request +import io/tempfile import io/window import ips/editor import ips/forkserver @@ -29,7 +34,6 @@ import js/regex import types/buffersource import types/color import types/cookie -import types/dispatcher import types/url import utils/opt import utils/twtstr @@ -49,13 +53,16 @@ type config: Config container*: Container cookiejars: Table[string, CookieJar] - dispatcher*: Dispatcher display: FixedGrid + forkserver: ForkServer iregex: Result[Regex, string] isearchpromise: EmptyPromise lineedit*: Option[LineEdit] linehist: array[LineMode, LineHistory] linemode*: LineMode + mailcap: Mailcap + mainproc: Pid + mimeTypes: MimeTypes numload*: int omnirules: seq[OmniRule] procmap*: Table[Pid, Container] @@ -179,18 +186,26 @@ proc gotoLine[T: string|int](pager: Pager, s: T = "") {.jsfunc.} = return pager.container.gotoLine(s) +proc alert*(pager: Pager, msg: string) + proc newPager*(config: Config, attrs: WindowAttributes, - dispatcher: Dispatcher, ctx: JSContext): Pager = + forkserver: ForkServer, mainproc: Pid, ctx: JSContext): Pager = let pager = Pager( - dispatcher: dispatcher, config: config, display: newFixedGrid(attrs.width, attrs.height - 1), + forkserver: forkserver, + mainproc: mainproc, + omnirules: config.getOmniRules(ctx), + proxy: config.getProxy(), + siteconf: config.getSiteConfig(ctx), statusgrid: newFixedGrid(attrs.width), term: newTerminal(stdout, config, attrs), - siteconf: config.getSiteConfig(ctx), - omnirules: config.getOmniRules(ctx), - proxy: config.getProxy() + mimeTypes: config.getMimeTypes() ) + let (mcap, errs) = config.getMailcap() + pager.mailcap = mcap + for err in errs: + pager.alert("Error reading mailcap: " & err) return pager proc launchPager*(pager: Pager, tty: File) = @@ -248,6 +263,7 @@ proc writeStatusMessage(pager: Pager, str: string, pager.statusgrid[i].format = def inc i +# Note: should only be called directly after user interaction. proc refreshStatusMsg*(pager: Pager) = let container = pager.container if container == nil: return @@ -381,11 +397,39 @@ proc addContainer*(pager: Pager, container: Container) = pager.registerContainer(container) pager.setContainer(container) -proc dupeContainer(pager: Pager, container: Container, location: URL): Container = - return pager.dispatcher.dupeBuffer(container, pager.config, location, "") +proc newBuffer(pager: Pager, bufferConfig: BufferConfig, source: BufferSource, + title = "", redirectdepth = 0): Container = + return newBuffer( + pager.forkserver, + pager.mainproc, + bufferConfig, + source, + title, + redirectdepth + ) + +proc dupeBuffer(pager: Pager, container: Container, location: URL, + contentType = ""): Container = + let contentType = if contentType != "": + some(contentType) + else: + container.contenttype + let location = if location != nil: + location + else: + container.source.location + let source = BufferSource( + t: CLONE, + location: location, + contenttype: contentType, + clonepid: container.process, + ) + let pipeTo = pager.newBuffer(container.config, source, container.title) + container.pipeBuffer(pipeTo) + return pipeTo -proc dupeBuffer*(pager: Pager, location: URL = nil) {.jsfunc.} = - pager.addContainer(pager.dupeContainer(pager.container, location)) +proc dupeBuffer(pager: Pager, location: URL = nil) {.jsfunc.} = + pager.addContainer(pager.dupeBuffer(pager.container, location)) # The prevBuffer and nextBuffer procedures emulate w3m's PREV and NEXT # commands by traversing the container tree in a depth-first order. @@ -491,7 +535,7 @@ proc deleteContainer(pager: Pager, container: Container) = container.parent = nil container.children.setLen(0) pager.unreg.add((container.process, SocketStream(container.iface.stream))) - pager.dispatcher.forkserver.removeChild(container.process) + pager.forkserver.removeChild(container.process) proc discardBuffer(pager: Pager, container = none(Container)) {.jsfunc.} = let c = container.get(pager.container) @@ -512,11 +556,11 @@ proc toggleSource(pager: Pager) {.jsfunc.} = if pager.container.sourcepair != nil: pager.setContainer(pager.container.sourcepair) else: - let contenttype = if pager.container.contenttype.get("") == "text/html": + let contenttype = if pager.container.contentType.get("") == "text/html": "text/plain" else: "text/html" - let container = pager.dispatcher.dupeBuffer(pager.container, pager.config, nil, contenttype) + let container = pager.dupeBuffer(pager.container, nil, contenttype) container.sourcepair = pager.container pager.container.sourcepair = container pager.addContainer(container) @@ -529,7 +573,7 @@ proc windowChange*(pager: Pager, attrs: WindowAttributes) = container.windowChange(attrs) if pager.askprompt != "": pager.writeAskPrompt() - pager.refreshStatusMsg() + pager.showAlerts() # Apply siteconf settings to a request. # Note that this may modify the URL passed. @@ -543,6 +587,7 @@ proc applySiteconf(pager: Pager, url: var URL): BufferConfig = var charsets = pager.config.encoding.document_charset var userstyle = pager.config.css.stylesheet var proxy = pager.proxy + let mimeTypes = pager.mimeTypes for sc in pager.siteconf: if sc.url.isSome and not sc.url.get.match($url): continue @@ -575,12 +620,12 @@ proc applySiteconf(pager: Pager, url: var URL): BufferConfig = userstyle &= sc.stylesheet.get if sc.proxy.isSome: proxy = sc.proxy.get - return pager.config.getBufferConfig(url, cookiejar, headers, - referer_from, scripting, charsets, images, userstyle, proxy) + return pager.config.getBufferConfig(url, cookiejar, headers, referer_from, + scripting, charsets, images, userstyle, proxy, mimeTypes) # Load request in a new buffer. proc gotoURL(pager: Pager, request: Request, prevurl = none(URL), - ctype = none(string), cs = none(Charset), replace: Container = nil, + ctype = none(string), cs = CHARSET_UNKNOWN, replace: Container = nil, redirectdepth = 0, referrer: Container = nil) = if referrer != nil and referrer.config.referer_from: request.referer = referrer.source.location @@ -602,7 +647,11 @@ proc gotoURL(pager: Pager, request: Request, prevurl = none(URL), ) if referrer != nil: bufferconfig.referrerpolicy = referrer.config.referrerpolicy - let container = pager.dispatcher.newBuffer(bufferconfig, source, redirectdepth = redirectdepth) + let container = pager.newBuffer( + bufferconfig, + source, + redirectdepth = redirectdepth + ) if replace != nil: container.replace = replace container.copyCursorPos(container.replace) @@ -627,7 +676,7 @@ proc omniRewrite(pager: Pager, s: string): string = # * https://<url> # So we attempt to load both, and see what works. proc loadURL*(pager: Pager, url: string, ctype = none(string), - cs = none(Charset)) = + cs = CHARSET_UNKNOWN) = let url0 = pager.omniRewrite(url) let url = if url[0] == '~': expandPath(url0) else: url0 let firstparse = parseURL(url) @@ -656,7 +705,7 @@ proc loadURL*(pager: Pager, url: string, ctype = none(string), if pager.container != prevc: pager.container.retry = urls -proc readPipe0*(pager: Pager, ctype: Option[string], cs: Option[Charset], +proc readPipe0*(pager: Pager, ctype: Option[string], cs: Charset, fd: FileHandle, location: Option[URL], title: string): Container = var location = location.get(newURL("file://-").get) let bufferconfig = pager.applySiteconf(location) @@ -667,9 +716,9 @@ proc readPipe0*(pager: Pager, ctype: Option[string], cs: Option[Charset], charset: cs, location: location ) - return pager.dispatcher.newBuffer(bufferconfig, source, title = title) + return pager.newBuffer(bufferconfig, source, title = title) -proc readPipe*(pager: Pager, ctype: Option[string], cs: Option[Charset], +proc readPipe*(pager: Pager, ctype: Option[string], cs: Charset, fd: FileHandle) = let container = pager.readPipe0(ctype, cs, fd, none(URL), "*pipe*") pager.addContainer(container) @@ -784,6 +833,206 @@ proc reload(pager: Pager) {.jsfunc.} = proc authorize(pager: Pager) = pager.setLineEdit("Username: ", USERNAME) +# Pipe input into the mailcap command, then read its output into a buffer. +# needsterminal is ignored. +proc runMailcapReadPipe(pager: Pager, container: Container, + entry: MailcapEntry, cmd: string): (EmptyPromise, bool) = + var pipefd_in: array[2, cint] + if pipe(pipefd_in) == -1: + raise newException(Defect, "Failed to open pipe.") + var pipefd_out: array[2, cint] + if pipe(pipefd_out) == -1: + raise newException(Defect, "Failed to open pipe.") + let pid = fork() + if pid == -1: + return (nil, false) + elif pid == 0: + # child process + discard close(pipefd_in[1]) + discard close(pipefd_out[0]) + stdout.flushFile() + discard dup2(pipefd_in[0], stdin.getFileHandle()) + discard dup2(pipefd_out[1], stdout.getFileHandle()) + let devnull = open("/dev/null", O_WRONLY) + discard dup2(devnull, stderr.getFileHandle()) + discard close(devnull) + discard close(pipefd_in[0]) + discard close(pipefd_out[1]) + discard execCmd(cmd) + discard close(stdin.getFileHandle()) + discard close(stdout.getFileHandle()) + quit(0) + # parent + discard close(pipefd_in[0]) + discard close(pipefd_out[1]) + let fdin = pipefd_in[1] + let fdout = pipefd_out[0] + let p = container.redirectToFd(fdin, wait = false) + let p2 = p.then(proc(): auto = + discard close(fdin) + let ishtml = HTMLOUTPUT in entry.flags + if ishtml: + #TODO this is a hack for dupe buffer and should be reconsidered. + container.source.contenttype = some("text/html") + return container.readFromFd(fdout, ishtml) + ).then(proc() = + discard close(fdout) + ) + return (p2, true) + +# Pipe input into the mailcap command, and discard its output. +# If needsterminal, leave stderr and stdout open and wait for the process. +proc runMailcapWritePipe(pager: Pager, container: Container, + entry: MailcapEntry, cmd: string): (EmptyPromise, bool) = + let needsterminal = NEEDSTERMINAL in entry.flags + var pipefd: array[2, cint] + if pipe(pipefd) == -1: + raise newException(Defect, "Failed to open pipe.") + if needsterminal: + pager.term.quit() + let pid = fork() + if pid == -1: + return (nil, false) + elif pid == 0: + # child process + discard close(pipefd[1]) + discard dup2(pipefd[0], stdin.getFileHandle()) + if not needsterminal: + let devnull = open("/dev/null", O_WRONLY) + discard dup2(devnull, stdout.getFileHandle()) + discard dup2(devnull, stderr.getFileHandle()) + discard close(devnull) + discard close(pipefd[0]) + discard execCmd(cmd) + discard close(stdin.getFileHandle()) + quit(0) + else: + # parent + discard close(pipefd[0]) + let fd = pipefd[1] + let p = container.redirectToFd(fd, wait = false) + discard close(fd) + if needsterminal: + var x: cint + discard waitpid(pid, x, 0) + pager.term.restart() + return (p, false) + +# Save input in a file, run the command, and redirect its output to a +# new buffer. +# needsterminal is ignored. +proc runMailcapReadFile(pager: Pager, container: Container, + entry: MailcapEntry, cmd, outpath: string): (EmptyPromise, bool) = + let fd = open(outpath, O_WRONLY or O_CREAT, 0o666) + if fd == -1: + return (nil, false) + let p = container.redirectToFd(fd, wait = true).then(proc(): auto = + var pipefd: array[2, cint] # redirect stdout here + if pipe(pipefd) == -1: + raise newException(Defect, "Failed to open pipe.") + let pid = fork() + if pid == 0: + # child process + discard close(pipefd[0]) + discard dup2(pipefd[1], stdout.getFileHandle()) + discard close(pipefd[1]) + let devnull = open("/dev/null", O_WRONLY) + discard dup2(devnull, stderr.getFileHandle()) + discard close(devnull) + discard execCmd(cmd) + discard tryRemoveFile(outpath) + quit(0) + # parent + discard close(pipefd[1]) + let fdout = pipefd[0] + let ishtml = HTMLOUTPUT in entry.flags + if ishtml: + #TODO this is a hack for dupe buffer and should be reconsidered. + container.source.contenttype = some("text/html") + return container.readFromFd(fdout, ishtml).then(proc() = + discard close(fdout) + ) + ) + return (p, true) + +# Save input in a file, run the command, and discard its output. +# If needsterminal, leave stderr and stdout open and wait for the process. +proc runMailcapWriteFile(pager: Pager, container: Container, + entry: MailcapEntry, cmd, outpath: string): (EmptyPromise, bool) = + let needsterminal = NEEDSTERMINAL in entry.flags + let fd = open(outpath, O_WRONLY or O_CREAT, 0o666) + if fd == -1: + return (nil, false) + let p = container.redirectToFd(fd, wait = true).then(proc() = + if needsterminal: + pager.term.quit() + discard execCmd(cmd) + discard tryRemoveFile(outpath) + pager.term.restart() + else: + # don't block + let pid = fork() + if pid == 0: + # child process + let devnull = open("/dev/null", O_WRONLY) + discard dup2(devnull, stdin.getFileHandle()) + discard dup2(devnull, stdout.getFileHandle()) + discard dup2(devnull, stderr.getFileHandle()) + discard close(devnull) + discard execCmd(cmd) + discard tryRemoveFile(outpath) + quit(0) + ) + return (p, false) + +# Search for a mailcap entry, and if found, execute the specified command +# and pipeline the input and output appropriately. +# There is four possible outcomes: +# * pipe stdin, discard stdout +# * pipe stdin, read stdout +# * write to file, run, discard stdout +# * write to file, run, read stdout +# If needsterminal is specified, and stdout is not being read, then the +# pager is suspended until the command exits. +#TODO add support for edit/compose, better error handling (use Promise[bool] +# instead of tuple[EmptyPromise, bool]) +proc checkMailcap(pager: Pager, container: Container): (EmptyPromise, bool) = + if container.contenttype.isNone: + return (nil, true) + if container.source.t == CLONE: + return (nil, true) # clone cannot use mailcap + let contentType = container.contenttype.get + if contentType == "text/html": + # We support HTML natively, so it would make little sense to execute + # mailcap filters for it. + return (nil, true) + #TODO callback for outpath or something + let url = container.location + let cs = container.source.charset + let entry = pager.mailcap.getMailcapEntry(contentType, "", url, cs) + if entry != nil: + let tmpdir = pager.config.external.tmpdir + let ext = container.location.pathname.afterLast('.') + let tempfile = getTempfile(tmpdir, ext) + let outpath = if entry.nametemplate != "": + unquoteCommand(entry.nametemplate, contentType, tempfile, url, cs) + else: + tempfile + var canpipe = true + let cmd = unquoteCommand(entry.cmd, contentType, outpath, url, cs, canpipe) + if {COPIOUSOUTPUT, HTMLOUTPUT} * entry.flags == {}: + # no output. + if canpipe: + return pager.runMailcapWritePipe(container, entry[], cmd) + else: + return pager.runMailcapWriteFile(container, entry[], cmd, outpath) + else: + if canpipe: + return pager.runMailcapReadPipe(container, entry[], cmd) + else: + return pager.runMailcapReadFile(container, entry[], cmd, outpath) + return (nil, true) + proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bool = case event.t of FAIL: @@ -838,7 +1087,7 @@ proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bo of ANCHOR: var url2 = newURL(container.source.location) url2.setHash(event.anchor) - pager.addContainer(pager.dupeContainer(container, url2)) + pager.addContainer(pager.dupeBuffer(container, url2)) of NO_ANCHOR: pager.alert("Couldn't find anchor " & event.anchor) of UPDATE: @@ -866,14 +1115,28 @@ proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bo of INVALID_COMMAND: discard of STATUS: if pager.container == container: - pager.refreshStatusMsg() + pager.showAlerts() of TITLE: if pager.container == container: - pager.refreshStatusMsg() + pager.showAlerts() pager.term.setTitle(container.getTitle()) of ALERT: if pager.container == container: pager.alert(event.msg) + of CHECK_MAILCAP: + var (cm, connect) = pager.checkMailcap(container) + if cm == nil: + cm = container.connect2() + if connect: + cm.then(proc() = + container.startload()) + else: + cm.then(proc(): auto = + container.quit()) + of QUIT: + dec pager.numload + pager.deleteContainer(container) + return false of NO_EVENT: discard return true diff --git a/src/html/dom.nim b/src/html/dom.nim index 11254c56..c0dc066c 100644 --- a/src/html/dom.nim +++ b/src/html/dom.nim @@ -27,10 +27,10 @@ import js/timeout import types/blob import types/color import types/matrix -import types/mime import types/referer import types/url import types/vector +import utils/mimeguess import utils/twtstr type @@ -794,6 +794,7 @@ const ReflectTable0 = [ # Forward declarations func attrb*(element: Element, s: string): bool proc attr*(element: Element, name, value: string) +func baseURL*(document: Document): URL proc tostr(ftype: enum): string = return ($ftype).split('_')[1..^1].join("-").tolower() @@ -1802,8 +1803,8 @@ func target0*(element: Element): string = # HTMLHyperlinkElementUtils (for <a> and <area>) func href0[T: HTMLAnchorElement|HTMLAreaElement](element: T): string = if element.attrb("href"): - let url = parseUrl(element.attr("href"), some(element.document.url)) - if url.issome: + let url = parseURL(element.attr("href"), some(element.document.baseURL)) + if url.isSome: return $url.get # <base> @@ -2030,7 +2031,7 @@ func isHostIncludingInclusiveAncestor*(a, b: Node): bool = return true return false -func baseURL*(document: Document): Url = +func baseURL*(document: Document): URL = #TODO frozen base url... var href = "" for base in document.elements(TAG_BASE): diff --git a/src/io/about.nim b/src/io/about.nim index 97e01133..737a291b 100644 --- a/src/io/about.nim +++ b/src/io/about.nim @@ -1,9 +1,9 @@ -import streams import tables +import io/connecterror import io/headers +import io/loaderhandle import io/request -import ips/serialize import types/url const chawan = staticRead"res/chawan.html" @@ -11,19 +11,18 @@ const HeaderTable = { "Content-Type": "text/html" }.toTable() -proc loadAbout*(request: Request, ostream: Stream) = +proc loadAbout*(handle: LoaderHandle, request: Request) = + template t(body: untyped) = + if not body: + return if request.url.pathname == "blank": - ostream.swrite(0) - ostream.swrite(200) # ok - let headers = newHeaders(HeaderTable) - ostream.swrite(headers) + t handle.sendResult(0) + t handle.sendStatus(200) # ok + t handle.sendHeaders(newHeaders(HeaderTable)) elif request.url.pathname == "chawan": - ostream.swrite(0) - ostream.swrite(200) # ok - let headers = newHeaders(HeaderTable) - ostream.swrite(headers) - ostream.write(chawan) + t handle.sendResult(0) + t handle.sendStatus(200) # ok + t handle.sendHeaders(newHeaders(HeaderTable)) + t handle.sendData(chawan) else: - ostream.swrite(-1) - ostream.flush() - + t handle.sendResult(ERROR_ABOUT_PAGE_NOT_FOUND) diff --git a/src/io/connecterror.nim b/src/io/connecterror.nim new file mode 100644 index 00000000..563a4291 --- /dev/null +++ b/src/io/connecterror.nim @@ -0,0 +1,17 @@ +import bindings/curl + +type ConnectErrorCode* = enum + ERROR_ABOUT_PAGE_NOT_FOUND = (-6, "about page not found") + ERROR_FILE_NOT_FOUND = (-5, "file not found") + ERROR_SOURCE_NOT_FOUND = (-4, "clone source could not be found"), + ERROR_LOADER_KILLED = (-3, "loader killed during transfer"), + ERROR_DISALLOWED_URL = (-2, "url not allowed by filter"), + ERROR_UNKNOWN_SCHEME = (-1, "unknown scheme") + +converter toInt*(code: ConnectErrorCode): int = + return int(code) + +func getLoaderErrorMessage*(code: int): string = + if code < 0: + return $ConnectErrorCode(code) + return $curl_easy_strerror(CURLcode(cint(code))) diff --git a/src/io/file.nim b/src/io/file.nim index b9fe0ce2..fe732d6c 100644 --- a/src/io/file.nim +++ b/src/io/file.nim @@ -3,18 +3,28 @@ import os import streams import tables +import io/connecterror import io/headers -import ips/serialize +import io/loaderhandle import types/url -proc loadDir(url: URL, path: string, ostream: Stream) = - ostream.swrite(0) - ostream.swrite(200) # ok - ostream.swrite(newHeaders({"Content-Type": "text/html"}.toTable())) - ostream.write(""" +proc loadDir(handle: LoaderHandle, url: URL, path: string) = + template t(body: untyped) = + if not body: + return + var path = path + if path[^1] != '/': #TODO dos/windows + path &= '/' + var base = $url + if base[^1] != '/': #TODO dos/windows + base &= '/' + t handle.sendResult(0) + t handle.sendStatus(200) # ok + t handle.sendHeaders(newHeaders({"Content-Type": "text/html"}.toTable())) + t handle.sendData(""" <HTML> <HEAD> -<BASE HREF="""" & $url & """"> +<BASE HREF="""" & base & """"> <TITLE>Directory list of """ & path & """</TITLE> </HEAD> <BODY> @@ -28,29 +38,31 @@ proc loadDir(url: URL, path: string, ostream: Stream) = for (pc, file) in fs: case pc of pcDir: - ostream.write("[DIR] ") + t handle.sendData("[DIR] ") of pcFile: - ostream.write("[FILE] ") + t handle.sendData("[FILE] ") of pcLinkToDir, pcLinkToFile: - ostream.write("[LINK] ") + t handle.sendData("[LINK] ") var fn = file if pc == pcDir: fn &= '/' - ostream.write("<A HREF=\"" & fn & "\">" & fn & "</A>") + t handle.sendData("<A HREF=\"" & fn & "\">" & fn & "</A>") if pc in {pcLinkToDir, pcLinkToFile}: - ostream.write(" -> " & expandSymlink(path / file)) - ostream.write("<br>") - ostream.write(""" + discard handle.sendData(" -> " & expandSymlink(path / file)) + t handle.sendData("<br>") + t handle.sendData(""" </BODY> </HTML>""") - ostream.flush() -proc loadSymlink(path: string, ostream: Stream) = - ostream.swrite(0) - ostream.swrite(200) # ok - ostream.swrite(newHeaders({"Content-Type": "text/html"}.toTable())) +proc loadSymlink(handle: LoaderHandle, path: string) = + template t(body: untyped) = + if not body: + return + t handle.sendResult(0) + t handle.sendStatus(200) # ok + t handle.sendHeaders(newHeaders({"Content-Type": "text/html"}.toTable())) let sl = expandSymlink(path) - ostream.write(""" + t handle.sendData(""" <HTML> <HEAD> <TITLE>Symlink view<TITLE> @@ -59,10 +71,26 @@ proc loadSymlink(path: string, ostream: Stream) = Symbolic link to <A HREF="""" & sl & """">""" & sl & """</A></br> </BODY> </HTML>""") - ostream.flush() +proc loadFile(handle: LoaderHandle, istream: Stream) = + template t(body: untyped) = + if not body: + return + t handle.sendResult(0) + t handle.sendStatus(200) # ok + t handle.sendHeaders(newHeaders()) + while not istream.atEnd: + const bufferSize = 4096 + var buffer {.noinit.}: array[bufferSize, char] + while true: + let n = readData(istream, addr buffer[0], bufferSize) + if n == 0: + break + t handle.sendData(addr buffer[0], n) + if n < bufferSize: + break -proc loadFile*(url: URL, ostream: Stream) = +proc loadFilePath*(handle: LoaderHandle, url: URL) = when defined(windows) or defined(OS2) or defined(DOS): let path = url.path.serialize_unicode_dos() else: @@ -70,24 +98,10 @@ proc loadFile*(url: URL, ostream: Stream) = let istream = newFileStream(path, fmRead) if istream == nil: if dirExists(path): - loadDir(url, path, ostream) + handle.loadDir(url, path) elif symlinkExists(path): - loadSymlink(path, ostream) + handle.loadSymlink(path) else: - ostream.swrite(-1) # error - ostream.flush() + discard handle.sendResult(ERROR_FILE_NOT_FOUND) else: - ostream.swrite(0) - ostream.swrite(200) # ok - ostream.swrite(newHeaders()) - while not istream.atEnd: - const bufferSize = 4096 - var buffer {.noinit.}: array[bufferSize, char] - while true: - let n = readData(istream, addr buffer[0], bufferSize) - if n == 0: - break - ostream.writeData(addr buffer[0], n) - ostream.flush() - if n < bufferSize: - break + handle.loadFile(istream) diff --git a/src/io/http.nim b/src/io/http.nim index 1ebcaf72..0a5a6d79 100644 --- a/src/io/http.nim +++ b/src/io/http.nim @@ -1,11 +1,10 @@ import options -import streams import strutils import bindings/curl import io/headers +import io/loaderhandle import io/request -import ips/serialize import types/blob import types/formdata import types/url @@ -13,26 +12,27 @@ import utils/opt import utils/twtstr type - HandleData* = ref HandleDataObj - HandleDataObj = object + CurlHandle* = ref CurlHandleObj + CurlHandleObj = object curl*: CURL statusline: bool headers: Headers request: Request - ostream*: Stream + handle*: LoaderHandle mime: curl_mime slist: curl_slist -func newHandleData(curl: CURL, request: Request, ostream: Stream): HandleData = - let handleData = HandleData( +func newCurlHandle(curl: CURL, request: Request, handle: LoaderHandle): + CurlHandle = + return CurlHandle( headers: newHeaders(), curl: curl, - ostream: ostream, + handle: handle, request: request ) - return handleData -proc cleanup*(handleData: HandleData) = +proc cleanup*(handleData: CurlHandle) = + handleData.handle.close() if handleData.mime != nil: curl_mime_free(handleData.mime) if handleData.slist != nil: @@ -48,58 +48,51 @@ template setopt(curl: CURL, opt: CURLoption, arg: string) = template getinfo(curl: CURL, info: CURLINFO, arg: typed) = discard curl_easy_getinfo(curl, info, arg) -proc curlWriteHeader(p: cstring, size: csize_t, nitems: csize_t, userdata: pointer): csize_t {.cdecl.} = +proc curlWriteHeader(p: cstring, size: csize_t, nitems: csize_t, + userdata: pointer): csize_t {.cdecl.} = var line = newString(nitems) for i in 0..<nitems: line[i] = p[i] - let op = cast[HandleData](userdata) + let op = cast[CurlHandle](userdata) if not op.statusline: op.statusline = true - try: - op.ostream.swrite(int(CURLE_OK)) - except IOError: # Broken pipe + if not op.handle.sendResult(int(CURLE_OK)): return 0 var status: clong op.curl.getinfo(CURLINFO_RESPONSE_CODE, addr status) - op.ostream.swrite(cast[int](status)) + if not op.handle.sendStatus(cast[int](status)): + return 0 return nitems let k = line.until(':') if k.len == line.len: # empty line (last, before body) or invalid (=> error) - op.ostream.swrite(op.headers) + if not op.handle.sendHeaders(op.headers): + return 0 return nitems let v = line.substr(k.len + 1).strip() op.headers.add(k, v) return nitems -proc curlWriteBody(p: cstring, size: csize_t, nmemb: csize_t, userdata: pointer): csize_t {.cdecl.} = - let handleData = cast[HandleData](userdata) +# From the documentation: size is always 1. +proc curlWriteBody(p: cstring, size: csize_t, nmemb: csize_t, + userdata: pointer): csize_t {.cdecl.} = + let handleData = cast[CurlHandle](userdata) if nmemb > 0: - try: - handleData.ostream.writeData(p, int(nmemb)) - except IOError: # Broken pipe + if not handleData.handle.sendData(p, int(nmemb)): return 0 return nmemb -proc applyPostBody(curl: CURL, request: Request, handleData: HandleData) = +proc applyPostBody(curl: CURL, request: Request, handleData: CurlHandle) = if request.multipart.isOk: handleData.mime = curl_mime_init(curl) - if handleData.mime == nil: - # fail (TODO: raise?) - handleData.ostream.swrite(-1) - handleData.ostream.flush() - return + doAssert handleData.mime != nil for entry in request.multipart.get: let part = curl_mime_addpart(handleData.mime) - if part == nil: - # fail (TODO: raise?) - handleData.ostream.swrite(-1) - handleData.ostream.flush() - return + doAssert part != nil curl_mime_name(part, cstring(entry.name)) if entry.isstr: curl_mime_data(part, cstring(entry.svalue), csize_t(entry.svalue.len)) @@ -116,15 +109,13 @@ proc applyPostBody(curl: CURL, request: Request, handleData: HandleData) = curl.setopt(CURLOPT_POSTFIELDS, cstring(request.body.get)) curl.setopt(CURLOPT_POSTFIELDSIZE, request.body.get.len) -proc loadHttp*(curlm: CURLM, request: Request, ostream: Stream): HandleData = +proc loadHttp*(handle: LoaderHandle, curlm: CURLM, + request: Request): CurlHandle = let curl = curl_easy_init() - if curl == nil: - ostream.swrite(-1) - ostream.flush() - return # fail + doAssert curl != nil let surl = request.url.serialize() curl.setopt(CURLOPT_URL, surl) - let handleData = curl.newHandleData(request, ostream) + let handleData = curl.newCurlHandle(request, handle) curl.setopt(CURLOPT_WRITEDATA, handleData) curl.setopt(CURLOPT_WRITEFUNCTION, curlWriteBody) curl.setopt(CURLOPT_HEADERDATA, handleData) @@ -146,8 +137,6 @@ proc loadHttp*(curlm: CURLM, request: Request, ostream: Stream): HandleData = curl.setopt(CURLOPT_HTTPHEADER, handleData.slist) let res = curl_multi_add_handle(curlm, curl) if res != CURLM_OK: - ostream.swrite(int(res)) - ostream.flush() - #TODO: raise here? - return + discard handle.sendResult(int(res)) + return nil return handleData diff --git a/src/io/loader.nim b/src/io/loader.nim index 46f694e6..8e125b31 100644 --- a/src/io/loader.nim +++ b/src/io/loader.nim @@ -12,18 +12,21 @@ # The body is passed to the stream as-is, so effectively nothing can follow it. import nativesockets +import net import options +import posix import streams +import strutils import tables -import net -when defined(posix): - import posix import bindings/curl +import data/charset import io/about +import io/connecterror import io/file import io/headers import io/http +import io/loaderhandle import io/posixstream import io/promise import io/request @@ -34,9 +37,9 @@ import ips/serversocket import ips/socketstream import js/javascript import types/cookie -import types/mime import types/referer import types/url +import utils/mimeguess import utils/twtstr export request @@ -62,14 +65,9 @@ type response: Response bodyRead: Promise[string] - ConnectErrorCode* = enum - ERROR_SOURCE_NOT_FOUND = (-4, "clone source could not be found"), - ERROR_LOADER_KILLED = (-3, "loader killed during transfer"), - ERROR_DISALLOWED_URL = (-2, "url not allowed by filter"), - ERROR_UNKNOWN_SCHEME = (-1, "unknown scheme") - LoaderCommand = enum - LOAD, QUIT + LOAD + QUIT LoaderContext = ref object ssock: ServerSocket @@ -77,7 +75,7 @@ type curlm: CURLM config: LoaderConfig extra_fds: seq[curl_waitfd] - handleList: seq[HandleData] + handleList: seq[CurlHandle] LoaderConfig* = object defaultheaders*: Headers @@ -91,39 +89,36 @@ type FetchPromise* = Promise[Result[Response, JSError]] -converter toInt*(code: ConnectErrorCode): int = - return int(code) - proc addFd(ctx: LoaderContext, fd: int, flags: int) = ctx.extra_fds.add(curl_waitfd( fd: cast[cint](fd), events: cast[cshort](flags) )) -proc loadResource(ctx: LoaderContext, request: Request, ostream: Stream) = +proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) = case request.url.scheme of "file": - loadFile(request.url, ostream) - ostream.close() + handle.loadFilePath(request.url) + handle.close() of "http", "https": - let handleData = loadHttp(ctx.curlm, request, ostream) + let handleData = handle.loadHttp(ctx.curlm, request) if handleData != nil: ctx.handleList.add(handleData) of "about": - loadAbout(request, ostream) - ostream.close() + handle.loadAbout(request) + handle.close() else: - ostream.swrite(ERROR_UNKNOWN_SCHEME) # error - ostream.close() + discard handle.sendResult(ERROR_UNKNOWN_SCHEME) + handle.close() proc onLoad(ctx: LoaderContext, stream: Stream) = var request: Request stream.sread(request) if not ctx.config.filter.match(request.url): - stream.swrite(ERROR_DISALLOWED_URL) # error - stream.flush() + stream.swrite(ERROR_DISALLOWED_URL) stream.close() else: + let handle = newLoaderHandle(stream, request.canredir) for k, v in ctx.config.defaultHeaders.table: if k notin request.headers.table: request.headers.table[k] = v @@ -138,7 +133,7 @@ proc onLoad(ctx: LoaderContext, stream: Stream) = request.headers["Referer"] = r if request.proxy == nil or not ctx.config.acceptProxy: request.proxy = ctx.config.proxy - ctx.loadResource(request, stream) + ctx.loadResource(request, handle) proc acceptConnection(ctx: LoaderContext) = #TODO TODO TODO acceptSocketStream should be non-blocking here, @@ -160,15 +155,10 @@ proc acceptConnection(ctx: LoaderContext) = # (TODO: this is probably not a very good idea.) stream.close() -proc finishCurlTransfer(ctx: LoaderContext, handleData: HandleData, res: int) = +proc finishCurlTransfer(ctx: LoaderContext, handleData: CurlHandle, res: int) = if res != int(CURLE_OK): - try: - handleData.ostream.swrite(int(res)) - handleData.ostream.flush() - except IOError: # Broken pipe - discard + discard handleData.handle.sendResult(int(res)) discard curl_multi_remove_handle(ctx.curlm, handleData.curl) - handleData.ostream.close() handleData.cleanup() proc exitLoader(ctx: LoaderContext) = @@ -192,7 +182,9 @@ proc initLoaderContext(fd: cint, config: LoaderConfig): LoaderContext = config: config ) gctx = ctx - ctx.ssock = initServerSocket() + #TODO ideally, buffered would be true. Unfortunately this conflicts with + # sendFileHandle/recvFileHandle. + ctx.ssock = initServerSocket(buffered = false) # The server has been initialized, so the main process can resume execution. var writef: File if not open(writef, FileHandle(fd), fmWrite): @@ -235,11 +227,39 @@ proc runFileLoader*(fd: cint, config: LoaderConfig) = ctx.handleList.del(idx) ctx.exitLoader() -proc applyHeaders(request: Request, response: Response) = +proc getAttribute(contentType, attrname: string): string = + let kvs = contentType.after(';') + var i = kvs.find(attrname) + var s = "" + if i != -1 and kvs.len > i + attrname.len and + kvs[i + attrname.len] == '=': + i += attrname.len + 1 + while i < kvs.len and kvs[i] in AsciiWhitespace: + inc i + var q = false + for j in i ..< kvs.len: + if q: + s &= kvs[j] + else: + if kvs[j] == '\\': + q = true + elif kvs[j] == ';' or kvs[j] in AsciiWhitespace: + break + else: + s &= kvs[j] + return s + +proc applyHeaders(loader: FileLoader, request: Request, response: Response) = if "Content-Type" in response.headers.table: - response.contenttype = response.headers.table["Content-Type"][0].until(';') + #TODO this is inefficient and broken on several levels. (In particular, + # it breaks mailcap named attributes other than charset.) + # Ideally, contentType would be a separate object type. + let header = response.headers.table["Content-Type"][0].toLowerAscii() + response.contenttype = header.until(';').strip().toLowerAscii() + response.charset = getCharset(header.getAttribute("charset")) else: - response.contenttype = guessContentType($response.url.path) + response.contenttype = guessContentType($response.url.path, + "application/octet-stream", DefaultGuess) if "Location" in response.headers.table: if response.status in 301u16..303u16 or response.status in 307u16..308u16: let location = response.headers.table["Location"][0] @@ -276,6 +296,18 @@ proc fetch*(loader: FileLoader, input: Request): FetchPromise = const BufferSize = 4096 +proc handleHeaders(loader: FileLoader, request: Request, response: Response, + stream: Stream): bool = + var status: int + stream.sread(status) + response.status = cast[uint16](status) + response.headers = newHeaders() + stream.sread(response.headers) + loader.applyHeaders(request, response) + # Only a stream of the response body may arrive after this point. + response.body = stream + return true # success + proc onConnected*(loader: FileLoader, fd: int) = let connectData = loader.connecting[fd] let stream = connectData.stream @@ -283,8 +315,8 @@ proc onConnected*(loader: FileLoader, fd: int) = let request = connectData.request var res: int stream.sread(res) - if res == 0: - let response = newResponse(res, request, fd, stream) + let response = newResponse(res, request, fd, stream) + if res == 0 and loader.handleHeaders(request, response, stream): assert loader.unregisterFun != nil let realCloseImpl = stream.closeImpl stream.closeImpl = nil @@ -293,12 +325,6 @@ proc onConnected*(loader: FileLoader, fd: int) = loader.unregistered.add(fd) loader.unregisterFun(fd) realCloseImpl(stream) - var status: int - stream.sread(status) - response.status = cast[uint16](status) - stream.sread(response.headers) - applyHeaders(request, response) - response.body = stream loader.ongoing[fd] = OngoingData( response: response, readbufsize: BufferSize, @@ -339,31 +365,23 @@ proc onRead*(loader: FileLoader, fd: int) = proc onError*(loader: FileLoader, fd: int) = loader.onRead(fd) -proc doRequest*(loader: FileLoader, request: Request, blocking = true): Response = - new(result) - result.url = request.url +proc doRequest*(loader: FileLoader, request: Request, blocking = true, + canredir = false): Response = + let response = Response(url: request.url) let stream = connectSocketStream(loader.process, false, blocking = true) + if canredir: + request.canredir = true #TODO set this somewhere else? stream.swrite(LOAD) stream.swrite(request) stream.flush() - stream.sread(result.res) - if result.res == 0: - var status: int - stream.sread(status) - result.status = cast[uint16](status) - stream.sread(result.headers) - applyHeaders(request, result) - # Only a stream of the response body may arrive after this point. - result.body = stream - if not blocking: - stream.source.getFd().setBlocking(blocking) + stream.sread(response.res) + if response.res == 0: + if loader.handleHeaders(request, response, stream): + if not blocking: + stream.source.getFd().setBlocking(blocking) + return response proc quit*(loader: FileLoader) = let stream = connectSocketStream(loader.process) if stream != nil: stream.swrite(QUIT) - -func getLoaderErrorMessage*(code: int): string = - if code < 0: - return $ConnectErrorCode(code) - return $curl_easy_strerror(CURLcode(cint(code))) diff --git a/src/io/loaderhandle.nim b/src/io/loaderhandle.nim new file mode 100644 index 00000000..077b1a2a --- /dev/null +++ b/src/io/loaderhandle.nim @@ -0,0 +1,73 @@ +import net +import streams + +import io/posixstream +import io/headers +import ips/serialize +import ips/socketstream + +type LoaderHandle* = ref object + ostream: Stream + # Only the first handle can be redirected, because a) mailcap can only + # redirect the first handle and b) async redirects would result in race + # conditions that would be difficult to untangle. + canredir: bool + sostream: Stream # saved ostream when redirected + +# Create a new loader handle, with the output stream ostream. +proc newLoaderHandle*(ostream: Stream, canredir: bool): LoaderHandle = + return LoaderHandle(ostream: ostream, canredir: canredir) + +proc getFd*(handle: LoaderHandle): int = + return int(SocketStream(handle.ostream).source.getFd()) + +proc sendResult*(handle: LoaderHandle, res: int): bool = + try: + handle.ostream.swrite(res) + return true + except IOError: # broken pipe + return false + +proc sendStatus*(handle: LoaderHandle, status: int): bool = + try: + handle.ostream.swrite(status) + return true + except IOError: # broken pipe + return false + +proc sendHeaders*(handle: LoaderHandle, headers: Headers): bool = + try: + handle.ostream.swrite(headers) + if handle.canredir: + var redir: bool + handle.ostream.sread(redir) + if redir: + let fd = SocketStream(handle.ostream).recvFileHandle() + handle.sostream = handle.ostream + let stream = newPosixStream(fd) + handle.ostream = stream + return true + except IOError: # broken pipe + return false + +proc sendData*(handle: LoaderHandle, p: pointer, nmemb: int): bool = + try: + handle.ostream.writeData(p, nmemb) + return true + except IOError: # broken pipe + return false + +proc sendData*(handle: LoaderHandle, s: string): bool = + if s.len > 0: + return handle.sendData(unsafeAddr s[0], s.len) + return true + +proc close*(handle: LoaderHandle) = + if handle.sostream != nil: + try: + handle.sostream.swrite(true) + except IOError: + # ignore error, that just means the buffer has already closed the stream + discard + handle.sostream.close() + handle.ostream.close() diff --git a/src/io/posixstream.nim b/src/io/posixstream.nim index 10fd2237..e24facde 100644 --- a/src/io/posixstream.nim +++ b/src/io/posixstream.nim @@ -30,6 +30,10 @@ proc raisePosixIOError*() = else: raise newException(IOError, $strerror(errno)) +proc psClose(s: Stream) = + let s = cast[PosixStream](s) + discard close(s.fd) + proc psReadData(s: Stream, buffer: pointer, len: int): int = assert len != 0 let s = cast[PosixStream](s) @@ -63,6 +67,7 @@ proc psAtEnd(s: Stream): bool = proc newPosixStream*(fd: FileHandle): PosixStream = return PosixStream( fd: fd, + closeImpl: psClose, readDataImpl: psReadData, writeDataImpl: psWriteData, atEndImpl: psAtEnd diff --git a/src/io/request.nim b/src/io/request.nim index 4ddd5d6d..f609360b 100644 --- a/src/io/request.nim +++ b/src/io/request.nim @@ -73,6 +73,7 @@ type destination* {.jsget.}: RequestDestination credentialsMode* {.jsget.}: CredentialsMode proxy*: URL #TODO do something with this + canredir*: bool ReadableStream* = ref object of Stream isource*: Stream @@ -154,7 +155,8 @@ proc newReadableStream*(isource: Stream): ReadableStream = func newRequest*(url: URL, httpmethod = HTTP_GET, headers = newHeaders(), body = opt(string), multipart = opt(FormData), mode = RequestMode.NO_CORS, credentialsMode = CredentialsMode.SAME_ORIGIN, - destination = RequestDestination.NO_DESTINATION, proxy: URL = nil): Request = + destination = RequestDestination.NO_DESTINATION, proxy: URL = nil, + canredir = false): Request = return Request( url: url, httpmethod: httpmethod, @@ -169,7 +171,8 @@ func newRequest*(url: URL, httpmethod = HTTP_GET, headers = newHeaders(), func newRequest*(url: URL, httpmethod = HTTP_GET, headers: seq[(string, string)] = @[], body = opt(string), - multipart = opt(FormData), mode = RequestMode.NO_CORS, proxy: URL = nil): + multipart = opt(FormData), mode = RequestMode.NO_CORS, proxy: URL = nil, + canredir = false): Request = let hl = newHeaders() for pair in headers: diff --git a/src/io/response.nim b/src/io/response.nim index b64f1504..dedddbcd 100644 --- a/src/io/response.nim +++ b/src/io/response.nim @@ -1,6 +1,7 @@ import streams import bindings/quickjs +import data/charset import io/headers import io/promise import io/request @@ -20,6 +21,7 @@ type url*: URL #TODO should be urllist? unregisterFun*: proc() bodyRead*: Promise[string] + charset*: Charset jsDestructor(Response) diff --git a/src/io/tempfile.nim b/src/io/tempfile.nim new file mode 100644 index 00000000..d99ea4dc --- /dev/null +++ b/src/io/tempfile.nim @@ -0,0 +1,18 @@ +import os + +var tmpf_seq: int +proc getTempFile*(tmpdir: string, ext = ""): string = + if not dirExists(tmpdir): + createDir(tmpdir) + var tmpf = tmpdir / "chatmp" & $tmpf_seq + if ext != "": + tmpf &= "." + tmpf &= ext + while fileExists(tmpf): + inc tmpf_seq + tmpf = tmpdir / "chatmp" & $tmpf_seq + if ext != "": + tmpf &= "." + tmpf &= ext + inc tmpf_seq + return tmpf diff --git a/src/ips/editor.nim b/src/ips/editor.nim index a6a6623c..19ba965a 100644 --- a/src/ips/editor.nim +++ b/src/ips/editor.nim @@ -3,6 +3,7 @@ import posix import config/config import display/term +import io/tempfile func formatEditorName(editor, file: string, line: int): string = result = newStringOfCap(editor.len + file.len) @@ -55,17 +56,10 @@ proc openEditor*(term: Terminal, config: Config, file: string, line = 1): bool = result = WIFSIGNALED(wstatus) and WTERMSIG(wstatus) == SIGINT term.restart() -var tmpf_seq: int proc openInEditor*(term: Terminal, config: Config, input: var string): bool = try: let tmpdir = config.external.tmpdir - if not dirExists(tmpdir): - createDir(tmpdir) - var tmpf = tmpdir / "chatmp" & $tmpf_seq - while fileExists(tmpf): - inc tmpf_seq - tmpf = tmpdir / "chatmp" & $tmpf_seq - inc tmpf_seq + let tmpf = getTempFile(tmpdir) if input != "": writeFile(tmpf, input) if openEditor(term, config, tmpf): diff --git a/src/ips/forkserver.nim b/src/ips/forkserver.nim index 62d41198..ec0c60d1 100644 --- a/src/ips/forkserver.nim +++ b/src/ips/forkserver.nim @@ -1,5 +1,6 @@ import options import streams +import tables when defined(posix): import posix @@ -36,7 +37,6 @@ type proc newFileLoader*(forkserver: ForkServer, defaultHeaders: Headers = nil, filter = newURLFilter(default = true), cookiejar: CookieJar = nil, proxy: URL = nil, acceptProxy = false): FileLoader = - new(result) forkserver.ostream.swrite(FORK_LOADER) var defaultHeaders = defaultHeaders if defaultHeaders == nil: @@ -51,7 +51,9 @@ proc newFileLoader*(forkserver: ForkServer, defaultHeaders: Headers = nil, ) forkserver.ostream.swrite(config) forkserver.ostream.flush() - forkserver.istream.sread(result.process) + var process: Pid + forkserver.istream.sread(process) + return FileLoader(process: process) proc loadForkServerConfig*(forkserver: ForkServer, config: Config) = forkserver.ostream.swrite(LOAD_CONFIG) @@ -117,7 +119,8 @@ proc forkBuffer(ctx: var ForkServerContext): Pid = filter: config.filter, cookiejar: config.cookiejar, referrerpolicy: config.referrerpolicy, - proxy: config.proxy + #TODO these should be in a separate config I think + proxy: config.proxy, ) ) let pid = fork() @@ -190,7 +193,6 @@ proc runForkServer() = quit(0) proc newForkServer*(): ForkServer = - new(result) var pipefd_in: array[2, cint] # stdin in forkserver var pipefd_out: array[2, cint] # stdout in forkserver var pipefd_err: array[2, cint] # stderr in forkserver @@ -230,7 +232,9 @@ proc newForkServer*(): ForkServer = raise newException(Defect, "Failed to open output handle") if not open(readf, pipefd_out[0], fmRead): raise newException(Defect, "Failed to open input handle") - result.ostream = newFileStream(writef) - result.istream = newFileStream(readf) - result.estream = newPosixStream(pipefd_err[0]) discard fcntl(pipefd_err[0], F_SETFL, fcntl(pipefd_err[0], F_GETFL, 0) or O_NONBLOCK) + return ForkServer( + ostream: newFileStream(writef), + istream: newFileStream(readf), + estream: newPosixStream(pipefd_err[0]) + ) diff --git a/src/ips/socketstream.nim b/src/ips/socketstream.nim index 88023f07..09a3ef5c 100644 --- a/src/ips/socketstream.nim +++ b/src/ips/socketstream.nim @@ -57,6 +57,7 @@ proc sockClose(s: Stream) = {.cast(tags: []).}: #...sigh # See https://stackoverflow.com/a/4491203 proc sendFileHandle*(s: SocketStream, fd: FileHandle) = + assert not s.source.hasDataBuffered var hdr: Tmsghdr var iov: IOVec var space: csize_t @@ -90,6 +91,7 @@ proc sendFileHandle*(s: SocketStream, fd: FileHandle) = assert n == int(iov.iov_len) #TODO remove this proc recvFileHandle*(s: SocketStream): FileHandle = + assert not s.source.hasDataBuffered var iov: IOVec var hdr: Tmsghdr var buf: char @@ -119,6 +121,9 @@ func newSocketStream*(): SocketStream = result.atEndImpl = sockAtEnd result.closeImpl = sockClose +proc setBlocking*(ss: SocketStream, blocking: bool) = + ss.source.getFd().setBlocking(blocking) + proc connectSocketStream*(path: string, buffered = true, blocking = true): SocketStream = result = newSocketStream() result.blk = blocking diff --git a/src/main.nim b/src/main.nim index 225a7578..73ea1eed 100644 --- a/src/main.nim +++ b/src/main.nim @@ -1,9 +1,10 @@ -import types/dispatcher -let disp = newDispatcher() +import ips/forkserver +let forks = newForkServer() import options import os import terminal +import posix when defined(profile): import nimprof @@ -11,7 +12,6 @@ when defined(profile): import config/config import data/charset import display/client -import ips/forkserver import utils/opt import utils/twtstr @@ -51,7 +51,7 @@ Options: var i = 0 var ctype = none(string) -var cs = none(Charset) +var cs = CHARSET_UNKNOWN var pages: seq[string] var dump = false var visual = false @@ -79,11 +79,10 @@ while i < params.len: of "-I", "--input-charset": inc i if i < params.len: - let c = getCharset(params[i]) - if c == CHARSET_UNKNOWN: + cs = getCharset(params[i]) + if cs == CHARSET_UNKNOWN: stderr.write("Unknown charset " & params[i] & "\n") quit(1) - cs = some(c) else: help(1) of "-O", "--output-charset": @@ -147,9 +146,9 @@ if pages.len == 0 and not conf.start.headless: conf.page = constructActionTable(conf.page) conf.line = constructActionTable(conf.line) -disp.forkserver.loadForkServerConfig(conf) +forks.loadForkServerConfig(conf) -let c = newClient(conf, disp) +let c = newClient(conf, forks, getpid()) try: c.launchClient(pages, ctype, cs, dump) except CatchableError: diff --git a/src/types/blob.nim b/src/types/blob.nim index 3af362d4..8bc96db7 100644 --- a/src/types/blob.nim +++ b/src/types/blob.nim @@ -1,7 +1,7 @@ import options import js/javascript -import types/mime +import utils/mimeguess import utils/twtstr type diff --git a/src/types/buffersource.nim b/src/types/buffersource.nim index 109e8361..9235e07f 100644 --- a/src/types/buffersource.nim +++ b/src/types/buffersource.nim @@ -14,7 +14,7 @@ type BufferSource* = object location*: URL contenttype*: Option[string] # override - charset*: Option[Charset] # override + charset*: Charset # fallback case t*: BufferSourceType of CLONE: clonepid*: Pid diff --git a/src/types/dispatcher.nim b/src/types/dispatcher.nim deleted file mode 100644 index cd668a15..00000000 --- a/src/types/dispatcher.nim +++ /dev/null @@ -1,12 +0,0 @@ -import posix - -import ips/forkserver - -type Dispatcher* = ref object - forkserver*: ForkServer - mainproc*: Pid - -proc newDispatcher*(): Dispatcher = - new(result) - result.forkserver = newForkServer() - result.mainproc = getpid() diff --git a/src/types/mime.nim b/src/utils/mimeguess.nim index 96742337..7477c399 100644 --- a/src/types/mime.nim +++ b/src/utils/mimeguess.nim @@ -1,33 +1,29 @@ import algorithm +import streams import tables -const DefaultGuess = [ - ("html", "text/html"), - ("htm", "text/html"), - ("xhtml", "application/xhtml+xml"), - ("xhtm", "application/xhtml+xml"), - ("xht", "application/xhtml+xml"), - ("txt", "text/plain"), - ("css", "text/css"), - ("png", "image/png"), - ("", "text/plain") -].toTable() +import config/mimetypes -proc guessContentType*(path: string, def = DefaultGuess[""]): string = +const DefaultGuess* = block: + let ss = newStringStream(staticRead"res/mime.types") + parseMimeTypes(ss) + +proc guessContentType*(path: string, fallback = "text/plain", + guess = DefaultGuess): string = var i = path.len - 1 var n = 0 while i > 0: if path[i] == '/': - return def + return fallback if path[i] == '.': n = i break dec i if n > 0: let ext = path.substr(n + 1) - if ext in DefaultGuess: - return DefaultGuess[ext] - return def + if ext in guess: + return guess[ext] + return fallback const JavaScriptTypes = [ "application/ecmascript", |