diff options
| -rw-r--r-- | src/config/mailcap.nim | 59 | ||||
| -rw-r--r-- | src/html/dom.nim | 4 | ||||
| -rw-r--r-- | src/loader/loader.nim | 42 | ||||
| -rw-r--r-- | src/loader/response.nim | 38 | ||||
| -rw-r--r-- | src/local/client.nim | 5 | ||||
| -rw-r--r-- | src/local/container.nim | 19 | ||||
| -rw-r--r-- | src/local/pager.nim | 37 | ||||
| -rw-r--r-- | src/types/blob.nim | 2 | ||||
| -rw-r--r-- | src/utils/mimeguess.nim | 19 | ||||
| -rw-r--r-- | src/utils/twtstr.nim | 21 |
10 files changed, 112 insertions, 134 deletions
diff --git a/src/config/mailcap.nim b/src/config/mailcap.nim index 90815249..b108ca20 100644 --- a/src/config/mailcap.nim +++ b/src/config/mailcap.nim @@ -8,8 +8,6 @@ import types/url import types/opt import utils/twtstr -import chagashi/charset - type MailcapParser = object stream: Stream @@ -51,11 +49,11 @@ proc consume(state: var MailcapParser): char = inc state.line return c -proc reconsume(state: var MailcapParser, c: char) = +proc reconsume(state: var MailcapParser; c: char) = state.buf = c state.hasbuf = true -proc skipBlanks(state: var MailcapParser, c: var char): bool = +proc skipBlanks(state: var MailcapParser; c: var char): bool = while state.has(): c = state.consume() if c notin AsciiWhitespace - {'\n'}: @@ -127,7 +125,7 @@ proc consumeCommand(state: var MailcapParser): Result[string, string] = type NamedField = enum NO_NAMED_FIELD, NAMED_FIELD_TEST, NAMED_FIELD_NAMETEMPLATE, NAMED_FIELD_EDIT -proc parseFieldKey(entry: var MailcapEntry, k: string): NamedField = +proc parseFieldKey(entry: var MailcapEntry; k: string): NamedField = case k of "needsterminal": entry.flags.incl(NEEDSTERMINAL) @@ -145,7 +143,7 @@ proc parseFieldKey(entry: var MailcapEntry, k: string): NamedField = return NAMED_FIELD_EDIT return NO_NAMED_FIELD -proc consumeField(state: var MailcapParser, entry: var MailcapEntry): +proc consumeField(state: var MailcapParser; entry: var MailcapEntry): Result[bool, string] = state.skipBlanks() if not state.has(): @@ -218,7 +216,7 @@ type UnquoteResult* = object type QuoteState = enum QS_NORMAL, QS_DQUOTED, QS_SQUOTED -proc quoteFile(file: string, qs: QuoteState): string = +proc quoteFile(file: string; qs: QuoteState): string = var s = "" for c in file: case c @@ -238,8 +236,8 @@ proc quoteFile(file: string, qs: QuoteState): string = s &= c return s -proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL, - charset: Charset, canpipe: var bool): string = +proc unquoteCommand*(ecmd, contentType, outpath: string; url: URL; + canpipe: var bool): string = var cmd = "" var attrname = "" var state: UnquoteState @@ -307,24 +305,7 @@ proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL, state = STATE_NORMAL of STATE_ATTR: if c == '}': - if attrname == "charset": - cmd &= quoteFile($charset, qs) - continue - #TODO this is broken, because content-type is stripped of ; fields - let kvs = contentType.after(';').toLowerAscii() - var i = kvs.find(attrname) - var s = "" - if i != -1 and kvs.len > i + attrname.len and - kvs[i + attrname.len] == '=': - i = skipBlanks(kvs, i + attrname.len + 1) - var q = false - for j in i ..< kvs.len: - if not q and kvs[j] == '\\': - q = true - elif not q and (kvs[j] == ';' or kvs[j] in AsciiWhitespace): - break - else: - s &= kvs[j] + let s = contentType.getContentTypeAttr(attrname) cmd &= quoteFile(s, qs) attrname = "" elif c == '\\': @@ -333,28 +314,24 @@ proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL, attrname &= c return cmd -proc unquoteCommand*(ecmd, contentType, outpath: string, url: URL, - charset: Charset): string = +proc unquoteCommand*(ecmd, contentType, outpath: string; url: URL): string = var canpipe: bool - return unquoteCommand(ecmd, contentType, outpath, url, charset, canpipe) + return unquoteCommand(ecmd, contentType, outpath, url, canpipe) -proc getMailcapEntry*(mailcap: Mailcap; mimeType, outpath: string; - url: URL; charset: Charset): ptr MailcapEntry = - let mt = mimeType.until('/') - if mt.len + 1 >= mimeType.len: +proc getMailcapEntry*(mailcap: Mailcap; contentType, outpath: string; url: URL): + ptr MailcapEntry = + let mt = contentType.until('/') + if mt.len + 1 >= contentType.len: return nil - let st = mimeType[mt.len + 1 .. ^1] + let st = contentType.until(AsciiWhitespace + {';'}, mt.len + 1) for entry in mailcap: - if not (entry.mt.len == 1 and entry.mt[0] == '*') and - entry.mt != mt: + if not (entry.mt.len == 1 and entry.mt[0] == '*') and entry.mt != mt: continue - if not (entry.subt.len == 1 and entry.subt[0] == '*') and - entry.subt != st: + if not (entry.subt.len == 1 and entry.subt[0] == '*') and entry.subt != st: continue if entry.test != "": var canpipe = true - let cmd = unquoteCommand(entry.test, mimeType, outpath, url, charset, - canpipe) + let cmd = unquoteCommand(entry.test, contentType, outpath, url, canpipe) if not canpipe: continue if execCmd(cmd) != 0: diff --git a/src/html/dom.nim b/src/html/dom.nim index 23dd49f2..81504d56 100644 --- a/src/html/dom.nim +++ b/src/html/dom.nim @@ -2776,7 +2776,7 @@ proc loadResource(window: Window, link: HTMLLinkElement) = let res = res.get #TODO we should use ReadableStreams for this (which would allow us to # parse CSS asynchronously) - if res.contentType == "text/css": + if res.getContentType() == "text/css": return res.text() res.unregisterFun() ).then(proc(s: JSResult[string]) = @@ -2803,7 +2803,7 @@ proc loadResource(window: Window, image: HTMLImageElement) = if res.isErr: return let res = res.get - if res.contentType == "image/png": + if res.getContentType() == "image/png": return res.blob() ).then(proc(pngData: JSResult[Blob]) = if pngData.isErr: diff --git a/src/loader/loader.nim b/src/loader/loader.nim index 70f84043..904069a6 100644 --- a/src/loader/loader.nim +++ b/src/loader/loader.nim @@ -43,11 +43,8 @@ import types/cookie import types/referrer import types/urimethodmap import types/url -import utils/mimeguess import utils/twtstr -import chagashi/charset - export request export response @@ -760,38 +757,7 @@ proc runFileLoader*(fd: cint; config: LoaderConfig) = ctx.finishCycle(unregRead, unregWrite) ctx.exitLoader() -proc getAttribute(contentType, attrname: string): string = - let kvs = contentType.after(';') - var i = kvs.find(attrname) - var s = "" - if i != -1 and kvs.len > i + attrname.len and - kvs[i + attrname.len] == '=': - i += attrname.len + 1 - while i < kvs.len and kvs[i] in AsciiWhitespace: - inc i - var q = false - for j, c in kvs.toOpenArray(i, kvs.high): - if q: - s &= c - elif c == '\\': - q = true - elif c == ';' or c in AsciiWhitespace: - break - else: - s &= c - return s - -proc applyHeaders(response: Response; request: Request) = - if "Content-Type" in response.headers.table: - #TODO this is inefficient and broken on several levels. (In particular, - # it breaks mailcap named attributes other than charset.) - # Ideally, contentType would be a separate object type. - let header = response.headers.table["Content-Type"][0].toLowerAscii() - response.contentType = header.until(';').strip().toLowerAscii() - response.charset = getCharset(header.getAttribute("charset")) - else: - response.contentType = guessContentType($response.url.path, - "application/octet-stream", DefaultGuess) +proc getRedirect*(response: Response; request: Request): Request = if "Location" in response.headers.table: if response.status in 301u16..303u16 or response.status in 307u16..308u16: let location = response.headers.table["Location"][0] @@ -801,14 +767,15 @@ proc applyHeaders(response: Response; request: Request) = request.httpMethod notin {HTTP_GET, HTTP_HEAD}) or (response.status == 301 or response.status == 302 and request.httpMethod == HTTP_POST): - response.redirect = newRequest(url.get, HTTP_GET, + return newRequest(url.get, HTTP_GET, mode = request.mode, credentialsMode = request.credentialsMode, destination = request.destination) else: - response.redirect = newRequest(url.get, request.httpMethod, + return newRequest(url.get, request.httpMethod, body = request.body, multipart = request.multipart, mode = request.mode, credentialsMode = request.credentialsMode, destination = request.destination) + return nil proc connect(loader: FileLoader; buffered = true): SocketStream = let stream = connectSocketStream(loader.process, buffered, blocking = true) @@ -899,7 +866,6 @@ proc handleHeaders(response: Response; request: Request; stream: SocketStream) = stream.sread(response.outputId) stream.sread(response.status) stream.sread(response.headers) - response.applyHeaders(request) # Only a stream of the response body may arrive after this point. response.body = stream diff --git a/src/loader/response.nim b/src/loader/response.nim index 6add4928..f419c432 100644 --- a/src/loader/response.nim +++ b/src/loader/response.nim @@ -1,4 +1,6 @@ import std/streams +import std/strutils +import std/tables import bindings/quickjs import io/promise @@ -10,6 +12,8 @@ import loader/headers import loader/request import types/blob import types/url +import utils/mimeguess +import utils/twtstr import chagashi/charset import chagashi/decoder @@ -37,15 +41,12 @@ type res*: int body*: SocketStream bodyUsed* {.jsget.}: bool - contentType*: string status* {.jsget.}: uint16 headers* {.jsget.}: Headers headersGuard: HeadersGuard - redirect*: Request url*: URL #TODO should be urllist? unregisterFun*: proc() bodyRead*: Promise[string] - charset*: Charset internalMessage*: string # should NOT be exposed to JS! outputId*: int @@ -87,6 +88,23 @@ proc close*(response: Response) {.jsfunc.} = if response.body != nil: response.body.close() +func getCharset*(this: Response; fallback: Charset): Charset = + if "Content-Type" notin this.headers.table: + return fallback + let header = this.headers.table["Content-Type"][0].toLowerAscii() + let cs = header.getContentTypeAttr("charset").getCharset() + if cs == CHARSET_UNKNOWN: + return fallback + return cs + +func getContentType*(this: Response): string = + if "Content-Type" in this.headers.table: + let header = this.headers.table["Content-Type"][0].toLowerAscii() + return header.until(';').strip() + # also use DefaultGuess for container, so that local mime.types cannot + # override buffer mime.types + return DefaultGuess.guessContentType(this.url.pathname) + proc text*(response: Response): Promise[JSResult[string]] {.jsfunc.} = if response.body == nil: let p = newPromise[JSResult[string]]() @@ -101,16 +119,13 @@ proc text*(response: Response): Promise[JSResult[string]] {.jsfunc.} = let bodyRead = response.bodyRead response.bodyRead = nil return bodyRead.then(proc(s: string): JSResult[string] = - let cs = if response.charset == CHARSET_UNKNOWN: - CHARSET_UTF_8 - else: - response.charset + let charset = response.getCharset(CHARSET_UTF_8) #TODO this is inefficient # maybe add a JS type that turns a seq[char] into JS strings - if cs in {CHARSET_UTF_8, CHARSET_UNKNOWN}: + if charset == CHARSET_UTF_8: ok(s.toValidUTF8()) else: - ok(newTextDecoder(cs).decodeAll(s)) + ok(newTextDecoder(charset).decodeAll(s)) ) proc blob*(response: Response): Promise[JSResult[Blob]] {.jsfunc.} = @@ -122,13 +137,14 @@ proc blob*(response: Response): Promise[JSResult[Blob]] {.jsfunc.} = return p let bodyRead = response.bodyRead response.bodyRead = nil + let contentType = response.getContentType() return bodyRead.then(proc(s: string): JSResult[Blob] = if s.len == 0: - return ok(newBlob(nil, 0, response.contentType, nil)) + return ok(newBlob(nil, 0, contentType, nil)) GC_ref(s) let deallocFun = proc() = GC_unref(s) - let blob = newBlob(unsafeAddr s[0], s.len, response.contentType, deallocFun) + let blob = newBlob(unsafeAddr s[0], s.len, contentType, deallocFun) ok(blob)) proc json(ctx: JSContext, this: Response): Promise[JSResult[JSValue]] diff --git a/src/local/client.nim b/src/local/client.nim index 0d1bf063..6b605311 100644 --- a/src/local/client.nim +++ b/src/local/client.nim @@ -518,8 +518,9 @@ proc handleRead(client: Client, fd: int) = let response = stream.readResponse(container.request) if response.body == nil: client.pager.fail(container, response.getErrorMessage()) - elif response.redirect != nil: - client.pager.redirect(container, response) + elif (let redirect = response.getRedirect(container.request); + redirect != nil): + client.pager.redirect(container, response, redirect) response.body.close() else: client.pager.connected(container, response) diff --git a/src/local/container.nim b/src/local/container.nim index 49d8885e..fc3384d9 100644 --- a/src/local/container.nim +++ b/src/local/container.nim @@ -1402,15 +1402,11 @@ proc applyResponse*(container: Container; response: Response) = container.setLoadInfo("Connected to " & $response.url & ". Downloading...") # setup content type; note that isSome means an override so we skip it if container.contentType.isNone: - if response.contentType == "application/octet-stream": - let contentType = guessContentType(container.url.pathname, - "application/octet-stream", container.config.mimeTypes) - if contentType != "application/octet-stream": - container.contentType = some(contentType) - else: - container.contentType = some(response.contentType) - else: - container.contentType = some(response.contentType) + var contentType = response.getContentType() + if contentType == "application/octet-stream": + contentType = container.config.mimeTypes + .guessContentType(container.url.pathname) + container.contentType = some(contentType) # setup charsets: # * override charset # * network charset @@ -1418,8 +1414,9 @@ proc applyResponse*(container: Container; response: Response) = # HTML may override the last two (but not the override charset). if container.config.charsetOverride != CHARSET_UNKNOWN: container.charsetStack = @[container.config.charsetOverride] - elif response.charset != CHARSET_UNKNOWN: - container.charsetStack = @[response.charset] + elif (let charset = response.getCharset(CHARSET_UNKNOWN); + charset != CHARSET_UNKNOWN): + container.charsetStack = @[charset] else: container.charsetStack = @[] for i in countdown(container.config.charsets.high, 0): diff --git a/src/local/pager.nim b/src/local/pager.nim index cba5a5d3..a0f63aca 100644 --- a/src/local/pager.nim +++ b/src/local/pager.nim @@ -1087,11 +1087,10 @@ type CheckMailcapResult = object ishtml: bool # Pipe output of an x-ansioutput mailcap command to the text/x-ansi handler. -proc ansiDecode(pager: Pager; url: URL; charset: Charset; ishtml: var bool; - fdin: cint): cint = - let entry = pager.mailcap.getMailcapEntry("text/x-ansi", "", url, charset) +proc ansiDecode(pager: Pager; url: URL; ishtml: var bool; fdin: cint): cint = + let entry = pager.mailcap.getMailcapEntry("text/x-ansi", "", url) var canpipe = true - let cmd = unquoteCommand(entry.cmd, "text/x-ansi", "", url, charset, canpipe) + let cmd = unquoteCommand(entry.cmd, "text/x-ansi", "", url, canpipe) if not canpipe: pager.alert("Error: could not pipe to text/x-ansi, decoding as text/plain") return -1 @@ -1283,35 +1282,34 @@ proc filterBuffer(pager: Pager; stream: SocketStream; cmd: string; # pager is suspended until the command exits. #TODO add support for edit/compose, better error handling proc checkMailcap(pager: Pager; container: Container; stream: SocketStream; - istreamOutputId: int): CheckMailcapResult = + istreamOutputId: int; contentType: string): CheckMailcapResult = if container.filter != nil: return pager.filterBuffer(stream, container.filter.cmd, container.ishtml) # contentType must exist, because we set it in applyResponse - let contentType = container.contentType.get - if contentType == "text/html": + let shortContentType = container.contentType.get + if shortContentType == "text/html": # We support text/html natively, so it would make little sense to execute # mailcap filters for it. return CheckMailcapResult(connect: true, fdout: stream.fd, ishtml: true) - if contentType == "text/plain": + if shortContentType == "text/plain": # text/plain could potentially be useful. Unfortunately, many mailcaps # include a text/plain entry with less by default, so it's probably better # to ignore this. return CheckMailcapResult(connect: true, fdout: stream.fd) #TODO callback for outpath or something let url = container.url - let cs = container.charset - let entry = pager.mailcap.getMailcapEntry(contentType, "", url, cs) + let entry = pager.mailcap.getMailcapEntry(contentType, "", url) if entry == nil: return CheckMailcapResult(connect: true, fdout: stream.fd) let tmpdir = pager.tmpdir let ext = url.pathname.afterLast('.') let tempfile = getTempFile(tmpdir, ext) let outpath = if entry.nametemplate != "": - unquoteCommand(entry.nametemplate, contentType, tempfile, url, cs) + unquoteCommand(entry.nametemplate, contentType, tempfile, url) else: tempfile var canpipe = true - let cmd = unquoteCommand(entry.cmd, contentType, outpath, url, cs, canpipe) + let cmd = unquoteCommand(entry.cmd, contentType, outpath, url, canpipe) var ishtml = HTMLOUTPUT in entry.flags let needsterminal = NEEDSTERMINAL in entry.flags putEnv("MAILCAP_URL", $url) @@ -1336,7 +1334,7 @@ proc checkMailcap(pager: Pager; container: Container; stream: SocketStream; pager.runMailcapReadFile(stream, cmd, outpath, pipefdOut) discard close(pipefdOut[1]) # close write let fdout = if not ishtml and ANSIOUTPUT in entry.flags: - pager.ansiDecode(url, cs, ishtml, pipefdOut[0]) + pager.ansiDecode(url, ishtml, pipefdOut[0]) else: pipefdOut[0] delEnv("MAILCAP_URL") @@ -1368,10 +1366,10 @@ proc fail*(pager: Pager; container: Container; errorMessage: string) = else: pager.alert("Can't load " & $container.url & " (" & errorMessage & ")") -proc redirect*(pager: Pager; container: Container; response: Response) = +proc redirect*(pager: Pager; container: Container; response: Response; + request: Request) = # still need to apply response, or we lose cookie jars. container.applyResponse(response) - let request = response.redirect if container.redirectdepth < pager.config.network.max_redirect: if container.url.scheme == request.url.scheme or container.url.scheme == "cgi-bin" or @@ -1395,10 +1393,15 @@ proc connected*(pager: Pager; container: Container; response: Response) = pager.authorize() istream.close() return - let mailcapRes = pager.checkMailcap(container, istream, response.outputId) + let realContentType = if "Content-Type" in response.headers: + response.headers["Content-Type"] + else: + # both contentType and charset must be set by applyResponse. + container.contentType.get & ";charset=" & $container.charset + let mailcapRes = pager.checkMailcap(container, istream, response.outputId, + realContentType) if mailcapRes.connect: container.ishtml = mailcapRes.ishtml - container.applyResponse(response) # buffer now actually exists; create a process for it container.process = pager.forkserver.forkBuffer( container.config, diff --git a/src/types/blob.nim b/src/types/blob.nim index 63fb5b84..5b1efa65 100644 --- a/src/types/blob.nim +++ b/src/types/blob.nim @@ -53,7 +53,7 @@ proc newWebFile*(path: string, webkitRelativePath = ""): WebFile = isfile: true, path: path, file: file, - ctype: guessContentType(path), + ctype: DefaultGuess.guessContentType(path), webkitRelativePath: webkitRelativePath ) diff --git a/src/utils/mimeguess.nim b/src/utils/mimeguess.nim index 0a65c909..4b8df086 100644 --- a/src/utils/mimeguess.nim +++ b/src/utils/mimeguess.nim @@ -8,22 +8,19 @@ const DefaultGuess* = block: let ss = newStringStream(staticRead"res/mime.types") parseMimeTypes(ss) -proc guessContentType*(path: string, fallback = "text/plain", - guess = DefaultGuess): string = - var i = path.len - 1 +func guessContentType*(mimeTypes: MimeTypes; path: string): string = var n = 0 - while i > 0: + for i in countdown(path.high, 0): if path[i] == '/': - return fallback + break if path[i] == '.': n = i break - dec i if n > 0: let ext = path.substr(n + 1) - if ext in guess: - return guess[ext] - return fallback + if ext in mimeTypes: + return mimeTypes[ext] + return "application/octet-stream" const JavaScriptTypes = [ "application/ecmascript", @@ -44,5 +41,5 @@ const JavaScriptTypes = [ "text/x-javascript" ] -proc isJavaScriptType*(s: string): bool = - return binarySearch(JavaScriptTypes, s) != -1 +func isJavaScriptType*(s: string): bool = + return JavaScriptTypes.binarySearch(s) != -1 diff --git a/src/utils/twtstr.nim b/src/utils/twtstr.nim index 4d6b48ea..fe6ba236 100644 --- a/src/utils/twtstr.nim +++ b/src/utils/twtstr.nim @@ -693,3 +693,24 @@ func strictParseEnum*[T: enum](s: string): Opt[T] = if s in tab: return ok(tab[s]) return err() + +proc getContentTypeAttr*(contentType, attrname: string): string = + let kvs = contentType.after(';') + var i = kvs.find(attrname) + var s = "" + if i != -1 and kvs.len > i + attrname.len and + kvs[i + attrname.len] == '=': + i += attrname.len + 1 + while i < kvs.len and kvs[i] in AsciiWhitespace: + inc i + var q = false + for j, c in kvs.toOpenArray(i, kvs.high): + if q: + s &= c + elif c == '\\': + q = true + elif c == ';' or c in AsciiWhitespace: + break + else: + s &= c + return s |