diff options
| m--------- | lib/chame | 0 | ||||
| -rw-r--r-- | src/html/chadombuilder.nim | 256 | ||||
| -rw-r--r-- | src/html/dom.nim | 9 | ||||
| -rw-r--r-- | src/io/posixstream.nim | 10 | ||||
| -rw-r--r-- | src/io/socketstream.nim | 11 | ||||
| -rw-r--r-- | src/loader/loader.nim | 93 | ||||
| -rw-r--r-- | src/loader/loaderhandle.nim | 110 | ||||
| -rw-r--r-- | src/server/buffer.nim | 124 |
8 files changed, 389 insertions, 224 deletions
diff --git a/lib/chame b/lib/chame -Subproject 946af631dc066b4cd291fc5a96ee0db4b075866 +Subproject f6018b45b8d3d41e31d5ab003813504333c42eb diff --git a/src/html/chadombuilder.nim b/src/html/chadombuilder.nim index 21e94901..a2e80f3b 100644 --- a/src/html/chadombuilder.nim +++ b/src/html/chadombuilder.nim @@ -20,14 +20,23 @@ import chame/tags # DOMBuilder implementation for Chawan. type CharsetConfidence = enum - CONFIDENCE_TENTATIVE, CONFIDENCE_CERTAIN, CONFIDENCE_IRRELEVANT + ccTentative, ccCertain, ccIrrelevant type + HTML5ParserWrapper* = ref object + parser: HTML5Parser[Node, CAtom] + charsetStack: seq[Charset] + seekable: bool + builder*: ChaDOMBuilder + opts: HTML5ParserOpts[Node, CAtom] + inputStream: Stream + encoder: EncoderStream + decoder: DecoderStream + ChaDOMBuilder = ref object of DOMBuilder[Node, CAtom] charset: Charset confidence: CharsetConfidence - document: Document - isFragment: bool + document*: Document factory: CAtomFactory poppedScript: HTMLScriptElement @@ -79,19 +88,21 @@ proc setQuirksModeImpl(builder: ChaDOMBuilder, quirksMode: QuirksMode) = proc setEncodingImpl(builder: ChaDOMBuilder, encoding: string): SetEncodingResult = - let charset = getCharset(encoding) - if charset == CHARSET_UNKNOWN: + if builder.confidence != ccTentative: return SET_ENCODING_CONTINUE if builder.charset in {CHARSET_UTF_16_LE, CHARSET_UTF_16_BE}: - builder.confidence = CONFIDENCE_CERTAIN + builder.confidence = ccCertain return SET_ENCODING_CONTINUE - builder.confidence = CONFIDENCE_CERTAIN + let charset = getCharset(encoding) + if charset == CHARSET_UNKNOWN: + return SET_ENCODING_CONTINUE + builder.confidence = ccCertain if charset == builder.charset: return SET_ENCODING_CONTINUE - if charset == CHARSET_X_USER_DEFINED: - builder.charset = CHARSET_WINDOWS_1252 + builder.charset = if charset == CHARSET_X_USER_DEFINED: + CHARSET_WINDOWS_1252 else: - builder.charset = charset + charset return SET_ENCODING_STOP proc getTemplateContentImpl(builder: ChaDOMBuilder, handle: Node): Node = @@ -189,7 +200,7 @@ proc elementPoppedImpl(builder: ChaDOMBuilder, element: Node) = builder.poppedScript = HTMLScriptElement(element) proc newChaDOMBuilder(url: URL, window: Window, factory: CAtomFactory, - isFragment = false): ChaDOMBuilder = + confidence: CharsetConfidence): ChaDOMBuilder = let document = newDocument(factory) document.contentType = "text/html" document.url = url @@ -198,17 +209,15 @@ proc newChaDOMBuilder(url: URL, window: Window, factory: CAtomFactory, window.document = document return ChaDOMBuilder( document: document, - isFragment: isFragment, - factory: factory + factory: factory, + confidence: confidence ) # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments proc parseHTMLFragment*(element: Element, s: string): seq[Node] = let url = parseURL("about:blank").get let factory = element.document.factory - let builder = newChaDOMBuilder(url, nil, factory) - let inputStream = newStringStream(s) - builder.isFragment = true + let builder = newChaDOMBuilder(url, nil, factory, ccIrrelevant) let document = builder.document document.mode = element.document.mode let state = case element.tagType @@ -234,12 +243,9 @@ proc parseHTMLFragment*(element: Element, s: string): seq[Node] = pushInTemplate: element.tagType == TAG_TEMPLATE ) var parser = initHTML5Parser(builder, opts) - var buffer: array[4096, char] - while true: - let n = inputStream.readData(addr buffer[0], buffer.len) - if n == 0: break - let res = parser.parseChunk(buffer.toOpenArray(0, n - 1)) - assert res == PRES_CONTINUE # scripting is false, so this must be continue + let res = parser.parseChunk(s.toOpenArray(0, s.high)) + # scripting is false and confidence is certain -> this must be continue + assert res == PRES_CONTINUE parser.finish() builder.finish() return root.childList @@ -257,107 +263,126 @@ proc bomSniff(inputStream: Stream): Charset = inputStream.setPosition(0) return CHARSET_UNKNOWN -proc parseHTML*(inputStream: Stream, window: Window, url: URL, - factory: CAtomFactory, charsets: seq[Charset] = @[], - seekable = true): Document = +proc switchCharset(wrapper: HTML5ParserWrapper) = + let builder = wrapper.builder + builder.charset = wrapper.charsetStack.pop() + if wrapper.seekable: + builder.confidence = ccTentative # used in the next iteration + else: + builder.confidence = ccCertain + let em = if wrapper.charsetStack.len == 0 or not wrapper.seekable: + DECODER_ERROR_MODE_REPLACEMENT + else: + DECODER_ERROR_MODE_FATAL + wrapper.parser = initHTML5Parser(builder, wrapper.opts) + wrapper.decoder = newDecoderStream(wrapper.inputStream, builder.charset, + errormode = em) + wrapper.decoder.setInhibitCheckEnd(true) + wrapper.encoder = newEncoderStream(wrapper.decoder, CHARSET_UTF_8, + errormode = ENCODER_ERROR_MODE_FATAL) + +proc newHTML5ParserWrapper*(inputStream: Stream, window: Window, url: URL, + factory: CAtomFactory, charsets: seq[Charset] = @[], seekable = true): + HTML5ParserWrapper = let opts = HTML5ParserOpts[Node, CAtom]( isIframeSrcdoc: false, #TODO? scripting: window != nil and window.settings.scripting ) - let builder = newChaDOMBuilder(url, window, factory) - var charsetStack: seq[Charset] - for i in countdown(charsets.high, 0): - charsetStack.add(charsets[i]) - var seekable = seekable - var inputStream = inputStream - if seekable: - let scs = inputStream.bomSniff() - if scs != CHARSET_UNKNOWN: - charsetStack.add(scs) - builder.confidence = CONFIDENCE_CERTAIN - seekable = false - if charsetStack.len == 0: - charsetStack.add(DefaultCharset) # UTF-8 - while true: - builder.charset = charsetStack.pop() - if seekable: - builder.confidence = CONFIDENCE_TENTATIVE # used in the next iteration - else: - builder.confidence = CONFIDENCE_CERTAIN - let em = if charsetStack.len == 0 or not seekable: - DECODER_ERROR_MODE_REPLACEMENT + let builder = newChaDOMBuilder(url, window, factory, ccTentative) + let wrapper = HTML5ParserWrapper( + seekable: seekable, + builder: builder, + opts: opts, + inputStream: inputStream + ) + if seekable and (let scs = inputStream.bomSniff(); scs != CHARSET_UNKNOWN): + builder.confidence = ccCertain + wrapper.charsetStack = @[scs] + wrapper.seekable = false + elif charsets.len == 0: + wrapper.charsetStack = @[DefaultCharset] # UTF-8 + else: + for i in countdown(charsets.high, 0): + wrapper.charsetStack.add(charsets[i]) + wrapper.switchCharset() + return wrapper + +proc parseBuffer(wrapper: HTML5ParserWrapper, buffer: openArray[char]): + ParseResult = + let builder = wrapper.builder + let document = builder.document + var res = wrapper.parser.parseChunk(buffer) + # set insertion point for when it's needed + var ip = wrapper.parser.getInsertionPoint() + while res == PRES_SCRIPT: + if builder.poppedScript != nil: + #TODO microtask + document.writeBuffers.add(DocumentWriteBuffer()) + builder.poppedScript.prepare() + while document.parserBlockingScript != nil: + let script = document.parserBlockingScript + document.parserBlockingScript = nil + #TODO style sheet + script.execute() + assert document.parserBlockingScript != script + builder.poppedScript = nil + if document.writeBuffers.len == 0: + if ip == buffer.len: + # nothing left to re-parse. + break + # parse rest of input buffer + res = wrapper.parser.parseChunk(buffer.toOpenArray(ip, buffer.high)) + ip += wrapper.parser.getInsertionPoint() # move insertion point else: - DECODER_ERROR_MODE_FATAL - let decoder = newDecoderStream(inputStream, builder.charset, errormode = em) - let encoder = newEncoderStream(decoder, CHARSET_UTF_8, - errormode = ENCODER_ERROR_MODE_FATAL) - var parser = initHTML5Parser(builder, opts) - let document = builder.document - var buffer: array[4096, char] - while true: - let n = encoder.readData(addr buffer[0], buffer.len) - if n == 0: break - var res = parser.parseChunk(buffer.toOpenArray(0, n - 1)) - # set insertion point for when it's needed - var ip = parser.getInsertionPoint() - while res == PRES_SCRIPT: - if builder.poppedScript != nil: - #TODO microtask - document.writeBuffers.add(DocumentWriteBuffer()) - builder.poppedScript.prepare() - while document.parserBlockingScript != nil: - let script = document.parserBlockingScript - document.parserBlockingScript = nil - #TODO style sheet - script.execute() - assert document.parserBlockingScript != script - builder.poppedScript = nil - if document.writeBuffers.len == 0: - if ip == n: - # nothing left to re-parse. - break - # parse rest of input buffer - res = parser.parseChunk(buffer.toOpenArray(ip, n - 1)) - ip += parser.getInsertionPoint() # move insertion point + let writeBuffer = document.writeBuffers[^1] + let p = writeBuffer.i + let H = writeBuffer.data.high + res = wrapper.parser.parseChunk(writeBuffer.data.toOpenArray(p, H)) + case res + of PRES_CONTINUE: + discard document.writeBuffers.pop() + res = PRES_SCRIPT + of PRES_SCRIPT: + let pp = p + wrapper.parser.getInsertionPoint() + if pp == writeBuffer.data.len: + discard document.writeBuffers.pop() else: - let writeBuffer = document.writeBuffers[^1] - let p = writeBuffer.i - let n = writeBuffer.data.len - res = parser.parseChunk(writeBuffer.data.toOpenArray(p, n - 1)) - case res - of PRES_CONTINUE: - discard document.writeBuffers.pop() - res = PRES_SCRIPT - of PRES_SCRIPT: - let pp = p + parser.getInsertionPoint() - if pp == writeBuffer.data.len: - discard document.writeBuffers.pop() - else: - writeBuffer.i = pp - of PRES_STOP: - break - {.linearScanEnd.} - # PRES_STOP is returned when we return SET_ENCODING_STOP from - # setEncodingImpl. We immediately stop parsing in this case. - if res == PRES_STOP: + writeBuffer.i = pp + of PRES_STOP: break - parser.finish() - if builder.confidence == CONFIDENCE_CERTAIN and seekable: - # A meta tag describing the charset has been found; force use of this - # charset. + {.linearScanEnd.} + return res + +proc parseAll*(wrapper: HTML5ParserWrapper) = + let builder = wrapper.builder + while true: + let buffer = wrapper.encoder.readAll() + if wrapper.decoder.failed: + assert wrapper.seekable + # Retry with another charset. builder.restart() - inputStream.setPosition(0) - charsetStack.add(builder.charset) - seekable = false + wrapper.inputStream.setPosition(0) + wrapper.switchCharset() continue - if decoder.failed and seekable: - # Retry with another charset. + if buffer.len == 0: + break + let res = wrapper.parseBuffer(buffer) + if res == PRES_STOP: + # A meta tag describing the charset has been found; force use of this + # charset. builder.restart() - inputStream.setPosition(0) + wrapper.inputStream.setPosition(0) + wrapper.charsetStack.add(builder.charset) + wrapper.seekable = false + wrapper.switchCharset() continue break - builder.finish() - return builder.document + +proc finish*(wrapper: HTML5ParserWrapper) = + wrapper.decoder.setInhibitCheckEnd(false) + wrapper.parseAll() + wrapper.parser.finish() + wrapper.builder.finish() proc newDOMParser(): DOMParser {.jsctor.} = return DOMParser() @@ -378,8 +403,13 @@ proc parseFromString(ctx: JSContext, parser: DOMParser, str, t: string): newURL("about:blank").get #TODO this is probably broken in client (or at least sub-optimal) let factory = if window != nil: window.factory else: newCAtomFactory() - let res = parseHTML(newStringStream(str), Window(nil), url, factory) - return ok(res) + let builder = newChaDOMBuilder(url, window, factory, ccIrrelevant) + var parser = initHTML5Parser(builder, HTML5ParserOpts[Node, CAtom]()) + let res = parser.parseChunk(str) + assert res == PRES_CONTINUE + parser.finish() + builder.finish() + return ok(builder.document) of "text/xml", "application/xml", "application/xhtml+xml", "image/svg+xml": return err(newInternalError("XML parsing is not supported yet")) else: diff --git a/src/html/dom.nim b/src/html/dom.nim index 114634e6..cea593cb 100644 --- a/src/html/dom.nim +++ b/src/html/dom.nim @@ -268,7 +268,7 @@ type value* {.jsget.}: Option[int32] HTMLStyleElement* = ref object of HTMLElement - sheet*: CSSStylesheet + sheet: CSSStylesheet HTMLLinkElement* = ref object of HTMLElement sheet*: CSSStylesheet @@ -2315,6 +2315,11 @@ func form(label: HTMLLabelElement): HTMLFormElement {.jsfget.} = proc setRelList(link: HTMLLinkElement, s: string) {.jsfset: "relList".} = link.attr("rel", s) +proc setSheet*(link: HTMLLinkElement, sheet: CSSStylesheet) = + link.sheet = sheet + if link.document != nil: + link.document.cachedSheetsInvalid = true + # <form> proc setRelList(form: HTMLFormElement, s: string) {.jsfset: "relList".} = form.attr("rel", s) @@ -3276,7 +3281,7 @@ proc fetchClassicScript(element: HTMLScriptElement, url: URL, return let loader = window.loader.get let request = createPotentialCORSRequest(url, RequestDestination.SCRIPT, cors) - let response = loader.doRequest(request) + let response = loader.doRequest(request, canredir = false) if response.res != 0: element.onComplete(ScriptResult(t: RESULT_NULL)) return diff --git a/src/io/posixstream.nim b/src/io/posixstream.nim index 73a957f8..04fe0e5c 100644 --- a/src/io/posixstream.nim +++ b/src/io/posixstream.nim @@ -62,11 +62,19 @@ proc psReadData(s: Stream, buffer: pointer, len: int): int = if result == -1: raisePosixIOError() +method sendData*(s: PosixStream, buffer: pointer, len: int): int {.base.} = + #TODO use sendData instead + let n = write(s.fd, buffer, len) + if n < 0: + raisePosixIOError() + return n + proc psWriteData(s: Stream, buffer: pointer, len: int) = + #TODO use sendData instead let s = cast[PosixStream](s) let res = write(s.fd, buffer, len) if res == -1: - raise newException(IOError, $strerror(errno)) + raisePosixIOError() proc psAtEnd(s: Stream): bool = return cast[PosixStream](s).isend diff --git a/src/io/socketstream.nim b/src/io/socketstream.nim index 9426f7a7..fb378083 100644 --- a/src/io/socketstream.nim +++ b/src/io/socketstream.nim @@ -9,10 +9,9 @@ when defined(posix): import io/posixstream import io/serversocket -type SocketStream* = ref object of Stream +type SocketStream* = ref object of PosixStream source*: Socket blk*: bool - isend: bool proc sockReadData(s: Stream, buffer: pointer, len: int): int = assert len != 0 @@ -50,6 +49,12 @@ proc sockWriteData(s: Stream, buffer: pointer, len: int) = raisePosixIOError() i += n +method sendData*(s: SocketStream, buffer: pointer, len: int): int = + let n = s.source.send(buffer, len) + if n < 0: + raisePosixIOError() + return n + proc sockAtEnd(s: Stream): bool = SocketStream(s).isend @@ -124,3 +129,5 @@ proc acceptSocketStream*(ssock: ServerSocket, blocking = true): SocketStream = var sock: Socket ssock.sock.accept(sock, inheritable = true) result.source = sock + if not blocking: + sock.getFd().setBlocking(false) diff --git a/src/loader/loader.nim b/src/loader/loader.nim index 02585630..749f7d29 100644 --- a/src/loader/loader.nim +++ b/src/loader/loader.nim @@ -139,7 +139,9 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) = handle.close() else: let fd = handle.istream.fd + handle.setBlocking(false) ctx.selector.registerHandle(fd, {Read}, 0) + ctx.selector.registerHandle(handle.fd, {Write}, 0) let ofl = fcntl(fd, F_GETFL, 0) discard fcntl(fd, F_SETFL, ofl or O_NONBLOCK) # yes, this puts the istream fd in addition to the ostream fd in @@ -164,7 +166,7 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) = proc onLoad(ctx: LoaderContext, stream: SocketStream) = var request: Request stream.sread(request) - let handle = newLoaderHandle(stream, request.canredir) + let handle = newLoaderHandle(stream, request.canredir, request.url) if not ctx.config.filter.match(request.url): handle.sendResult(ERROR_DISALLOWED_URL) handle.close() @@ -188,9 +190,6 @@ proc onLoad(ctx: LoaderContext, stream: SocketStream) = ctx.loadResource(request, handle) proc acceptConnection(ctx: LoaderContext) = - #TODO TODO TODO acceptSocketStream should be non-blocking here, - # otherwise the client disconnecting between poll and accept could - # block this indefinitely. let stream = ctx.ssock.acceptSocketStream() try: var cmd: LoaderCommand @@ -250,7 +249,7 @@ proc initLoaderContext(fd: cint, config: LoaderConfig): LoaderContext = gctx = ctx #TODO ideally, buffered would be true. Unfortunately this conflicts with # sendFileHandle/recvFileHandle. - ctx.ssock = initServerSocket(buffered = false) + ctx.ssock = initServerSocket(buffered = false, blocking = false) ctx.fd = int(ctx.ssock.sock.getFd()) ctx.selector.registerHandle(ctx.fd, {Read}, 0) # The server has been initialized, so the main process can resume execution. @@ -271,41 +270,81 @@ proc initLoaderContext(fd: cint, config: LoaderConfig): LoaderContext = proc runFileLoader*(fd: cint, config: LoaderConfig) = var ctx = initLoaderContext(fd, config) - var buffer {.noinit.}: array[16384, uint8] while ctx.alive: let events = ctx.selector.select(-1) - var unreg: seq[int] + var unregRead: seq[LoaderHandle] + var unregWrite: seq[LoaderHandle] for event in events: if Read in event.events: if event.fd == ctx.fd: # incoming connection ctx.acceptConnection() else: let handle = ctx.handleMap[event.fd] - while not handle.istream.atEnd: + assert event.fd != handle.fd + while true: try: - let n = handle.istream.readData(addr buffer[0], buffer.len) - handle.sendData(addr buffer[0], n) + let buffer = newLoaderBuffer() + buffer.len = handle.istream.readData(addr buffer[0], buffer.cap) + if buffer.len == 0: + dealloc(buffer) + break + handle.addBuffer(buffer) + if buffer.len < buffer.cap: + break except ErrorAgain, ErrorWouldBlock: # retry later break - except ErrorBrokenPipe: # receiver died; stop streaming - unreg.add(event.fd) + except ErrorBrokenPipe: # sender died; stop streaming + unregRead.add(handle) break + if Write in event.events: + let handle = ctx.handleMap[event.fd] + assert event.fd == handle.fd + while handle.currentBuffer != nil: + let buffer = handle.currentBuffer + try: + let i = handle.currentBufferIdx + assert buffer.len - i > 0 + let n = handle.sendData(addr buffer[i], buffer.len - i) + handle.currentBufferIdx += n + if handle.currentBufferIdx < buffer.len: + break + handle.bufferCleared() # swap out buffer + except ErrorAgain, ErrorWouldBlock: # never mind + break + except ErrorBrokenPipe: # receiver died; stop streaming + unregWrite.add(handle) + break + if handle.istream == nil and handle.currentBuffer == nil and + (unregWrite.len == 0 or unregWrite[^1] != handle): + # after EOF, but not appended in this send cycle + unregWrite.add(handle) if Error in event.events: assert event.fd != ctx.fd - when defined(debug): - # sanity check - let handle = ctx.handleMap[event.fd] - if not handle.istream.atEnd(): - let n = handle.istream.readData(addr buffer[0], buffer.len) - assert n == 0 - assert handle.istream.atEnd() - unreg.add(event.fd) - for fd in unreg: - ctx.selector.unregister(fd) - let handle = ctx.handleMap[fd] - ctx.handleMap.del(fd) - ctx.handleMap.del(handle.getFd()) - handle.close() + let handle = ctx.handleMap[event.fd] + if handle.fd == event.fd: # ostream died + unregWrite.add(handle) + else: # istream died + unregRead.add(handle) + for handle in unregRead: + ctx.selector.unregister(handle.istream.fd) + ctx.handleMap.del(handle.istream.fd) + handle.istream.close() + handle.istream = nil + if handle.currentBuffer == nil: + unregWrite.add(handle) + #TODO TODO TODO what to do about sostream + for handle in unregWrite: + ctx.selector.unregister(handle.fd) + ctx.handleMap.del(handle.fd) + handle.ostream.close() + handle.ostream = nil + if handle.istream != nil: + handle.istream.close() + ctx.handleMap.del(handle.istream.fd) + ctx.selector.unregister(handle.istream.fd) + handle.istream.close() + handle.istream = nil + #TODO TODO TODO what to do about sostream ctx.exitLoader() proc getAttribute(contentType, attrname: string): string = @@ -478,6 +517,8 @@ proc onRead*(loader: FileLoader, fd: int) = buffer[].buf.setLen(olen + BufferSize) let n = response.body.readData(addr buffer[].buf[olen], BufferSize) buffer[].buf.setLen(olen + n) + if n == 0: + break except ErrorAgain, ErrorWouldBlock: break if response.body.atEnd(): diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim index 7a9b3434..5d2dee4e 100644 --- a/src/loader/loaderhandle.nim +++ b/src/loader/loaderhandle.nim @@ -1,3 +1,4 @@ +import std/deques import std/net import std/streams @@ -7,28 +8,72 @@ import io/serialize import io/socketstream import loader/headers -type LoaderHandle* = ref object - ostream: Stream - # Stream for taking input - istream*: PosixStream - # Only the first handle can be redirected, because a) mailcap can only - # redirect the first handle and b) async redirects would result in race - # conditions that would be difficult to untangle. - canredir: bool - sostream: Stream # saved ostream when redirected - sostream_suspend: Stream # saved ostream when suspended - fd: int +import types/url +type + LoaderBufferPage = array[4056, uint8] # 4096 - 8 - 32 + + LoaderBufferObj = object + page*: LoaderBufferPage + len: int + + LoaderBuffer* = ptr LoaderBufferObj + + LoaderHandle* = ref object + ostream*: PosixStream #TODO un-extern + # Stream for taking input + istream*: PosixStream + # Only the first handle can be redirected, because a) mailcap can only + # redirect the first handle and b) async redirects would result in race + # conditions that would be difficult to untangle. + canredir: bool + sostream: Stream # saved ostream when redirected + sostream_suspend: Stream # saved ostream when suspended + fd*: int # ostream fd + currentBuffer*: LoaderBuffer + currentBufferIdx*: int + buffers: Deque[LoaderBuffer] + url*: URL #TODO TODO TODO debug # Create a new loader handle, with the output stream ostream. -proc newLoaderHandle*(ostream: Stream, canredir: bool): LoaderHandle = +proc newLoaderHandle*(ostream: PosixStream, canredir: bool, url: URL): LoaderHandle = return LoaderHandle( ostream: ostream, canredir: canredir, - fd: int(SocketStream(ostream).source.getFd()) + fd: int(SocketStream(ostream).source.getFd()), + url: url ) -proc getFd*(handle: LoaderHandle): int = - return handle.fd +func `[]`*(buffer: LoaderBuffer, i: int): var uint8 {.inline.} = + return buffer[].page[i] + +func cap*(buffer: LoaderBuffer): int {.inline.} = + return buffer[].page.len + +func len*(buffer: LoaderBuffer): var int {.inline.} = + return buffer[].len + +proc `len=`*(buffer: LoaderBuffer, i: int) {.inline.} = + buffer[].len = i + +proc newLoaderBuffer*(): LoaderBuffer = + let buffer = cast[LoaderBuffer](alloc(sizeof(LoaderBufferObj))) + buffer.len = 0 + return buffer + +proc addBuffer*(handle: LoaderHandle, buffer: LoaderBuffer) = + if handle.currentBuffer == nil: + handle.currentBuffer = buffer + else: + handle.buffers.addLast(buffer) + +proc bufferCleared*(handle: LoaderHandle) = + assert handle.currentBuffer != nil + handle.currentBufferIdx = 0 + dealloc(handle.currentBuffer) + if handle.buffers.len > 0: + handle.currentBuffer = handle.buffers.popFirst() + else: + handle.currentBuffer = nil proc addOutputStream*(handle: LoaderHandle, stream: Stream) = if likely(handle.sostream_suspend != nil): @@ -43,8 +88,18 @@ proc addOutputStream*(handle: LoaderHandle, stream: Stream) = # sostream_suspend is never nil when the function is called. # (Feel free to remove this assertion if this changes.) doAssert false - let ms = newMultiStream(handle.ostream, stream) - handle.ostream = ms + #TODO TODO TODO fix this + #let ms = newMultiStream(handle.ostream, stream) + #handle.ostream = ms + +proc setBlocking*(handle: LoaderHandle, blocking: bool) = + #TODO this is stupid + if handle.sostream_suspend != nil and handle.sostream_suspend of SocketStream: + SocketStream(handle.sostream_suspend).setBlocking(blocking) + elif handle.sostream != nil and handle.sostream of SocketStream: + SocketStream(handle.sostream).setBlocking(blocking) + else: + SocketStream(handle.ostream).setBlocking(blocking) proc sendResult*(handle: LoaderHandle, res: int, msg = "") = handle.ostream.swrite(res) @@ -67,23 +122,25 @@ proc sendHeaders*(handle: LoaderHandle, headers: Headers) = let stream = newPosixStream(fd) handle.ostream = stream -proc sendData*(handle: LoaderHandle, p: pointer, nmemb: int) = - handle.ostream.writeData(p, nmemb) - -proc sendData*(handle: LoaderHandle, s: string) = - if s.len > 0: - handle.sendData(unsafeAddr s[0], s.len) +proc sendData*(handle: LoaderHandle, p: pointer, nmemb: int): int = + return handle.ostream.sendData(p, nmemb) proc suspend*(handle: LoaderHandle) = + #TODO TODO TODO fix suspend + doAssert false handle.sostream_suspend = handle.ostream - handle.ostream = newStringStream() + #handle.ostream = newStringStream() proc resume*(handle: LoaderHandle) = + #TODO TODO TODO fix resume + doAssert false + #[ let ss = handle.ostream handle.ostream = handle.sostream_suspend handle.sostream_suspend = nil handle.sendData(ss.readAll()) ss.close() + ]# proc close*(handle: LoaderHandle) = if handle.sostream != nil: @@ -93,6 +150,9 @@ proc close*(handle: LoaderHandle) = # ignore error, that just means the buffer has already closed the stream discard handle.sostream.close() - handle.ostream.close() + if handle.ostream != nil: + handle.ostream.close() + handle.ostream = nil if handle.istream != nil: handle.istream.close() + handle.istream = nil diff --git a/src/server/buffer.nim b/src/server/buffer.nim index abb116c6..b3b7f4a6 100644 --- a/src/server/buffer.nim +++ b/src/server/buffer.nim @@ -93,7 +93,6 @@ type rfd: int # file descriptor of command pipe fd: int # file descriptor of buffer source alive: bool - readbufsize: int lines: FlexibleGrid rendered: bool source: BufferSource @@ -124,6 +123,7 @@ type factory: CAtomFactory uastyle: CSSStylesheet quirkstyle: CSSStylesheet + htmlParser: HTML5ParserWrapper InterfaceOpaque = ref object stream: Stream @@ -644,7 +644,13 @@ proc do_reshape(buffer: Buffer) = buffer.prevstyled = styledRoot else: buffer.lines.renderStream(buffer.srenderer) - buffer.available = 0 + +proc processData(buffer: Buffer) = + if buffer.ishtml: + buffer.htmlParser.parseAll() + buffer.document = buffer.htmlParser.builder.document + else: + buffer.lines.renderStream(buffer.srenderer) proc windowChange*(buffer: Buffer, attrs: WindowAttributes) {.proxy.} = buffer.attrs = attrs @@ -695,14 +701,14 @@ proc updateHover*(buffer: Buffer, cursorx, cursory: int): UpdateHoverResult {.pr buffer.prevnode = thisnode -proc loadResource(buffer: Buffer, elem: HTMLLinkElement): EmptyPromise = +proc loadResource(buffer: Buffer, link: HTMLLinkElement): EmptyPromise = let document = buffer.document - let href = elem.attr("href") + let href = link.attr("href") if href == "": return let url = parseURL(href, document.url.some) if url.isSome: let url = url.get - let media = elem.media + let media = link.media if media != "": let cvals = parseListOfComponentValues(newStringStream(media)) let media = parseMediaQueryList(cvals) @@ -724,7 +730,8 @@ proc loadResource(buffer: Buffer, elem: HTMLLinkElement): EmptyPromise = #TODO non-utf-8 css let ds = newDecoderStream(ss, cs = CHARSET_UTF_8) let source = newEncoderStream(ds, cs = CHARSET_UTF_8) - elem.sheet = parseStylesheet(source, buffer.factory)) + link.setSheet(parseStylesheet(source, buffer.factory)) + ) proc loadResource(buffer: Buffer, elem: HTMLImageElement): EmptyPromise = let document = buffer.document @@ -803,6 +810,14 @@ proc setHTML(buffer: Buffer, ishtml: bool) = buffer.attrs, buffer.factory ) + buffer.htmlParser = newHTML5ParserWrapper( + buffer.sstream, + buffer.window, + buffer.url, + buffer.factory, + buffer.charsets, + seekable = true + ) const css = staticRead"res/ua.css" const quirk = css & staticRead"res/quirk.css" buffer.uastyle = css.parseStylesheet(factory) @@ -1138,17 +1153,9 @@ proc finishLoad(buffer: Buffer): EmptyPromise = return p var p: EmptyPromise if buffer.ishtml: - buffer.sstream.setPosition(0) - buffer.available = 0 - let document = parseHTML( - buffer.sstream, - charsets = buffer.charsets, - window = buffer.window, - url = buffer.url, - factory = buffer.factory - ) - buffer.document = document - document.readyState = READY_STATE_INTERACTIVE + buffer.htmlParser.finish() + buffer.document = buffer.htmlParser.builder.document + buffer.document.readyState = READY_STATE_INTERACTIVE buffer.state = LOADING_RESOURCES buffer.dispatchDOMContentLoadedEvent() p = buffer.loadResources() @@ -1194,34 +1201,47 @@ proc onload(buffer: Buffer) = return of LOADING_PAGE: discard - let op = buffer.sstream.getPosition() - var s {.noinit.}: array[BufferSize, uint8] - try: - buffer.sstream.setPosition(op + buffer.available) - let n = buffer.istream.readData(addr s[0], buffer.readbufsize) - if n != 0: # n can be 0 if we get EOF. (in which case we shouldn't reshape unnecessarily.) - buffer.sstream.writeData(addr s[0], n) - buffer.sstream.setPosition(op) - if buffer.readbufsize < BufferSize: - buffer.readbufsize = min(BufferSize, buffer.readbufsize * 2) - buffer.available += n - if buffer.ishtml: + while true: + let op = buffer.sstream.getPosition() + var s {.noinit.}: array[BufferSize, uint8] + try: + let n = buffer.istream.readData(addr s[0], s.len) + if n != 0: + buffer.sstream.writeData(addr s[0], n) + buffer.sstream.setPosition(op) + buffer.available += n + buffer.processData() res.bytes = buffer.available - else: - buffer.do_reshape() - if buffer.istream.atEnd(): - res.atend = true - buffer.finishLoad().then(proc() = - buffer.state = LOADED - if buffer.document != nil: # may be nil if not buffer.ishtml - buffer.document.readyState = READY_STATE_COMPLETE - buffer.dispatchLoadEvent() - buffer.resolveTask(LOAD, res)) - return - buffer.resolveTask(LOAD, res) - except ErrorAgain, ErrorWouldBlock: - if buffer.readbufsize > 1: - buffer.readbufsize = buffer.readbufsize div 2 + res.lines = buffer.lines.len + if buffer.istream.atEnd(): + # EOF + res.atend = true + buffer.finishLoad().then(proc() = + buffer.prevstyled = nil # for incremental rendering + buffer.do_reshape() + res.lines = buffer.lines.len + buffer.state = LOADED + if buffer.document != nil: # may be nil if not buffer.ishtml + buffer.document.readyState = READY_STATE_COMPLETE + buffer.dispatchLoadEvent() + buffer.resolveTask(LOAD, res) + ) + return # skip incr render + buffer.resolveTask(LOAD, res) + except ErrorAgain, ErrorWouldBlock: + break + if buffer.document != nil: + # incremental rendering: only if we cannot read the entire stream in one + # pass + #TODO this is too simplistic to be really useful + let uastyle = if buffer.document.mode != QUIRKS: + buffer.uastyle + else: + buffer.quirkstyle + let styledRoot = buffer.document.applyStylesheets(uastyle, + buffer.userstyle, buffer.prevstyled) + buffer.lines = renderDocument(styledRoot, buffer.attrs) + buffer.prevstyled = styledRoot proc getTitle*(buffer: Buffer): string {.proxy.} = if buffer.document != nil: @@ -1237,16 +1257,10 @@ proc cancel*(buffer: Buffer): int {.proxy.} = buffer.istream.close() buffer.state = LOADED if buffer.ishtml: - buffer.sstream.setPosition(0) - buffer.available = 0 - buffer.document = parseHTML( - buffer.sstream, - charsets = buffer.charsets, - window = buffer.window, - url = buffer.url, - factory = buffer.factory, - seekable = false - ) + buffer.htmlParser.finish() + buffer.document = buffer.htmlParser.builder.document + buffer.document.readyState = READY_STATE_INTERACTIVE + buffer.state = LOADING_RESOURCES buffer.do_reshape() return buffer.lines.len @@ -1784,6 +1798,7 @@ proc handleRead(buffer: Buffer, fd: int) = buffer.onload() elif fd in buffer.loader.connecting: buffer.loader.onConnected(fd) + buffer.loader.onRead(fd) if buffer.config.scripting: buffer.window.runJSJobs() elif fd in buffer.loader.ongoing: @@ -1848,7 +1863,6 @@ proc launchBuffer*(config: BufferConfig, source: BufferSource, sstream: newStringStream(), width: attrs.width, height: attrs.height - 1, - readbufsize: BufferSize, selector: newSelector[int](), estream: newFileStream(stderr), pstream: socks, |