xhr: more progress

* add responseText, response
* add net tests
	-> currently sync XHR only; should find a way to do async
	   tests...
* update monoucha
	-> simplified & updated some related code that no longer worked
	   properly

4 files changed, 156 insertions, 48 deletions

diff --git a/src/html/chadombuilder.nim b/src/html/chadombuilder.nim
index 5470deb2..bedd0d66 100644
--- a/src/html/chadombuilder.nim
+++ b/src/html/chadombuilder.nim
@@ -349,10 +349,10 @@ proc finish*(wrapper: HTML5ParserWrapper) =
     r.parser = nil
   wrapper.refs.setLen(0)
 
-proc newDOMParser(): DOMParser {.jsctor.} =
+proc newDOMParser*(): DOMParser {.jsctor.} =
   return DOMParser()
 
-proc parseFromString(ctx: JSContext; parser: DOMParser; str, t: string):
+proc parseFromString*(ctx: JSContext; parser: DOMParser; str, t: string):
     JSResult[Document] {.jsfunc.} =
   case t
   of "text/html":
@@ -361,7 +361,6 @@ proc parseFromString(ctx: JSContext; parser: DOMParser; str, t: string):
       window.document.url
     else:
       newURL("about:blank").get
-    #TODO this is probably broken in client (or at least sub-optimal)
     let builder = newChaDOMBuilder(url, window, window.factory, ccIrrelevant)
     var parser = initHTML5Parser(builder, HTML5ParserOpts[Node, CAtom]())
     let res = parser.parseChunk(str)
diff --git a/src/html/dom.nim b/src/html/dom.nim
index 208f8168..7bd3ff9a 100644
--- a/src/html/dom.nim
+++ b/src/html/dom.nim
@@ -464,7 +464,7 @@ proc resetState(state: var DrawingState) =
   state.path = newPath()
 
 proc create2DContext*(jctx: JSContext; target: HTMLCanvasElement;
-    options: Option[JSValue]): CanvasRenderingContext2D =
+    options = JS_UNDEFINED): CanvasRenderingContext2D =
   let ctx = CanvasRenderingContext2D(
     bitmap: target.bitmap,
     canvas: target
@@ -1273,10 +1273,10 @@ func childNodes(node: Node): NodeList {.jsfget.} =
 func length(tokenList: DOMTokenList): uint32 {.jsfget.} =
   return uint32(tokenList.toks.len)
 
-func item(tokenList: DOMTokenList; i: int): Option[string] {.jsfunc.} =
+proc item(ctx: JSContext; tokenList: DOMTokenList; i: int): JSValue {.jsfunc.} =
   if i < tokenList.toks.len:
-    return some(tokenList.element.document.toStr(tokenList.toks[i]))
-  return none(string)
+    return ctx.toJS(tokenList.toks[i])
+  return JS_NULL
 
 func contains*(tokenList: DOMTokenList; a: CAtom): bool =
   return a in tokenList.toks
@@ -1381,8 +1381,12 @@ func supports(tokenList: DOMTokenList; token: string):
 func value(tokenList: DOMTokenList): string {.jsfget.} =
   return $tokenList
 
-func getter(tokenList: DOMTokenList; i: uint32): Option[string] {.jsgetprop.} =
-  return tokenList.item(int(i))
+proc getter(ctx: JSContext; this: DOMTokenList; atom: JSAtom): JSValue
+    {.jsgetprop.} =
+  var u: uint32
+  if ctx.fromJS(atom, u).isSome:
+    return ctx.item(this, int(u))
+  return JS_NULL
 
 # DOMStringMap
 func validateAttributeName(name: string): Err[DOMException] =
@@ -1450,9 +1454,14 @@ func hasprop(nodeList: NodeList; i: uint32): bool {.jshasprop.} =
 func item(nodeList: NodeList; i: int): Node {.jsfunc.} =
   if i < nodeList.len:
     return nodeList.snapshot[i]
+  return nil
 
-func getter(nodeList: NodeList; i: uint32): Option[Node] {.jsgetprop.} =
-  return option(nodeList.item(int(i)))
+func getter(ctx: JSContext; nodeList: NodeList; atom: JSAtom): Node
+    {.jsgetprop.} =
+  var u: uint32
+  if ctx.fromJS(atom, u).isSome:
+    return nodeList.item(int(u))
+  return nil
 
 func names(ctx: JSContext; nodeList: NodeList): JSPropertyEnumList
     {.jspropnames.} =
@@ -1482,21 +1491,21 @@ func namedItem(collection: HTMLCollection; s: string): Element {.jsfunc.} =
       return it
   return nil
 
-func getter(ctx: JSContext; collection: HTMLCollection; atom: JSAtom):
-    Opt[Option[Element]] {.jsgetprop.} =
+func getter(ctx: JSContext; collection: HTMLCollection; atom: JSAtom): Element
+    {.jsgetprop.} =
   var u: uint32
   if ctx.fromJS(atom, u).isSome:
-    return ok(option(collection.item(u)))
+    return collection.item(u)
   var s: string
   if ctx.fromJS(atom, s).isSome:
-    return ok(option(collection.namedItem(s)))
-  return err()
+    return collection.namedItem(s)
+  return nil
 
 func names(ctx: JSContext; collection: HTMLCollection): JSPropertyEnumList
     {.jspropnames.} =
   let L = collection.length
   var list = newJSPropertyEnumList(ctx, L)
-  var ids: OrderedSet[CAtom]
+  var ids = initOrderedSet[CAtom]()
   for u in 0 ..< L:
     list.add(u)
     let elem = collection.item(u)
@@ -1509,25 +1518,32 @@ func names(ctx: JSContext; collection: HTMLCollection): JSPropertyEnumList
   return list
 
 # HTMLAllCollection
-proc length(collection: HTMLAllCollection): uint32 {.jsfget.} =
-  return uint32(collection.len)
+proc length(this: HTMLAllCollection): uint32 {.jsfget.} =
+  return uint32(this.len)
 
-func hasprop(collection: HTMLAllCollection; i: uint32): bool {.jshasprop.} =
-  return int(i) < collection.len
+func hasprop(ctx: JSContext; this: HTMLAllCollection; atom: JSAtom): bool
+    {.jshasprop.} =
+  var u: uint32
+  if ctx.fromJS(atom, u).isSome:
+    return int(u) < this.len
+  return false
 
-func item(collection: HTMLAllCollection; i: uint32): Element {.jsfunc.} =
-  let i = int(i)
-  if i < collection.len:
-    return Element(collection.snapshot[i])
+func item(this: HTMLAllCollection; u: uint32): Element {.jsfunc.} =
+  let i = int(u)
+  if i < this.len:
+    return Element(this.snapshot[i])
   return nil
 
-func getter(collection: HTMLAllCollection; i: uint32): Option[Element]
+func getter(ctx: JSContext; this: HTMLAllCollection; atom: JSAtom): Element
     {.jsgetprop.} =
-  return option(collection.item(i))
+  var u: uint32
+  if ctx.fromJS(atom, u).isSome:
+    return this.item(u)
+  return nil
 
-func names(ctx: JSContext; collection: HTMLAllCollection): JSPropertyEnumList
+func names(ctx: JSContext; this: HTMLAllCollection): JSPropertyEnumList
     {.jspropnames.} =
-  let L = collection.length
+  let L = this.length
   var list = newJSPropertyEnumList(ctx, L)
   for u in 0 ..< L:
     list.add(u)
@@ -3822,7 +3838,7 @@ proc fetchClassicScript(element: HTMLScriptElement; url: URL;
   if response.res != 0:
     element.onComplete(ScriptResult(t: srtNull))
     return
-  window.loader.resume(response.outputId)
+  response.resume()
   let s = response.body.recvAll()
   let cs = if cs == CHARSET_UNKNOWN: CHARSET_UTF_8 else: cs
   let source = s.decodeAll(cs)
@@ -4409,7 +4425,7 @@ func getElementReflectFunctions(): seq[TabGetSet] =
     ))
 
 proc getContext*(jctx: JSContext; this: HTMLCanvasElement; contextId: string;
-    options = none(JSValue)): RenderingContext {.jsfunc.} =
+    options = JS_UNDEFINED): RenderingContext {.jsfunc.} =
   if contextId == "2d":
     if this.ctx2d != nil:
       return this.ctx2d
diff --git a/src/html/env.nim b/src/html/env.nim
index 4ecdc08b..ea55dcbc 100644
--- a/src/html/env.nim
+++ b/src/html/env.nim
@@ -71,12 +71,11 @@ proc item(pluginArray: var PluginArray): JSValue {.jsfunc.} = JS_NULL
 proc length(pluginArray: var PluginArray): uint32 {.jsfget.} = 0
 proc item(mimeTypeArray: var MimeTypeArray): JSValue {.jsfunc.} = JS_NULL
 proc length(mimeTypeArray: var MimeTypeArray): uint32 {.jsfget.} = 0
-proc getter(pluginArray: var PluginArray; i: uint32): Option[JSValue]
+proc getter(pluginArray: var PluginArray; atom: JSAtom): JSValue {.jsgetprop.} =
+  return JS_NULL
+proc getter(mimeTypeArray: var MimeTypeArray; atom: JSAtom): JSValue
     {.jsgetprop.} =
-  discard
-proc getter(mimeTypeArray: var MimeTypeArray; i: uint32): Option[JSValue]
-    {.jsgetprop.} =
-  discard
+  return JS_NULL
 
 # Screen
 proc availWidth(screen: var Screen): int64 {.jsfget.} =
@@ -116,8 +115,7 @@ func key(this: var Storage; i: uint32): Option[string] {.jsfunc.} =
     return some(this.map[int(i)].value)
   return none(string)
 
-func getItem(this: var Storage; s: string): Option[string]
-    {.jsfunc.} =
+func getItem(this: var Storage; s: string): Option[string] {.jsfunc.} =
   let i = this.find(s)
   if i != -1:
     return some(this.map[i].value)
diff --git a/src/html/xmlhttprequest.nim b/src/html/xmlhttprequest.nim
index ed1b7035..68039978 100644
--- a/src/html/xmlhttprequest.nim
+++ b/src/html/xmlhttprequest.nim
@@ -2,8 +2,10 @@ import std/options
 import std/strutils
 import std/tables
 
+import chagashi/charset
 import chagashi/decoder
 import html/catom
+import html/chadombuilder
 import html/dom
 import html/event
 import html/script
@@ -19,6 +21,7 @@ import monoucha/javascript
 import monoucha/jserror
 import monoucha/quickjs
 import monoucha/tojs
+import types/blob
 import types/opt
 import types/url
 import utils/twtstr
@@ -56,7 +59,9 @@ type
     response: Response
     responseType {.jsget.}: XMLHttpRequestResponseType
     timeout {.jsget.}: uint32
+    responseObject: JSValue
     received: string
+    contentTypeOverride: string
 
   ProgressEvent = ref object of Event
     lengthComputable {.jsget.}: bool
@@ -77,9 +82,13 @@ func newXMLHttpRequest(): XMLHttpRequest {.jsctor.} =
   let upload = XMLHttpRequestUpload()
   return XMLHttpRequest(
     upload: upload,
-    headers: newHeaders()
+    headers: newHeaders(),
+    responseObject: JS_UNDEFINED
   )
 
+proc finalize(rt: JSRuntime; this: XMLHttpRequest) {.jsfin.} =
+  JS_FreeValueRT(rt, this.responseObject)
+
 proc newProgressEvent(ctype: CAtom; init = ProgressEventInit()): ProgressEvent
     {.jsctor.} =
   let event = ProgressEvent(
@@ -170,6 +179,14 @@ proc setRequestHeader(this: XMLHttpRequest; name, value: string):
   this.headers.table[name.toHeaderCase()] = @[value]
   ok()
 
+proc setTimeout(ctx: JSContext; this: XMLHttpRequest; value: uint32):
+    Err[DOMException] {.jsfset: "timeout".} =
+  if ctx.getWindow() != nil and xhrfSync in this.flags:
+    return errDOMException("timeout may not be set on synchronous XHR",
+      "InvalidAccessError")
+  this.timeout = value
+  ok()
+
 proc fireProgressEvent(window: Window; target: EventTarget; name: StaticAtom;
     loaded, length: int64) =
   let event = newProgressEvent(window.factory.toAtom(name), ProgressEventInit(
@@ -316,6 +333,7 @@ proc send(ctx: JSContext; this: XMLHttpRequest; body = JS_NULL): JSResult[void]
       response.opaque = XHROpaque(this: this, window: window, len: len)
       response.onRead = onReadXHR
       response.onFinish = onFinishXHR
+      response.resume()
       #TODO timeout
     )
   else: # sync
@@ -324,6 +342,7 @@ proc send(ctx: JSContext; this: XMLHttpRequest; body = JS_NULL): JSResult[void]
       let response = window.loader.doRequest(request)
       if response.res == 0:
         #TODO timeout
+        response.resume()
         try:
           this.received = response.body.recvAll()
           #TODO report timing
@@ -352,11 +371,40 @@ proc statusText(this: XMLHttpRequest): string {.jsfget.} =
 
 proc getResponseHeader(ctx: JSContext; this: XMLHttpRequest; name: string):
     JSValue {.jsfunc.} =
-  #TODO this can throw, but I don't think the standard allows that?
-  return ctx.get(this.response.headers, name)
+  let res = ctx.get(this.response.headers, name)
+  if JS_IsException(res):
+    return JS_NULL
+  return res
 
 #TODO getAllResponseHeaders
 
+func getCharset(this: XMLHttpRequest): Charset =
+  let override = this.contentTypeOverride.toLowerAscii()
+  let cs = override.getContentTypeAttr("charset").getCharset()
+  if cs != CHARSET_UNKNOWN:
+    return cs
+  return this.response.getCharset(CHARSET_UTF_8)
+
+proc responseText(ctx: JSContext; this: XMLHttpRequest): JSValue {.jsfget.} =
+  if this.responseType notin {xhrtUnknown, xhrtText}:
+    let ex = newDOMException("response type was expected to be '' or 'text'",
+      "InvalidStateError")
+    return JS_Throw(ctx, ctx.toJS(ex))
+  if this.readyState notin {xhrsLoading, xhrsDone}:
+    return ctx.toJS("")
+  let charset = this.getCharset()
+  #TODO XML encoding stuff?
+  return ctx.toJS(this.received.decodeAll(charset))
+
+proc overrideMimeType(this: XMLHttpRequest; s: string): DOMResult[void]
+    {.jsfunc.} =
+  if this.readyState notin {xhrsLoading, xhrsDone}:
+    return errDOMException("readyState must not be loading or done",
+      "InvalidStateError")
+  #TODO parse
+  this.contentTypeOverride = s
+  return ok()
+
 proc setResponseType(ctx: JSContext; this: XMLHttpRequest;
     value: XMLHttpRequestResponseType): Err[DOMException]
     {.jsfset: "responseType".} =
@@ -372,13 +420,60 @@ proc setResponseType(ctx: JSContext; this: XMLHttpRequest;
   this.responseType = value
   ok()
 
-proc setTimeout(ctx: JSContext; this: XMLHttpRequest; value: uint32):
-    Err[DOMException] {.jsfset: "timeout".} =
-  if ctx.getWindow() != nil and xhrfSync in this.flags:
-    return errDOMException("timeout may not be set on synchronous XHR",
-      "InvalidAccessError")
-  this.timeout = value
-  ok()
+func getContentType(this: XMLHttpRequest): string =
+  if this.contentTypeOverride != "":
+    return this.contentTypeOverride
+  return this.response.getContentType()
+
+proc ptrify(s: var string): pointer =
+  if s.len == 0:
+    return nil
+  var sr = new(string)
+  sr[] = move(s)
+  GC_ref(sr)
+  return cast[pointer](sr)
+
+proc deallocPtrified(p: pointer) =
+  if p != nil:
+    let sr = cast[ref string](p)
+    GC_unref(sr)
+
+proc abufFree(rt: JSRuntime; opaque, p: pointer) {.cdecl.} =
+  deallocPtrified(opaque)
+
+proc blobFree(opaque, p: pointer) {.nimcall.} =
+  deallocPtrified(opaque)
+
+proc response(ctx: JSContext; this: XMLHttpRequest): JSValue {.jsfget.} =
+  if this.responseType in {xhrtText, xhrtUnknown}:
+    return ctx.responseText(this)
+  if this.readyState != xhrsDone:
+    return JS_NULL
+  if JS_IsUndefined(this.responseObject):
+    case this.responseType
+    of xhrtArraybuffer:
+      let len = csize_t(this.received.len)
+      let p = cast[ptr UncheckedArray[uint8]](cstring(this.received))
+      let opaque = this.received.ptrify()
+      this.responseObject = JS_NewArrayBuffer(ctx, p, len, abufFree, opaque,
+        false)
+    of xhrtBlob:
+      let len = this.received.len
+      let p = cast[ptr UncheckedArray[uint8]](cstring(this.received))
+      let opaque = this.received.ptrify()
+      let blob = newBlob(p, len, this.getContentType(), blobFree, opaque)
+      this.responseObject = ctx.toJS(blob)
+    of xhrtDocument:
+      #TODO this is certainly not compliant
+      let res = ctx.parseFromString(newDOMParser(), this.received, "text/html")
+      this.responseObject = ctx.toJS(res)
+    of xhrtJSON:
+      this.responseObject = JS_ParseJSON(ctx, cstring(this.received),
+        csize_t(this.received.len), cstring"<input>")
+    else: discard
+  if JS_IsException(this.responseObject):
+    this.responseObject = JS_UNDEFINED
+  return this.responseObject
 
 # Event reflection