dom, xhr: slight progress on modules, fix an XHR bug

* actually download & compile modules (but don't run them yet)
* fix a bug in XHR (on some older Nim versions, move() doesn't
  actually move)

5 files changed, 118 insertions, 59 deletions

diff --git a/src/html/dom.nim b/src/html/dom.nim
index 7bd3ff9a..707c35f5 100644
--- a/src/html/dom.nim
+++ b/src/html/dom.nim
@@ -903,6 +903,21 @@ func baseURL*(document: Document): URL
 proc delAttr(element: Element; i: int; keep = false)
 proc reflectAttr(element: Element; name: CAtom; value: Option[string])
 
+# Forward declaration hacks
+# set in css/cascade
+var appliesFwdDecl*: proc(mqlist: MediaQueryList; window: Window): bool
+  {.nimcall, noSideEffect.}
+# set in css/match
+var doqsa*: proc (node: Node; q: string): seq[Element] {.nimcall.} = nil
+var doqs*: proc (node: Node; q: string): Element {.nimcall.} = nil
+# set in html/chadombuilder
+var domParseHTMLFragment*: proc(element: Element; s: string): seq[Node]
+  {.nimcall.}
+# set in html/env
+var windowFetch*: proc(window: Window; input: JSValue;
+  init = RequestInit(window: JS_UNDEFINED)): JSResult[FetchPromise]
+  {.nimcall.} = nil
+
 # For now, these are the same; on an API level however, getGlobal is guaranteed
 # to be non-null, while getWindow may return null in the future. (This is in
 # preparation for Worker support.)
@@ -2992,10 +3007,6 @@ proc style*(element: Element): CSSStyleDeclaration {.jsfget.} =
     element.cachedStyle = CSSStyleDeclaration(element: element)
   return element.cachedStyle
 
-# Forward declaration hack
-var appliesFwdDecl*: proc(mqlist: MediaQueryList; window: Window): bool
-  {.nimcall, noSideEffect.}
-
 # see https://html.spec.whatwg.org/multipage/links.html#link-type-stylesheet
 #TODO make this somewhat compliant with ^this
 proc loadResource(window: Window; link: HTMLLinkElement) =
@@ -3811,17 +3822,6 @@ proc markAsReady(element: HTMLScriptElement; res: ScriptResult) =
     element.onReady = nil
   element.delayingTheLoadEvent = false
 
-proc createClassicScript(ctx: JSContext; source: string; baseURL: URL;
-    options: ScriptOptions; mutedErrors = false): Script =
-  let urls = baseURL.serialize(excludepassword = true)
-  let record = compileScript(ctx, source, urls)
-  return Script(
-    record: record,
-    baseURL: baseURL,
-    options: options,
-    mutedErrors: mutedErrors
-  )
-
 type OnCompleteProc = proc(element: HTMLScriptElement, res: ScriptResult)
 
 proc fetchClassicScript(element: HTMLScriptElement; url: URL;
@@ -3843,8 +3843,8 @@ proc fetchClassicScript(element: HTMLScriptElement; url: URL;
   let cs = if cs == CHARSET_UNKNOWN: CHARSET_UTF_8 else: cs
   let source = s.decodeAll(cs)
   response.body.sclose()
-  let script = window.jsctx.createClassicScript(source, url, options, false)
-  element.onComplete(ScriptResult(t: srtScript, script: script))
+  let script = window.jsctx.newClassicScript(source, url, options, false)
+  element.onComplete(script)
 
 #TODO settings object
 proc fetchDescendantsAndLink(element: HTMLScriptElement; script: Script;
@@ -3883,10 +3883,10 @@ proc fetchSingleModule(element: HTMLScriptElement; url: URL;
     destination: RequestDestination; options: ScriptOptions,
     referrer: URL; isTopLevel: bool; onComplete: OnCompleteProc) =
   discard #TODO implement
-  #[
   let moduleType = "javascript"
   #TODO moduleRequest
-  let settings = element.document.window.settings
+  let window = element.document.window
+  let settings = window.settings
   let i = settings.moduleMap.find(url, moduleType)
   if i != -1:
     if settings.moduleMap[i].value.t == srtFetching:
@@ -3894,21 +3894,55 @@ proc fetchSingleModule(element: HTMLScriptElement; url: URL;
       assert false
     element.onComplete(settings.moduleMap[i].value)
     return
-  let destination = fetchDestinationFromModuleType(destination, moduleType)
+  let destination = moduleType.moduleTypeToRequestDest(destination)
   let mode = if destination in {rdWorker, rdSharedworker, rdServiceworker}:
     rmSameOrigin
   else:
     rmCors
   #TODO client
   #TODO initiator type
-  let request = newRequest(
-    url,
-    mode = mode,
-    referrer = referrer,
-    destination = destination
+  let request = JSRequest(
+    request: newRequest(
+      url,
+      referrer = referrer,
+    ),
+    destination: destination,
+    mode: mode
   )
-  discard request #TODO
-  ]#
+  #TODO set up module script request
+  #TODO performFetch
+  let ctx = window.jsctx
+  let v = ctx.toJS(request)
+  let p = window.windowFetch(v)
+  JS_FreeValue(ctx, v)
+  if p.isSome:
+    p.get.then(proc(res: JSResult[Response]) =
+      if res.isNone:
+        let res = ScriptResult(t: srtNull)
+        settings.moduleMap.set(url, moduleType, res, ctx)
+        element.onComplete(res)
+        return
+      let res = res.get
+      let contentType = res.getContentType()
+      let referrerPolicy = res.getReferrerPolicy()
+      res.text().then(proc(s: JSResult[string]) =
+        if s.isNone:
+          let res = ScriptResult(t: srtNull)
+          settings.moduleMap.set(url, moduleType, res, ctx)
+          element.onComplete(res)
+          return
+        if contentType.isJavaScriptType():
+          let res = ctx.newJSModuleScript(s.get, element.document.baseURL,
+            options)
+          if referrerPolicy.isSome:
+            res.script.options.referrerPolicy = referrerPolicy
+          settings.moduleMap.set(url, moduleType, res, ctx)
+          element.onComplete(res)
+        else:
+          #TODO non-JS modules
+          discard
+      )
+    )
 
 proc execute*(element: HTMLScriptElement) =
   let document = element.document
@@ -4042,8 +4076,8 @@ proc prepare*(element: HTMLScriptElement) =
     let baseURL = element.document.baseURL
     if element.ctype == stClassic:
       let ctx = element.document.window.jsctx
-      let script = ctx.createClassicScript(sourceText, baseURL, options)
-      element.markAsReady(ScriptResult(t: srtScript, script: script))
+      let script = ctx.newClassicScript(sourceText, baseURL, options)
+      element.markAsReady(script)
     else:
       #TODO stModule, stImportMap
       element.markAsReady(ScriptResult(t: srtNull))
@@ -4290,10 +4324,6 @@ func isEqualNode(node, other: Node): bool {.jsfunc.} =
 func isSameNode(node, other: Node): bool {.jsfunc.} =
   return node == other
 
-# Forward declaration hack (these are set in selectors.nim)
-var doqsa*: proc (node: Node; q: string): seq[Element] {.nimcall.} = nil
-var doqs*: proc (node: Node; q: string): Element {.nimcall.} = nil
-
 proc querySelectorAll(node: Node; q: string): seq[Element] {.jsfunc.} =
   return doqsa(node, q)
 
@@ -4498,10 +4528,6 @@ proc toBlob(ctx: JSContext; this: HTMLCanvasElement; callback: JSValue;
   )
   return JS_UNDEFINED
 
-# Forward declaration hack
-var domParseHTMLFragment*: proc(element: Element; s: string): seq[Node]
-  {.nimcall.}
-
 # https://w3c.github.io/DOM-Parsing/#dfn-fragment-parsing-algorithm
 proc fragmentParsingAlgorithm*(element: Element; s: string): DocumentFragment =
   #TODO xml
diff --git a/src/html/script.nim b/src/html/script.nim
index ab445516..1cc80c5d 100644
--- a/src/html/script.nim
+++ b/src/html/script.nim
@@ -46,7 +46,7 @@ type
     moduleMap*: ModuleMap
     origin*: Origin
 
-  Script* = object
+  Script* = ref object
     #TODO setings
     baseURL*: URL
     options*: ScriptOptions
@@ -77,6 +77,10 @@ type
 
   ModuleMap* = seq[ModuleMapEntry]
 
+# Forward declaration hack
+# set in html/dom
+var windowConsoleError*: proc(ctx: JSContext; ss: varargs[string]) {.nimcall.}
+
 proc find*(moduleMap: ModuleMap; url: URL; moduleType: string): int =
   let surl = $url
   for i, entry in moduleMap:
@@ -84,15 +88,50 @@ proc find*(moduleMap: ModuleMap; url: URL; moduleType: string): int =
       return i
   return -1
 
-func fetchDestinationFromModuleType*(default: RequestDestination;
-    moduleType: string): RequestDestination =
+proc set*(moduleMap: var ModuleMap; url: URL; moduleType: string;
+    value: ScriptResult; ctx: JSContext) =
+  let i = moduleMap.find(url, moduleType)
+  if i != -1:
+    if moduleMap[i].value.t == srtScript:
+      JS_FreeValue(ctx, moduleMap[i].value.script.record)
+    moduleMap[i].value = value
+  else:
+    moduleMap.add(ModuleMapEntry(key: ($url, moduleType), value: value))
+
+func moduleTypeToRequestDest*(moduleType: string; default: RequestDestination):
+    RequestDestination =
   if moduleType == "json":
     return rdJson
   if moduleType == "css":
     return rdStyle
   return default
 
-var windowConsoleError*: proc(ctx: JSContext; ss: varargs[string]) {.nimcall.}
+proc newClassicScript*(ctx: JSContext; source: string; baseURL: URL;
+    options: ScriptOptions; mutedErrors = false): ScriptResult =
+  let urls = baseURL.serialize(excludepassword = true)
+  let record = ctx.compileScript(source, urls)
+  return ScriptResult(
+    t: srtScript,
+    script: Script(
+      record: record,
+      baseURL: baseURL,
+      options: options,
+      mutedErrors: mutedErrors
+    )
+  )
+
+proc newJSModuleScript*(ctx: JSContext; source: string; baseURL: URL;
+    options: ScriptOptions): ScriptResult =
+  let urls = baseURL.serialize(excludepassword = true)
+  let record = ctx.compileModule(source, urls)
+  return ScriptResult(
+    t: srtScript,
+    script: Script(
+      record: record,
+      baseURL: baseURL,
+      options: options
+    )
+  )
 
 proc logException*(ctx: JSContext) =
   windowConsoleError(ctx, ctx.getExceptionMsg())
diff --git a/src/html/xmlhttprequest.nim b/src/html/xmlhttprequest.nim
index 68039978..cffd2357 100644
--- a/src/html/xmlhttprequest.nim
+++ b/src/html/xmlhttprequest.nim
@@ -196,11 +196,6 @@ proc fireProgressEvent(window: Window; target: EventTarget; name: StaticAtom;
   ))
   discard window.jsctx.dispatch(target, event)
 
-# Forward declaration hack
-var windowFetch*: proc(window: Window; input: JSValue;
-  init = RequestInit(window: JS_UNDEFINED)): JSResult[FetchPromise]
-  {.nimcall.} = nil
-
 proc errorSteps(window: Window; this: XMLHttpRequest; name: StaticAtom) =
   this.readyState = xhrsDone
   this.response = makeNetworkError()
@@ -425,13 +420,14 @@ func getContentType(this: XMLHttpRequest): string =
     return this.contentTypeOverride
   return this.response.getContentType()
 
-proc ptrify(s: var string): pointer =
+proc ptrify(s: var string):
+    tuple[opaque: pointer; p: ptr UncheckedArray[uint8]] =
   if s.len == 0:
-    return nil
+    return (nil, nil)
   var sr = new(string)
   sr[] = move(s)
   GC_ref(sr)
-  return cast[pointer](sr)
+  return (cast[pointer](sr), cast[ptr UncheckedArray[uint8]](addr sr[0]))
 
 proc deallocPtrified(p: pointer) =
   if p != nil:
@@ -453,14 +449,12 @@ proc response(ctx: JSContext; this: XMLHttpRequest): JSValue {.jsfget.} =
     case this.responseType
     of xhrtArraybuffer:
       let len = csize_t(this.received.len)
-      let p = cast[ptr UncheckedArray[uint8]](cstring(this.received))
-      let opaque = this.received.ptrify()
+      let (opaque, p) = this.received.ptrify()
       this.responseObject = JS_NewArrayBuffer(ctx, p, len, abufFree, opaque,
         false)
     of xhrtBlob:
       let len = this.received.len
-      let p = cast[ptr UncheckedArray[uint8]](cstring(this.received))
-      let opaque = this.received.ptrify()
+      let (opaque, p) = this.received.ptrify()
       let blob = newBlob(p, len, this.getContentType(), blobFree, opaque)
       this.responseObject = ctx.toJS(blob)
     of xhrtDocument:
diff --git a/src/loader/response.nim b/src/loader/response.nim
index 6463080a..d41f62eb 100644
--- a/src/loader/response.nim
+++ b/src/loader/response.nim
@@ -15,6 +15,7 @@ import monoucha/tojs
 import types/blob
 import types/color
 import types/opt
+import types/referrer
 import types/url
 import utils/mimeguess
 import utils/twtstr
@@ -112,6 +113,10 @@ func getContentLength*(this: Response): int64 =
         return int64(u.get)
   return -1
 
+func getReferrerPolicy*(this: Response): Option[ReferrerPolicy] =
+  this.headers.table.withValue("Referrer-Policy", p):
+    return strictParseEnum[ReferrerPolicy](p[][0])
+
 type TextOpaque = ref object of RootObj
   buf: string
   bodyRead: Promise[JSResult[string]]
diff --git a/src/local/container.nim b/src/local/container.nim
index a01d5e3b..a21af3af 100644
--- a/src/local/container.nim
+++ b/src/local/container.nim
@@ -1740,11 +1740,6 @@ proc extractCookies(response: Response): seq[Cookie] =
       if cookie.isSome:
         result.add(cookie.get)
 
-proc extractReferrerPolicy(response: Response): Option[ReferrerPolicy] =
-  if "Referrer-Policy" in response.headers:
-    return strictParseEnum[ReferrerPolicy](response.headers["Referrer-Policy"])
-  return none(ReferrerPolicy)
-
 # Apply data received in response.
 # Note: pager must call this before checkMailcap.
 proc applyResponse*(container: Container; response: Response;
@@ -1755,7 +1750,7 @@ proc applyResponse*(container: Container; response: Response;
   if cookieJar != nil:
     cookieJar.add(response.extractCookies())
   # set referrer policy, if any
-  let referrerPolicy = response.extractReferrerPolicy()
+  let referrerPolicy = response.getReferrerPolicy()
   if container.config.refererFrom:
     if referrerPolicy.isSome:
       container.loaderConfig.referrerPolicy = referrerPolicy.get