headers, request: bring it closer to the standard

* add standard interfaces to headers
* use window base URL for newRequest
* remove pointless generic in newRequest

7 files changed, 233 insertions, 128 deletions

diff --git a/src/html/dom.nim b/src/html/dom.nim
index 0e3c83ee..60a74d17 100644
--- a/src/html/dom.nim
+++ b/src/html/dom.nim
@@ -2465,6 +2465,9 @@ getFactory = proc(ctx: JSContext): CAtomFactory =
 windowConsoleError = proc(ctx: JSContext; ss: varargs[string]) =
   ctx.getGlobal().console.error(ss)
 
+getAPIBaseURLImpl = func(ctx: JSContext): URL =
+  return ctx.getGlobal().document.baseURL
+
 proc fireEvent*(window: Window; name: StaticAtom; target: EventTarget) =
   let event = newEvent(window.toAtom(name), target)
   discard window.jsctx.dispatch(target, event)
diff --git a/src/html/env.nim b/src/html/env.nim
index 377c2fbf..131d7b4d 100644
--- a/src/html/env.nim
+++ b/src/html/env.nim
@@ -98,12 +98,12 @@ proc addNavigatorModule*(ctx: JSContext) =
   ctx.registerType(MimeTypeArray)
   ctx.registerType(Screen)
 
-proc fetch[T: JSRequest|string](window: Window; input: T;
-    init = none(RequestInit)): JSResult[FetchPromise] {.jsfunc.} =
+proc fetch(window: Window; input: JSValue; init = none(RequestInit)):
+    JSResult[FetchPromise] {.jsfunc.} =
   let input = ?newRequest(window.jsctx, input, init)
   #TODO cors requests?
   if not window.settings.origin.isSameOrigin(input.request.url.origin):
-    let promise = FetchPromise()
+    let promise = newPromise[JSResult[Response]]()
     let err = newTypeError("NetworkError when attempting to fetch resource")
     promise.resolve(JSResult[Response].err(err))
     return ok(promise)
diff --git a/src/html/xmlhttprequest.nim b/src/html/xmlhttprequest.nim
index 1fc2e8d7..8ac94792 100644
--- a/src/html/xmlhttprequest.nim
+++ b/src/html/xmlhttprequest.nim
@@ -152,7 +152,7 @@ proc setRequestHeader(this: XMLHttpRequest; name, value: string):
   ?this.checkSendFlag()
   if not name.isValidHeaderName() or not value.isValidHeaderValue():
     return errDOMException("Invalid header name or value", "SyntaxError")
-  if isForbiddenHeader(name, value):
+  if isForbiddenRequestHeader(name, value):
     return ok()
   this.headers.table[name.toHeaderCase()] = @[value]
   ok()
@@ -167,7 +167,7 @@ proc fireProgressEvent(window: Window; target: EventTarget; name: StaticAtom;
   discard window.jsctx.dispatch(target, event)
 
 # Forward declaration hack
-var windowFetch*: proc(window: Window; input: JSRequest;
+var windowFetch*: proc(window: Window; input: JSValue;
   init = none(RequestInit)): JSResult[FetchPromise] {.nimcall.} = nil
 
 proc errorSteps(window: Window; this: XMLHttpRequest; name: StaticAtom) =
@@ -236,7 +236,9 @@ proc send(ctx: JSContext; this: XMLHttpRequest; body = JS_NULL): DOMResult[void]
   let window = ctx.getWindow()
   if xhrfSync notin this.flags: # async
     window.fireProgressEvent(this, satLoadstart, 0, 0)
-    let p = window.windowFetch(jsRequest)
+    let v = ctx.toJS(jsRequest)
+    let p = window.windowFetch(v)
+    JS_FreeValue(ctx, v)
     if p.isSome:
       p.get.then(proc(res: JSResult[Response]) =
         if res.isNone:
diff --git a/src/loader/headers.nim b/src/loader/headers.nim
index cd69251c..bed8f8e8 100644
--- a/src/loader/headers.nim
+++ b/src/loader/headers.nim
@@ -9,59 +9,215 @@ import types/opt
 import utils/twtstr
 
 type
+  HeaderGuard* = enum
+    hgNone = "none"
+    hgImmutable = "immutable"
+    hgRequest = "request"
+    hgRequestNoCors = "request-no-cors"
+    hgResponse = "response"
+
   Headers* = ref object
-    table* {.jsget.}: Table[string, seq[string]]
+    table*: Table[string, seq[string]]
+    guard*: HeaderGuard
 
   HeadersInitType = enum
-    HEADERS_INIT_SEQUENCE, HEADERS_INIT_TABLE
+    hitSequence, hitTable
 
   HeadersInit* = object
     case t: HeadersInitType
-    of HEADERS_INIT_SEQUENCE:
+    of hitSequence:
       s: seq[(string, string)]
-    of HEADERS_INIT_TABLE:
+    of hitTable:
       tab: Table[string, string]
 
 jsDestructor(Headers)
 
+const HTTPWhitespace = {'\n', '\r', '\t', ' '}
+
 proc fromJSHeadersInit(ctx: JSContext; val: JSValue): JSResult[HeadersInit] =
   if JS_IsUndefined(val) or JS_IsNull(val):
     return err(nil)
-  if isSequence(ctx, val):
+  if (let x = fromJS[Headers](ctx, val); x.isSome):
+    var s: seq[(string, string)] = @[]
+    for k, v in x.get.table:
+      for vv in v:
+        s.add((k, vv))
+    return ok(HeadersInit(t: hitSequence, s: s))
+  if ctx.isSequence(val):
     let x = fromJS[seq[(string, string)]](ctx, val)
     if x.isSome:
-      return ok(HeadersInit(t: HEADERS_INIT_SEQUENCE, s: x.get))
+      return ok(HeadersInit(t: hitSequence, s: x.get))
   let x = ?fromJS[Table[string, string]](ctx, val)
-  return ok(HeadersInit(t: HEADERS_INIT_TABLE, tab: x))
+  return ok(HeadersInit(t: hitTable, tab: x))
 
-proc fill*(headers: Headers; s: seq[(string, string)]) =
-  for (k, v) in s:
-    if k in headers.table:
-      headers.table[k].add(v)
+const TokenChars = {
+  '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~'
+} + AsciiAlphaNumeric
+
+func isValidHeaderName*(s: string): bool =
+  return s.len > 0 and AllChars - TokenChars notin s
+
+func isValidHeaderValue*(s: string): bool =
+  return s.len == 0 or s[0] notin {' ', '\t'} and s[^1] notin {' ', '\t'} and
+    '\n' notin s
+
+func isForbiddenRequestHeader*(name, value: string): bool =
+  const ForbiddenNames = [
+    "Accept-Charset",
+    "Accept-Encoding",
+    "Access-Control-Request-Headers",
+    "Access-Control-Request-Method",
+    "Connection",
+    "Content-Length",
+    "Cookie",
+    "Cookie2",
+    "Date",
+    "DNT",
+    "Expect",
+    "Host",
+    "Keep-Alive",
+    "Origin",
+    "Referer",
+    "Set-Cookie",
+    "TE",
+    "Trailer",
+    "Transfer-Encoding",
+    "Upgrade",
+    "Via"
+  ]
+  for x in ForbiddenNames:
+    if name.equalsIgnoreCase(x):
+      return true
+  if name.startsWithIgnoreCase("proxy-") or name.startsWithIgnoreCase("sec-"):
+    return true
+  if name.equalsIgnoreCase("X-HTTP-Method") or
+      name.equalsIgnoreCase("X-HTTP-Method-Override") or
+      name.equalsIgnoreCase("X-Method-Override"):
+    return true # meh
+  return false
+
+func isForbiddenResponseHeaderName*(name: string): bool =
+  return name in ["Set-Cookie", "Set-Cookie2"]
+
+proc validate(this: Headers; name, value: string): JSResult[bool] =
+  if not name.isValidHeaderName() or not value.isValidHeaderValue():
+    return errTypeError("Invalid header name or value")
+  if this.guard == hgImmutable:
+    return errTypeError("Tried to modify immutable Headers object")
+  if this.guard == hgRequest and isForbiddenRequestHeader(name, value):
+    return ok(false)
+  if this.guard == hgResponse and name.isForbiddenResponseHeaderName():
+    return ok(false)
+  return ok(true)
+
+func isNoCorsSafelistedName(name: string): bool =
+  return name.equalsIgnoreCase("Accept") or
+    name.equalsIgnoreCase("Accept-Language") or
+    name.equalsIgnoreCase("Content-Language") or
+    name.equalsIgnoreCase("Content-Type")
+
+
+const CorsUnsafeRequestByte = {
+  char(0x00)..char(0x08), char(0x10)..char(0x1F), '"', '(', ')', ':', '<', '>',
+  '?', '@', '[', '\\', ']', '{', '}', '\e'
+}
+
+func isNoCorsSafelisted(name, value: string): bool =
+  if value.len > 128:
+    return false
+  if name.equalsIgnoreCase("Accept"):
+    return CorsUnsafeRequestByte notin value
+  if name.equalsIgnoreCase("Accept-Language") or
+      name.equalsIgnoreCase("Content-Language"):
+    const Forbidden = AllChars - AsciiAlphaNumeric -
+      {' ', '*', ',', '-', '.', ';', '='}
+    return Forbidden notin value
+  if name.equalsIgnoreCase("Content-Type"):
+    return value.strip(chars = AsciiWhitespace).toLowerAscii() in [
+      "multipart/form-data",
+      "text/plain",
+      "application-x-www-form-urlencoded"
+    ]
+  return false
+
+func get0(this: Headers; name: string): string =
+  return this.table[name].join(", ")
+
+proc get(this: Headers; name: string): JSResult[Option[string]] {.jsfunc.} =
+  if not name.isValidHeaderName():
+    return errTypeError("Invalid header name")
+  if name notin this.table:
+    return ok(none(string))
+  return ok(some(this.get0(name)))
+
+proc removeRange(this: Headers) =
+  if this.guard == hgRequestNoCors:
+    #TODO do this case insensitively
+    this.table.del("Range") # privileged no-CORS request headers
+    this.table.del("range")
+
+proc append(this: Headers; name, value: string): JSResult[void] {.jsfunc.} =
+  let value = value.strip(chars = HTTPWhitespace)
+  if not ?this.validate(name, value):
+    return ok()
+  if this.guard == hgRequestNoCors:
+    if name in this.table:
+      let tmp = this.get0(name) & ", " & value
+      if not name.isNoCorsSafelisted(tmp):
+        return ok()
+      this.table[name].add(value)
     else:
-      headers.table[k] = @[v]
+      this.table[name] = @[value]
+  this.removeRange()
+  ok()
+
+proc delete(this: Headers; name: string): JSResult[void] {.jsfunc.} =
+  if not ?this.validate(name, "") or name notin this.table:
+    return ok()
+  if not name.isNoCorsSafelistedName() and not name.equalsIgnoreCase("Range"):
+    return ok()
+  this.table.del(name)
+  this.removeRange()
+  ok()
+
+proc has(this: Headers; name: string): JSResult[bool] {.jsfunc.} =
+  if not name.isValidHeaderName():
+    return errTypeError("Invalid header name")
+  return ok(name in this.table)
+
+proc set(this: Headers; name, value: string): JSResult[void] {.jsfunc.} =
+  let value = value.strip(chars = HTTPWhitespace)
+  if not ?this.validate(name, value):
+    return
+  if this.guard == hgRequestNoCors and not name.isNoCorsSafelisted(value):
+    return
+  #TODO do this case insensitively
+  this.table[name] = @[value]
+  this.removeRange()
+
+proc fill(headers: Headers; s: seq[(string, string)]): JSResult[void] =
+  for (k, v) in s:
+    ?headers.append(k, v)
+  ok()
 
-proc fill*(headers: Headers; tab: Table[string, string]) =
+proc fill(headers: Headers; tab: Table[string, string]): JSResult[void] =
   for k, v in tab:
-    if k in headers.table:
-      headers.table[k].add(v)
-    else:
-      headers.table[k] = @[v]
+    ?headers.append(k, v)
+  ok()
 
-proc fill*(headers: Headers; init: HeadersInit) =
-  if init.t == HEADERS_INIT_SEQUENCE:
-    headers.fill(init.s)
-  else: # table
-    headers.fill(init.tab)
+proc fill*(headers: Headers; init: HeadersInit): JSResult[void] =
+  case init.t
+  of hitSequence: return headers.fill(init.s)
+  of hitTable: return headers.fill(init.tab)
 
-func newHeaders*(): Headers =
-  return Headers()
+func newHeaders*(guard = hgNone): Headers =
+  return Headers(guard: guard)
 
-func newHeaders(obj = none(HeadersInit)): Headers {.jsctor.} =
-  let headers = Headers()
+func newHeaders(obj = none(HeadersInit)): JSResult[Headers] {.jsctor.} =
+  let headers = Headers(guard: hgNone)
   if obj.isSome:
-    headers.fill(obj.get)
-  return headers
+    ?headers.fill(obj.get)
+  return ok(headers)
 
 func newHeaders*(table: openArray[(string, string)]): Headers =
   let headers = Headers()
@@ -84,9 +240,7 @@ func newHeaders*(table: Table[string, string]): Headers =
   return headers
 
 func clone*(headers: Headers): Headers =
-  return Headers(
-    table: headers.table
-  )
+  return Headers(table: headers.table)
 
 proc add*(headers: Headers; k, v: string) =
   let k = k.toHeaderCase()
@@ -114,51 +268,5 @@ func getOrDefault*(headers: Headers; k: static string; default = ""): string =
   do:
     return default
 
-const TokenChars = {
-  '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~'
-} + AsciiAlphaNumeric
-
-#TODO maybe assert these are valid on insertion?
-func isValidHeaderName*(s: string): bool =
-  return s.len > 0 and AllChars - TokenChars notin s
-
-func isValidHeaderValue*(s: string): bool =
-  return s.len == 0 or s[0] notin {' ', '\t'} and s[^1] notin {' ', '\t'} and
-    '\n' notin s
-
-func isForbiddenHeader*(name, value: string): bool =
-  const ForbiddenNames = [
-    "Accept-Charset",
-    "Accept-Encoding",
-    "Access-Control-Request-Headers",
-    "Access-Control-Request-Method",
-    "Connection",
-    "Content-Length",
-    "Cookie",
-    "Cookie2",
-    "Date",
-    "DNT",
-    "Expect",
-    "Host",
-    "Keep-Alive",
-    "Origin",
-    "Referer",
-    "Set-Cookie",
-    "TE",
-    "Trailer",
-    "Transfer-Encoding",
-    "Upgrade",
-    "Via",
-  ]
-  if name in ForbiddenNames:
-    return true
-  if name.startsWith("proxy-") or name.startsWith("sec-"):
-    return true
-  if name.equalsIgnoreCase("X-HTTP-Method") or
-      name.equalsIgnoreCase("X-HTTP-Method-Override") or
-      name.equalsIgnoreCase("X-Method-Override"):
-    return true # meh
-  return false
-
 proc addHeadersModule*(ctx: JSContext) =
   ctx.registerType(Headers)
diff --git a/src/loader/request.nim b/src/loader/request.nim
index f92098cb..23be8ba8 100644
--- a/src/loader/request.nim
+++ b/src/loader/request.nim
@@ -1,5 +1,4 @@
 import std/options
-import std/strutils
 import std/tables
 
 import html/script
@@ -169,7 +168,6 @@ type
     referrer: Option[string]
     referrerPolicy: Option[ReferrerPolicy]
     credentials: Option[CredentialsMode]
-    proxyUrl: URL
     mode: Option[RequestMode]
     window: Option[JSValue]
 
@@ -194,33 +192,36 @@ proc fromJSBodyInit(ctx: JSContext; val: JSValue): JSResult[BodyInit] =
       return ok(BodyInit(t: bitString, str: x.get))
   return errTypeError("Invalid body init type")
 
-func newRequest*[T: string|JSRequest](ctx: JSContext; resource: T;
-    init = none(RequestInit)): JSResult[JSRequest] {.jsctor.} =
+var getAPIBaseURLImpl*: proc(ctx: JSContext): URL {.noSideEffect, nimcall.}
+
+proc newRequest*(ctx: JSContext; resource: JSValue; init = none(RequestInit)):
+    JSResult[JSRequest] {.jsctor.} =
   defer:
     if init.isSome and init.get.window.isSome:
       JS_FreeValue(ctx, init.get.window.get)
-  when T is string:
-    let url = ?newURL(resource)
-    if url.username != "" or url.password != "":
-      return errTypeError("Input URL contains a username or password")
-    var httpMethod = hmGet
-    var headers = newHeaders()
-    let referrer: URL = nil
-    var credentials = cmSameOrigin
-    var body = RequestBody()
-    var proxyUrl: URL #TODO?
-    let fallbackMode = opt(rmCors)
-    var window = RequestWindow(t: rwtClient)
+  let headers = newHeaders(hgRequest)
+  var fallbackMode = opt(rmCors)
+  var window = RequestWindow(t: rwtClient)
+  var body = RequestBody()
+  var credentials = cmSameOrigin
+  var httpMethod = hmGet
+  var referrer: URL = nil
+  var url: URL = nil
+  if JS_IsString(resource):
+    let s = ?fromJS[string](ctx, resource)
+    url = ?parseJSURL(s, option(ctx.getAPIBaseURLImpl()))
   else:
-    let url = resource.url
-    var httpMethod = resource.request.httpMethod
-    var headers = resource.headers.clone()
-    let referrer = resource.request.referrer
-    var credentials = resource.credentialsMode
-    var body = resource.request.body
-    var proxyUrl = resource.request.proxy #TODO?
-    let fallbackMode = none(RequestMode)
-    var window = resource.window
+    let resource = ?fromJS[JSRequest](ctx, resource)
+    url = resource.url
+    httpMethod = resource.request.httpMethod
+    headers.table = resource.headers.table
+    referrer = resource.request.referrer
+    credentials = resource.credentialsMode
+    body = resource.request.body
+    fallbackMode = opt(RequestMode)
+    window = resource.window
+  if url.username != "" or url.password != "":
+    return errTypeError("Input URL contains a username or password")
   var mode = fallbackMode.get(rmNoCors)
   let destination = rdNone
   #TODO origin, window
@@ -246,20 +247,19 @@ func newRequest*[T: string|JSRequest](ctx: JSContext; resource: T;
       if httpMethod in {hmGet, hmHead}:
         return errTypeError("HEAD or GET Request cannot have a body.")
     if init.headers.isSome:
-      headers.fill(init.headers.get)
+      ?headers.fill(init.headers.get)
     if init.credentials.isSome:
       credentials = init.credentials.get
     if init.mode.isSome:
       mode = init.mode.get
-    #TODO find a standard compatible way to implement this
-    proxyUrl = init.proxyUrl
+  if mode == rmNoCors:
+    headers.guard = hgRequestNoCors
   return ok(JSRequest(
     request: newRequest(
       url,
       httpMethod,
       headers,
       body,
-      proxy = proxyUrl,
       referrer = referrer
     ),
     mode: mode,
diff --git a/src/loader/response.nim b/src/loader/response.nim
index 9c173188..bbf41741 100644
--- a/src/loader/response.nim
+++ b/src/loader/response.nim
@@ -28,14 +28,6 @@ type
     rtOpaque = "opaque"
     rtOpaquedirect = "opaqueredirect"
 
-  #TODO fully implement headers guards
-  HeadersGuard* = enum
-    hgImmutable = "immutable"
-    hgRequest = "request"
-    hgRequestNoCors = "request-no-cors"
-    hgResponse = "response"
-    hgNone = "none"
-
   ResponseFlag* = enum
     rfAborted
 
@@ -46,7 +38,6 @@ type
     bodyUsed* {.jsget.}: bool
     status* {.jsget.}: uint16
     headers* {.jsget.}: Headers
-    headersGuard: HeadersGuard
     url*: URL #TODO should be urllist?
     unregisterFun*: proc()
     resumeFun*: proc(outputId: int)
@@ -72,13 +63,11 @@ proc newResponse*(res: int; request: Request; stream: SocketStream;
 
 func makeNetworkError*(): Response {.jsstfunc: "Response.error".} =
   #TODO use "create" function
-  #TODO headers immutable
   return Response(
     res: 0,
     responseType: rtError,
     status: 0,
-    headers: newHeaders(),
-    headersGuard: hgImmutable,
+    headers: newHeaders(hgImmutable),
     bodyUsed: true
   )
 
diff --git a/src/types/url.nim b/src/types/url.nim
index 1ec2eca0..23485602 100644
--- a/src/types/url.nim
+++ b/src/types/url.nim
@@ -914,6 +914,12 @@ proc parseURL*(input: string; base = none(URL); override = none(URLState)):
     url.get.blob = some(BlobURLEntry())
   return url
 
+proc parseJSURL*(s: string; base = none(URL)): JSResult[URL] =
+  let url = parseURL(s, base)
+  if url.isNone:
+    return errTypeError(s & " is not a valid URL")
+  return ok(url.get)
+
 func serializeip(ipv4: uint32): string =
   var n = ipv4
   for i in 1..4:
@@ -1168,10 +1174,7 @@ proc parseAPIURL(s: string; base: Option[string]): JSResult[URL] =
     x
   else:
     none(URL)
-  let url = parseURL(s, baseURL)
-  if url.isNone:
-    return errTypeError(s & " is not a valid URL")
-  return ok(url.get)
+  return parseJSURL(s, baseURL)
 
 proc newURL*(s: string; base: Option[string] = none(string)):
     JSResult[URL] {.jsctor.} =