headers, request: bring it closer to the standard

* add standard interfaces to headers
* use window base URL for newRequest
* remove pointless generic in newRequest

3 files changed, 215 insertions, 118 deletions

diff --git a/src/loader/headers.nim b/src/loader/headers.nim
index cd69251c..bed8f8e8 100644
--- a/src/loader/headers.nim
+++ b/src/loader/headers.nim
@@ -9,59 +9,215 @@ import types/opt
 import utils/twtstr
 
 type
+  HeaderGuard* = enum
+    hgNone = "none"
+    hgImmutable = "immutable"
+    hgRequest = "request"
+    hgRequestNoCors = "request-no-cors"
+    hgResponse = "response"
+
   Headers* = ref object
-    table* {.jsget.}: Table[string, seq[string]]
+    table*: Table[string, seq[string]]
+    guard*: HeaderGuard
 
   HeadersInitType = enum
-    HEADERS_INIT_SEQUENCE, HEADERS_INIT_TABLE
+    hitSequence, hitTable
 
   HeadersInit* = object
     case t: HeadersInitType
-    of HEADERS_INIT_SEQUENCE:
+    of hitSequence:
       s: seq[(string, string)]
-    of HEADERS_INIT_TABLE:
+    of hitTable:
       tab: Table[string, string]
 
 jsDestructor(Headers)
 
+const HTTPWhitespace = {'\n', '\r', '\t', ' '}
+
 proc fromJSHeadersInit(ctx: JSContext; val: JSValue): JSResult[HeadersInit] =
   if JS_IsUndefined(val) or JS_IsNull(val):
     return err(nil)
-  if isSequence(ctx, val):
+  if (let x = fromJS[Headers](ctx, val); x.isSome):
+    var s: seq[(string, string)] = @[]
+    for k, v in x.get.table:
+      for vv in v:
+        s.add((k, vv))
+    return ok(HeadersInit(t: hitSequence, s: s))
+  if ctx.isSequence(val):
     let x = fromJS[seq[(string, string)]](ctx, val)
     if x.isSome:
-      return ok(HeadersInit(t: HEADERS_INIT_SEQUENCE, s: x.get))
+      return ok(HeadersInit(t: hitSequence, s: x.get))
   let x = ?fromJS[Table[string, string]](ctx, val)
-  return ok(HeadersInit(t: HEADERS_INIT_TABLE, tab: x))
+  return ok(HeadersInit(t: hitTable, tab: x))
 
-proc fill*(headers: Headers; s: seq[(string, string)]) =
-  for (k, v) in s:
-    if k in headers.table:
-      headers.table[k].add(v)
+const TokenChars = {
+  '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~'
+} + AsciiAlphaNumeric
+
+func isValidHeaderName*(s: string): bool =
+  return s.len > 0 and AllChars - TokenChars notin s
+
+func isValidHeaderValue*(s: string): bool =
+  return s.len == 0 or s[0] notin {' ', '\t'} and s[^1] notin {' ', '\t'} and
+    '\n' notin s
+
+func isForbiddenRequestHeader*(name, value: string): bool =
+  const ForbiddenNames = [
+    "Accept-Charset",
+    "Accept-Encoding",
+    "Access-Control-Request-Headers",
+    "Access-Control-Request-Method",
+    "Connection",
+    "Content-Length",
+    "Cookie",
+    "Cookie2",
+    "Date",
+    "DNT",
+    "Expect",
+    "Host",
+    "Keep-Alive",
+    "Origin",
+    "Referer",
+    "Set-Cookie",
+    "TE",
+    "Trailer",
+    "Transfer-Encoding",
+    "Upgrade",
+    "Via"
+  ]
+  for x in ForbiddenNames:
+    if name.equalsIgnoreCase(x):
+      return true
+  if name.startsWithIgnoreCase("proxy-") or name.startsWithIgnoreCase("sec-"):
+    return true
+  if name.equalsIgnoreCase("X-HTTP-Method") or
+      name.equalsIgnoreCase("X-HTTP-Method-Override") or
+      name.equalsIgnoreCase("X-Method-Override"):
+    return true # meh
+  return false
+
+func isForbiddenResponseHeaderName*(name: string): bool =
+  return name in ["Set-Cookie", "Set-Cookie2"]
+
+proc validate(this: Headers; name, value: string): JSResult[bool] =
+  if not name.isValidHeaderName() or not value.isValidHeaderValue():
+    return errTypeError("Invalid header name or value")
+  if this.guard == hgImmutable:
+    return errTypeError("Tried to modify immutable Headers object")
+  if this.guard == hgRequest and isForbiddenRequestHeader(name, value):
+    return ok(false)
+  if this.guard == hgResponse and name.isForbiddenResponseHeaderName():
+    return ok(false)
+  return ok(true)
+
+func isNoCorsSafelistedName(name: string): bool =
+  return name.equalsIgnoreCase("Accept") or
+    name.equalsIgnoreCase("Accept-Language") or
+    name.equalsIgnoreCase("Content-Language") or
+    name.equalsIgnoreCase("Content-Type")
+
+
+const CorsUnsafeRequestByte = {
+  char(0x00)..char(0x08), char(0x10)..char(0x1F), '"', '(', ')', ':', '<', '>',
+  '?', '@', '[', '\\', ']', '{', '}', '\e'
+}
+
+func isNoCorsSafelisted(name, value: string): bool =
+  if value.len > 128:
+    return false
+  if name.equalsIgnoreCase("Accept"):
+    return CorsUnsafeRequestByte notin value
+  if name.equalsIgnoreCase("Accept-Language") or
+      name.equalsIgnoreCase("Content-Language"):
+    const Forbidden = AllChars - AsciiAlphaNumeric -
+      {' ', '*', ',', '-', '.', ';', '='}
+    return Forbidden notin value
+  if name.equalsIgnoreCase("Content-Type"):
+    return value.strip(chars = AsciiWhitespace).toLowerAscii() in [
+      "multipart/form-data",
+      "text/plain",
+      "application-x-www-form-urlencoded"
+    ]
+  return false
+
+func get0(this: Headers; name: string): string =
+  return this.table[name].join(", ")
+
+proc get(this: Headers; name: string): JSResult[Option[string]] {.jsfunc.} =
+  if not name.isValidHeaderName():
+    return errTypeError("Invalid header name")
+  if name notin this.table:
+    return ok(none(string))
+  return ok(some(this.get0(name)))
+
+proc removeRange(this: Headers) =
+  if this.guard == hgRequestNoCors:
+    #TODO do this case insensitively
+    this.table.del("Range") # privileged no-CORS request headers
+    this.table.del("range")
+
+proc append(this: Headers; name, value: string): JSResult[void] {.jsfunc.} =
+  let value = value.strip(chars = HTTPWhitespace)
+  if not ?this.validate(name, value):
+    return ok()
+  if this.guard == hgRequestNoCors:
+    if name in this.table:
+      let tmp = this.get0(name) & ", " & value
+      if not name.isNoCorsSafelisted(tmp):
+        return ok()
+      this.table[name].add(value)
     else:
-      headers.table[k] = @[v]
+      this.table[name] = @[value]
+  this.removeRange()
+  ok()
+
+proc delete(this: Headers; name: string): JSResult[void] {.jsfunc.} =
+  if not ?this.validate(name, "") or name notin this.table:
+    return ok()
+  if not name.isNoCorsSafelistedName() and not name.equalsIgnoreCase("Range"):
+    return ok()
+  this.table.del(name)
+  this.removeRange()
+  ok()
+
+proc has(this: Headers; name: string): JSResult[bool] {.jsfunc.} =
+  if not name.isValidHeaderName():
+    return errTypeError("Invalid header name")
+  return ok(name in this.table)
+
+proc set(this: Headers; name, value: string): JSResult[void] {.jsfunc.} =
+  let value = value.strip(chars = HTTPWhitespace)
+  if not ?this.validate(name, value):
+    return
+  if this.guard == hgRequestNoCors and not name.isNoCorsSafelisted(value):
+    return
+  #TODO do this case insensitively
+  this.table[name] = @[value]
+  this.removeRange()
+
+proc fill(headers: Headers; s: seq[(string, string)]): JSResult[void] =
+  for (k, v) in s:
+    ?headers.append(k, v)
+  ok()
 
-proc fill*(headers: Headers; tab: Table[string, string]) =
+proc fill(headers: Headers; tab: Table[string, string]): JSResult[void] =
   for k, v in tab:
-    if k in headers.table:
-      headers.table[k].add(v)
-    else:
-      headers.table[k] = @[v]
+    ?headers.append(k, v)
+  ok()
 
-proc fill*(headers: Headers; init: HeadersInit) =
-  if init.t == HEADERS_INIT_SEQUENCE:
-    headers.fill(init.s)
-  else: # table
-    headers.fill(init.tab)
+proc fill*(headers: Headers; init: HeadersInit): JSResult[void] =
+  case init.t
+  of hitSequence: return headers.fill(init.s)
+  of hitTable: return headers.fill(init.tab)
 
-func newHeaders*(): Headers =
-  return Headers()
+func newHeaders*(guard = hgNone): Headers =
+  return Headers(guard: guard)
 
-func newHeaders(obj = none(HeadersInit)): Headers {.jsctor.} =
-  let headers = Headers()
+func newHeaders(obj = none(HeadersInit)): JSResult[Headers] {.jsctor.} =
+  let headers = Headers(guard: hgNone)
   if obj.isSome:
-    headers.fill(obj.get)
-  return headers
+    ?headers.fill(obj.get)
+  return ok(headers)
 
 func newHeaders*(table: openArray[(string, string)]): Headers =
   let headers = Headers()
@@ -84,9 +240,7 @@ func newHeaders*(table: Table[string, string]): Headers =
   return headers
 
 func clone*(headers: Headers): Headers =
-  return Headers(
-    table: headers.table
-  )
+  return Headers(table: headers.table)
 
 proc add*(headers: Headers; k, v: string) =
   let k = k.toHeaderCase()
@@ -114,51 +268,5 @@ func getOrDefault*(headers: Headers; k: static string; default = ""): string =
   do:
     return default
 
-const TokenChars = {
-  '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~'
-} + AsciiAlphaNumeric
-
-#TODO maybe assert these are valid on insertion?
-func isValidHeaderName*(s: string): bool =
-  return s.len > 0 and AllChars - TokenChars notin s
-
-func isValidHeaderValue*(s: string): bool =
-  return s.len == 0 or s[0] notin {' ', '\t'} and s[^1] notin {' ', '\t'} and
-    '\n' notin s
-
-func isForbiddenHeader*(name, value: string): bool =
-  const ForbiddenNames = [
-    "Accept-Charset",
-    "Accept-Encoding",
-    "Access-Control-Request-Headers",
-    "Access-Control-Request-Method",
-    "Connection",
-    "Content-Length",
-    "Cookie",
-    "Cookie2",
-    "Date",
-    "DNT",
-    "Expect",
-    "Host",
-    "Keep-Alive",
-    "Origin",
-    "Referer",
-    "Set-Cookie",
-    "TE",
-    "Trailer",
-    "Transfer-Encoding",
-    "Upgrade",
-    "Via",
-  ]
-  if name in ForbiddenNames:
-    return true
-  if name.startsWith("proxy-") or name.startsWith("sec-"):
-    return true
-  if name.equalsIgnoreCase("X-HTTP-Method") or
-      name.equalsIgnoreCase("X-HTTP-Method-Override") or
-      name.equalsIgnoreCase("X-Method-Override"):
-    return true # meh
-  return false
-
 proc addHeadersModule*(ctx: JSContext) =
   ctx.registerType(Headers)
diff --git a/src/loader/request.nim b/src/loader/request.nim
index f92098cb..23be8ba8 100644
--- a/src/loader/request.nim
+++ b/src/loader/request.nim
@@ -1,5 +1,4 @@
 import std/options
-import std/strutils
 import std/tables
 
 import html/script
@@ -169,7 +168,6 @@ type
     referrer: Option[string]
     referrerPolicy: Option[ReferrerPolicy]
     credentials: Option[CredentialsMode]
-    proxyUrl: URL
     mode: Option[RequestMode]
     window: Option[JSValue]
 
@@ -194,33 +192,36 @@ proc fromJSBodyInit(ctx: JSContext; val: JSValue): JSResult[BodyInit] =
       return ok(BodyInit(t: bitString, str: x.get))
   return errTypeError("Invalid body init type")
 
-func newRequest*[T: string|JSRequest](ctx: JSContext; resource: T;
-    init = none(RequestInit)): JSResult[JSRequest] {.jsctor.} =
+var getAPIBaseURLImpl*: proc(ctx: JSContext): URL {.noSideEffect, nimcall.}
+
+proc newRequest*(ctx: JSContext; resource: JSValue; init = none(RequestInit)):
+    JSResult[JSRequest] {.jsctor.} =
   defer:
     if init.isSome and init.get.window.isSome:
       JS_FreeValue(ctx, init.get.window.get)
-  when T is string:
-    let url = ?newURL(resource)
-    if url.username != "" or url.password != "":
-      return errTypeError("Input URL contains a username or password")
-    var httpMethod = hmGet
-    var headers = newHeaders()
-    let referrer: URL = nil
-    var credentials = cmSameOrigin
-    var body = RequestBody()
-    var proxyUrl: URL #TODO?
-    let fallbackMode = opt(rmCors)
-    var window = RequestWindow(t: rwtClient)
+  let headers = newHeaders(hgRequest)
+  var fallbackMode = opt(rmCors)
+  var window = RequestWindow(t: rwtClient)
+  var body = RequestBody()
+  var credentials = cmSameOrigin
+  var httpMethod = hmGet
+  var referrer: URL = nil
+  var url: URL = nil
+  if JS_IsString(resource):
+    let s = ?fromJS[string](ctx, resource)
+    url = ?parseJSURL(s, option(ctx.getAPIBaseURLImpl()))
   else:
-    let url = resource.url
-    var httpMethod = resource.request.httpMethod
-    var headers = resource.headers.clone()
-    let referrer = resource.request.referrer
-    var credentials = resource.credentialsMode
-    var body = resource.request.body
-    var proxyUrl = resource.request.proxy #TODO?
-    let fallbackMode = none(RequestMode)
-    var window = resource.window
+    let resource = ?fromJS[JSRequest](ctx, resource)
+    url = resource.url
+    httpMethod = resource.request.httpMethod
+    headers.table = resource.headers.table
+    referrer = resource.request.referrer
+    credentials = resource.credentialsMode
+    body = resource.request.body
+    fallbackMode = opt(RequestMode)
+    window = resource.window
+  if url.username != "" or url.password != "":
+    return errTypeError("Input URL contains a username or password")
   var mode = fallbackMode.get(rmNoCors)
   let destination = rdNone
   #TODO origin, window
@@ -246,20 +247,19 @@ func newRequest*[T: string|JSRequest](ctx: JSContext; resource: T;
       if httpMethod in {hmGet, hmHead}:
         return errTypeError("HEAD or GET Request cannot have a body.")
     if init.headers.isSome:
-      headers.fill(init.headers.get)
+      ?headers.fill(init.headers.get)
     if init.credentials.isSome:
       credentials = init.credentials.get
     if init.mode.isSome:
       mode = init.mode.get
-    #TODO find a standard compatible way to implement this
-    proxyUrl = init.proxyUrl
+  if mode == rmNoCors:
+    headers.guard = hgRequestNoCors
   return ok(JSRequest(
     request: newRequest(
       url,
       httpMethod,
       headers,
       body,
-      proxy = proxyUrl,
       referrer = referrer
     ),
     mode: mode,
diff --git a/src/loader/response.nim b/src/loader/response.nim
index 9c173188..bbf41741 100644
--- a/src/loader/response.nim
+++ b/src/loader/response.nim
@@ -28,14 +28,6 @@ type
     rtOpaque = "opaque"
     rtOpaquedirect = "opaqueredirect"
 
-  #TODO fully implement headers guards
-  HeadersGuard* = enum
-    hgImmutable = "immutable"
-    hgRequest = "request"
-    hgRequestNoCors = "request-no-cors"
-    hgResponse = "response"
-    hgNone = "none"
-
   ResponseFlag* = enum
     rfAborted
 
@@ -46,7 +38,6 @@ type
     bodyUsed* {.jsget.}: bool
     status* {.jsget.}: uint16
     headers* {.jsget.}: Headers
-    headersGuard: HeadersGuard
     url*: URL #TODO should be urllist?
     unregisterFun*: proc()
     resumeFun*: proc(outputId: int)
@@ -72,13 +63,11 @@ proc newResponse*(res: int; request: Request; stream: SocketStream;
 
 func makeNetworkError*(): Response {.jsstfunc: "Response.error".} =
   #TODO use "create" function
-  #TODO headers immutable
   return Response(
     res: 0,
     responseType: rtError,
     status: 0,
-    headers: newHeaders(),
-    headersGuard: hgImmutable,
+    headers: newHeaders(hgImmutable),
     bodyUsed: true
   )