about summary refs log tree commit diff stats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/config.md12
-rw-r--r--src/buffer/buffer.nim22
-rw-r--r--src/buffer/container.nim26
-rw-r--r--src/config/config.nim38
-rw-r--r--src/display/pager.nim50
-rw-r--r--src/io/loader.nim30
-rw-r--r--src/io/request.nim1
-rw-r--r--src/io/urlfilter.nim11
-rw-r--r--src/ips/forkserver.nim39
-rw-r--r--src/ips/serialize.nim15
-rw-r--r--src/js/regex.nim4
-rw-r--r--src/types/cookie.nim5
-rw-r--r--src/types/referer.nim71
-rw-r--r--src/types/url.nim42
14 files changed, 274 insertions, 92 deletions
diff --git a/doc/config.md b/doc/config.md
index a044f719..efc60226 100644
--- a/doc/config.md
+++ b/doc/config.md
@@ -326,6 +326,18 @@ this only works for buffers which share the same cookie jar.</td>
 subdomains.</td>
 </tr>
 
+<tr>
+<td>referer-from</td>
+<td>boolean</td>
+<td>Whether or not we should send a Referer header when opening requests
+originating from this domain. Simplified example: if you click a link on a.com
+that refers to b.com, and referer-from is true, b.com is sent "a.com" as the
+Referer header.  
+Defaults to false.
+</td>
+</tr>
+
+
 </table>
 
 ## Stylesheets
diff --git a/src/buffer/buffer.nim b/src/buffer/buffer.nim
index 11492834..5d69325f 100644
--- a/src/buffer/buffer.nim
+++ b/src/buffer/buffer.nim
@@ -37,6 +37,7 @@ import render/rendertext
 import types/buffersource
 import types/color
 import types/cookie
+import types/referer
 import types/url
 import utils/twtstr
 
@@ -137,6 +138,17 @@ proc then*[T](promise: Promise[T], cb: (proc(x: T))): EmptyPromise {.discardable
       promise.stream.sread(promise.res)
     cb(promise.res))
 
+proc then*[T](promise: EmptyPromise, cb: (proc(): Promise[T])): Promise[T] {.discardable.} =
+  if promise == nil: return
+  let next = Promise[T]()
+  promise.then(proc() =
+    let p2 = cb()
+    if p2 != nil:
+      p2.then(proc(x: T) =
+        next.res = x
+        next.cb()))
+  return next
+
 proc then*[T, U](promise: Promise[T], cb: (proc(x: T): Promise[U])): Promise[U] {.discardable.} =
   if promise == nil: return
   let next = Promise[U]()
@@ -530,7 +542,13 @@ proc loadResources(buffer: Buffer, document: Document) =
     for child in elem.children_rev:
       stack.add(child)
 
-type ConnectResult* = tuple[code: int, needsAuth: bool, redirect: Option[URL], contentType: string, cookies: seq[Cookie]] 
+type ConnectResult* = object
+  code*: int
+  needsAuth*: bool
+  redirect*: Option[URL]
+  contentType*: string
+  cookies*: seq[Cookie]
+  referrerpolicy*: Option[ReferrerPolicy]
 
 proc setupSource(buffer: Buffer): ConnectResult =
   if buffer.loaded:
@@ -571,6 +589,8 @@ proc setupSource(buffer: Buffer): ConnectResult =
         let cookie = newCookie(s)
         if cookie != nil:
           result.cookies.add(cookie)
+    if "Referrer-Policy" in response.headers.table:
+      result.referrerpolicy = getReferrerPolicy(response.headers.table["Referrer-Policy"][0])
   buffer.istream = newTeeStream(buffer.istream, buffer.sstream, closedest = false)
   if setct:
     result.contentType = buffer.contenttype
diff --git a/src/buffer/container.nim b/src/buffer/container.nim
index 5c579c27..8fc2261a 100644
--- a/src/buffer/container.nim
+++ b/src/buffer/container.nim
@@ -63,7 +63,7 @@ type
     clear*: bool
 
   Container* = ref object
-    config: BufferConfig
+    config*: BufferConfig
     iface*: BufferInterface
     attrs: WindowAttributes
     width*: int
@@ -96,11 +96,9 @@ type
     hasstart: bool
     redirectdepth*: int
 
-proc newBuffer*(dispatcher: Dispatcher, config: Config, source: BufferSource,
-                cookiejar: CookieJar, title = "",
-                redirectdepth = 0): Container =
+proc newBuffer*(dispatcher: Dispatcher, config: BufferConfig,
+                source: BufferSource, title = "", redirectdepth = 0): Container =
   let attrs = getWindowAttributes(stdout)
-  let config = config.getBufferConfig(source.location, cookiejar)
   let ostream = dispatcher.forkserver.ostream
   let istream = dispatcher.forkserver.istream
   ostream.swrite(FORK_BUFFER)
@@ -652,6 +650,8 @@ proc load*(container: Container) =
         container.triggerEvent(SUCCESS)
         if res.cookies.len > 0 and container.config.cookiejar != nil: # accept cookies
           container.config.cookiejar.cookies.add(res.cookies)
+        if res.referrerpolicy.isSome and container.config.refererfrom:
+          container.config.referrerpolicy = res.referrerpolicy.get
         container.setLoadInfo("Connected to " & $container.source.location & ". Downloading...")
         if res.needsAuth:
           container.triggerEvent(NEEDS_AUTH)
@@ -704,7 +704,7 @@ proc dupeBuffer*(dispatcher: Dispatcher, container: Container, config: Config, l
     contenttype: if contenttype.isSome: contenttype else: container.contenttype,
     clonepid: container.process,
   )
-  container.pipeto = dispatcher.newBuffer(config, source, container.config.cookiejar, container.title)
+  container.pipeto = dispatcher.newBuffer(container.config, source, container.title)
   container.iface.getSource().then(proc() =
     if container.pipeto != nil:
       container.pipeto.load()
@@ -736,10 +736,16 @@ proc click*(container: Container) {.jsfunc.} =
 
 proc windowChange*(container: Container, attrs: WindowAttributes) =
   container.attrs = attrs
-  container.width = attrs.width
-  container.height = attrs.height - 1
-  container.iface.windowChange(attrs).then(proc() =
-    container.needslines = true)
+  if attrs.width != container.width or attrs.height - 1 != container.height:
+    container.width = attrs.width
+    container.height = attrs.height - 1
+    container.iface.windowChange(attrs).then(proc(): auto =
+      container.needslines = true
+      return container.iface.render()
+    ).then(proc(lines: int) =
+      if lines != container.numLines:
+        container.setNumLines(lines, true)
+      container.needslines = true)
 
 proc peek*(container: Container) {.jsfunc.} =
   container.alert($container.source.location)
diff --git a/src/config/config.nim b/src/config/config.nim
index 850ccb71..220b3948 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -11,6 +11,7 @@ import js/javascript
 import js/regex
 import types/color
 import types/cookie
+import types/referer
 import types/url
 import utils/twtstr
 
@@ -26,9 +27,10 @@ type
     url: Option[string]
     host: Option[string]
     subst: Option[string]
-    cookie: bool
+    cookie: Option[bool]
     thirdpartycookie: seq[string]
     sharecookiejar: Option[string]
+    refererfrom*: Option[bool]
 
   StaticOmniRule = object
     match: string
@@ -38,9 +40,10 @@ type
     url*: Option[Regex]
     host*: Option[Regex]
     subst*: (proc(s: URL): Option[URL])
-    cookie*: bool
+    cookie*: Option[bool]
     thirdpartycookie*: seq[Regex]
     sharecookiejar*: Option[string]
+    refererfrom*: Option[bool]
 
   OmniRule* = object
     match*: Regex
@@ -76,27 +79,45 @@ type
     filter*: URLFilter
     cookiejar*: CookieJar
     headers*: HeaderList
+    refererfrom*: bool
+    referrerpolicy*: ReferrerPolicy
 
   ForkServerConfig* = object
     tmpdir*: string
     ambiguous_double*: bool
 
+const DefaultHeaders* = {
+  "User-Agent": "chawan",
+  "Accept": "text/html,text/*;q=0.5",
+  "Accept-Language": "en;q=1.0",
+  "Pragma": "no-cache",
+  "Cache-Control": "no-cache",
+}.toTable().newHeaderList()[]
+
 func getForkServerConfig*(config: Config): ForkServerConfig =
   return ForkServerConfig(
     tmpdir: config.tmpdir,
     ambiguous_double: config.ambiguous_double
   )
 
-func getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar): BufferConfig =
-  result.userstyle = config.stylesheet
-  result.filter = newURLFilter(scheme = some(location.scheme), default = true)
-  result.cookiejar = cookiejar
+proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar = nil,
+                      headers: HeaderList = nil, refererfrom = false): BufferConfig =
+  result = BufferConfig(
+    userstyle: config.stylesheet,
+    filter: newURLFilter(scheme = some(location.scheme), default = true),
+    cookiejar: cookiejar,
+    headers: headers,
+    refererfrom: refererfrom
+  )
+  new(result.headers)
+  result.headers[] = DefaultHeaders
 
 proc getSiteConfig*(config: Config, jsctx: JSContext): seq[SiteConfig] =
   for sc in config.siteconf:
     var conf = SiteConfig(
       cookie: sc.cookie,
-      sharecookiejar: sc.sharecookiejar
+      sharecookiejar: sc.sharecookiejar,
+      refererfrom: sc.refererfrom
     )
     if sc.url.isSome:
       conf.url = compileRegex(sc.url.get, 0)
@@ -327,7 +348,8 @@ proc parseConfig(config: Config, dir: string, t: TomlValue) =
           of "url": conf.url = some(v.s)
           of "host": conf.host = some(v.s)
           of "rewrite-url": conf.subst = some(v.s)
-          of "cookie": conf.cookie = v.b
+          of "referer-from": conf.refererfrom = some(v.b)
+          of "cookie": conf.cookie = some(v.b)
           of "third-party-cookie":
             if v.vt == VALUE_STRING:
               conf.thirdpartycookie = @[v.s]
diff --git a/src/display/pager.nim b/src/display/pager.nim
index 3825f088..9837d2fa 100644
--- a/src/display/pager.nim
+++ b/src/display/pager.nim
@@ -467,30 +467,43 @@ proc windowChange*(pager: Pager, attrs: WindowAttributes) =
   pager.statusgrid = newFixedGrid(attrs.width)
   for container in pager.containers:
     container.windowChange(attrs)
+  pager.refreshStatusMsg()
 
-proc applySiteconf(pager: Pager, request: Request) =
+proc applySiteconf(pager: Pager, request: Request): BufferConfig =
   let url = $request.url
   let host = request.url.host
+  var refererfrom: bool
+  var cookiejar: CookieJar
+  var headers: HeaderList
   for sc in pager.siteconf:
-    if sc.url.isSome and not sc.url.get.exec(url).success:
+    if sc.url.isSome and not sc.url.get.match(url):
       continue
-    elif sc.host.isSome and not sc.host.get.exec(host).success:
+    elif sc.host.isSome and not sc.host.get.match(host):
       continue
     if sc.subst != nil:
       let s = sc.subst(request.url)
       if s.isSome and s.get != nil:
         request.url = s.get
-    if sc.cookie:
-      # host/url might have changed by now
-      let jarid = sc.sharecookiejar.get(request.url.host)
-      if jarid notin pager.cookiejars:
-        pager.cookiejars[jarid] = newCookieJar(request.url, sc.thirdpartycookie)
+    if sc.cookie.isSome:
+      if sc.cookie.get:
+        # host/url might have changed by now
+        let jarid = sc.sharecookiejar.get(request.url.host)
+        if jarid notin pager.cookiejars:
+          pager.cookiejars[jarid] = newCookieJar(request.url, sc.thirdpartycookie)
+        cookiejar = pager.cookiejars[jarid]
+      else:
+        cookiejar = nil # override
+    if sc.refererfrom.isSome:
+      refererfrom = sc.refererfrom.get
+  return pager.config.getBufferConfig(request.url, cookiejar, headers, refererfrom)
 
 # Load request in a new buffer.
 proc gotoURL*(pager: Pager, request: Request, prevurl = none(URL),
               ctype = none(string), replace: Container = nil,
-              redirectdepth = 0) =
-  pager.applySiteconf(request)
+              redirectdepth = 0, referrer: Container = nil) =
+  if referrer != nil and referrer.config.refererfrom:
+    request.referer = referrer.source.location
+  var bufferconfig = pager.applySiteconf(request)
   if prevurl.isnone or not prevurl.get.equals(request.url, true) or
       request.url.hash == "" or request.httpmethod != HTTP_GET:
     # Basically, we want to reload the page *only* when
@@ -505,8 +518,9 @@ proc gotoURL*(pager: Pager, request: Request, prevurl = none(URL),
       contenttype: ctype,
       location: request.url
     )
-    let cookiejar = pager.cookiejars.getOrDefault(request.url.host)
-    let container = pager.dispatcher.newBuffer(pager.config, source, cookiejar, redirectdepth = redirectdepth)
+    if referrer != nil:
+      bufferconfig.referrerpolicy = referrer.config.referrerpolicy
+    let container = pager.dispatcher.newBuffer(bufferconfig, source, redirectdepth = redirectdepth)
     if replace != nil:
       container.replace = replace
       container.copyCursorPos(container.replace)
@@ -517,7 +531,7 @@ proc gotoURL*(pager: Pager, request: Request, prevurl = none(URL),
 
 proc omniRewrite(pager: Pager, s: string): string =
   for rule in pager.omnirules:
-    if rule.match.exec(s).success:
+    if rule.match.match(s):
       return rule.subst(s).get
   return s
 
@@ -562,8 +576,8 @@ proc readPipe0*(pager: Pager, ctype: Option[string], fd: FileHandle, location: O
     contenttype: some(ctype.get("text/plain")),
     location: location.get(newURL("file://-"))
   )
-  let container = pager.dispatcher.newBuffer(pager.config, source, nil, title)
-  return container
+  let bufferconfig = pager.config.getBufferConfig(source.location)
+  return pager.dispatcher.newBuffer(bufferconfig, source, title = title)
 
 proc readPipe*(pager: Pager, ctype: Option[string], fd: FileHandle) =
   let container = pager.readPipe0(ctype, fd, none(URL), "*pipe*")
@@ -620,7 +634,7 @@ proc updateReadLine*(pager: Pager) =
         url.username = pager.username
         url.password = s
         pager.username = ""
-        pager.gotoURL(newRequest(url), some(pager.container.source.location), replace = pager.container)
+        pager.gotoURL(newRequest(url), some(pager.container.source.location), replace = pager.container, referrer = pager.container)
       of COMMAND:
         pager.scommand = s
         if pager.commandmode:
@@ -715,7 +729,7 @@ proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bo
     if container.redirectdepth < pager.config.maxredirect:
       let redirect = event.location
       pager.alert("Redirecting to " & $redirect)
-      pager.gotoURL(newRequest(redirect), some(container.source.location), replace = container, redirectdepth = container.redirectdepth + 1)
+      pager.gotoURL(newRequest(redirect), some(container.source.location), replace = container, redirectdepth = container.redirectdepth + 1, referrer = pager.container)
     else:
       pager.alert("Error: maximum redirection depth reached")
       pager.deleteContainer(container)
@@ -743,7 +757,7 @@ proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bo
         pager.container.readCanceled()
       pager.redraw = true
   of OPEN:
-    pager.gotoURL(event.request, some(container.source.location))
+    pager.gotoURL(event.request, some(container.source.location), referrer = pager.container)
   of INVALID_COMMAND: discard
   of STATUS:
     if pager.container == container:
diff --git a/src/io/loader.nim b/src/io/loader.nim
index 3e8baaf0..a1d449c7 100644
--- a/src/io/loader.nim
+++ b/src/io/loader.nim
@@ -28,18 +28,10 @@ import ips/serversocket
 import ips/socketstream
 import types/cookie
 import types/mime
+import types/referer
 import types/url
 import utils/twtstr
 
-const DefaultHeaders0 = {
-  "User-Agent": "chawan",
-  "Accept": "text/html,text/*;q=0.5",
-  "Accept-Language": "en;q=1.0",
-  "Pragma": "no-cache",
-  "Cache-Control": "no-cache",
-}.toTable()
-let DefaultHeaders* = DefaultHeaders0.newHeaderList()
-
 type
   FileLoader* = object
     process*: Pid
@@ -47,6 +39,12 @@ type
   LoaderCommand = enum
     LOAD, QUIT
 
+  LoaderConfig* = object
+    defaultheaders*: HeaderList
+    filter*: URLFilter
+    cookiejar*: CookieJar
+    referrerpolicy*: ReferrerPolicy
+
 proc loadFile(url: Url, ostream: Stream) =
   when defined(windows) or defined(OS2) or defined(DOS):
     let path = url.path.serialize_unicode_dos()
@@ -85,7 +83,7 @@ proc loadResource(request: Request, ostream: Stream) =
     ostream.flush()
 
 var ssock: ServerSocket
-proc runFileLoader*(fd: cint, defaultHeaders: HeaderList, filter: URLFilter, cookiejar: CookieJar) =
+proc runFileLoader*(fd: cint, config: LoaderConfig) =
   if curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK:
     raise newException(Defect, "Failed to initialize libcurl.")
   ssock = initServerSocket()
@@ -110,18 +108,22 @@ proc runFileLoader*(fd: cint, defaultHeaders: HeaderList, filter: URLFilter, coo
       of LOAD:
         var request: Request
         stream.sread(request)
-        if not filter.match(request.url):
+        if not config.filter.match(request.url):
           stream.swrite(-1) # error
           stream.flush()
         else:
-          for k, v in defaultHeaders.table:
+          for k, v in config.defaultHeaders.table:
             if k notin request.headers.table:
               request.headers.table[k] = v
-          if cookiejar != nil and cookiejar.cookies.len > 0:
+          if config.cookiejar != nil and config.cookiejar.cookies.len > 0:
             if "Cookie" notin request.headers.table:
-              let cookie = cookiejar.serialize(request.url)
+              let cookie = config.cookiejar.serialize(request.url)
               if cookie != "":
                 request.headers["Cookie"] = cookie
+          if request.referer != nil and "Referer" notin request.headers.table:
+            let r = getReferer(request.referer, request.url, config.referrerpolicy)
+            if r != "":
+              request.headers["Referer"] = r
           loadResource(request, stream)
         stream.close()
       of QUIT:
diff --git a/src/io/request.nim b/src/io/request.nim
index 7587b2e0..4a0888f3 100644
--- a/src/io/request.nim
+++ b/src/io/request.nim
@@ -25,6 +25,7 @@ type
     headers*: HeaderList
     body*: Option[string]
     multipart*: Option[MimeData]
+    referer*: URL
 
   Response* = ref object
     body*: Stream
diff --git a/src/io/urlfilter.nim b/src/io/urlfilter.nim
index 28bdef28..9541a699 100644
--- a/src/io/urlfilter.nim
+++ b/src/io/urlfilter.nim
@@ -7,11 +7,11 @@ import types/url
 type URLFilter* = object
   scheme: Option[string]
   allowhost*: Option[string]
-  allowhosts: Option[seq[Regex]]
+  allowhosts: seq[Regex]
   default: bool
 
 proc newURLFilter*(scheme = none(string), allowhost = none(string),
-                   allowhosts = none(seq[Regex]), default = false): URLFilter =
+                   allowhosts: seq[Regex] = @[], default = false): URLFilter =
   return URLFilter(
     scheme: scheme,
     allowhost: allowhost,
@@ -29,8 +29,7 @@ proc match*(filter: URLFilter, url: URL): bool =
   let host = url.host
   if filter.allowhost.isSome and filter.allowhost.get == host:
     return true
-  if filter.allowhosts.isSome:
-    for regex in filter.allowhosts.get:
-      if regex.match(host):
-        return true
+  for regex in filter.allowhosts:
+    if regex.match(host):
+      return true
   return filter.default
diff --git a/src/ips/forkserver.nim b/src/ips/forkserver.nim
index 29463248..e6622213 100644
--- a/src/ips/forkserver.nim
+++ b/src/ips/forkserver.nim
@@ -1,3 +1,4 @@
+import options
 import streams
 
 when defined(posix):
@@ -30,11 +31,18 @@ type
     ostream: Stream
     children: seq[(Pid, Pid)]
 
-proc newFileLoader*(forkserver: ForkServer, defaultHeaders: HeaderList = DefaultHeaders, filter = newURLFilter(), cookiejar: CookieJar = nil): FileLoader =
+proc newFileLoader*(forkserver: ForkServer, defaultHeaders: HeaderList = nil, filter = newURLFilter(), cookiejar: CookieJar = nil): FileLoader =
   forkserver.ostream.swrite(FORK_LOADER)
-  forkserver.ostream.swrite(defaultHeaders)
-  forkserver.ostream.swrite(filter)
-  forkserver.ostream.swrite(cookiejar)
+  var defaultHeaders = defaultHeaders
+  if defaultHeaders == nil:
+    new(defaultHeaders)
+    defaultHeaders[] = DefaultHeaders
+  let config = LoaderConfig(
+    defaultHeaders: defaultHeaders,
+    filter: filter,
+    cookiejar: cookiejar
+  )
+  forkserver.ostream.swrite(config)
   forkserver.ostream.flush()
   forkserver.istream.sread(result)
 
@@ -48,7 +56,7 @@ proc removeChild*(forkserver: Forkserver, pid: Pid) =
   forkserver.ostream.swrite(pid)
   forkserver.ostream.flush()
 
-proc forkLoader(ctx: var ForkServerContext, defaultHeaders: HeaderList, filter: URLFilter, cookiejar: CookieJar): FileLoader =
+proc forkLoader(ctx: var ForkServerContext, config: LoaderConfig): FileLoader =
   var pipefd: array[2, cint]
   if pipe(pipefd) == -1:
     raise newException(Defect, "Failed to open pipe.")
@@ -59,7 +67,7 @@ proc forkLoader(ctx: var ForkServerContext, defaultHeaders: HeaderList, filter:
     ctx.children.setLen(0)
     zeroMem(addr ctx, sizeof(ctx))
     discard close(pipefd[0]) # close read
-    runFileLoader(pipefd[1], defaultHeaders, filter, cookiejar)
+    runFileLoader(pipefd[1], config)
     assert false
   let readfd = pipefd[0] # get read
   discard close(pipefd[1]) # close write
@@ -80,7 +88,14 @@ proc forkBuffer(ctx: var ForkServerContext): Pid =
   ctx.istream.sread(config)
   ctx.istream.sread(attrs)
   ctx.istream.sread(mainproc)
-  let loader = ctx.forkLoader(DefaultHeaders, config.filter, config.cookiejar) #TODO make this configurable
+  let loader = ctx.forkLoader(
+    LoaderConfig(
+      defaultHeaders: config.headers,
+      filter: config.filter,
+      cookiejar: config.cookiejar,
+      referrerpolicy: config.referrerpolicy
+    )
+  )
   let pid = fork()
   if pid == 0:
     for i in 0 ..< ctx.children.len: ctx.children[i] = (Pid(0), Pid(0))
@@ -110,13 +125,9 @@ proc runForkServer() =
       of FORK_BUFFER:
         ctx.ostream.swrite(ctx.forkBuffer())
       of FORK_LOADER:
-        var defaultHeaders: HeaderList
-        var filter: URLFilter
-        var cookiejar: CookieJar
-        ctx.istream.sread(defaultHeaders)
-        ctx.istream.sread(filter)
-        ctx.istream.sread(cookiejar)
-        let loader = ctx.forkLoader(defaultHeaders, filter, cookiejar)
+        var config: LoaderConfig
+        ctx.istream.sread(config)
+        let loader = ctx.forkLoader(config)
         ctx.ostream.swrite(loader)
         ctx.children.add((loader.process, Pid(-1)))
       of LOAD_CONFIG:
diff --git a/src/ips/serialize.nim b/src/ips/serialize.nim
index 05f05ac9..652e5164 100644
--- a/src/ips/serialize.nim
+++ b/src/ips/serialize.nim
@@ -142,19 +142,24 @@ proc sread*(stream: Stream, b: var bool) =
 func slen*(b: bool): int =
   return sizeof(uint8)
 
-proc swrite*(stream: Stream, url: Url) =
+proc swrite*(stream: Stream, url: URL) =
   if url != nil:
     stream.swrite(url.serialize())
   else:
     stream.swrite("")
 
-proc sread*(stream: Stream, url: var Url) =
+proc sread*(stream: Stream, url: var URL) =
   var s: string
   stream.sread(s)
-  url = newURL(s)
+  if s == "":
+    url = nil
+  else:
+    url = newURL(s)
 
-func slen*(url: Url): int =
-  slen(url.serialize())
+func slen*(url: URL): int =
+  if url == nil:
+    return slen("")
+  return slen(url.serialize())
 
 proc swrite*(stream: Stream, tup: tuple) =
   for f in tup.fields:
diff --git a/src/js/regex.nim b/src/js/regex.nim
index 391cb46b..2a205130 100644
--- a/src/js/regex.nim
+++ b/src/js/regex.nim
@@ -91,7 +91,7 @@ template fastRuneAt(s: string16, i: int, r: untyped, doInc = true, be = false) =
 var dummyRuntime = newJSRuntime()
 var dummyContext = dummyRuntime.newJSContextRaw()
 
-proc `=destroy`(regex: var Regex) =
+proc `=destroy`*(regex: var Regex) =
   if regex.bytecode != nil:
     if regex.clone:
       dealloc(regex.bytecode)
@@ -99,7 +99,7 @@ proc `=destroy`(regex: var Regex) =
       dummyRuntime.js_free_rt(regex.bytecode)
     regex.bytecode = nil
 
-proc `=copy`(dest: var Regex, source: Regex) =
+proc `=copy`*(dest: var Regex, source: Regex) =
   if dest.bytecode != source.bytecode:
     `=destroy`(dest)
     wasMoved(dest)
diff --git a/src/types/cookie.nim b/src/types/cookie.nim
index e5e50d3e..41de0f4c 100644
--- a/src/types/cookie.nim
+++ b/src/types/cookie.nim
@@ -123,7 +123,7 @@ proc serialize*(cookiejar: CookieJar, location: URL): string =
   let t = now().toTime().toUnix()
   for i in countdown(cookiejar.cookies.high, 0):
     let cookie = cookiejar.cookies[i]
-    if cookie.expires <= t:
+    if cookie.expires != -1 and cookie.expires <= t:
       cookiejar.cookies.delete(i)
     elif cookie.domain == "" or location.host.endsWith(cookie.domain):
       result.percentEncode(cookie.name, UserInfoPercentEncodeSet)
@@ -133,6 +133,7 @@ proc serialize*(cookiejar: CookieJar, location: URL): string =
 
 proc newCookie*(str: string): Cookie {.jsctor.} =
   let cookie = new(Cookie)
+  cookie.expires = -1
   var first = true
   for part in str.split(';'):
     if first:
@@ -169,7 +170,7 @@ proc newCookieJar*(location: URL, allowhosts: seq[Regex]): CookieJar =
     filter: newURLFilter(
       scheme = some(location.scheme),
       allowhost = some(location.host),
-      allowhosts = some(allowhosts)
+      allowhosts = allowhosts
     )
   )
 
diff --git a/src/types/referer.nim b/src/types/referer.nim
new file mode 100644
index 00000000..77cb4e4d
--- /dev/null
+++ b/src/types/referer.nim
@@ -0,0 +1,71 @@
+import url
+import options
+
+type ReferrerPolicy* = enum
+  STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+  NO_REFERRER
+  NO_REFERRER_WHEN_DOWNGRADE
+  STRICT_ORIGIN
+  ORIGIN
+  SAME_ORIGIN
+  ORIGIN_WHEN_CROSS_ORIGIN
+  UNSAFE_URL
+
+const DefaultPolicy* = STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+
+proc getReferrerPolicy*(s: string): Option[ReferrerPolicy] =
+  case s
+  of "no-referrer":
+    return some(NO_REFERRER)
+  of "no-referrer-when-downgrade":
+    return some(NO_REFERRER_WHEN_DOWNGRADE)
+  of "origin":
+    return some(ORIGIN)
+  of "origin-when-cross-origin":
+    return some(ORIGIN_WHEN_CROSS_ORIGIN)
+  of "same-origin":
+    return some(SAME_ORIGIN)
+  of "strict-origin":
+    return some(STRICT_ORIGIN)
+  of "strict-origin-when-cross-origin":
+    return some(STRICT_ORIGIN_WHEN_CROSS_ORIGIN)
+  of "unsafe-url":
+    return some(UNSAFE_URL)
+
+proc getReferer*(prev, target: URL, policy: ReferrerPolicy): string =
+  let origin = prev.origin0
+  if origin.isNone:
+    return ""
+  if prev.scheme != "http" and prev.scheme != "https":
+    return ""
+  if target.scheme != "http" and target.scheme != "https":
+    return ""
+  case policy
+  of NO_REFERRER:
+    return ""
+  of NO_REFERRER_WHEN_DOWNGRADE:
+    if prev.scheme == "https" and target.scheme == "http":
+      return ""
+    return $origin & prev.pathname & prev.search
+  of SAME_ORIGIN:
+    if origin == target.origin0:
+      return $origin
+    return ""
+  of ORIGIN:
+    return $origin
+  of STRICT_ORIGIN:
+    if prev.scheme == "https" and target.scheme == "http":
+      return ""
+    return $origin
+  of ORIGIN_WHEN_CROSS_ORIGIN:
+    if origin != target.origin0:
+      return $origin
+    return $origin & prev.pathname & prev.search
+  of STRICT_ORIGIN_WHEN_CROSS_ORIGIN:
+    if prev.scheme == "https" and target.scheme == "http":
+      return $origin
+    if origin != target.origin0:
+      return $origin
+    return $origin & prev.pathname & prev.search
+  of UNSAFE_URL:
+    return $origin & prev.pathname & prev.search
diff --git a/src/types/url.nim b/src/types/url.nim
index da27d96b..0b4471e3 100644
--- a/src/types/url.nim
+++ b/src/types/url.nim
@@ -53,6 +53,13 @@ type
     blob: Option[BlobUrlEntry]
     searchParams* {.jsget.}: URLSearchParams
 
+  Origin* = Option[tuple[
+    scheme: string,
+    host: Host,
+    port: Option[uint16],
+    domain: Option[string]
+  ]]
+
 const EmptyPath = UrlPath(opaque: true, s: "")
 const EmptyHost = Host(domain: "").some
 
@@ -973,7 +980,7 @@ proc newURL*(s: string, base: Option[string] = none(string)): URL {.jserr, jscto
   url.get.searchParams.initURLSearchParams(url.get.query.get(""))
   return url.get
 
-proc origin*(url: URL): string {.jsfget.} =
+proc origin0*(url: URL): Origin =
   case url.scheme
   of "blob":
     if url.blob.issome:
@@ -981,22 +988,33 @@ proc origin*(url: URL): string {.jsfget.} =
       discard
     let pathURL = parseURL($url.path)
     if pathURL.isnone:
-      return "null"
-    return pathURL.get.origin
+      return # opaque
+    return pathURL.get.origin0
   of "ftp", "http", "https", "ws", "wss":
-    # return the tuple origin.
-    #TODO split this out
-    result = url.scheme
-    result &= "://"
-    result &= url.host.get.serialize()
-    if url.port.issome:
-      result &= ":"
-      result &= $url.port.get
+    return some((url.scheme, url.host.get, url.port, none(string)))
   of "file":
     #???
-    return "null"
+    return # opaque
   else:
+    return # opaque
+
+proc `==`*(a, b: Origin): bool =
+  if a.isNone or b.isNone: return false
+  return a.get == b.get
+
+proc `$`*(origin: Origin): string =
+  if origin.isNone:
     return "null"
+  let origin = origin.get
+  result = origin.scheme
+  result &= "://"
+  result &= origin.host.serialize()
+  if origin.port.isSome:
+    result &= ':'
+    result &= $origin.port.get
+
+proc origin*(url: URL): string {.jsfget.} =
+  return $url.origin0
 
 proc protocol*(url: URL): string {.jsfget.} =
   return url.scheme & ':'
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-04-17 02:29:57 +0000