move around more modules

* ips -> io/
* loader related stuff -> loader/
* tempfile -> extern/
* buffer, forkserver -> server/
* lineedit, window -> display/
* cell -> types/
* opt -> types/

10 files changed, 1311 insertions, 0 deletions

diff --git a/src/loader/about.nim b/src/loader/about.nim
new file mode 100644
index 00000000..1bbe9625
--- /dev/null
+++ b/src/loader/about.nim
@@ -0,0 +1,28 @@
+import tables
+
+import loader/connecterror
+import loader/headers
+import loader/loaderhandle
+import loader/request
+import types/url
+
+const chawan = staticRead"res/chawan.html"
+const HeaderTable = {
+  "Content-Type": "text/html"
+}.toTable()
+
+proc loadAbout*(handle: LoaderHandle, request: Request) =
+  template t(body: untyped) =
+    if not body:
+      return
+  if request.url.pathname == "blank":
+    t handle.sendResult(0)
+    t handle.sendStatus(200) # ok
+    t handle.sendHeaders(newHeaders(HeaderTable))
+  elif request.url.pathname == "chawan":
+    t handle.sendResult(0)
+    t handle.sendStatus(200) # ok
+    t handle.sendHeaders(newHeaders(HeaderTable))
+    t handle.sendData(chawan)
+  else:
+    t handle.sendResult(ERROR_ABOUT_PAGE_NOT_FOUND)
diff --git a/src/loader/connecterror.nim b/src/loader/connecterror.nim
new file mode 100644
index 00000000..d2af5762
--- /dev/null
+++ b/src/loader/connecterror.nim
@@ -0,0 +1,18 @@
+import bindings/curl
+
+type ConnectErrorCode* = enum
+  ERROR_INVALID_DATA_URL = (-7, "invalid data URL")
+  ERROR_ABOUT_PAGE_NOT_FOUND = (-6, "about page not found")
+  ERROR_FILE_NOT_FOUND = (-5, "file not found")
+  ERROR_SOURCE_NOT_FOUND = (-4, "clone source could not be found"),
+  ERROR_LOADER_KILLED = (-3, "loader killed during transfer"),
+  ERROR_DISALLOWED_URL = (-2, "url not allowed by filter"),
+  ERROR_UNKNOWN_SCHEME = (-1, "unknown scheme")
+
+converter toInt*(code: ConnectErrorCode): int =
+  return int(code)
+
+func getLoaderErrorMessage*(code: int): string =
+  if code < 0:
+    return $ConnectErrorCode(code)
+  return $curl_easy_strerror(CURLcode(cint(code)))
diff --git a/src/loader/data.nim b/src/loader/data.nim
new file mode 100644
index 00000000..effd9ce9
--- /dev/null
+++ b/src/loader/data.nim
@@ -0,0 +1,43 @@
+import base64
+import strutils
+import tables
+
+import loader/connecterror
+import loader/headers
+import loader/loaderhandle
+import loader/request
+import types/url
+
+proc loadData*(handle: LoaderHandle, request: Request) =
+  template t(body: untyped) =
+    if not body:
+      return
+  var str = $request.url
+  let si = "data:".len # start index
+  var ct = ""
+  for i in si ..< str.len:
+    if str[i] == ',':
+      break
+    ct &= str[i]
+  let sd = si + ct.len + 1 # data start
+  if ct.endsWith(";base64"):
+    try:
+      let d = base64.decode(str[sd .. ^1]) # decode from ct end + 1
+      t handle.sendResult(0)
+      t handle.sendStatus(200)
+      ct.setLen(ct.len - ";base64".len) # remove base64 indicator
+      t handle.sendHeaders(newHeaders({
+        "Content-Type": ct
+      }.toTable()))
+      if d.len > 0:
+        t handle.sendData(d)
+    except ValueError:
+      discard handle.sendResult(ERROR_INVALID_DATA_URL)
+  else:
+    t handle.sendResult(0)
+    t handle.sendStatus(200)
+    t handle.sendHeaders(newHeaders({
+      "Content-Type": ct
+    }.toTable()))
+    if ct.len + 1 < str.len:
+      t handle.sendData(addr str[sd], str.len - sd)
diff --git a/src/loader/file.nim b/src/loader/file.nim
new file mode 100644
index 00000000..b627e0c7
--- /dev/null
+++ b/src/loader/file.nim
@@ -0,0 +1,107 @@
+import algorithm
+import os
+import streams
+import tables
+
+import loader/connecterror
+import loader/headers
+import loader/loaderhandle
+import types/url
+
+proc loadDir(handle: LoaderHandle, url: URL, path: string) =
+  template t(body: untyped) =
+    if not body:
+      return
+  var path = path
+  if path[^1] != '/': #TODO dos/windows
+    path &= '/'
+  var base = $url
+  if base[^1] != '/': #TODO dos/windows
+    base &= '/'
+  t handle.sendResult(0)
+  t handle.sendStatus(200) # ok
+  t handle.sendHeaders(newHeaders({"Content-Type": "text/html"}.toTable()))
+  t handle.sendData("""
+<HTML>
+<HEAD>
+<BASE HREF="""" & base & """">
+<TITLE>Directory list of """ & path & """</TITLE>
+</HEAD>
+<BODY>
+<H1>Directory list of """ & path & """</H1>
+[DIR]&nbsp; <A HREF="../">../</A></br>
+""")
+  var fs: seq[(PathComponent, string)]
+  for pc, file in walkDir(path, relative = true):
+    fs.add((pc, file))
+  fs.sort(cmp = proc(a, b: (PathComponent, string)): int = cmp(a[1], b[1]))
+  for (pc, file) in fs:
+    case pc
+    of pcDir:
+      t handle.sendData("[DIR]&nbsp; ")
+    of pcFile:
+      t handle.sendData("[FILE] ")
+    of pcLinkToDir, pcLinkToFile:
+      t handle.sendData("[LINK] ")
+    var fn = file
+    if pc == pcDir:
+      fn &= '/'
+    t handle.sendData("<A HREF=\"" & fn & "\">" & fn & "</A>")
+    if pc in {pcLinkToDir, pcLinkToFile}:
+      discard handle.sendData(" -> " & expandSymlink(path / file))
+    t handle.sendData("<br>")
+  t handle.sendData("""
+</BODY>
+</HTML>""")
+
+proc loadSymlink(handle: LoaderHandle, path: string) =
+  template t(body: untyped) =
+    if not body:
+      return
+  t handle.sendResult(0)
+  t handle.sendStatus(200) # ok
+  t handle.sendHeaders(newHeaders({"Content-Type": "text/html"}.toTable()))
+  let sl = expandSymlink(path)
+  t handle.sendData("""
+<HTML>
+<HEAD>
+<TITLE>Symlink view<TITLE>
+</HEAD>
+<BODY>
+Symbolic link to <A HREF="""" & sl & """">""" & sl & """</A></br>
+</BODY>
+</HTML>""")
+
+proc loadFile(handle: LoaderHandle, istream: Stream) =
+  template t(body: untyped) =
+    if not body:
+      return
+  t handle.sendResult(0)
+  t handle.sendStatus(200) # ok
+  t handle.sendHeaders(newHeaders())
+  while not istream.atEnd:
+    const bufferSize = 4096
+    var buffer {.noinit.}: array[bufferSize, char]
+    while true:
+      let n = readData(istream, addr buffer[0], bufferSize)
+      if n == 0:
+        break
+      t handle.sendData(addr buffer[0], n)
+      if n < bufferSize:
+        break
+
+proc loadFilePath*(handle: LoaderHandle, url: URL) =
+  when defined(windows) or defined(OS2) or defined(DOS):
+    let path = url.path.serialize_unicode_dos()
+  else:
+    let path = url.path.serialize_unicode()
+  let istream = newFileStream(path, fmRead)
+  if istream == nil:
+    if dirExists(path):
+      handle.loadDir(url, path)
+    elif symlinkExists(path):
+      handle.loadSymlink(path)
+    else:
+      discard handle.sendResult(ERROR_FILE_NOT_FOUND)
+  else:
+    handle.loadFile(istream)
diff --git a/src/loader/headers.nim b/src/loader/headers.nim
new file mode 100644
index 00000000..b02f30df
--- /dev/null
+++ b/src/loader/headers.nim
@@ -0,0 +1,100 @@
+import tables
+
+import bindings/quickjs
+import js/error
+import js/fromjs
+import js/javascript
+import utils/twtstr
+
+type
+  Headers* = ref object
+    table* {.jsget.}: Table[string, seq[string]]
+
+  HeadersInitType = enum
+    HEADERS_INIT_SEQUENCE, HEADERS_INIT_TABLE
+
+  HeadersInit* = object
+    case t: HeadersInitType
+    of HEADERS_INIT_SEQUENCE:
+      s: seq[(string, string)]
+    of HEADERS_INIT_TABLE:
+      tab: Table[string, string]
+
+jsDestructor(Headers)
+
+proc fromJS2*(ctx: JSContext, val: JSValue, res: var JSResult[HeadersInit]) =
+  if JS_IsUndefined(val) or JS_IsNull(val):
+    res.err(nil)
+    return
+  if isSequence(ctx, val):
+    let x = fromJS[seq[(string, string)]](ctx, val)
+    if x.isSome:
+      res.ok(HeadersInit(t: HEADERS_INIT_SEQUENCE, s: x.get))
+  else:
+    let x = fromJS[Table[string, string]](ctx, val)
+    if x.isSome:
+      res.ok(HeadersInit(t: HEADERS_INIT_TABLE, tab: x.get))
+
+proc fill*(headers: Headers, s: seq[(string, string)]) =
+  for (k, v) in s:
+    if k in headers.table:
+      headers.table[k].add(v)
+    else:
+      headers.table[k] = @[v]
+
+proc fill*(headers: Headers, tab: Table[string, string]) =
+  for k, v in tab:
+    if k in headers.table:
+      headers.table[k].add(v)
+    else:
+      headers.table[k] = @[v]
+
+proc fill*(headers: Headers, init: HeadersInit) =
+  if init.t == HEADERS_INIT_SEQUENCE:
+    headers.fill(init.s)
+  else: # table
+    headers.fill(init.tab)
+
+func newHeaders*(): Headers =
+  return Headers()
+
+func newHeaders(obj = none(HeadersInit)): Headers {.jsctor.} =
+  let headers = Headers()
+  if obj.isSome:
+    headers.fill(obj.get)
+  return headers
+
+func newHeaders*(table: Table[string, string]): Headers =
+  let headers = Headers()
+  for k, v in table:
+    let k = k.toHeaderCase()
+    if k in headers.table:
+      headers.table[k].add(v)
+    else:
+      headers.table[k] = @[v]
+  return headers
+
+func clone*(headers: Headers): Headers =
+  return Headers(
+    table: headers.table
+  )
+
+proc add*(headers: var Headers, k, v: string) =
+  let k = k.toHeaderCase()
+  if k notin headers.table:
+    headers.table[k] = @[v]
+  else:
+    headers.table[k].add(v)
+
+proc `[]=`*(headers: var Headers, k, v: string) =
+  headers.table[k.toHeaderCase()] = @[v]
+
+func getOrDefault*(headers: Headers, k: string, default = ""): string =
+  let k = k.toHeaderCase()
+  if k in headers.table:
+    headers.table[k][0]
+  else:
+    default
+
+proc addHeadersModule*(ctx: JSContext) =
+  ctx.registerType(Headers)
diff --git a/src/loader/http.nim b/src/loader/http.nim
new file mode 100644
index 00000000..97fd9690
--- /dev/null
+++ b/src/loader/http.nim
@@ -0,0 +1,142 @@
+import options
+import strutils
+
+import bindings/curl
+import loader/headers
+import loader/loaderhandle
+import loader/request
+import types/blob
+import types/formdata
+import types/url
+import types/opt
+import utils/twtstr
+
+type
+  CurlHandle* = ref CurlHandleObj
+  CurlHandleObj = object
+    curl*: CURL
+    statusline: bool
+    headers: Headers
+    request: Request
+    handle*: LoaderHandle
+    mime: curl_mime
+    slist: curl_slist
+
+func newCurlHandle(curl: CURL, request: Request, handle: LoaderHandle):
+    CurlHandle =
+  return CurlHandle(
+    headers: newHeaders(),
+    curl: curl,
+    handle: handle,
+    request: request
+  )
+
+proc cleanup*(handleData: CurlHandle) =
+  handleData.handle.close()
+  if handleData.mime != nil:
+    curl_mime_free(handleData.mime)
+  if handleData.slist != nil:
+    curl_slist_free_all(handleData.slist)
+  curl_easy_cleanup(handleData.curl)
+
+template setopt(curl: CURL, opt: CURLoption, arg: typed) =
+  discard curl_easy_setopt(curl, opt, arg)
+
+template setopt(curl: CURL, opt: CURLoption, arg: string) =
+  discard curl_easy_setopt(curl, opt, cstring(arg))
+
+template getinfo(curl: CURL, info: CURLINFO, arg: typed) =
+  discard curl_easy_getinfo(curl, info, arg)
+
+proc curlWriteHeader(p: cstring, size: csize_t, nitems: csize_t,
+    userdata: pointer): csize_t {.cdecl.} =
+  var line = newString(nitems)
+  for i in 0..<nitems:
+    line[i] = p[i]
+
+  let op = cast[CurlHandle](userdata)
+  if not op.statusline:
+    op.statusline = true
+    if not op.handle.sendResult(int(CURLE_OK)):
+      return 0
+    var status: clong
+    op.curl.getinfo(CURLINFO_RESPONSE_CODE, addr status)
+    if not op.handle.sendStatus(cast[int](status)):
+      return 0
+    return nitems
+
+  let k = line.until(':')
+
+  if k.len == line.len:
+    # empty line (last, before body) or invalid (=> error)
+    if not op.handle.sendHeaders(op.headers):
+      return 0
+    return nitems
+
+  let v = line.substr(k.len + 1).strip()
+  op.headers.add(k, v)
+  return nitems
+
+# From the documentation: size is always 1.
+proc curlWriteBody(p: cstring, size: csize_t, nmemb: csize_t,
+    userdata: pointer): csize_t {.cdecl.} =
+  let handleData = cast[CurlHandle](userdata)
+  if nmemb > 0:
+    if not handleData.handle.sendData(p, int(nmemb)):
+      return 0
+  return nmemb
+
+proc applyPostBody(curl: CURL, request: Request, handleData: CurlHandle) =
+  if request.multipart.isOk:
+    handleData.mime = curl_mime_init(curl)
+    doAssert handleData.mime != nil
+    for entry in request.multipart.get:
+      let part = curl_mime_addpart(handleData.mime)
+      doAssert part != nil
+      curl_mime_name(part, cstring(entry.name))
+      if entry.isstr:
+        curl_mime_data(part, cstring(entry.svalue), csize_t(entry.svalue.len))
+      else:
+        let blob = entry.value
+        if blob.isfile: #TODO ?
+          curl_mime_filedata(part, cstring(WebFile(blob).path))
+        else:
+          curl_mime_data(part, blob.buffer, csize_t(blob.size))
+        # may be overridden by curl_mime_filedata, so set it here
+        curl_mime_filename(part, cstring(entry.filename))
+    curl.setopt(CURLOPT_MIMEPOST, handleData.mime)
+  elif request.body.issome:
+    curl.setopt(CURLOPT_POSTFIELDS, cstring(request.body.get))
+    curl.setopt(CURLOPT_POSTFIELDSIZE, request.body.get.len)
+
+proc loadHttp*(handle: LoaderHandle, curlm: CURLM,
+    request: Request): CurlHandle =
+  let curl = curl_easy_init()
+  doAssert curl != nil
+  let surl = request.url.serialize()
+  curl.setopt(CURLOPT_URL, surl)
+  let handleData = curl.newCurlHandle(request, handle)
+  curl.setopt(CURLOPT_WRITEDATA, handleData)
+  curl.setopt(CURLOPT_WRITEFUNCTION, curlWriteBody)
+  curl.setopt(CURLOPT_HEADERDATA, handleData)
+  curl.setopt(CURLOPT_HEADERFUNCTION, curlWriteHeader)
+  if request.proxy != nil:
+    let purl = request.proxy.serialize()
+    curl.setopt(CURLOPT_PROXY, purl)
+  case request.httpmethod
+  of HTTP_GET:
+    curl.setopt(CURLOPT_HTTPGET, 1)
+  of HTTP_POST:
+    curl.setopt(CURLOPT_POST, 1)
+    curl.applyPostBody(request, handleData)
+  else: discard #TODO
+  for k, v in request.headers:
+    let header = k & ": " & v
+    handleData.slist = curl_slist_append(handleData.slist, cstring(header))
+  if handleData.slist != nil:
+    curl.setopt(CURLOPT_HTTPHEADER, handleData.slist)
+  let res = curl_multi_add_handle(curlm, curl)
+  if res != CURLM_OK:
+    discard handle.sendResult(int(res))
+    return nil
+  return handleData
diff --git a/src/loader/loader.nim b/src/loader/loader.nim
new file mode 100644
index 00000000..162dd120
--- /dev/null
+++ b/src/loader/loader.nim
@@ -0,0 +1,394 @@
+# A file loader server (?)
+# The idea here is that we receive requests with a socket, then respond to each
+# with a response (ideally a document.)
+# For now, the protocol looks like:
+# C: Request
+# S: res (0 => success, _ => error)
+# if success:
+#  S: status code
+#  S: headers
+#  S: response body
+#
+# The body is passed to the stream as-is, so effectively nothing can follow it.
+
+import nativesockets
+import net
+import options
+import posix
+import streams
+import strutils
+import tables
+
+import bindings/curl
+import io/posixstream
+import io/promise
+import io/serialize
+import io/serversocket
+import io/socketstream
+import io/urlfilter
+import js/error
+import js/javascript
+import loader/about
+import loader/connecterror
+import loader/data
+import loader/file
+import loader/headers
+import loader/http
+import loader/loaderhandle
+import loader/request
+import loader/response
+import types/cookie
+import types/referer
+import types/url
+import utils/mimeguess
+import utils/twtstr
+
+import chakasu/charset
+
+export request
+export response
+
+type
+  FileLoader* = ref object
+    process*: Pid
+    connecting*: Table[int, ConnectData]
+    ongoing*: Table[int, OngoingData]
+    unregistered*: seq[int]
+    registerFun*: proc(fd: int)
+    unregisterFun*: proc(fd: int)
+
+  ConnectData = object
+    promise: Promise[JSResult[Response]]
+    stream: Stream
+    request: Request
+
+  OngoingData = object
+    buf: string
+    readbufsize: int
+    response: Response
+    bodyRead: Promise[string]
+
+  LoaderCommand = enum
+    LOAD
+    QUIT
+
+  LoaderContext = ref object
+    ssock: ServerSocket
+    alive: bool
+    curlm: CURLM
+    config: LoaderConfig
+    extra_fds: seq[curl_waitfd]
+    handleList: seq[CurlHandle]
+
+  LoaderConfig* = object
+    defaultheaders*: Headers
+    filter*: URLFilter
+    cookiejar*: CookieJar
+    referrerpolicy*: ReferrerPolicy
+    proxy*: URL
+    # When set to false, requests with a proxy URL are overridden by the
+    # loader proxy.
+    acceptProxy*: bool
+
+  FetchPromise* = Promise[JSResult[Response]]
+
+proc addFd(ctx: LoaderContext, fd: int, flags: int) =
+  ctx.extra_fds.add(curl_waitfd(
+    fd: cast[cint](fd),
+    events: cast[cshort](flags)
+  ))
+
+proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
+  case request.url.scheme
+  of "file":
+    handle.loadFilePath(request.url)
+    handle.close()
+  of "http", "https":
+    let handleData = handle.loadHttp(ctx.curlm, request)
+    if handleData != nil:
+      ctx.handleList.add(handleData)
+  of "about":
+    handle.loadAbout(request)
+    handle.close()
+  of "data":
+    handle.loadData(request)
+    handle.close()
+  else:
+    discard handle.sendResult(ERROR_UNKNOWN_SCHEME)
+    handle.close()
+
+proc onLoad(ctx: LoaderContext, stream: Stream) =
+  var request: Request
+  stream.sread(request)
+  if not ctx.config.filter.match(request.url):
+    stream.swrite(ERROR_DISALLOWED_URL)
+    stream.close()
+  else:
+    let handle = newLoaderHandle(stream, request.canredir)
+    for k, v in ctx.config.defaultHeaders.table:
+      if k notin request.headers.table:
+        request.headers.table[k] = v
+    if ctx.config.cookiejar != nil and ctx.config.cookiejar.cookies.len > 0:
+      if "Cookie" notin request.headers.table:
+        let cookie = ctx.config.cookiejar.serialize(request.url)
+        if cookie != "":
+          request.headers["Cookie"] = cookie
+    if request.referer != nil and "Referer" notin request.headers.table:
+      let r = getReferer(request.referer, request.url, ctx.config.referrerpolicy)
+      if r != "":
+        request.headers["Referer"] = r
+    if request.proxy == nil or not ctx.config.acceptProxy:
+      request.proxy = ctx.config.proxy
+    ctx.loadResource(request, handle)
+
+proc acceptConnection(ctx: LoaderContext) =
+  #TODO TODO TODO acceptSocketStream should be non-blocking here,
+  # otherwise the client disconnecting between poll and accept could
+  # block this indefinitely.
+  let stream = ctx.ssock.acceptSocketStream()
+  try:
+    var cmd: LoaderCommand
+    stream.sread(cmd)
+    case cmd
+    of LOAD:
+      ctx.onLoad(stream)
+    of QUIT:
+      ctx.alive = false
+      stream.close()
+  except IOError:
+    # End-of-file, broken pipe, or something else. For now we just
+    # ignore it and pray nothing breaks.
+    # (TODO: this is probably not a very good idea.)
+    stream.close()
+
+proc finishCurlTransfer(ctx: LoaderContext, handleData: CurlHandle, res: int) =
+  if res != int(CURLE_OK):
+    discard handleData.handle.sendResult(int(res))
+  discard curl_multi_remove_handle(ctx.curlm, handleData.curl)
+  handleData.cleanup()
+
+proc exitLoader(ctx: LoaderContext) =
+  for handleData in ctx.handleList:
+    ctx.finishCurlTransfer(handleData, ERROR_LOADER_KILLED)
+  discard curl_multi_cleanup(ctx.curlm)
+  curl_global_cleanup()
+  ctx.ssock.close()
+  quit(0)
+
+var gctx: LoaderContext
+proc initLoaderContext(fd: cint, config: LoaderConfig): LoaderContext =
+  if curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK:
+    raise newException(Defect, "Failed to initialize libcurl.")
+  let curlm = curl_multi_init()
+  if curlm == nil:
+    raise newException(Defect, "Failed to initialize multi handle.")
+  var ctx = LoaderContext(
+    alive: true,
+    curlm: curlm,
+    config: config
+  )
+  gctx = ctx
+  #TODO ideally, buffered would be true. Unfortunately this conflicts with
+  # sendFileHandle/recvFileHandle.
+  ctx.ssock = initServerSocket(buffered = false)
+  # The server has been initialized, so the main process can resume execution.
+  var writef: File
+  if not open(writef, FileHandle(fd), fmWrite):
+    raise newException(Defect, "Failed to open input handle.")
+  writef.write(char(0u8))
+  writef.flushFile()
+  close(writef)
+  discard close(fd)
+  onSignal SIGTERM, SIGINT:
+    discard sig
+    gctx.exitLoader()
+  ctx.addFd(int(ctx.ssock.sock.getFd()), CURL_WAIT_POLLIN)
+  return ctx
+
+proc runFileLoader*(fd: cint, config: LoaderConfig) =
+  var ctx = initLoaderContext(fd, config)
+  while ctx.alive:
+    var numfds: cint = 0
+    #TODO do not discard
+    discard curl_multi_poll(ctx.curlm, addr ctx.extra_fds[0],
+      cuint(ctx.extra_fds.len), 30_000, addr numfds)
+    discard curl_multi_perform(ctx.curlm, addr numfds)
+    for extra_fd in ctx.extra_fds.mitems:
+      # For now, this is always ssock.sock.getFd().
+      if extra_fd.events == extra_fd.revents:
+        ctx.acceptConnection()
+        extra_fd.revents = 0
+    var msgs_left: cint = 1
+    while msgs_left > 0:
+      let msg = curl_multi_info_read(ctx.curlm, addr msgs_left)
+      if msg == nil:
+        break
+      if msg.msg == CURLMSG_DONE: # the only possible value atm
+        var idx = -1
+        for i in 0 ..< ctx.handleList.len:
+          if ctx.handleList[i].curl == msg.easy_handle:
+            idx = i
+            break
+        assert idx != -1
+        ctx.finishCurlTransfer(ctx.handleList[idx], int(msg.data.result))
+        ctx.handleList.del(idx)
+  ctx.exitLoader()
+
+proc getAttribute(contentType, attrname: string): string =
+  let kvs = contentType.after(';')
+  var i = kvs.find(attrname)
+  var s = ""
+  if i != -1 and kvs.len > i + attrname.len and
+      kvs[i + attrname.len] == '=':
+    i += attrname.len + 1
+    while i < kvs.len and kvs[i] in AsciiWhitespace:
+      inc i
+    var q = false
+    for j in i ..< kvs.len:
+      if q:
+        s &= kvs[j]
+      else:
+        if kvs[j] == '\\':
+          q = true
+        elif kvs[j] == ';' or kvs[j] in AsciiWhitespace:
+          break
+        else:
+          s &= kvs[j]
+  return s
+
+proc applyHeaders(loader: FileLoader, request: Request, response: Response) =
+  if "Content-Type" in response.headers.table:
+    #TODO this is inefficient and broken on several levels. (In particular,
+    # it breaks mailcap named attributes other than charset.)
+    # Ideally, contentType would be a separate object type.
+    let header = response.headers.table["Content-Type"][0].toLowerAscii()
+    response.contenttype = header.until(';').strip().toLowerAscii()
+    response.charset = getCharset(header.getAttribute("charset"))
+  else:
+    response.contenttype = guessContentType($response.url.path,
+      "application/octet-stream", DefaultGuess)
+  if "Location" in response.headers.table:
+    if response.status in 301u16..303u16 or response.status in 307u16..308u16:
+      let location = response.headers.table["Location"][0]
+      let url = parseUrl(location, option(request.url))
+      if url.isSome:
+        if (response.status == 303 and
+            request.httpmethod notin {HTTP_GET, HTTP_HEAD}) or
+            (response.status == 301 or response.status == 302 and
+            request.httpmethod == HTTP_POST):
+          response.redirect = newRequest(url.get, HTTP_GET,
+            mode = request.mode, credentialsMode = request.credentialsMode,
+            destination = request.destination)
+        else:
+          response.redirect = newRequest(url.get, request.httpmethod,
+            body = request.body, multipart = request.multipart,
+            mode = request.mode, credentialsMode = request.credentialsMode,
+            destination = request.destination)
+
+#TODO: add init
+proc fetch*(loader: FileLoader, input: Request): FetchPromise =
+  let stream = connectSocketStream(loader.process, false, blocking = true)
+  stream.swrite(LOAD)
+  stream.swrite(input)
+  stream.flush()
+  let fd = int(stream.source.getFd())
+  loader.registerFun(fd)
+  let promise = FetchPromise()
+  loader.connecting[fd] = ConnectData(
+    promise: promise,
+    request: input,
+    stream: stream
+  )
+  return promise
+
+const BufferSize = 4096
+
+proc handleHeaders(loader: FileLoader, request: Request, response: Response,
+    stream: Stream): bool =
+  var status: int
+  stream.sread(status)
+  response.status = cast[uint16](status)
+  response.headers = newHeaders()
+  stream.sread(response.headers)
+  loader.applyHeaders(request, response)
+  # Only a stream of the response body may arrive after this point.
+  response.body = stream
+  return true # success
+
+proc onConnected*(loader: FileLoader, fd: int) =
+  let connectData = loader.connecting[fd]
+  let stream = connectData.stream
+  let promise = connectData.promise
+  let request = connectData.request
+  var res: int
+  stream.sread(res)
+  let response = newResponse(res, request, fd, stream)
+  if res == 0 and loader.handleHeaders(request, response, stream):
+    assert loader.unregisterFun != nil
+    let realCloseImpl = stream.closeImpl
+    stream.closeImpl = nil
+    response.unregisterFun = proc() =
+      loader.ongoing.del(fd)
+      loader.unregistered.add(fd)
+      loader.unregisterFun(fd)
+      realCloseImpl(stream)
+    loader.ongoing[fd] = OngoingData(
+      response: response,
+      readbufsize: BufferSize,
+      bodyRead: response.bodyRead
+    )
+    SocketStream(stream).source.getFd().setBlocking(false)
+    promise.resolve(JSResult[Response].ok(response))
+  else:
+    loader.unregisterFun(fd)
+    loader.unregistered.add(fd)
+    let err = newTypeError("NetworkError when attempting to fetch resource")
+    promise.resolve(JSResult[Response].err(err))
+  loader.connecting.del(fd)
+
+proc onRead*(loader: FileLoader, fd: int) =
+  loader.ongoing.withValue(fd, buffer):
+    let response = buffer[].response
+    while true:
+      let olen = buffer[].buf.len
+      buffer[].buf.setLen(olen + buffer.readbufsize)
+      try:
+        let n = response.body.readData(addr buffer[].buf[olen],
+          buffer.readbufsize)
+        if n != 0:
+          if buffer[].readbufsize < BufferSize:
+            buffer[].readbufsize = min(BufferSize, buffer[].readbufsize * 2)
+        buffer[].buf.setLen(olen + n)
+        if response.body.atEnd():
+          buffer[].bodyRead.resolve(buffer[].buf)
+          buffer[].bodyRead = nil
+          buffer[].buf = ""
+          response.unregisterFun()
+        break
+      except ErrorAgain, ErrorWouldBlock:
+        assert buffer.readbufsize > 1
+        buffer.readbufsize = buffer.readbufsize div 2
+
+proc onError*(loader: FileLoader, fd: int) =
+  loader.onRead(fd)
+
+proc doRequest*(loader: FileLoader, request: Request, blocking = true,
+    canredir = false): Response =
+  let response = Response(url: request.url)
+  let stream = connectSocketStream(loader.process, false, blocking = true)
+  if canredir:
+    request.canredir = true #TODO set this somewhere else?
+  stream.swrite(LOAD)
+  stream.swrite(request)
+  stream.flush()
+  stream.sread(response.res)
+  if response.res == 0:
+    if loader.handleHeaders(request, response, stream):
+      if not blocking:
+        stream.source.getFd().setBlocking(blocking)
+  return response
+
+proc quit*(loader: FileLoader) =
+  let stream = connectSocketStream(loader.process)
+  if stream != nil:
+    stream.swrite(QUIT)
diff --git a/src/loader/loaderhandle.nim b/src/loader/loaderhandle.nim
new file mode 100644
index 00000000..2c999813
--- /dev/null
+++ b/src/loader/loaderhandle.nim
@@ -0,0 +1,73 @@
+import net
+import streams
+
+import io/posixstream
+import io/serialize
+import io/socketstream
+import loader/headers
+
+type LoaderHandle* = ref object
+  ostream: Stream
+  # Only the first handle can be redirected, because a) mailcap can only
+  # redirect the first handle and b) async redirects would result in race
+  # conditions that would be difficult to untangle.
+  canredir: bool
+  sostream: Stream # saved ostream when redirected
+
+# Create a new loader handle, with the output stream ostream.
+proc newLoaderHandle*(ostream: Stream, canredir: bool): LoaderHandle =
+  return LoaderHandle(ostream: ostream, canredir: canredir)
+
+proc getFd*(handle: LoaderHandle): int =
+  return int(SocketStream(handle.ostream).source.getFd())
+
+proc sendResult*(handle: LoaderHandle, res: int): bool =
+  try:
+    handle.ostream.swrite(res)
+    return true
+  except IOError: # broken pipe
+    return false
+
+proc sendStatus*(handle: LoaderHandle, status: int): bool =
+  try:
+    handle.ostream.swrite(status)
+    return true
+  except IOError: # broken pipe
+    return false
+
+proc sendHeaders*(handle: LoaderHandle, headers: Headers): bool =
+  try:
+    handle.ostream.swrite(headers)
+    if handle.canredir:
+      var redir: bool
+      handle.ostream.sread(redir)
+      if redir:
+        let fd = SocketStream(handle.ostream).recvFileHandle()
+        handle.sostream = handle.ostream
+        let stream = newPosixStream(fd)
+        handle.ostream = stream
+    return true
+  except IOError: # broken pipe
+    return false
+
+proc sendData*(handle: LoaderHandle, p: pointer, nmemb: int): bool =
+  try:
+    handle.ostream.writeData(p, nmemb)
+    return true
+  except IOError: # broken pipe
+    return false
+
+proc sendData*(handle: LoaderHandle, s: string): bool =
+  if s.len > 0:
+    return handle.sendData(unsafeAddr s[0], s.len)
+  return true
+
+proc close*(handle: LoaderHandle) =
+  if handle.sostream != nil:
+    try:
+      handle.sostream.swrite(true)
+    except IOError:
+      # ignore error, that just means the buffer has already closed the stream
+      discard
+    handle.sostream.close()
+  handle.ostream.close()
diff --git a/src/loader/request.nim b/src/loader/request.nim
new file mode 100644
index 00000000..e5f8f013
--- /dev/null
+++ b/src/loader/request.nim
@@ -0,0 +1,332 @@
+import options
+import streams
+import strutils
+import tables
+
+import bindings/quickjs
+import js/dict
+import js/error
+import js/fromjs
+import js/javascript
+import loader/headers
+import types/blob
+import types/formdata
+import types/referer
+import types/url
+
+type
+  HttpMethod* = enum
+    HTTP_GET = "GET"
+    HTTP_CONNECT = "CONNECT"
+    HTTP_DELETE = "DELETE"
+    HTTP_HEAD = "HEAD"
+    HTTP_OPTIONS = "OPTIONS"
+    HTTP_PATCH = "PATCH"
+    HTTP_POST = "POST"
+    HTTP_PUT = "PUT"
+    HTTP_TRACE = "TRACE"
+
+  RequestMode* = enum
+    NO_CORS = "no-cors"
+    SAME_ORIGIN = "same-origin"
+    CORS = "cors"
+    NAVIGATE = "navigate"
+    WEBSOCKET = "websocket"
+
+  RequestDestination* = enum
+    NO_DESTINATION = ""
+    AUDIO = "audio"
+    AUDIOWORKLET = "audioworklet"
+    DOCUMENT = "document"
+    EMBED = "embed"
+    FONT = "font"
+    FRAME = "frame"
+    IFRAME = "iframe"
+    IMAGE = "image"
+    MANIFEST = "manifest"
+    OBJECT = "object"
+    PAINTWORKLET = "paintworklet"
+    REPORT = "report"
+    SCRIPT = "script"
+    SERVICEWORKER = "serviceworker"
+    SHAREDWORKER = "sharedworker"
+    STYLE = "style"
+    TRACK = "track"
+    WORKER = "worker"
+    XSLT = "xslt"
+
+  CredentialsMode* = enum
+    SAME_ORIGIN = "same-origin"
+    OMIT = "omit"
+    INCLUDE = "include"
+
+  CORSAttribute* = enum
+    NO_CORS = "no-cors"
+    ANONYMOUS = "anonymous"
+    USE_CREDENTIALS = "use-credentials"
+
+type
+  Request* = ref RequestObj
+  RequestObj* = object
+    httpmethod*: HttpMethod
+    url*: Url
+    headers* {.jsget.}: Headers
+    body*: Opt[string]
+    multipart*: Opt[FormData]
+    referer*: URL
+    mode* {.jsget.}: RequestMode
+    destination* {.jsget.}: RequestDestination
+    credentialsMode* {.jsget.}: CredentialsMode
+    proxy*: URL #TODO do something with this
+    canredir*: bool
+ 
+  ReadableStream* = ref object of Stream
+    isource*: Stream
+    buf: string
+    isend: bool
+
+jsDestructor(Request)
+
+proc js_url(this: Request): string {.jsfget: "url".} =
+  return $this.url
+
+#TODO pretty sure this is incorrect
+proc js_referrer(this: Request): string {.jsfget: "referrer".} =
+  if this.referer != nil:
+    return $this.referer
+  return ""
+
+iterator pairs*(headers: Headers): (string, string) =
+  for k, vs in headers.table:
+    for v in vs:
+      yield (k, v)
+
+proc rsReadData(s: Stream, buffer: pointer, bufLen: int): int =
+  var s = ReadableStream(s)
+  if s.atEnd:
+    return 0
+  while s.buf.len < bufLen:
+    var len: int
+    s.isource.read(len)
+    if len == 0:
+      result = s.buf.len
+      copyMem(buffer, addr(s.buf[0]), result)
+      s.buf = s.buf.substr(result)
+      s.isend = true
+      return
+    var nbuf: string
+    s.isource.readStr(len, nbuf)
+    s.buf &= nbuf
+  assert s.buf.len >= bufLen
+  result = bufLen
+  copyMem(buffer, addr(s.buf[0]), result)
+  s.buf = s.buf.substr(result)
+  if s.buf.len == 0:
+    var len: int
+    s.isource.read(len)
+    if len == 0:
+      s.isend = true
+    else:
+      s.isource.readStr(len, s.buf)
+
+proc rsAtEnd(s: Stream): bool =
+  ReadableStream(s).isend
+
+proc rsClose(s: Stream) = {.cast(tags: [WriteIOEffect]).}: #TODO TODO TODO ew.
+  var s = ReadableStream(s)
+  if s.isend: return
+  s.buf = ""
+  while true:
+    var len: int
+    s.isource.read(len)
+    if len == 0:
+      s.isend = true
+      break
+    s.isource.setPosition(s.isource.getPosition() + len)
+
+proc newReadableStream*(isource: Stream): ReadableStream =
+  new(result)
+  result.isource = isource
+  result.readDataImpl = rsReadData
+  result.atEndImpl = rsAtEnd
+  result.closeImpl = rsClose
+  var len: int
+  result.isource.read(len)
+  if len == 0:
+    result.isend = true
+  else:
+    result.isource.readStr(len, result.buf)
+
+func newRequest*(url: URL, httpmethod = HTTP_GET, headers = newHeaders(),
+    body = opt(string), multipart = opt(FormData), mode = RequestMode.NO_CORS,
+    credentialsMode = CredentialsMode.SAME_ORIGIN,
+    destination = RequestDestination.NO_DESTINATION, proxy: URL = nil,
+    canredir = false): Request =
+  return Request(
+    url: url,
+    httpmethod: httpmethod,
+    headers: headers,
+    body: body,
+    multipart: multipart,
+    mode: mode,
+    credentialsMode: credentialsMode,
+    destination: destination,
+    proxy: proxy
+  )
+
+func newRequest*(url: URL, httpmethod = HTTP_GET,
+    headers: seq[(string, string)] = @[], body = opt(string),
+    multipart = opt(FormData), mode = RequestMode.NO_CORS, proxy: URL = nil,
+    canredir = false):
+    Request =
+  let hl = newHeaders()
+  for pair in headers:
+    let (k, v) = pair
+    hl.table[k] = @[v]
+  return newRequest(url, httpmethod, hl, body, multipart, mode, proxy = proxy)
+
+func createPotentialCORSRequest*(url: URL, destination: RequestDestination, cors: CORSAttribute, fallbackFlag = false): Request =
+  var mode = if cors == NO_CORS:
+    RequestMode.NO_CORS
+  else:
+    RequestMode.CORS
+  if fallbackFlag and mode == NO_CORS:
+    mode = SAME_ORIGIN
+  let credentialsMode = if cors == ANONYMOUS:
+    CredentialsMode.SAME_ORIGIN
+  else: CredentialsMode.INCLUDE
+  return newRequest(url, destination = destination, mode = mode, credentialsMode = credentialsMode)
+
+type
+  BodyInitType = enum
+    BODY_INIT_BLOB, BODY_INIT_FORM_DATA, BODY_INIT_URL_SEARCH_PARAMS,
+    BODY_INIT_STRING
+
+  BodyInit = object
+    #TODO ReadableStream, BufferSource
+    case t: BodyInitType
+    of BODY_INIT_BLOB:
+      blob: Blob
+    of BODY_INIT_FORM_DATA:
+      formData: FormData
+    of BODY_INIT_URL_SEARCH_PARAMS:
+      searchParams: URLSearchParams
+    of BODY_INIT_STRING:
+      str: string
+
+  RequestInit* = object of JSDict
+    #TODO aliasing in dicts
+    `method`: HttpMethod # default: GET
+    headers: Opt[HeadersInit]
+    body: Opt[BodyInit]
+    referrer: Opt[string]
+    referrerPolicy: Opt[ReferrerPolicy]
+    credentials: Opt[CredentialsMode]
+    proxyUrl: URL
+    mode: Opt[RequestMode]
+
+proc fromJS2*(ctx: JSContext, val: JSValue, res: var JSResult[BodyInit]) =
+  if JS_IsUndefined(val) or JS_IsNull(val):
+    res.err(nil)
+    return
+  if not JS_IsObject(val):
+    res.err(newTypeError("Not an object"))
+    return
+  block formData:
+    let x = fromJS[FormData](ctx, val)
+    if x.isSome:
+      res.ok(BodyInit(t: BODY_INIT_FORM_DATA, formData: x.get))
+      return
+  block blob:
+    let x = fromJS[Blob](ctx, val)
+    if x.isSome:
+      res.ok(BodyInit(t: BODY_INIT_BLOB, blob: x.get))
+      return
+  block searchParams:
+    let x = fromJS[URLSearchParams](ctx, val)
+    if x.isSome:
+      res.ok(BodyInit(t: BODY_INIT_URL_SEARCH_PARAMS, searchParams: x.get))
+      return
+  block str:
+    let x = fromJS[string](ctx, val)
+    if x.isSome:
+      res.ok(BodyInit(t: BODY_INIT_STRING, str: x.get))
+      return
+  res.err(newTypeError("Invalid body init type"))
+
+func newRequest*[T: string|Request](ctx: JSContext, resource: T,
+    init = none(RequestInit)): JSResult[Request] {.jsctor.} =
+  when T is string:
+    let url = ?newURL(resource)
+    if url.username != "" or url.password != "":
+      return err(newTypeError("Input URL contains a username or password"))
+    var httpMethod = HTTP_GET
+    var headers = newHeaders()
+    let referer: URL = nil
+    var credentials = CredentialsMode.SAME_ORIGIN
+    var body: Opt[string]
+    var multipart: Opt[FormData]
+    var proxyUrl: URL #TODO?
+    let fallbackMode = opt(RequestMode.CORS)
+  else:
+    let url = resource.url
+    var httpMethod = resource.httpMethod
+    var headers = resource.headers.clone()
+    let referer = resource.referer
+    var credentials = resource.credentialsMode
+    var body = resource.body
+    var multipart = resource.multipart
+    var proxyUrl = resource.proxy #TODO?
+    let fallbackMode = opt(RequestMode)
+    #TODO window
+  var mode = fallbackMode.get(RequestMode.NO_CORS)
+  let destination = NO_DESTINATION
+  #TODO origin, window
+  if init.isSome:
+    if mode == RequestMode.NAVIGATE:
+      mode = RequestMode.SAME_ORIGIN
+    #TODO flags?
+    #TODO referrer
+    let init = init.get
+    httpMethod = init.`method`
+    if init.body.isSome:
+      let ibody = init.body.get
+      case ibody.t
+      of BODY_INIT_FORM_DATA:
+        multipart = opt(ibody.formData)
+      of BODY_INIT_STRING:
+        body = opt(ibody.str)
+      else:
+        discard #TODO
+      if httpMethod in {HTTP_GET, HTTP_HEAD}:
+        return err(newTypeError("HEAD or GET Request cannot have a body."))
+    if init.headers.isSome:
+      headers.fill(init.headers.get)
+    if init.credentials.isSome:
+      credentials = init.credentials.get
+    if init.mode.isSome:
+      mode = init.mode.get
+    #TODO find a standard compatible way to implement this
+    proxyUrl = init.proxyUrl
+  return ok(Request(
+    url: url,
+    httpmethod: httpmethod,
+    headers: headers,
+    body: body,
+    multipart: multipart,
+    mode: mode,
+    credentialsMode: credentials,
+    destination: destination,
+    proxy: proxyUrl,
+    referer: referer
+  ))
+
+func credentialsMode*(attribute: CORSAttribute): CredentialsMode =
+  case attribute
+  of NO_CORS, ANONYMOUS:
+    return SAME_ORIGIN
+  of USE_CREDENTIALS:
+    return INCLUDE
+
+proc addRequestModule*(ctx: JSContext) =
+  ctx.registerType(Request)
diff --git a/src/loader/response.nim b/src/loader/response.nim
new file mode 100644
index 00000000..5c95e3af
--- /dev/null
+++ b/src/loader/response.nim
@@ -0,0 +1,74 @@
+import streams
+
+import bindings/quickjs
+import io/promise
+import js/error
+import js/javascript
+import loader/headers
+import loader/request
+import types/url
+
+import chakasu/charset
+
+type
+  Response* = ref object
+    res*: int
+    fd*: int
+    body*: Stream
+    bodyUsed* {.jsget.}: bool
+    contenttype* {.jsget.}: string
+    status* {.jsget.}: uint16
+    headers* {.jsget.}: Headers
+    redirect*: Request
+    url*: URL #TODO should be urllist?
+    unregisterFun*: proc()
+    bodyRead*: Promise[string]
+    charset*: Charset
+
+jsDestructor(Response)
+
+proc newResponse*(res: int, request: Request, fd = -1, stream: Stream = nil):
+    Response =
+  return Response(
+    res: res,
+    url: request.url,
+    body: stream,
+    bodyRead: Promise[string](),
+    fd: fd
+  )
+
+func sok(response: Response): bool {.jsfget: "ok".} =
+  return response.status in 200u16 .. 299u16
+
+func surl(response: Response): string {.jsfget: "url".} =
+  return $response.url
+
+#TODO: this should be a property of body
+proc close*(response: Response) {.jsfunc.} =
+  response.bodyUsed = true
+  if response.unregisterFun != nil:
+    response.unregisterFun()
+  if response.body != nil:
+    response.body.close()
+
+proc text*(response: Response): Promise[JSResult[string]] {.jsfunc.} =
+  if response.bodyRead == nil:
+    let p = newPromise[JSResult[string]]()
+    let err = JSResult[string]
+      .err(newTypeError("Body has already been consumed"))
+    p.resolve(err)
+    return p
+  let bodyRead = response.bodyRead
+  response.bodyRead = nil
+  return bodyRead.then(proc(s: string): JSResult[string] =
+    ok(s))
+
+proc json(ctx: JSContext, this: Response): Promise[JSResult[JSValue]]
+    {.jsfunc.} =
+  return this.text().then(proc(s: JSResult[string]): JSResult[JSValue] =
+    let s = ?s
+    return ok(JS_ParseJSON(ctx, cstring(s), cast[csize_t](s.len),
+      cstring"<input>")))
+
+proc addResponseModule*(ctx: JSContext) =
+  ctx.registerType(Response)