Refactor fileloader

4 files changed, 263 insertions, 212 deletions

diff --git a/src/io/buffer.nim b/src/io/buffer.nim
index c9cb64a0..08feb73f 100644
--- a/src/io/buffer.nim
+++ b/src/io/buffer.nim
@@ -933,8 +933,8 @@ type
     url*: string
     httpmethod*: HttpMethod
     mimetype*: string
-    body*: string
-    multipart*: MimeData
+    body*: Option[string]
+    multipart*: Option[MimeData]
 
 # https://html.spec.whatwg.org/multipage/form-control-infrastructure.html#constructing-the-form-data-set
 proc constructEntryList(form: HTMLFormElement, submitter: Element = nil, encoding: string = ""): Table[string, string] =
@@ -1029,7 +1029,6 @@ proc makeCRLF(s: string): string =
     inc i
 
 proc serializeMultipartFormData(kvs: Table[string, string]): MimeData =
-  new(result)
   for name, value in kvs:
     let name = makeCRLF(name)
     let value = makeCRLF(value)
@@ -1080,18 +1079,18 @@ proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction]
     return ClickAction(url: $parsedaction, httpmethod: httpmethod).some
 
   template submitAsEntityBody() =
-    var body: string
     var mimetype: string
-    var multipart: MimeData
+    var body = none(string)
+    var multipart = none(MimeData)
     case enctype
     of FORM_ENCODING_TYPE_URLENCODED:
-      body = serializeApplicationXWWFormUrlEncoded(entrylist)
+      body = serializeApplicationXWWFormUrlEncoded(entrylist).some
       mimeType = $enctype
     of FORM_ENCODING_TYPE_MULTIPART:
-      multipart = serializeMultipartFormData(entrylist) 
+      multipart = serializeMultipartFormData(entrylist).some
       mimetype = $enctype
     of FORM_ENCODING_TYPE_TEXT_PLAIN:
-      body = serializePlainTextFormData(entrylist)
+      body = serializePlainTextFormData(entrylist).some
       mimetype = $enctype
     return ClickAction(url: $parsedaction, httpmethod: httpmethod, body: body, mimetype: mimetype, multipart: multipart).some
 
diff --git a/src/io/http.nim b/src/io/http.nim
new file mode 100644
index 00000000..0cba2c29
--- /dev/null
+++ b/src/io/http.nim
@@ -0,0 +1,127 @@
+import options
+import streams
+import strutils
+
+import bindings/curl
+import io/loadertypes
+import types/url
+import types/mime
+import utils/twtstr
+
+type
+  HeaderResult* = ref object
+    statusline: bool
+    headers: HeaderList
+
+template setopt(curl: CURL, opt: CURLoption, arg: typed) =
+  discard curl_easy_setopt(curl, opt, arg)
+
+template setopt(curl: CURL, opt: CURLoption, arg: string) =
+  discard curl_easy_setopt(curl, opt, cstring(arg))
+
+template getinfo(curl: CURL, info: CURLINFO, arg: typed) =
+  discard curl_easy_getinfo(curl, info, arg)
+
+proc curlWriteHeader(p: cstring, size: csize_t, nitems: csize_t, userdata: pointer): csize_t {.cdecl.} =
+  var line = newString(nitems)
+  for i in 0..<nitems:
+    line[i] = p[i]
+
+  let headers = cast[HeaderResult](userdata)
+  if not headers.statusline:
+    headers.statusline = true
+    return nitems #TODO handle status line?
+
+  let k = line.until(':')
+
+  if k.len == line.len:
+    return nitems # empty line (last, before body) or invalid (=> error)
+
+  let v = line.substr(k.len + 1).strip()
+  headers.headers.add(k, v)
+  return nitems
+
+proc curlWriteBody(p: cstring, size: csize_t, nmemb: csize_t, userdata: pointer): csize_t {.cdecl.} =
+  var s = newString(nmemb)
+  for i in 0..<nmemb:
+    s[i] = p[i]
+  let stream = cast[Stream](userdata)
+  stream.write(s)
+  stream.flush()
+  return nmemb
+
+proc getPageHttp*(request: Request): LoadResult =
+  let curl = curl_easy_init()
+
+  if curl == nil: return # fail
+
+  let surl = request.url.serialize()
+  curl.setopt(CURLOPT_URL, surl)
+
+  var cs = newStringStream()
+  curl.setopt(CURLOPT_WRITEDATA, cs)
+  curl.setopt(CURLOPT_WRITEFUNCTION, curlWriteBody)
+
+  let headerres = HeaderResult(headers: newHeaderList())
+  curl.setopt(CURLOPT_HEADERDATA, headerres)
+  curl.setopt(CURLOPT_HEADERFUNCTION, curlWriteHeader)
+
+  var mime: curl_mime = nil
+
+  case request.httpmethod
+  of HTTP_GET: curl.setopt(CURLOPT_HTTPGET, 1)
+  of HTTP_POST:
+    curl.setopt(CURLOPT_POST, 1)
+    if request.multipart.issome:
+      mime = curl_mime_init(curl)
+      if mime == nil: return # fail
+      for entry in request.multipart.get.content:
+        let part = curl_mime_addpart(mime)
+        if part == nil: return # fail
+        curl_mime_name(part, cstring(entry.name))
+        if entry.isFile:
+          if entry.isStream:
+            curl_mime_filedata(part, cstring(entry.filename))
+          else:
+            let fd = readFile(entry.filename)
+            curl_mime_data(part, cstring(fd), csize_t(fd.len))
+          # may be overridden by curl_mime_filedata, so set it here
+          curl_mime_filename(part, cstring(entry.filename))
+        else:
+          curl_mime_data(part, cstring(entry.content), csize_t(entry.content.len))
+      curl.setopt(CURLOPT_MIMEPOST, mime)
+    elif request.body.issome:
+      curl.setopt(CURLOPT_POSTFIELDS, cstring(request.body.get))
+      curl.setopt(CURLOPT_POSTFIELDSIZE, request.body.get.len)
+  else: discard #TODO
+
+  var slist: curl_slist = nil
+  for k, v in request.headers:
+    let header = k & ": " & v
+    slist = curl_slist_append(slist, cstring(header))
+  if slist != nil:
+    curl.setopt(CURLOPT_HTTPHEADER, slist)
+
+  let res = curl_easy_perform(curl)
+  if res == CURLE_OK: # TODO handle errors
+    cs.setPosition(0)
+    result.s = cs
+
+    let ct = headerres.headers.getOrDefault("Content-Type")
+    if ct != "":
+      result.contenttype = ct.until(';')
+    else:
+      result.contenttype = guessContentType(request.url.path.serialize())
+    curl.getinfo(CURLINFO_RESPONSE_CODE, addr result.status)
+    if result.status in {301, 302, 303}: #TODO 300, 304, 307
+      var urlp: cstring
+      curl.getinfo(CURLINFO_REDIRECT_URL, addr urlp)
+      if urlp != nil:
+        let urls = $urlp
+        result.redirect = parseUrl(urls, some(request.url))
+
+  curl_easy_cleanup(curl)
+  if mime != nil:
+    curl_mime_free(mime)
+  if slist != nil:
+    curl_slist_free_all(slist)
diff --git a/src/io/loader.nim b/src/io/loader.nim
index 2efc15c2..f7ac06e6 100644
--- a/src/io/loader.nim
+++ b/src/io/loader.nim
@@ -1,35 +1,15 @@
 import options
 import streams
-import strutils
 import tables
+when defined(posix):
+  import posix
 
-import bindings/curl
+import io/http
+import io/loadertypes
 import types/mime
 import types/url
-import utils/twtstr
 
-type
-  HttpMethod* = enum
-    HTTP_CONNECT, HTTP_DELETE, HTTP_GET, HTTP_HEAD, HTTP_OPTIONS, HTTP_PATCH,
-    HTTP_POST, HTTP_PUT, HTTP_TRACE
-
-type
-  FileLoader* = ref object
-    headers*: HttpHeaderList
-
-  HttpHeaderList = ref object
-    table: Table[string, seq[string]]
-
-  LoadResult* = object
-    s*: Stream
-    contenttype*: string
-    status*: int
-    headers*: HttpHeaderList
-    redirect*: Option[Url]
-
-  HeaderResult = ref object
-    statusline: bool
-    headers: HttpHeaderList
+export loadertypes
 
 const DefaultHeaders = {
   "User-Agent": "chawan",
@@ -37,192 +17,41 @@ const DefaultHeaders = {
   "Accept-Language": "en;q=1.0",
   "Pragma": "no-cache",
   "Cache-Control": "no-cache",
-}
-
-proc newFileLoader*(headers: HttpHeaderList): FileLoader =
+}.toTable().newHeaderList()
+
+proc doFork(): Pid =
+  result = fork()
+  if result == -1:
+    eprint "Failed to fork child process."
+    quit(1)
+  elif result != 0:
+    return result
+  discard setsid()
+  let pid = fork()
+  if pid != 0:
+    quit(0)
+  return 0
+
+proc newFileLoader*(defaultHeaders: HeaderList): FileLoader =
   new(result)
-  result.headers = headers
-
-proc newHttpHeaderList*(): HttpHeaderList =
-  new(result)
-
-proc add(headers: HttpHeaderList, k, v: string) =
-  let k = k.toHeaderCase()
-  if k notin headers.table:
-    headers.table[k] = @[v]
-  else:
-    headers.table[k].add(v)
-
-proc `[]=`(headers: HttpHeaderList, k, v: string) =
-  headers.table[k.toHeaderCase()] = @[v]
-
-iterator pairs(headers: HttpHeaderList): (string, string) =
-  for k, vs in headers.table:
-    for v in vs:
-      yield (k, v)
+  result.defaultHeaders = defaultHeaders
 
 proc newFileLoader*(): FileLoader =
-  var headers = new(HttpHeaderList)
-  for header in DefaultHeaders:
-    headers[header[0]] = header[1]
-  newFileLoader(headers)
-
-proc getOrDefault*(headers: HttpHeaderList, k: string): string =
-  let k = k.toHeaderCase()
-  if k in headers.table:
-    headers.table[k][0]
-  else:
-    k
-
-# Originally from the stdlib
-type
-  MimePart* = object
-    name, content: string
-    case isFile: bool
-    of true:
-      filename, contentType: string
-      fileSize: int64
-      isStream: bool
-    else: discard
-
-  MimeData* = ref object
-    content: seq[MimePart]
-
-proc `[]=`*(multipart: MimeData, k, v: string) =
-  multipart.content.add(MimePart(name: k, content: v))
-
-proc curlWriteHeader(p: cstring, size: csize_t, nitems: csize_t, userdata: pointer): csize_t {.cdecl.} =
-  var line = newString(nitems)
-  for i in 0..<nitems:
-    line[i] = p[i]
-
-  let headers = cast[HeaderResult](userdata)
-  if not headers.statusline:
-    headers.statusline = true
-    return nitems #TODO handle status line?
-
-  let k = line.until(':')
-
-  if k.len == line.len:
-    return nitems # empty line (last, before body) or invalid (=> error)
-
-  let v = line.substr(k.len + 1).strip()
-  headers.headers.add(k, v)
-  return nitems
+  newFileLoader(DefaultHeaders)
 
-proc curlWriteBody(p: cstring, size: csize_t, nmemb: csize_t, userdata: pointer): csize_t {.cdecl.} =
-  var s = newString(nmemb)
-  for i in 0..<nmemb:
-    s[i] = p[i]
-  let stream = cast[Stream](userdata)
-  stream.write(s)
-  stream.flush()
-  return nmemb
-
-template setopt(curl: CURL, opt: CURLoption, arg: typed) =
-  discard curl_easy_setopt(curl, opt, arg)
-
-template setopt(curl: CURL, opt: CURLoption, arg: string) =
-  discard curl_easy_setopt(curl, opt, cstring(arg))
-
-template getinfo(curl: CURL, info: CURLINFO, arg: typed) =
-  discard curl_easy_getinfo(curl, info, arg)
-
-proc getPageLibcurl(loader: FileLoader, url: Url, smethod: HttpMethod = HTTP_GET, mimetype = "", body: string = "", multipart: MimeData = nil): LoadResult =
-  let curl = curl_easy_init()
-
-  if curl == nil: return # fail
-
-  let surl = url.serialize()
-  curl.setopt(CURLOPT_URL, surl)
-
-  var cs = newStringStream()
-  curl.setopt(CURLOPT_WRITEDATA, cs)
-  curl.setopt(CURLOPT_WRITEFUNCTION, curlWriteBody)
-
-  let headers = newHttpHeaderList()
-  let headerres = HeaderResult(headers: headers)
-  curl.setopt(CURLOPT_HEADERDATA, headerres)
-  curl.setopt(CURLOPT_HEADERFUNCTION, curlWriteHeader)
-
-  var mime: curl_mime = nil
-
-  case smethod
-  of HTTP_GET: curl.setopt(CURLOPT_HTTPGET, 1)
-  of HTTP_POST:
-    curl.setopt(CURLOPT_POST, 1)
-    if multipart != nil:
-      mime = curl_mime_init(curl)
-      if mime == nil: return # fail
-      for entry in multipart.content:
-        let part = curl_mime_addpart(mime)
-        if part == nil: return # fail
-        curl_mime_name(part, cstring(entry.name))
-        if entry.isFile:
-          if entry.isStream:
-            curl_mime_filedata(part, cstring(entry.filename))
-          else:
-            let fd = readFile(entry.filename)
-            curl_mime_data(part, cstring(fd), csize_t(fd.len))
-          # may be overridden by curl_mime_filedata, so set it here
-          curl_mime_filename(part, cstring(entry.filename))
-        else:
-          curl_mime_data(part, cstring(entry.content), csize_t(entry.content.len))
-      curl.setopt(CURLOPT_MIMEPOST, mime)
-    elif body != "":
-      curl.setopt(CURLOPT_POSTFIELDS, cstring(body))
-      curl.setopt(CURLOPT_POSTFIELDSIZE, body.len)
-  else: discard #TODO
-
-  var requestHeaders = newHttpHeaderList()
-  requestHeaders.table = loader.headers.table
-  if mimetype != "":
-    requestHeaders["Content-Type"] = mimetype
-  var slist: curl_slist = nil
-  for k, v in requestHeaders:
-    let header = k & ": " & v
-    slist = curl_slist_append(slist, cstring(header))
-  if slist != nil:
-    curl.setopt(CURLOPT_HTTPHEADER, slist)
-
-  let res = curl_easy_perform(curl)
-  if res == CURLE_OK: # TODO handle errors
-    cs.setPosition(0)
-    result.s = cs
-
-    let ct = headers.getOrDefault("Content-Type")
-    if ct != "":
-      result.contenttype = ct.until(';')
-    else:
-      result.contenttype = guessContentType(url.path.serialize())
-    discard curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, addr result.status)
-    if result.status in {301, 302, 303}: #TODO 300, 304, 307
-      var urlp: cstring
-      curl.getinfo(CURLINFO_REDIRECT_URL, addr urlp)
-      if urlp != nil:
-        let urls = $urlp
-        result.redirect = parseUrl(urls, some(url))
-
-  curl_easy_cleanup(curl)
-  if mime != nil:
-    curl_mime_free(mime)
-  if slist != nil:
-    curl_slist_free_all(slist)
-
-proc getPage*(loader: FileLoader, url: Url, smethod: HttpMethod = HTTP_GET, mimetype = "", body: string = "", multipart: MimeData = nil): LoadResult =
-  if url.scheme == "file":
+proc getPage*(loader: FileLoader, url: Url, smethod: HttpMethod = HTTP_GET, mimetype = "", body = none(string), multipart = none(MimeData)): LoadResult =
+  case url.scheme
+  of "file":
     when defined(windows) or defined(OS2) or defined(DOS):
       let path = url.path.serialize_unicode_dos()
     else:
       let path = url.path.serialize_unicode()
     result.contenttype = guessContentType(path)
     result.s = newFileStream(path, fmRead)
-    result.status = 200 # doesn't make much sense...
-  elif url.scheme == "http" or url.scheme == "https":
-    return getPageLibcurl(loader, url, smethod, mimetype, body, multipart)
-
-proc getPage*(loader: FileLoader, url: string, smethod: HttpMethod = HTTP_GET, mimetype = "", body: string = "", multipart: MimeData = nil): LoadResult =
-  let url = parseUrl(url)
-  if url.isnone:
-    raise newException(Exception, "Invalid URL")
-  loader.getPage(url.get, smethod, mimetype, body, multipart)
+    if result.s != nil:
+      result.status = 200 # ok
+    else:
+      result.status = 404 # file not found
+  of "http", "https":
+    let request = loader.newRequest(url, smethod, {"Content-Type": mimetype}, body, multipart)
+    return getPageHttp(request)
diff --git a/src/io/loadertypes.nim b/src/io/loadertypes.nim
new file mode 100644
index 00000000..4c910b44
--- /dev/null
+++ b/src/io/loadertypes.nim
@@ -0,0 +1,96 @@
+import options
+import streams
+import tables
+
+import utils/twtstr
+import types/url
+
+type HttpMethod* = enum
+  HTTP_CONNECT, HTTP_DELETE, HTTP_GET, HTTP_HEAD, HTTP_OPTIONS, HTTP_PATCH,
+  HTTP_POST, HTTP_PUT, HTTP_TRACE
+
+type
+  Request* = ref object
+    httpmethod*: HttpMethod
+    url*: Url
+    headers*: HeaderList
+    body*: Option[string]
+    multipart*: Option[MimeData]
+
+  FileLoader* = ref object
+    defaultHeaders*: HeaderList
+
+  LoadResult* = object
+    s*: Stream
+    contenttype*: string
+    status*: int
+    headers*: HeaderList
+    redirect*: Option[Url]
+
+  HeaderList* = object
+    table*: Table[string, seq[string]]
+
+# Originally from the stdlib
+  MimePart* = object
+    name*, content*: string
+    case isFile*: bool
+    of true:
+      filename*, contentType*: string
+      fileSize*: int64
+      isStream*: bool
+    else: discard
+
+  MimeData* = object
+    content*: seq[MimePart]
+
+iterator pairs*(headers: HeaderList): (string, string) =
+  for k, vs in headers.table:
+    for v in vs:
+      yield (k, v)
+
+func newHeaderList*(): HeaderList =
+  discard
+
+func newHeaderList*(table: Table[string, string]): HeaderList =
+  for k, v in table:
+    let k = k.toHeaderCase()
+    if k in result.table:
+      result.table[k].add(v)
+    else:
+      result.table[k] = @[v]
+
+func newRequest*(loader: FileLoader,
+                 url: Url,
+                 httpmethod = HTTP_GET,
+                 headers: openarray[(string, string)] = [],
+                 body = none(string),
+                 multipart = none(MimeData)): Request =
+  new(result)
+  result.httpmethod = httpmethod
+  result.url = url
+  result.headers.table = loader.defaultHeaders.table
+  for it in headers:
+    if it[1] != "": #TODO not sure if this is a good idea, options would probably work better
+      result.headers.table[it[0]] = @[it[1]]
+  result.body = body
+  result.multipart = multipart
+
+proc `[]=`*(multipart: var MimeData, k, v: string) =
+  multipart.content.add(MimePart(name: k, content: v))
+
+proc add*(headers: var HeaderList, k, v: string) =
+  let k = k.toHeaderCase()
+  if k notin headers.table:
+    headers.table[k] = @[v]
+  else:
+    headers.table[k].add(v)
+
+proc `[]=`*(headers: var HeaderList, k, v: string) =
+  headers.table[k.toHeaderCase()] = @[v]
+
+func getOrDefault*(headers: HeaderList, k: string, default = ""): string =
+  let k = k.toHeaderCase()
+  if k in headers.table:
+    headers.table[k][0]
+  else:
+    default