Clean up client request methods, add DUPE_BUFFER

5 files changed, 93 insertions, 90 deletions

diff --git a/src/client.nim b/src/client.nim
index 1df1b3d6..202090ca 100644
--- a/src/client.nim
+++ b/src/client.nim
@@ -10,6 +10,7 @@ import io/buffer
 import io/cell
 import io/lineedit
 import io/loader
+import io/serialize
 import io/term
 import js/javascript
 import js/regex
@@ -94,6 +95,8 @@ proc addBuffer(client: Client) =
     client.buffer.next.next = oldnext
     client.buffer = client.buffer.next
   client.buffer.loader = client.loader
+  client.buffer.userstyle = client.userstyle
+  client.buffer.markcolor = gconfig.markcolor
 
 proc prevBuffer(client: Client) =
   if client.buffer.prev != nil:
@@ -135,16 +138,26 @@ proc discardBuffer(client: Client) =
 
 proc setupBuffer(client: Client) =
   let buffer = client.buffer
-  buffer.userstyle = client.userstyle
-  buffer.markcolor = gconfig.markcolor
   buffer.load()
   buffer.render()
   buffer.gotoAnchor()
   buffer.redraw = true
 
+proc dupeBuffer(client: Client, location = none(Url)) =
+  let prev = client.buffer
+  client.addBuffer()
+  client.buffer.contenttype = prev.contenttype
+  client.buffer.ispipe = prev.ispipe
+  client.buffer.istream = newStringStream(prev.source)
+  if location.issome:
+    client.buffer.location = location.get
+  else:
+    client.buffer.location = prev.location
+  client.buffer.document = prev.document
+  client.setupBuffer()
+
 proc readPipe(client: Client, ctype: string) =
-  client.buffer = newBuffer()
-  client.buffer.loader = client.loader
+  client.addBuffer()
   client.buffer.contenttype = if ctype != "": ctype else: "text/plain"
   client.buffer.ispipe = true
   client.buffer.istream = newFileStream(stdin)
@@ -157,51 +170,53 @@ proc readPipe(client: Client, ctype: string) =
   else:
     client.buffer.drawBuffer()
 
+# Load request in a new buffer.
 var g_client: Client
-proc getPage(client: Client, url: Url, click = none(ClickAction)): LoadResult =
-  let page = if click.isnone:
-    client.loader.getPage(newRequest(url))
-  else:
-    client.loader.getPage(newRequest(url, click.get.httpmethod, {"Content-Type": click.get.mimetype}, click.get.body, click.get.multipart))
-  return page
-
-# Load url in a new buffer.
-proc gotoUrl(client: Client, url: Url, click = none(ClickAction), prevurl = none(Url), force = false, ctype = "") =
+proc gotoUrl(client: Client, request: Request, prevurl = none(Url), force = false, ctype = "") =
   setControlCHook(proc() {.noconv.} =
     raise newException(InterruptError, "Interrupted"))
-  if force or prevurl.issome or not prevurl.get.equals(url, true):
+  if force or prevurl.isnone or not prevurl.get.equals(request.url, true):
     try:
-      let page = client.getPage(url, click)
+      let page = client.loader.getPage(request)
       client.needsauth = page.status == 401 # Unauthorized
       client.redirecturl = page.redirect
-      if page.s != nil:
+      var buf: string
+      page.s.sread(buf)
+      if buf != "": #TODO what about pages with an empty body?
         client.addBuffer()
         g_client = client
         setControlCHook(proc() {.noconv.} =
           if g_client.buffer.prev != nil or g_client.buffer.next != nil:
             g_client.discardBuffer()
           interruptError())
-        client.buffer.istream = page.s
         client.buffer.contenttype = if ctype != "": ctype else: page.contenttype
+        client.buffer.istream = newStringStream()
+        while true:
+          client.buffer.istream.write(buf)
+          page.s.sread(buf)
+          if buf == "": break
+        client.buffer.istream.setPosition(0)
+        client.buffer.location = request.url
+        client.setupBuffer()
       else:
-        loadError("Couldn't load " & $url)
+        loadError("Couldn't load " & $request.url)
     except IOError, OSError:
-      loadError("Couldn't load " & $url)
-  elif client.buffer != nil and prevurl.isnone or not prevurl.get.equals(url):
-    if not client.buffer.hasAnchor(url.anchor):
-      loadError("Couldn't find anchor " & url.anchor)
-  client.buffer.location = url
-  client.setupBuffer()
+      loadError("Couldn't load " & $request.url)
+  elif client.buffer != nil and prevurl.issome and prevurl.get.equals(request.url, true):
+    if client.buffer.hasAnchor(request.url.anchor):
+      client.dupeBuffer(request.url.some)
+    else:
+      loadError("Couldn't find anchor " & request.url.anchor)
 
 # Relative gotoUrl: either to prevurl, or if that's none, client.buffer.url.
-proc gotoUrl(client: Client, url: string, click = none(ClickAction), prevurl = none(Url), force = false, ctype = "") =
+proc gotoUrl(client: Client, url: string, prevurl = none(Url), force = false, ctype = "") =
   var prevurl = prevurl
   if prevurl.isnone and client.buffer != nil:
     prevurl = client.buffer.location.some
   let newurl = parseUrl(url, prevurl)
   if newurl.isnone:
     loadError("Invalid URL " & url)
-  client.gotoUrl(newurl.get, click, prevurl, force, ctype)
+  client.gotoUrl(newRequest(newurl.get), prevurl, force, ctype)
 
 # When the user has passed a partial URL as an argument, they might've meant
 # either:
@@ -212,24 +227,24 @@ proc gotoUrl(client: Client, url: string, click = none(ClickAction), prevurl = n
 proc loadUrl(client: Client, url: string, ctype = "") =
   let firstparse = parseUrl(url)
   if firstparse.issome:
-    client.gotoUrl(url, none(ClickAction), none(Url), true, ctype)
+    client.gotoUrl(newRequest(firstparse.get), none(Url), true, ctype)
   else:
     let cdir = parseUrl("file://" & getCurrentDir() & DirSep)
     try:
       # attempt to load local file
-      client.gotoUrl(url, none(ClickAction), cdir, true, ctype)
+      client.gotoUrl(url, cdir, true, ctype)
     except LoadError:
       try:
         # attempt to load local file (this time percent encoded)
-        client.gotoUrl(percentEncode(url, LocalPathPercentEncodeSet), none(ClickAction), cdir, true, ctype)
+        client.gotoUrl(percentEncode(url, LocalPathPercentEncodeSet), cdir, true, ctype)
       except LoadError:
         # attempt to load remote page
-        client.gotoUrl("https://" & url, none(ClickAction), none(Url), true, ctype)
+        client.gotoUrl("https://" & url, none(Url), true, ctype)
 
 # Reload the page in a new buffer, then kill the previous buffer.
 proc reloadPage(client: Client) =
   let buf = client.buffer
-  client.gotoUrl(client.buffer.location, none(ClickAction), none(Url), true, client.buffer.contenttype)
+  client.gotoUrl(newRequest(client.buffer.location), none(Url), true, client.buffer.contenttype)
   discardBuffer(buf)
 
 # Open a URL prompt and visit the specified URL.
@@ -242,9 +257,9 @@ proc changeLocation(client: Client) =
     client.loadUrl(url)
 
 proc click(client: Client) =
-  let s = client.buffer.click()
-  if s.issome and s.get.url != "":
-    client.gotoUrl(s.get.url, s)
+  let req = client.buffer.click()
+  if req.issome:
+    client.gotoUrl(req.get, client.buffer.location.some)
 
 proc toggleSource*(client: Client) =
   let buffer = client.buffer
@@ -456,6 +471,7 @@ proc input(client: Client) =
   of ACTION_SCROLL_RIGHT: buffer.scrollRight()
   of ACTION_CLICK: client.click()
   of ACTION_CHANGE_LOCATION: client.changeLocation()
+  of ACTION_DUPE_BUFFER: client.dupeBuffer()
   of ACTION_LINE_INFO: buffer.lineInfo()
   of ACTION_FEED_NEXT: client.feednext = true
   of ACTION_RELOAD: client.reloadPage()
@@ -495,7 +511,7 @@ proc checkAuth(client: Client) =
     url.username = username
     url.password = password
     var buf = client.buffer
-    client.gotoUrl(url, prevurl = some(client.buffer.location))
+    client.gotoUrl(newRequest(url), prevurl = some(client.buffer.location))
     discardBuffer(buf)
     client.followRedirect()
 
@@ -505,7 +521,7 @@ proc followRedirect(client: Client) =
     var buf = client.buffer
     let redirecturl = client.redirecturl.get
     client.redirecturl = none(Url)
-    client.gotoUrl(redirecturl, prevurl = some(client.buffer.location))
+    client.gotoUrl(newRequest(redirecturl), prevurl = some(client.buffer.location))
     discardBuffer(buf)
     if client.needsauth:
       client.checkAuth()
diff --git a/src/config/config.nim b/src/config/config.nim
index b35b03f5..9151628f 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -21,7 +21,7 @@ type
     ACTION_HALF_PAGE_DOWN, ACTION_HALF_PAGE_UP,
     ACTION_SCROLL_DOWN, ACTION_SCROLL_UP, ACTION_SCROLL_LEFT, ACTION_SCROLL_RIGHT,
     ACTION_CLICK,
-    ACTION_CHANGE_LOCATION,
+    ACTION_CHANGE_LOCATION, ACTION_DUPE_BUFFER,
     ACTION_PREV_BUFFER, ACTION_NEXT_BUFFER, ACTION_DISCARD_BUFFER,
     ACTION_RELOAD, ACTION_RESHAPE, ACTION_REDRAW, ACTION_TOGGLE_SOURCE,
     ACTION_CURSOR_FIRST_LINE, ACTION_CURSOR_LAST_LINE,
diff --git a/src/io/buffer.nim b/src/io/buffer.nim
index e6c290c7..13add309 100644
--- a/src/io/buffer.nim
+++ b/src/io/buffer.nim
@@ -696,9 +696,10 @@ proc scrollLeft*(buffer: Buffer) =
     buffer.redraw = true
 
 proc gotoAnchor*(buffer: Buffer) =
+  if buffer.document == nil: return
   let anchor = buffer.document.getElementById(buffer.location.anchor)
   if anchor == nil: return
-  for y in 0..(buffer.numLines - 1):
+  for y in 0..<buffer.numLines:
     let line = buffer.lines[y]
     var i = 0
     while i < line.formats.len:
@@ -863,11 +864,8 @@ proc load*(buffer: Buffer) =
   case buffer.contenttype
   of "text/html":
     if not buffer.streamclosed:
-      while true:
-        var s: string
-        buffer.istream.sread(s)
-        if s == "": break
-        buffer.source &= s
+      buffer.source = buffer.istream.readAll()
+      buffer.istream.close()
       buffer.istream = newStringStream(buffer.source)
       buffer.document = parseHTML5(buffer.istream)
       buffer.streamclosed = true
@@ -940,14 +938,6 @@ proc displayStatusMessage*(buffer: Buffer) =
   print(buffer.generateStatusMessage())
   print(SGR())
 
-type
-  ClickAction* = object
-    url*: string
-    httpmethod*: HttpMethod
-    mimetype*: string
-    body*: Option[string]
-    multipart*: Option[MimeData]
-
 # https://html.spec.whatwg.org/multipage/form-control-infrastructure.html#constructing-the-form-data-set
 proc constructEntryList(form: HTMLFormElement, submitter: Element = nil, encoding: string = ""): Table[string, string] =
   if form.constructingentrylist:
@@ -1054,7 +1044,7 @@ proc serializePlainTextFormData(kvs: Table[string, string]): string =
     result &= '\r'
     result &= '\n'
 
-proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction] =
+proc submitForm(form: HTMLFormElement, submitter: Element): Option[Request] =
   let entrylist = form.constructEntryList(submitter)
 
   let action = if submitter.action() == "":
@@ -1064,7 +1054,7 @@ proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction]
 
   let url = parseUrl(action, submitter.document.baseUrl.some)
   if url.isnone:
-    return none(ClickAction)
+    return none(Request)
 
   var parsedaction = url.get
   let scheme = parsedaction.scheme
@@ -1072,7 +1062,7 @@ proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction]
   let formmethod = submitter.formmethod()
   if formmethod == FORM_METHOD_DIALOG:
     #TODO
-    return none(ClickAction)
+    return none(Request)
   let httpmethod = if formmethod == FORM_METHOD_GET:
     HTTP_GET
   else:
@@ -1088,7 +1078,7 @@ proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction]
   template mutateActionUrl() =
     let query = serializeApplicationXWWFormUrlEncoded(entrylist)
     parsedaction.query = query.some
-    return ClickAction(url: $parsedaction, httpmethod: httpmethod).some
+    return newRequest(parsedaction, httpmethod).some
 
   template submitAsEntityBody() =
     var mimetype: string
@@ -1104,10 +1094,10 @@ proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction]
     of FORM_ENCODING_TYPE_TEXT_PLAIN:
       body = serializePlainTextFormData(entrylist).some
       mimetype = $enctype
-    return ClickAction(url: $parsedaction, httpmethod: httpmethod, body: body, mimetype: mimetype, multipart: multipart).some
+    return newRequest(parsedaction, httpmethod, {"Content-Type": mimetype}, body, multipart).some
 
   template getActionUrl() =
-    return ClickAction(url: $parsedaction).some
+    return newRequest(parsedaction).some
 
   case scheme
   of "http", "https":
@@ -1125,7 +1115,7 @@ proc submitForm(form: HTMLFormElement, submitter: Element): Option[ClickAction]
       assert formmethod == FORM_METHOD_POST
       getActionUrl
 
-proc click*(buffer: Buffer): Option[ClickAction] =
+proc click*(buffer: Buffer): Option[Request] =
   let clickable = buffer.getCursorClickable()
   if clickable != nil:
     template set_focus(e: Element) =
@@ -1141,7 +1131,9 @@ proc click*(buffer: Buffer): Option[ClickAction] =
       set_focus clickable
     of TAG_A:
       restore_focus
-      return ClickAction(url: HTMLAnchorElement(clickable).href, httpmethod: HTTP_GET).some
+      let url = parseUrl(HTMLAnchorElement(clickable).href, clickable.document.baseUrl.some)
+      if url.issome:
+        return newRequest(url.get, HTTP_GET).some
     of TAG_OPTION:
       let option = HTMLOptionElement(clickable)
       let select = option.select
diff --git a/src/io/loader.nim b/src/io/loader.nim
index 74a692fe..1fb6e7b7 100644
--- a/src/io/loader.nim
+++ b/src/io/loader.nim
@@ -40,27 +40,30 @@ proc loadFile(url: Url, ostream: Stream) =
   else:
     let path = url.path.serialize_unicode()
   let istream = newFileStream(path, fmRead)
-  ostream.swrite(if istream != nil:
-    200 # ok
-  else:
-    404 # file not found
-  )
-  ostream.swrite(guessContentType(path))
-  ostream.swrite(none(Url))
-  while not istream.atEnd:
-    const bufferSize = 4096
-    var buffer {.noinit.}: array[bufferSize, char]
-    while true:
-      let n = readData(istream, addr buffer[0], bufferSize)
-      if n == 0:
-        break
-      ostream.swrite(n)
-      ostream.writeData(addr buffer[0], n)
-      ostream.flush()
-      if n < bufferSize:
-        break
+  if istream == nil:
+    ostream.swrite(404) # file not found
+    ostream.swrite("")
+    ostream.swrite(none(Url))
     ostream.swrite("")
     ostream.flush()
+  else:
+    ostream.swrite(200) # ok
+    ostream.swrite(guessContentType(path))
+    ostream.swrite(none(Url))
+    while not istream.atEnd:
+      const bufferSize = 4096
+      var buffer {.noinit.}: array[bufferSize, char]
+      while true:
+        let n = readData(istream, addr buffer[0], bufferSize)
+        if n == 0:
+          break
+        ostream.swrite(n)
+        ostream.writeData(addr buffer[0], n)
+        ostream.flush()
+        if n < bufferSize:
+          break
+      ostream.swrite("")
+      ostream.flush()
 
 proc loadResource(loader: FileLoader, request: Request, ostream: Stream) =
   case request.url.scheme
@@ -78,6 +81,9 @@ proc runFileLoader(loader: FileLoader) =
   while true:
     try:
       let request = istream.readRequest()
+      for k, v in loader.defaultHeaders:
+        if k notin request.headers.table:
+          request.headers[k] = v
       loader.loadResource(request, ostream)
     except IOError:
       # End-of-file, quit.
diff --git a/src/io/request.nim b/src/io/request.nim
index 461a3680..68706c49 100644
--- a/src/io/request.nim
+++ b/src/io/request.nim
@@ -67,27 +67,16 @@ func newRequest*(url: Url,
                  httpmethod = HTTP_GET,
                  headers: openarray[(string, string)] = [],
                  body = none(string),
-                 multipart = none(MimeData),
-                 defaultHeaders = none(HeaderList)): Request =
+                 multipart = none(MimeData)): Request =
   new(result)
   result.httpmethod = httpmethod
   result.url = url
-  if defaultHeaders.issome:
-    result.headers.table = defaultHeaders.get.table
   for it in headers:
     if it[1] != "": #TODO not sure if this is a good idea, options would probably work better
       result.headers.table[it[0]] = @[it[1]]
   result.body = body
   result.multipart = multipart
 
-func newRequest*(loader: FileLoader,
-                 url: Url,
-                 httpmethod = HTTP_GET,
-                 headers: openarray[(string, string)] = [],
-                 body = none(string),
-                 multipart = none(MimeData)): Request =
-  newRequest(url, httpmethod, headers, body, multipart, some(loader.defaultHeaders))
-
 proc `[]=`*(multipart: var MimeData, k, v: string) =
   multipart.content.add(MimePart(name: k, content: v))