removing old readline

9 files changed, 524 insertions, 210 deletions

diff --git a/buffer.nim b/buffer.nim
index e6be7975..1897cdc4 100644
--- a/buffer.nim
+++ b/buffer.nim
@@ -298,7 +298,7 @@ proc cursorPrevWord*(buffer: Buffer): bool =
       fastRuneAt(buffer.rawtext[y], x, r, false)
 
   if x == 0:
-    if y < buffer.lastLine():
+    if y > 0:
       dec y
       x = buffer.rawtext[y].runeLen() - 1
   return buffer.cursorTo(x, y)
diff --git a/config b/config
index 5724dbc8..a40c4e16 100644
--- a/config
+++ b/config
@@ -1,56 +1,56 @@
 #normal mode keybindings
-nmap q ACTION_QUIT
-nmap h ACTION_CURSOR_LEFT
-nmap j ACTION_CURSOR_DOWN
-nmap k ACTION_CURSOR_UP
-nmap l ACTION_CURSOR_RIGHT
-nmap \e[D ACTION_CURSOR_LEFT
-nmap \e[B ACTION_CURSOR_DOWN
-nmap \e[A ACTION_CURSOR_UP
-nmap \e[C ACTION_CURSOR_RIGHT
-nmap ^ ACTION_CURSOR_LINEBEGIN
-nmap $ ACTION_CURSOR_LINEEND
-nmap b ACTION_CURSOR_PREV_WORD
-nmap w ACTION_CURSOR_NEXT_WORD
-nmap [ ACTION_CURSOR_PREV_LINK
-nmap ] ACTION_CURSOR_NEXT_LINK
-nmap H ACTION_CURSOR_TOP
-nmap M ACTION_CURSOR_MIDDLE
-nmap L ACTION_CURSOR_BOTTOM
-nmap C-d ACTION_HALF_PAGE_DOWN
-nmap C-u ACTION_HALF_PAGE_UP
-nmap C-f ACTION_PAGE_DOWN
-nmap C-b ACTION_PAGE_UP
-nmap \e[6~ ACTION_PAGE_DOWN
-nmap \e[5~ ACTION_PAGE_UP
-nmap C-e ACTION_SCROLL_DOWN
-nmap C-y ACTION_SCROLL_UP
-nmap J ACTION_SCROLL_DOWN
-nmap K ACTION_SCROLL_UP
-nmap C-m ACTION_CLICK
-nmap C-j ACTION_CLICK
-nmap C-l ACTION_CHANGE_LOCATION
-nmap U ACTION_RELOAD
-nmap r ACTION_RESHAPE
-nmap R ACTION_REDRAW
-nmap gg ACTION_CURSOR_FIRST_LINE
-nmap G ACTION_CURSOR_LAST_LINE
-nmap \e[H ACTION_CURSOR_FIRST_LINE
-nmap \e[F ACTION_CURSOR_LAST_LINE
-nmap z ACTION_CENTER_LINE
-nmap C-g ACTION_LINE_INFO
+nmap q QUIT
+nmap h CURSOR_LEFT
+nmap j CURSOR_DOWN
+nmap k CURSOR_UP
+nmap l CURSOR_RIGHT
+nmap \e[D CURSOR_LEFT
+nmap \e[B CURSOR_DOWN
+nmap \e[A CURSOR_UP
+nmap \e[C CURSOR_RIGHT
+nmap ^ CURSOR_LINEBEGIN
+nmap $ CURSOR_LINEEND
+nmap b CURSOR_PREV_WORD
+nmap w CURSOR_NEXT_WORD
+nmap [ CURSOR_PREV_LINK
+nmap ] CURSOR_NEXT_LINK
+nmap H CURSOR_TOP
+nmap M CURSOR_MIDDLE
+nmap L CURSOR_BOTTOM
+nmap C-d HALF_PAGE_DOWN
+nmap C-u HALF_PAGE_UP
+nmap C-f PAGE_DOWN
+nmap C-b PAGE_UP
+nmap \e[6~ PAGE_DOWN
+nmap \e[5~ PAGE_UP
+nmap C-e SCROLL_DOWN
+nmap C-y SCROLL_UP
+nmap J SCROLL_DOWN
+nmap K SCROLL_UP
+nmap C-m CLICK
+nmap C-j CLICK
+nmap C-l CHANGE_LOCATION
+nmap U RELOAD
+nmap r RESHAPE
+nmap R REDRAW
+nmap gg CURSOR_FIRST_LINE
+nmap G CURSOR_LAST_LINE
+nmap \e[H CURSOR_FIRST_LINE
+nmap \e[F CURSOR_LAST_LINE
+nmap z CENTER_LINE
+nmap C-g LINE_INFO
 
 #line editing keybindings
-lemap C-m ACTION_LINED_SUBMIT
-lemap C-j ACTION_LINED_SUBMIT
-lemap C-h ACTION_LINED_BACKSPACE
-lemap C-? ACTION_LINED_BACKSPACE
-lemap C-c ACTION_LINED_CANCEL
-lemap \eb ACTION_LINED_PREV_WORD
-lemap \ef ACTION_LINED_NEXT_WORD
-lemap C-b ACTION_LINED_BACK
-lemap C-f ACTION_LINED_FORWARD
-lemap C-u ACTION_LINED_CLEAR
-lemap C-k ACTION_LINED_KILL
-lemap C-w ACTION_LINED_KILL_WORD
-lemap C-v ACTION_LINED_ESC
+lemap C-m LINED_SUBMIT
+lemap C-j LINED_SUBMIT
+lemap C-h LINED_BACKSPACE
+lemap C-? LINED_BACKSPACE
+lemap C-c LINED_CANCEL
+lemap \eb LINED_PREV_WORD
+lemap \ef LINED_NEXT_WORD
+lemap C-b LINED_BACK
+lemap C-f LINED_FORWARD
+lemap C-u LINED_CLEAR
+lemap C-k LINED_KILL
+lemap C-w LINED_KILL_WORD
+lemap C-v LINED_ESC
diff --git a/config.nim b/config.nim
index abdd74db..0e7aa92b 100644
--- a/config.nim
+++ b/config.nim
@@ -13,6 +13,7 @@ type
     ACTION_CURSOR_UP, ACTION_CURSOR_DOWN, ACTION_CURSOR_LEFT, ACTION_CURSOR_RIGHT,
     ACTION_CURSOR_LINEEND, ACTION_CURSOR_LINEBEGIN,
     ACTION_CURSOR_NEXT_WORD, ACTION_CURSOR_PREV_WORD,
+    ACTION_CURSOR_NEXT_NODE, ACTION_CURSOR_PREV_NODE,
     ACTION_CURSOR_NEXT_LINK, ACTION_CURSOR_PREV_LINK,
     ACTION_PAGE_DOWN, ACTION_PAGE_UP,
     ACTION_HALF_PAGE_DOWN, ACTION_HALF_PAGE_UP,
@@ -93,9 +94,9 @@ macro staticReadKeymap(): untyped =
     let cmd = line.split(' ')
     if cmd.len == 3:
       if cmd[0] == "nmap":
-        normalActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction](cmd[2])
+        normalActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction]("ACTION_" & cmd[2])
       elif cmd[0] == "lemap":
-        linedActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction](cmd[2])
+        linedActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction]("ACTION_" & cmd[2])
 
   normalActionMap = constructActionTable(normalActionMap)
   linedActionMap = constructActionTable(linedActionMap)
@@ -139,9 +140,9 @@ proc readConfig*(filename: string): bool =
       let cmd = line.split(' ')
       if cmd.len == 3:
         if cmd[0] == "nmap":
-          normalActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction](cmd[2])
+          normalActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction]("ACTION_" & cmd[2])
         elif cmd[0] == "lemap":
-          linedActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction](cmd[2])
+          linedActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction]("ACTION_" & cmd[2])
 
     normalActionRemap = constructActionTable(normalActionMap)
     linedActionRemap = constructActionTable(linedActionMap)
@@ -158,9 +159,9 @@ proc parseKeymap*(keymap: string) =
     let cmd = line.split(' ')
     if cmd.len == 3:
       if cmd[0] == "nmap":
-        normalActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction](cmd[2])
+        normalActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction]("ACTION_" & cmd[2])
       elif cmd[0] == "lemap":
-        linedActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction](cmd[2])
+        linedActionMap[getRealKey(cmd[1])] = parseEnum[TwtAction]("ACTION_" & cmd[2])
 
   normalActionRemap = constructActionTable(normalActionMap)
   linedActionRemap = constructActionTable(linedActionMap)
diff --git a/display.nim b/display.nim
index c7cd4bb4..4bf15921 100644
--- a/display.nim
+++ b/display.nim
@@ -158,8 +158,7 @@ proc postAlignNode(buffer: Buffer, node: HtmlNode, state: var RenderState) =
 
   if state.rawline.len > 0 and state.blanklines == 0:
     state.nextspaces += max(elem.margin, elem.marginright)
-    if node.closeblock and (node.isTextNode() or elem.numChildNodes == 0):
-      state.write($node.nodeAttr().tagType)
+    if node.closeblock and (node.isTextNode() or elem.childNodes.len == 0):
       buffer.flushLine(state)
 
   if node.closeblock:
@@ -355,7 +354,7 @@ proc inputLoop(attrs: TermAttributes, buffer: Buffer): bool =
         case selectedElem.get().tagType
         of TAG_INPUT:
           clearStatusMsg(buffer.height)
-          let status = readLine("TEXT:", HtmlInputElement(selectedElem.get()).value)
+          let status = readLine("TEXT: ", HtmlInputElement(selectedElem.get()).value, buffer.width)
           if status:
             reshape = true
             redraw = true
@@ -368,7 +367,7 @@ proc inputLoop(attrs: TermAttributes, buffer: Buffer): bool =
       var url = $buffer.document.location
 
       clearStatusMsg(buffer.height)
-      let status = readLine("URL:", url)
+      let status = readLine("URL: ", url, buffer.width)
       if status:
         buffer.setLocation(parseUri(url))
         return true
diff --git a/htmlelement.nim b/htmlelement.nim
index 1d4106a5..2b7c1c64 100644
--- a/htmlelement.nim
+++ b/htmlelement.nim
@@ -332,8 +332,6 @@ proc getHtmlElement*(xmlElement: XmlNode, parentNode: HtmlNode): HtmlElement =
     result.underscore = result.underscore or parent.underscore
     result.hidden = result.hidden or parent.hidden
     result.islink = result.islink or parent.islink
-  
-  result.numChildNodes = xmlElement.len
 
 proc getHtmlNode*(xmlElement: XmlNode, parent: HtmlNode): HtmlNode =
   case kind(xmlElement)
diff --git a/main.nim b/main.nim
index a9549165..b2ce1ace 100644
--- a/main.nim
+++ b/main.nim
@@ -3,15 +3,11 @@ import uri
 import os
 import streams
 
-import fusion/htmlparser
-import fusion/htmlparser/xmltree
-
 import display
 import termattrs
 import buffer
 import twtio
 import config
-import twtstr
 import parser
 
 let clientInstance = newHttpClient()
diff --git a/parser.nim b/parser.nim
index 3873566d..a8951368 100644
--- a/parser.nim
+++ b/parser.nim
@@ -2,6 +2,7 @@ import parsexml
 import htmlelement
 import streams
 import macros
+import unicode
 
 import twtio
 import enums
@@ -9,33 +10,56 @@ import strutils
 
 type
   ParseState = object
+    stream: Stream
     closed: bool
     parents: seq[HtmlNode]
     parsedNode: HtmlNode
+    a: string
+    attrs: seq[string]
+
+  ParseEvent =
+    enum
+    NO_EVENT, EVENT_COMMENT, EVENT_STARTELEM, EVENT_ENDELEM, EVENT_OPENELEM,
+    EVENT_CLOSEELEM, EVENT_ATTRIBUTE, EVENT_TEXT
 
 #> no I won't manually write all this down
-#> maybe todo to accept stuff other than tagtype (idk how useful that'd be)
-#still todo, it'd be very useful
-macro genEnumCase(s: string): untyped =
-  let casestmt = nnkCaseStmt.newTree() 
-  casestmt.add(ident("s"))
-  for i in low(TagType) .. high(TagType):
+#yes this is incredibly ugly
+#...but hey, so long as it works
+
+macro genEnumCase(s: string, t: typedesc) =
+  result = quote do:
+    let casestmt = nnkCaseStmt.newTree() 
+    casestmt.add(ident(`s`))
+    var first = true
+    for e in low(`t`) .. high(`t`):
+      if first:
+        first = false
+        continue
+      let ret = nnkReturnStmt.newTree()
+      ret.add(newLit(e))
+      let branch = nnkOfBranch.newTree()
+      let enumname = $e
+      let tagname = enumname.split('_')[1..^1].join("_").tolower()
+      branch.add(newLit(tagname))
+      branch.add(ret)
+      casestmt.add(branch)
     let ret = nnkReturnStmt.newTree()
-    ret.add(newLit(TagType(i)))
-    let branch = nnkOfBranch.newTree()
-    let enumname = $TagType(i)
-    let tagname = enumname.substr("TAG_".len, enumname.len - 1).tolower()
-    branch.add(newLit(tagname))
+    ret.add(newLit(low(`t`)))
+    let branch = nnkElse.newTree()
     branch.add(ret)
     casestmt.add(branch)
-  let ret = nnkReturnStmt.newTree()
-  ret.add(newLit(TAG_UNKNOWN))
-  let branch = nnkElse.newTree()
-  branch.add(ret)
-  casestmt.add(branch)
+
+macro genTagTypeCase() =
+  genEnumCase("s", TagType)
+
+macro genInputTypeCase() =
+  genEnumCase("s", InputType)
 
 func tagType(s: string): TagType =
-  genEnumCase(s)
+  genTagTypeCase
+
+func inputType(s: string): InputType =
+  genInputTypeCase
 
 func newHtmlElement(tagType: TagType, parentNode: HtmlNode): HtmlElement =
   case tagType
@@ -88,6 +112,8 @@ func newHtmlElement(tagType: TagType, parentNode: HtmlNode): HtmlElement =
     result.marginbottom = 1
   of TAG_A:
     result.islink = true
+  of TAG_INPUT:
+    HtmlInputElement(result).size = 20
   else: discard
 
   if parentNode.isElemNode():
@@ -99,32 +125,6 @@ func newHtmlElement(tagType: TagType, parentNode: HtmlNode): HtmlElement =
     result.hidden = result.hidden or parent.hidden
     result.islink = result.islink or parent.islink
 
-func toInputType*(str: string): InputType =
-  case str
-  of "button": INPUT_BUTTON
-  of "checkbox": INPUT_CHECKBOX
-  of "color": INPUT_COLOR
-  of "date": INPUT_DATE
-  of "datetime_local": INPUT_DATETIME_LOCAL
-  of "email": INPUT_EMAIL
-  of "file": INPUT_FILE
-  of "hidden": INPUT_HIDDEN
-  of "image": INPUT_IMAGE
-  of "month": INPUT_MONTH
-  of "number": INPUT_NUMBER
-  of "password": INPUT_PASSWORD
-  of "radio": INPUT_RADIO
-  of "range": INPUT_RANGE
-  of "reset": INPUT_RESET
-  of "search": INPUT_SEARCH
-  of "submit": INPUT_SUBMIT
-  of "tel": INPUT_TEL
-  of "text": INPUT_TEXT
-  of "time": INPUT_TIME
-  of "url": INPUT_URL
-  of "week": INPUT_WEEK
-  else: INPUT_UNKNOWN
-
 func toInputSize*(str: string): int =
   if str.len == 0:
     return 20
@@ -153,7 +153,7 @@ proc applyAttribute(htmlElement: HtmlElement, key: string, value: string) =
     else: discard
   of "type":
     case htmlElement.tagType
-    of TAG_INPUT: HtmlInputElement(htmlElement).itype = value.toInputType()
+    of TAG_INPUT: HtmlInputElement(htmlElement).itype = value.inputType()
     else: discard
   of "size":
     case htmlElement.tagType
@@ -162,6 +162,10 @@ proc applyAttribute(htmlElement: HtmlElement, key: string, value: string) =
   else: return
 
 proc closeNode(state: var ParseState) =
+  let node = state.parents[^1]
+  if node.childNodes.len > 0 and node.isElemNode() and HtmlElement(node).display == DISPLAY_BLOCK:
+    node.childNodes[0].openblock = true
+    node.childNodes[^1].closeblock = true
   state.parents.setLen(state.parents.len - 1)
   state.closed = true
 
@@ -169,76 +173,257 @@ proc closeSingleNodes(state: var ParseState) =
   if not state.closed and state.parents[^1].isElemNode() and HtmlElement(state.parents[^1]).tagType in SingleTagTypes:
     state.closeNode()
 
+proc applyNodeText(htmlNode: HtmlNode) =
+  htmlNode.rawtext = htmlNode.getRawText()
+  htmlNode.fmttext = htmlNode.getFmtText()
+
+proc setParent(state: var ParseState, htmlNode: HtmlNode) =
+  htmlNode.parentNode = state.parents[^1]
+  if state.parents[^1].isElemNode():
+    htmlNode.parentElement = HtmlElement(state.parents[^1])
+  if state.parents[^1].childNodes.len > 0:
+    htmlNode.previousSibling = state.parents[^1].childNodes[^1]
+    htmlNode.previousSibling.nextSibling = htmlNode
+  state.parents[^1].childNodes.add(htmlNode)
+
 proc processHtmlElement(state: var ParseState, htmlElement: HtmlElement) =
   state.closed = false
-  if state.parents[^1].childNodes.len > 0:
-    htmlElement.previousSibling = state.parents[^1].childNodes[^1]
-    htmlElement.previousSibling.nextSibling = htmlElement
-  state.parents[^1].childNodes.add(htmlElement)
+  state.setParent(htmlElement)
   state.parents.add(htmlElement)
 
-proc applyNodeText(htmlNode: HtmlNode) =
-  htmlNode.rawtext = htmlNode.getRawText()
-  htmlNode.fmttext = htmlNode.getFmtText()
+proc parsecomment(state: var ParseState) =
+  var s = ""
+  state.a = ""
+  var e = 0
+  while not state.stream.atEnd():
+    let c = cast[char](state.stream.readInt8())
+    if c > char(127):
+      s &= c
+      if s.validateUtf8() == -1:
+        state.a &= s
+        s = ""
+    else:
+      case e
+      of 0:
+        if c == '-': inc e
+      of 1:
+        if c == '-': inc e
+        else:
+          e = 0
+          state.a &= '-' & c
+      of 2:
+        if c == '>': return
+        else:
+          e = 0
+          state.a &= "--" & c
+      else: state.a &= c
+
+proc parsecdata(state: var ParseState) =
+  var s = ""
+  var e = 0
+  while not state.stream.atEnd():
+    let c = cast[char](state.stream.readInt8())
+    if c > char(127):
+      s &= c
+      if s.validateUtf8() == -1:
+        state.a &= s
+        s = ""
+    else:
+      case e
+      of 0:
+        if c == ']': inc e
+      of 1:
+        if c == ']': inc e
+        else: e = 0
+      of 2:
+        if c == '>': return
+        else: e = 0
+      else: discard
+      state.a &= c
+
+proc next(state: var ParseState): ParseEvent =
+  result = NO_EVENT
+  if state.stream.atEnd(): return result
+
+  var c = cast[char](state.stream.readInt8())
+  var cdata = false
+  var s = ""
+  state.a = ""
+  if c < char(128): #ascii
+    case c
+    of '<':
+      if state.stream.atEnd():
+        state.a = $c
+        return EVENT_TEXT
+      let d = char(state.stream.peekInt8())
+      case d
+      of '/': result = EVENT_ENDELEM
+      of '!':
+        state.a = state.stream.readStr(2)
+        case state.a
+        of "[C":
+          state.a &= state.stream.readStr(7)
+          if state.a == "[CDATA[":
+            state.parsecdata()
+            return EVENT_COMMENT
+          result = EVENT_TEXT
+        of "--":
+          state.parsecomment()
+          return EVENT_COMMENT
+        else:
+          while not state.stream.atEnd():
+            c = cast[char](state.stream.readInt8())
+            if s.len == 0 and c == '>':
+              break
+            elif c > char(127):
+              s &= c
+              if s.validateUtf8() == -1:
+                s = ""
+          return NO_EVENT
+      of Letters:
+        result = EVENT_STARTELEM
+      else:
+        result = EVENT_TEXT
+        state.a = c & d
+    of '>':
+      return EVENT_CLOSEELEM
+    else: result = EVENT_TEXT
+  else: result = EVENT_TEXT
+
+  case result
+  of EVENT_STARTELEM:
+    var atspace = false
+    var atattr = false
+    while not state.stream.atEnd():
+      c = cast[char](state.stream.peekInt8())
+      if s.len == 0 and c < char(128):
+        case c
+        of Whitespace: atspace = true
+        of '>':
+          discard state.stream.readInt8()
+          break
+        else:
+          if atspace:
+            return EVENT_OPENELEM
+          else:
+            state.a &= s
+      else:
+        if atspace:
+          return EVENT_OPENELEM
+        s &= c
+        if s.validateUtf8() == -1:
+          state.a &= s
+          s = ""
+      discard state.stream.readInt8()
+  of EVENT_ENDELEM:
+    while not state.stream.atEnd():
+      c = cast[char](state.stream.readInt8())
+      if s.len == 0 and c < char(128):
+        if c == '>': break
+        elif c in Whitespace: discard
+        else: state.a &= c
+      else:
+        s &= c
+        if s.validateUtf8() == -1:
+          state.a &= s
+          s = ""
+  of EVENT_TEXT:
+    while not state.stream.atEnd():
+      c = cast[char](state.stream.peekInt8())
+      if s.len == 0 and c < char(128):
+        if c in {'<', '>'}: break
+        state.a &= c
+      else:
+        s &= c
+        if s.validateUtf8() == -1:
+          state.a &= s
+          s = ""
+      discard state.stream.readInt8()
+  else: assert(false)
 
-#TODO honestly parsexml sucks I should just make my own
 proc nparseHtml*(inputStream: Stream): Document =
-  var x: XmlParser
-  let options = @[reportWhitespace, allowUnquotedAttribs, allowEmptyAttribs]
-  x.open(inputStream, "")
-  var state: ParseState
+  var state = ParseState(stream: inputStream)
   let document = newDocument()
   state.parents.add(document)
-  while state.parents.len > 0 and x.kind != xmlEof:
-    x.next()
-    case x.kind
-    of xmlComment: discard #TODO
-    of xmlElementStart:
-      eprint "<" & x.rawdata & ">"
+  while state.parents.len > 0 and not inputStream.atEnd():
+    let event = state.next()
+    case event
+    of EVENT_COMMENT: discard #TODO
+    of EVENT_STARTELEM:
       state.closeSingleNodes()
-      let parsedNode = newHtmlElement(tagType(x.rawData), state.parents[^1])
+      let parsedNode = newHtmlElement(tagType(state.a), state.parents[^1])
       parsedNode.applyNodeText()
       state.processHtmlElement(parsedNode)
-    of xmlElementEnd:
-      eprint "</" & x.rawdata & ">"
+    of EVENT_ENDELEM:
       state.closeNode()
-    of xmlElementOpen:
-      var s = "<" & x.rawdata
+    of EVENT_OPENELEM:
       state.closeSingleNodes()
-      let parsedNode = newHtmlElement(tagType(x.rawData), state.parents[^1])
-      x.next()
-      while x.kind != xmlElementClose and x.kind != xmlEof:
-        if x.kind == xmlAttribute:
-          HtmlElement(parsedNode).applyAttribute(x.rawData.tolower(), x.rawData2)
-          s &= " " & x.rawdata & "=\"" & x.rawdata2 & "\""
-        elif x.kind == xmlError:
-          HtmlElement(parsedNode).applyAttribute(x.rawData.tolower(), "")
-        elif x.kind == xmlCharData:
-          if x.rawData.strip() == "/>":
-            break
-        elif x.kind == xmlElementEnd:
-          break
-        elif x.kind == xmlElementOpen:
-          #wtf??? TODO
-          break
-        else:
-          assert(false, "wtf " & $x.kind & " " & x.rawdata) #TODO
-        x.next()
-      s &= ">"
-      eprint s
+      let parsedNode = newHtmlElement(tagType(state.a), state.parents[^1])
+      var next = state.next()
+      while next != EVENT_CLOSEELEM and not inputStream.atEnd():
+        #TODO
+        #if next == EVENT_ATTRIBUTE:
+        #  parsedNode.applyAttribute(state.a.tolower(), state.b)
+        #  s &= " " & x.rawdata & "=\"" & x.rawdata2 & "\""
+        #else:
+        #  assert(false, "wtf " & $x.kind & " " & x.rawdata) #TODO
+        next = state.next()
       parsedNode.applyNodeText()
       state.processHtmlElement(parsedNode)
-    of xmlCharData:
-      eprint x.rawdata
+    of EVENT_TEXT:
+      if unicode.strip(state.a).len == 0:
+        continue
       let textNode = new(HtmlNode)
       textNode.nodeType = NODE_TEXT
-      state.parents[^1].childNodes.add(textNode)
-      textNode.parentNode = state.parents[^1]
-      if state.parents[^1].isElemNode():
-        textNode.parentElement = HtmlElement(state.parents[^1])
-      textNode.rawtext = x.rawData
+      state.setParent(textNode)
+      textNode.rawtext = state.a
       textNode.applyNodeText()
-    of xmlEntity: discard #TODO
-    of xmlEof: break
     else: discard
   return document
+
+#old nparseHtml because I don't trust myself
+#proc nparseHtml*(inputStream: Stream): Document =
+#  var x: XmlParser
+#  let options = {reportWhitespace, allowUnquotedAttribs, allowEmptyAttribs}
+#  x.open(inputStream, "", options)
+#  var state = ParseState(stream: inputStream)
+#  let document = newDocument()
+#  state.parents.add(document)
+#  while state.parents.len > 0 and x.kind != xmlEof:
+#    #let event = state.next()
+#    x.next()
+#    case x.kind
+#    of xmlComment: discard #TODO
+#    of xmlElementStart:
+#      state.closeSingleNodes()
+#      let parsedNode = newHtmlElement(tagType(x.rawData), state.parents[^1])
+#      parsedNode.applyNodeText()
+#      state.processHtmlElement(parsedNode)
+#    of xmlElementEnd:
+#      state.closeNode()
+#    of xmlElementOpen:
+#      var s = "<" & x.rawdata
+#      state.closeSingleNodes()
+#      let parsedNode = newHtmlElement(tagType(x.rawData), state.parents[^1])
+#      x.next()
+#      while x.kind != xmlElementClose and x.kind != xmlEof:
+#        if x.kind == xmlAttribute:
+#          parsedNode.applyAttribute(x.rawData.tolower(), x.rawData2)
+#          s &= " " & x.rawdata & "=\"" & x.rawdata2 & "\""
+#        else:
+#          assert(false, "wtf " & $x.kind & " " & x.rawdata) #TODO
+#        x.next()
+#      s &= ">"
+#      parsedNode.applyNodeText()
+#      state.processHtmlElement(parsedNode)
+#    of xmlCharData:
+#      let textNode = new(HtmlNode)
+#      textNode.nodeType = NODE_TEXT
+#
+#      state.setParent(textNode)
+#      textNode.rawtext = x.rawData
+#      textNode.applyNodeText()
+#    of xmlEntity: discard #TODO
+#    of xmlEof: break
+#    else: discard
+#  return document
diff --git a/twtio.nim b/twtio.nim
index 0a19bfbf..acb4add7 100644
--- a/twtio.nim
+++ b/twtio.nim
@@ -40,23 +40,188 @@ proc getLinedAction*(s: string): TwtAction =
     return linedActionRemap[s]
   return NO_ACTION
 
-proc readLine*(prompt: string, current: var string): bool =
+const breakWord = [
+  Rune('\n'), Rune('/'), Rune('\\'), Rune(' '), Rune('&'), Rune('=')
+]
+
+#proc readLine*(prompt: string, current: var string, termwidth: int): bool =
+#  var new = current
+#  print(prompt)
+#  let maxlen = termwidth - prompt.len
+#  printesc(new)
+#  var s = ""
+#  var feedNext = false
+#  var escNext = false
+#  var cursor = new.runeLen
+#  var shift = 0
+#  while true:
+#    var rl = new.runeLen()
+#
+#    if cursor < shift:
+#      shift = cursor
+#    elif cursor - shift > maxlen:
+#      shift += cursor - maxlen
+#
+#    if not feedNext:
+#      s = ""
+#    else:
+#      feedNext = false
+#    let c = getch()
+#    s &= c
+#    var action = getLinedAction(s)
+#    if escNext:
+#      action = NO_ACTION
+#    case action
+#    of ACTION_LINED_CANCEL:
+#      return false
+#    of ACTION_LINED_SUBMIT:
+#      current = new
+#      return true
+#    of ACTION_LINED_BACKSPACE:
+#      if cursor > 0:
+#        print(' '.repeat(rl - cursor + 1))
+#        print('\b'.repeat(rl - cursor + 1))
+#        print("\b \b")
+#        new = new.runeSubstr(0, cursor - 1) & new.runeSubstr(cursor)
+#        rl = new.runeLen()
+#        cursor -= 1
+#        printesc(new.runeSubstr(cursor))
+#        print('\b'.repeat(rl - cursor))
+#    of ACTION_LINED_ESC:
+#      escNext = true
+#    of ACTION_LINED_CLEAR:
+#      print('\r')
+#      print(' '.repeat(termwidth))
+#      print('\r')
+#      new = new.runeSubstr(cursor)
+#      rl = new.runeLen()
+#      printesc(prompt)
+#      printesc(new.maxString(maxlen + 1))
+#      print('\r')
+#      printesc(prompt)
+#      cursor = 0
+#    of ACTION_LINED_KILL:
+#      print(' '.repeat(rl - cursor + 1))
+#      print('\b'.repeat(rl - cursor + 1))
+#      new = new.runeSubstr(0, cursor)
+#    of ACTION_LINED_BACK:
+#      if cursor > 0:
+#        if cursor < maxlen:
+#          print('\b')
+#        dec cursor
+#    of ACTION_LINED_FORWARD:
+#      if cursor < rl:
+#        if cursor + 1 < maxlen:
+#          var rune: Rune
+#          new.fastRuneAt(cursor, rune, false)
+#          printesc($rune)
+#        elif cursor + 1 == maxlen:
+#          print('$')
+#        inc cursor
+#    of ACTION_LINED_PREV_WORD:
+#      while cursor > 0:
+#        print('\b')
+#        cursor -= 1
+#        var rune: Rune
+#        new.fastRuneAt(cursor, rune, false)
+#        if rune in breakWord:
+#          break
+#    of ACTION_LINED_NEXT_WORD:
+#      while cursor < rl:
+#        var rune: Rune
+#        new.fastRuneAt(cursor, rune, false)
+#        printesc($rune)
+#        inc cursor
+#        if cursor < rl:
+#          new.fastRuneAt(cursor, rune, false)
+#          if rune in breakWord:
+#            break
+#    of ACTION_LINED_KILL_WORD:
+#      var chars = 0
+#      while cursor > chars:
+#        inc chars
+#        var rune: Rune
+#        new.fastRuneAt(cursor - chars, rune, false)
+#        if rune in breakWord:
+#          break
+#      if chars > 0:
+#        print(' '.repeat(rl - cursor + 1))
+#        print('\b'.repeat(rl - cursor + 1))
+#        print("\b \b".repeat(chars))
+#        new = new.runeSubstr(0, cursor - chars) & new.runeSubstr(cursor)
+#        rl = new.runeLen()
+#        cursor -= chars
+#        printesc(new.runeSubstr(cursor))
+#        print('\b'.repeat(rl - cursor))
+#    of ACTION_FEED_NEXT:
+#      feedNext = true
+#    elif validateUtf8(s) == -1:
+#      var cs = ""
+#      for c in s:
+#        if not c.isControlChar():
+#          cs &= c
+#        elif escNext:
+#          cs &= c
+#          escNext = false
+#      escNext = false
+#      if cs.len == 0:
+#        continue
+#      if rl + 1 < maxlen:
+#        print(' '.repeat(rl - cursor + 1))
+#        print('\b'.repeat(rl - cursor + 1))
+#      new = new.runeSubstr(0, cursor) & cs & new.runeSubstr(cursor)
+#      rl = new.runeLen()
+#      if cursor - shift > maxlen:
+#        shift += maxlen - cursor
+#      if shift == 0:
+#        printesc(new.runeSubstr(cursor, min(maxlen - cursor - 1, rl)))
+#        print('\b'.repeat(max(min(maxlen - cursor - 2, rl - cursor - 1), 0)))
+#      else:
+#        print('\r')
+#        print(' '.repeat(termwidth))
+#        print('\r')
+#        print(prompt)
+#        print(new.runeSubstr(shift, min(maxlen - 1, rl - shift)))
+#        if maxlen < rl - shift:
+#          print(new.runeSubstr(shift, maxlen - 1))
+#          print('\b'.repeat(maxlen - cursor + shift))
+#        else:
+#          print(new.runeSubstr(shift, rl - shift))
+#          print('\b'.repeat(rl + shift - cursor))
+#      inc cursor
+#    else:
+#      feedNext = true
+
+proc readLine*(prompt: string, current: var string, termwidth: int): bool =
   var new = current
   print(prompt)
-  print(' ')
+  let maxlen = termwidth - prompt.len
   printesc(new)
   var s = ""
   var feedNext = false
   var escNext = false
   var cursor = new.runeLen
+  var shift = 0
   while true:
+    var rl = new.runeLen()
+    print('\r')
+    print(' '.repeat(termwidth))
+    print('\r')
+    printesc(prompt & new)
+    print('\r')
+    cursorForward(prompt.len + cursor)
+
+    if cursor < shift:
+      shift = cursor
+    elif cursor - shift > maxlen:
+      shift += cursor - maxlen
+
     if not feedNext:
       s = ""
     else:
       feedNext = false
     let c = getch()
     s &= c
-    var rl = new.runeLen()
     var action = getLinedAction(s)
     if escNext:
       action = NO_ACTION
@@ -68,58 +233,37 @@ proc readLine*(prompt: string, current: var string): bool =
       return true
     of ACTION_LINED_BACKSPACE:
       if cursor > 0:
-        print(' '.repeat(rl - cursor + 1))
-        print('\b'.repeat(rl - cursor + 1))
-        print("\b \b")
         new = new.runeSubstr(0, cursor - 1) & new.runeSubstr(cursor)
         rl = new.runeLen()
-        cursor -= 1
-        printesc(new.runeSubstr(cursor))
-        print('\b'.repeat(rl - cursor))
+        dec cursor
     of ACTION_LINED_ESC:
       escNext = true
     of ACTION_LINED_CLEAR:
-      print(' '.repeat(rl - cursor + 1))
-      print('\b'.repeat(rl - cursor + 1))
-      print('\b'.repeat(cursor))
-      print(' '.repeat(cursor))
-      print('\b'.repeat(cursor))
       new = new.runeSubstr(cursor)
       rl = new.runeLen()
-      printesc(new)
-      print('\b'.repeat(rl))
       cursor = 0
     of ACTION_LINED_KILL:
-      print(' '.repeat(rl - cursor + 1))
-      print('\b'.repeat(rl - cursor + 1))
       new = new.runeSubstr(0, cursor)
     of ACTION_LINED_BACK:
       if cursor > 0:
-        cursor -= 1
-        print("\b")
+        dec cursor
     of ACTION_LINED_FORWARD:
       if cursor < rl:
-        var rune: Rune
-        new.fastRuneAt(cursor, rune, false)
-        printesc($rune)
         inc cursor
     of ACTION_LINED_PREV_WORD:
       while cursor > 0:
-        print('\b')
-        cursor -= 1
+        dec cursor
         var rune: Rune
         new.fastRuneAt(cursor, rune, false)
-        if rune == Rune(' '):
+        if rune in breakWord:
           break
     of ACTION_LINED_NEXT_WORD:
       while cursor < rl:
         var rune: Rune
-        new.fastRuneAt(cursor, rune, false)
-        printesc($rune)
         inc cursor
         if cursor < rl:
           new.fastRuneAt(cursor, rune, false)
-          if rune == Rune(' '):
+          if rune in breakWord:
             break
     of ACTION_LINED_KILL_WORD:
       var chars = 0
@@ -127,17 +271,12 @@ proc readLine*(prompt: string, current: var string): bool =
         inc chars
         var rune: Rune
         new.fastRuneAt(cursor - chars, rune, false)
-        if rune == Rune(' '):
+        if rune in breakWord:
           break
       if chars > 0:
-        print(' '.repeat(rl - cursor + 1))
-        print('\b'.repeat(rl - cursor + 1))
-        print("\b \b".repeat(chars))
         new = new.runeSubstr(0, cursor - chars) & new.runeSubstr(cursor)
         rl = new.runeLen()
         cursor -= chars
-        printesc(new.runeSubstr(cursor))
-        print('\b'.repeat(rl - cursor))
     of ACTION_FEED_NEXT:
       feedNext = true
     elif validateUtf8(s) == -1:
@@ -151,12 +290,8 @@ proc readLine*(prompt: string, current: var string): bool =
       escNext = false
       if cs.len == 0:
         continue
-      print(' '.repeat(rl - cursor + 1))
-      print('\b'.repeat(rl - cursor + 1))
       new = new.runeSubstr(0, cursor) & cs & new.runeSubstr(cursor)
       rl = new.runeLen()
-      printesc(new.runeSubstr(cursor))
-      print('\b'.repeat(rl - cursor - 1))
       inc cursor
     else:
       feedNext = true
diff --git a/twtstr.nim b/twtstr.nim
index 09b19cc0..63854879 100644
--- a/twtstr.nim
+++ b/twtstr.nim
@@ -24,8 +24,8 @@ func ansiReset*(str: seq[string]): seq[string] =
   return str & ansiResetCode
 
 func maxString*(str: string, max: int): string =
-  if max < str.len:
-    return str.substr(0, max - 2) & "$"
+  if max < str.runeLen():
+    return str.runeSubstr(0, max - 2) & "$"
   return str
 
 func fitValueToSize*(str: string, size: int): string =
@@ -49,10 +49,10 @@ func remove*(str: string, c: string): string =
       result &= $rune
 
 func isControlChar*(c: char): bool =
-  return int(c) <= 0x1F or int(c) == 0x7F
+  return c <= char(0x1F) or c == char(0x7F)
 
 func getControlChar*(c: char): char =
-  if int(c) >= int('a'):
+  if c >= 'a':
     return char(int(c) - int('a') + 1)
   elif c == '?':
     return char(127)