Update CSS tokenizer

It was slightly outdated and in many places incorrect.

6 files changed, 260 insertions, 246 deletions

diff --git a/src/css/cssparser.nim b/src/css/cssparser.nim
index 4321eb57..6b5599bf 100644
--- a/src/css/cssparser.nim
+++ b/src/css/cssparser.nim
@@ -18,7 +18,8 @@ type
   CSSTokenizerState = object
     at: int
     stream: Stream
-    buf: seq[Rune]
+    buf: string
+    curr: char
 
   CSSParseState = object
     tokens: seq[CSSParsedItem]
@@ -37,14 +38,14 @@ type
     case tokenType*: CSSTokenType
     of CSS_IDENT_TOKEN, CSS_FUNCTION_TOKEN, CSS_AT_KEYWORD_TOKEN,
        CSS_HASH_TOKEN, CSS_STRING_TOKEN, CSS_URL_TOKEN:
-      value*: seq[Rune]
+      value*: string
       tflaga*: tflaga
     of CSS_DELIM_TOKEN:
       rvalue*: Rune
     of CSS_NUMBER_TOKEN, CSS_PERCENTAGE_TOKEN, CSS_DIMENSION_TOKEN:
       nvalue*: float64
       tflagb*: tflagb
-      unit*: seq[Rune]
+      unit*: string
     else: discard
 
   CSSRule* = ref object of CSSParsedItem
@@ -52,17 +53,17 @@ type
     oblock*: CSSSimpleBlock
 
   CSSAtRule* = ref object of CSSRule
-    name*: seq[Rune]
+    name*: string
 
   CSSQualifiedRule* = ref object of CSSRule
 
   CSSDeclaration* = ref object of CSSComponentValue
-    name*: seq[Rune]
+    name*: string
     value*: seq[CSSComponentValue]
     important*: bool
 
   CSSFunction* = ref object of CSSComponentValue
-    name*: seq[Rune]
+    name*: string
     value*: seq[CSSComponentValue]
 
   CSSSimpleBlock* = ref object of CSSComponentValue
@@ -75,23 +76,23 @@ type
   SyntaxError = object of ValueError
 
 # For debugging
-template `$`*(c: CSSParsedItem): string =
+proc `$`*(c: CSSParsedItem): string =
   if c of CSSToken:
     case CSSToken(c).tokenType:
     of CSS_FUNCTION_TOKEN, CSS_AT_KEYWORD_TOKEN, CSS_URL_TOKEN:
-      result &= $CSSToken(c).tokenType & $CSSToken(c).value & '\n'
+      result &= $CSSToken(c).tokenType & CSSToken(c).value & '\n'
     of CSS_HASH_TOKEN:
-      result &= '#' & $CSSToken(c).value
+      result &= '#' & CSSToken(c).value
     of CSS_IDENT_TOKEN:
-      result &= $CSSToken(c).value
+      result &= CSSToken(c).value
     of CSS_STRING_TOKEN:
-      result &= ("\"" & $CSSToken(c).value & "\"")
+      result &= ("\"" & CSSToken(c).value & "\"")
     of CSS_DELIM_TOKEN:
-      result &= $CSSToken(c).rvalue
+      result &= CSSToken(c).rvalue
     of CSS_DIMENSION_TOKEN:
-      result &= $CSSToken(c).tokenType & $CSSToken(c).nvalue & "unit" & $CSSToken(c).unit & $CSSToken(c).tflagb
+      result &= $CSSToken(c).tokenType & $CSSToken(c).nvalue & "unit" & CSSToken(c).unit & $CSSToken(c).tflagb
     of CSS_NUMBER_TOKEN:
-      result &= $CSSToken(c).nvalue & $CSSToken(c).unit
+      result &= $CSSToken(c).nvalue & CSSToken(c).unit
     of CSS_PERCENTAGE_TOKEN:
       result &= $CSSToken(c).nvalue & "%"
     of CSS_COLON_TOKEN:
@@ -105,13 +106,13 @@ template `$`*(c: CSSParsedItem): string =
     else:
       result &= $CSSToken(c).tokenType & '\n'
   elif c of CSSDeclaration:
-    result &= $CSSDeclaration(c).name
+    result &= CSSDeclaration(c).name
     result &= ": "
     for s in CSSDeclaration(c).value:
       result &= $s
     result &= ";\n"
   elif c of CSSFunction:
-    result &= $CSSFunction(c).name & "("
+    result &= CSSFunction(c).name & "("
     for s in CSSFunction(c).value:
       result &= $s
     result &= ")"
@@ -130,243 +131,254 @@ template `$`*(c: CSSParsedItem): string =
     else: discard
   elif c of CSSRule:
     if c of CSSAtRule:
-      result &= $CSSAtRule(c).name & " "
+      result &= CSSAtRule(c).name & " "
     result &= $CSSRule(c).prelude & "\n"
     result &= $CSSRule(c).oblock
 
 func `==`*(a: CSSParsedItem, b: CSSTokenType): bool =
   return a of CSSToken and CSSToken(a).tokenType == b
 
-func isNameStartCodePoint(r: Rune): bool =
-  return not isAscii(r) or r == Rune('_') or isAlphaAscii(r)
+const IdentStart = AsciiAlpha + NonAscii + {'_'} 
+const Ident = IdentStart + AsciiDigit + {'-'}
 
-func isNameCodePoint(r: Rune): bool =
-  return isNameStartCodePoint(r) or isDigitAscii(r) or r == Rune('-')
-
-proc consume(state: var CSSTokenizerState): Rune =
-  result = state.buf[state.at]
+proc consume(state: var CSSTokenizerState): char =
+  state.curr = state.buf[state.at]
   inc state.at
+  return state.curr
+
+proc consumeRune(state: var CSSTokenizerState): Rune =
+  fastRuneAt(state.buf, state.at, result)
 
 proc reconsume(state: var CSSTokenizerState) =
   dec state.at
 
-func peek(state: CSSTokenizerState, i: int): Rune =
+func peek(state: CSSTokenizerState, i: int = 0): char =
   return state.buf[state.at + i]
 
 proc has(state: var CSSTokenizerState, i: int = 0): bool =
   if state.at + i >= state.buf.len and not state.stream.atEnd():
-    state.buf &= state.stream.readLine().toRunes() & Rune('\n')
+    state.buf &= state.stream.readLine() & '\n'
   return state.at + i < state.buf.len
 
-func curr(state: CSSTokenizerState): Rune =
-  return state.buf[state.at]
+proc isValidEscape(a, b: char): bool =
+  return a == '\\' and b != '\n'
 
 proc isValidEscape(state: var CSSTokenizerState): bool =
-  return state.has(1) and state.curr() == Rune('\\') and state.peek(1) != Rune('\n')
+  return state.has() and isValidEscape(state.curr, state.peek())
+
+# current + next + next(1)
+proc startsWithIdentSequence(state: var CSSTokenizerState): bool =
+  case state.curr
+  of '-':
+    return state.has() and state.peek() in IdentStart + {'-'} or state.has(1) and state.isValidEscape()
+  of IdentStart:
+    return true
+  of '\\':
+    return state.isValidEscape()
+  else:
+    return false
 
-proc startsWithIdentifier(state: var CSSTokenizerState): bool =
+# next, next(1), next(2)
+proc next3startsWithIdentSequence(state: var CSSTokenizerState): bool =
   if not state.has():
     return false
 
-  if isNameStartCodePoint(state.curr()):
+  case state.peek()
+  of '-':
+    return state.has(1) and state.peek(1) in IdentStart + {'-'} or state.has(2) and isValidEscape(state.peek(1), state.peek(2)):
+  of IdentStart:
     return true
-  if state.curr() == Rune('-'):
-    if state.has(1) and state.peek(1).isNameStartCodePoint():
-      return true
-    if state.isValidEscape():
-      return true
+  of '\\':
+    return state.has(1) and isValidEscape(state.peek(), state.peek(1))
+  else:
     return false
-  elif state.curr() == Rune('\\'):
-    return state.isValidEscape()
-
-  return false
 
 proc startsWithNumber(state: var CSSTokenizerState): bool =
   if state.has():
-    case state.curr()
-    of Rune('+'), Rune('-'):
+    case state.peek()
+    of '+', '-':
       if state.has(1):
-        if isDigitAscii(state.peek(1)):
+        if state.peek(1) in AsciiDigit:
           return true
-        elif state.peek(1) == Rune('.'):
-          if state.has(2) and isDigitAscii(state.peek(2)):
+        elif state.peek(1) == '.':
+          if state.has(2) and state.peek(2) in AsciiDigit:
             return true
-    of Rune('.'):
-      if isDigitAscii(state.peek(1)):
+    of '.':
+      if state.peek(1) in AsciiDigit:
         return true
-    elif isDigitAscii(state.curr()):
+    elif state.peek() in AsciiDigit:
       return true
     else:
       return false
   return false
 
-proc consumeEscape(state: var CSSTokenizerState): Rune =
-  let r = state.consume()
-  var num = hexValue(r)
-  if num != -1:
+proc consumeEscape(state: var CSSTokenizerState): string =
+  if not state.has():
+    return $Rune(0xFFFD)
+  let c = state.consume()
+  if c in AsciiHexDigit:
+    var num = hexValue(c)
     var i = 0
-    while state.has() and i <= 5:
-      let r = state.consume()
-      if hexValue(r) == -1:
+    while i <= 5 and state.has():
+      let c = state.consume()
+      if hexValue(c) == -1:
         state.reconsume()
         break
       num *= 0x10
-      num += hexValue(r)
+      num += hexValue(c)
       inc i
+    if state.peek().isWhitespace():
+      discard state.consume()
     if num == 0 or num > 0x10FFFF or num in {0xD800..0xDFFF}:
-      return Rune(0xFFFD)
+      return $Rune(0xFFFD)
     else:
-      return Rune(num)
+      return $Rune(num)
   else:
-    return r
+    return $c #NOTE this assumes the caller doesn't care about non-ascii
 
 proc consumeString(state: var CSSTokenizerState): CSSToken =
-  var s: seq[Rune]
-  state.reconsume()
-  let ending = state.consume()
+  var s: string
+  let ending = state.curr
 
   while state.has():
-    let r = state.consume()
-    case r
-    of Rune('\n'):
+    let c = state.consume()
+    case c
+    of '\n':
+      state.reconsume()
       return CSSToken(tokenType: CSS_BAD_STRING_TOKEN)
-    of Rune('\\'):
-      s &= consumeEscape(state)
-    elif r == ending:
+    of '\\':
+      if not state.has():
+        continue
+      elif state.peek() == '\n':
+        discard state.consume()
+      else:
+        s &= consumeEscape(state)
+    elif c == ending:
       break
     else:
-      s &= r
+      s &= c
   return CSSToken(tokenType: CSS_STRING_TOKEN, value: s)
 
-proc consumeName(state: var CSSTokenizerState): seq[Rune] =
+proc consumeIdentSequence(state: var CSSTokenizerState): string =
   while state.has():
-    let r = state.consume()
+    let c = state.consume()
     if state.isValidEscape():
       result &= state.consumeEscape()
-    elif isNameCodePoint(r):
-      result &= r
+    elif c in Ident:
+      result &= c
     else:
       state.reconsume()
       return result
 
-proc consumeNumberSign(state: var CSSTokenizerState): CSSToken =
-  if state.has():
-    let r = state.consume()
-    if isNameCodePoint(r) or state.isValidEscape():
-      result = CSSToken(tokenType: CSS_HASH_TOKEN)
-      if state.startsWithIdentifier():
-        result.tflaga = TFLAGA_ID
-      
-      state.reconsume()
-      result.value = consumeName(state)
-  else:
-    let r = state.consume()
-    result = CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
-
-proc consumeNumber(state: var CSSTokenizerState): tuple[t: tflagb, val: float64] =
+proc consumeNumber(state: var CSSTokenizerState): (tflagb, float64) =
   var t = TFLAGB_INTEGER
-  var repr: seq[Rune]
-  if state.has():
-    if state.curr() == Rune('+') or state.curr() == Rune('-'):
-      repr &= state.consume()
+  var repr: string
+  if state.has() and state.peek() in {'+', '-'}:
+    repr &= state.consume()
 
-  while state.has() and isDigitAscii(state.curr()):
+  while state.has() and state.peek() in AsciiDigit:
     repr &= state.consume()
 
-  if state.has(1):
-    if state.curr() == Rune('.') and isDigitAscii(state.peek(1)):
+  if state.has(1) and state.peek() == '.' and state.peek(1) in AsciiDigit:
+    repr &= state.consume()
+    repr &= state.consume()
+    t = TFLAGB_NUMBER
+    while state.has() and state.peek() in AsciiDigit:
+      repr &= state.consume()
+
+  if state.has(1) and state.peek() in {'E', 'e'} and state.peek(1) in AsciiDigit or
+      state.has(2) and state.peek() in {'E', 'e'} and state.peek(1) in {'-', '+'} and state.peek(2) in AsciiDigit:
+    repr &= state.consume()
+    if state.peek() in {'-', '+'}:
       repr &= state.consume()
       repr &= state.consume()
-      t = TFLAGB_NUMBER
-      while state.has() and isDigitAscii(state.curr()):
-        repr &= state.consume()
-
-  if state.has(1):
-    if state.curr() == Rune('E') or state.curr() == Rune('e'):
-      var j = 2
-      if state.peek(1) == Rune('-') or state.peek(1) == Rune('+'):
-        inc j
-      if state.has(j) and isDigitAscii(state.peek(j)):
-        while j > 0:
-          repr &= state.consume()
-          dec j
-
-        while state.has() and isDigitAscii(state.curr()):
-          repr &= state.consume()
+    else:
+      repr &= state.consume()
+    t = TFLAGB_NUMBER
+    while state.has() and state.peek() in AsciiDigit:
+      repr &= state.consume()
 
   let val = parseFloat64($repr)
   return (t, val)
 
 proc consumeNumericToken(state: var CSSTokenizerState): CSSToken =
-  let num = state.consumeNumber()
-  if state.startsWithIdentifier():
-    result = CSSToken(tokenType: CSS_DIMENSION_TOKEN, nvalue: num.val, tflagb: num.t)
-    result.unit = state.consumeName()
-  elif state.has() and state.curr() == Rune('%'):
+  let (t, val) = state.consumeNumber()
+  if state.next3startsWithIdentSequence():
+    result = CSSToken(tokenType: CSS_DIMENSION_TOKEN, nvalue: val, tflagb: t)
+    result.unit = state.consumeIdentSequence()
+  elif state.has() and state.peek() == '%':
     discard state.consume()
-    result = CSSToken(tokenType: CSS_PERCENTAGE_TOKEN, nvalue: num.val)
+    result = CSSToken(tokenType: CSS_PERCENTAGE_TOKEN, nvalue: val)
   else:
-    result = CSSToken(tokenType: CSS_NUMBER_TOKEN, nvalue: num.val, tflagb: num.t)
+    result = CSSToken(tokenType: CSS_NUMBER_TOKEN, nvalue: val, tflagb: t)
 
 proc consumeBadURL(state: var CSSTokenizerState) =
-  while state.has(1):
-    let r = state.consume()
-    case r
-    of Rune(')'):
+  while state.has():
+    let c = state.consume()
+    case c
+    of ')':
       return
     elif state.isValidEscape():
       discard state.consumeEscape()
     else: discard
 
+const NonPrintable = {char(0x00)..char(0x08), char(0x0B), char(0x0E)..char(0x1F), char(0x7F)}
+
 proc consumeURL(state: var CSSTokenizerState): CSSToken =
   result = CSSToken(tokenType: CSS_URL_TOKEN)
-  while state.has(1) and state.peek(1).isWhitespace():
+  while state.has() and state.peek().isWhitespace():
     discard state.consume()
 
-  while state.has(1):
-    let r = state.consume()
-    case r
-    of Rune(')'):
+  while state.has():
+    let c = state.consume()
+    case c
+    of ')':
       return result
-    of Rune('"'), Rune('\''), Rune('('):
+    of '"', '\'', '(', NonPrintable:
       state.consumeBadURL()
       return CSSToken(tokenType: CSS_BAD_URL_TOKEN)
-    of Rune('\\'):
+    of AsciiWhitespace:
+      while state.has() and state.peek().isWhitespace():
+        discard state.consume()
+      if not state.has():
+        return result
+      if state.peek() == ')':
+        discard state.consume()
+        return result
+      state.consumeBadURL()
+      return CSSToken(tokenType: CSS_BAD_URL_TOKEN)
+    of '\\':
       state.reconsume()
       if state.isValidEscape():
         result.value &= state.consumeEscape()
       else:
         state.consumeBadURL()
         return CSSToken(tokenType: CSS_BAD_URL_TOKEN)
-    elif r.isWhitespace():
-      while state.has(1) and state.peek(1).isWhitespace():
-        discard state.consume()
     else:
-      result.value &= r
+      result.value &= c
 
 proc consumeIdentLikeToken(state: var CSSTokenizerState): CSSToken =
-  let s = state.consumeName()
-  if s.equalsIgnoreCase("url") and state.has() and state.curr() == Rune('('):
+  let s = state.consumeIdentSequence()
+  if s.equalsIgnoreCase("url") and state.has() and state.peek() == '(':
     discard state.consume()
-    while state.has(1) and state.curr().isWhitespace() and state.peek(1).isWhitespace():
+    while state.has(1) and state.peek().isWhitespace() and state.peek(1).isWhitespace():
       discard state.consume()
-    if state.curr() == Rune('\'') or state.curr() == Rune('"') or state.curr().isWhitespace():
+    if state.has(1) and state.peek() in {'"', '\''} + AsciiWhitespace and state.peek(1) in {'"', '\''}:
       return CSSToken(tokenType: CSS_FUNCTION_TOKEN, value: s)
     else:
       return state.consumeURL()
-  elif state.has() and state.curr() == Rune('('):
+  elif state.has() and state.peek() == '(':
     discard state.consume()
     return CSSToken(tokenType: CSS_FUNCTION_TOKEN, value: s)
 
   return CSSToken(tokenType: CSS_IDENT_TOKEN, value: s)
 
 proc consumeComments(state: var CSSTokenizerState) =
-  if state.has(1) and state.curr() == Rune('/') and state.peek(1) == Rune('*'):
+  if state.has(1) and state.peek() == '/' and state.peek(1) == '*':
     discard state.consume()
     discard state.consume()
-    while state.has(1) and not (state.curr() == Rune('*') and state.peek(1) == Rune('/')):
+    while state.has() and not (state.has(1) and state.peek() == '*' and state.peek(1) == '/'):
       discard state.consume()
-
     if state.has(1):
       discard state.consume()
     if state.has():
@@ -376,87 +388,92 @@ proc consumeToken(state: var CSSTokenizerState): CSSToken =
   state.consumeComments()
   if not state.has():
     return
-  let r = state.consume()
-  case r
-  of Rune('\n'), Rune('\t'), Rune(' '), Rune('\f'), Rune('\r'):
-    while state.has() and state.curr().isWhitespace():
+  let c = state.consume()
+  case c
+  of AsciiWhitespace:
+    while state.has() and state.peek().isWhitespace():
       discard state.consume()
     return CSSToken(tokenType: CSS_WHITESPACE_TOKEN)
-  of Rune('"'), Rune('\''):
+  of '"', '\'':
     return consumeString(state)
-  of Rune('#'):
-    return consumeNumberSign(state)
-  of Rune('('):
-    return CSSToken(tokenType: CSS_LPAREN_TOKEN)
-  of Rune(')'):
-    return CSSToken(tokenType: CSS_RPAREN_TOKEN)
-  of Rune('['):
-    return CSSToken(tokenType: CSS_LBRACKET_TOKEN)
-  of Rune(']'):
-    return CSSToken(tokenType: CSS_RBRACKET_TOKEN)
-  of Rune('{'):
-    return CSSToken(tokenType: CSS_LBRACE_TOKEN)
-  of Rune('}'):
-    return CSSToken(tokenType: CSS_RBRACE_TOKEN)
-  of Rune(','):
-    return CSSToken(tokenType: CSS_COMMA_TOKEN)
-  of Rune(':'):
-    return CSSToken(tokenType: CSS_COLON_TOKEN)
-  of Rune(';'):
-    return CSSToken(tokenType: CSS_SEMICOLON_TOKEN)
-  of Rune('+'):
+  of '#':
+    if state.has() and state.peek() in Ident or state.isValidEscape():
+      result = CSSToken(tokenType: CSS_HASH_TOKEN)
+      if state.startsWithIdentSequence():
+        result.tflaga = TFLAGA_ID
+      result.value = consumeIdentSequence(state)
+    else:
+      state.reconsume()
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: state.consumeRune())
+  of '(': return CSSToken(tokenType: CSS_LPAREN_TOKEN)
+  of ')': return CSSToken(tokenType: CSS_RPAREN_TOKEN)
+  of '{': return CSSToken(tokenType: CSS_LBRACE_TOKEN)
+  of '}': return CSSToken(tokenType: CSS_RBRACE_TOKEN)
+  of '+':
     if state.startsWithNumber():
       state.reconsume()
       return state.consumeNumericToken()
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
-  of Rune('-'):
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+  of ',': return CSSToken(tokenType: CSS_COMMA_TOKEN)
+  of '-':
     if state.startsWithNumber():
       state.reconsume()
       return state.consumeNumericToken()
     else:
-      if state.has(2) and state.peek(1) == Rune('-') and state.peek(2) == Rune('>'):
+      if state.has(1) and state.peek() == '-' and state.peek(1) == '>':
         discard state.consume()
         discard state.consume()
         return CSSToken(tokenType: CSS_CDC_TOKEN)
-      elif state.startsWithIdentifier():
+      elif state.startsWithIdentSequence():
         state.reconsume()
-        result = state.consumeIdentLikeToken()
+        return state.consumeIdentLikeToken()
       else:
-        return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
-  of Rune('.'):
+        return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+  of '.':
     if state.startsWithNumber():
       state.reconsume()
       return state.consumeNumericToken()
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
-  of Rune('<'):
-    if state.has(3) and state.peek(1) == Rune('!') and state.peek(2) == Rune('-') and state.peek(3) == Rune('-'):
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+  of ':': return CSSToken(tokenType: CSS_COLON_TOKEN)
+  of ';': return CSSToken(tokenType: CSS_SEMICOLON_TOKEN)
+  of '<':
+    if state.has(2) and state.peek() == '!' and state.peek(1) == '-' and state.peek(2) == '-':
       discard state.consume()
       discard state.consume()
       discard state.consume()
       return CSSToken(tokenType: CSS_CDO_TOKEN)
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
-  of Rune('@'):
-    if state.startsWithIdentifier():
-      let name = state.consumeName()
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+  of '@':
+    if state.next3startsWithIdentSequence():
+      let name = state.consumeIdentSequence()
       return CSSToken(tokenType: CSS_AT_KEYWORD_TOKEN, value: name)
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
-  elif isDigitAscii(r):
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+  of '[': return CSSToken(tokenType: CSS_LBRACKET_TOKEN)
+  of '\\':
+    if state.isValidEscape():
+      state.reconsume()
+      return state.consumeIdentLikeToken()
+    else:
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+  of ']': return CSSToken(tokenType: CSS_RBRACKET_TOKEN)
+  of AsciiDigit:
     state.reconsume()
     return state.consumeNumericToken()
-  elif isNameStartCodePoint(r):
+  of IdentStart:
     state.reconsume()
     return state.consumeIdentLikeToken()
   else:
-    return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: r)
+    state.reconsume()
+    return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: state.consumeRune())
 
 proc tokenizeCSS*(inputStream: Stream): seq[CSSParsedItem] =
   var state: CSSTokenizerState
   state.stream = inputStream
-  state.buf = state.stream.readLine().toRunes()
+  state.buf = state.stream.readLine() & '\n'
   while state.has():
     let tok = state.consumeToken()
     if tok != nil:
@@ -474,7 +491,7 @@ proc reconsume(state: var CSSParseState) =
 func has(state: CSSParseState, i: int = 0): bool =
   return state.at + i < state.tokens.len
 
-func curr(state: CSSParseState): CSSParsedItem =
+func peek(state: CSSParseState): CSSParsedItem =
   return state.tokens[state.at]
 
 proc consumeComponentValue(state: var CSSParseState): CSSComponentValue
@@ -558,12 +575,12 @@ proc consumeAtRule(state: var CSSParseState): CSSAtRule =
 proc consumeDeclaration(state: var CSSParseState): Option[CSSDeclaration] =
   let t = CSSToken(state.consume())
   var decl = CSSDeclaration(name: t.value)
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
-  if not state.has() or state.curr() != CSS_COLON_TOKEN:
+  if not state.has() or state.peek() != CSS_COLON_TOKEN:
     return none(CSSDeclaration)
   discard state.consume()
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
 
   while state.has():
@@ -608,7 +625,7 @@ proc consumeListOfDeclarations(state: var CSSParseState): seq[CSSParsedItem] =
     elif t == CSS_IDENT_TOKEN:
       var tempList: seq[CSSParsedItem]
       tempList.add(CSSToken(t))
-      while state.has() and state.curr() != CSS_SEMICOLON_TOKEN:
+      while state.has() and state.peek() != CSS_SEMICOLON_TOKEN:
         tempList.add(state.consumeComponentValue())
 
       var tempState = CSSParseState(at: 0, tokens: tempList)
@@ -617,7 +634,7 @@ proc consumeListOfDeclarations(state: var CSSParseState): seq[CSSParsedItem] =
         result.add(decl.get)
     else:
       state.reconsume()
-      if state.curr() != CSS_SEMICOLON_TOKEN:
+      if state.peek() != CSS_SEMICOLON_TOKEN:
         discard state.consumeComponentValue()
 
 proc consumeListOfDeclarations2(state: var CSSParseState): seq[CSSDeclaration] =
@@ -631,7 +648,7 @@ proc consumeListOfDeclarations2(state: var CSSParseState): seq[CSSDeclaration] =
     elif t == CSS_IDENT_TOKEN:
       var tempList: seq[CSSParsedItem]
       tempList.add(CSSToken(t))
-      while state.has() and state.curr() != CSS_SEMICOLON_TOKEN:
+      while state.has() and state.peek() != CSS_SEMICOLON_TOKEN:
         tempList.add(state.consumeComponentValue())
 
       var tempState = CSSParseState(at: 0, tokens: tempList)
@@ -640,7 +657,7 @@ proc consumeListOfDeclarations2(state: var CSSParseState): seq[CSSDeclaration] =
         result.add(decl.get)
     else:
       state.reconsume()
-      if state.curr() != CSS_SEMICOLON_TOKEN:
+      if state.peek() != CSS_SEMICOLON_TOKEN:
         discard state.consumeComponentValue()
 
 proc consumeListOfRules(state: var CSSParseState): seq[CSSRule] =
@@ -686,12 +703,12 @@ proc parseListOfRules*(cvals: seq[CSSComponentValue]): seq[CSSRule] =
   return state.parseListOfRules()
 
 proc parseRule(state: var CSSParseState): CSSRule =
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
   if not state.has():
     raise newException(SyntaxError, "EOF reached!")
 
-  if state.curr() == CSS_AT_KEYWORD_TOKEN:
+  if state.peek() == CSS_AT_KEYWORD_TOKEN:
     result = state.consumeAtRule()
   else:
     let q = state.consumeQualifiedRule()
@@ -700,7 +717,7 @@ proc parseRule(state: var CSSParseState): CSSRule =
     else:
       raise newException(SyntaxError, "No qualified rule found!")
 
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
   if state.has():
     raise newException(SyntaxError, "EOF not reached!")
@@ -711,10 +728,10 @@ proc parseRule(inputStream: Stream): CSSRule =
   return state.parseRule()
 
 proc parseDeclaration(state: var CSSParseState): CSSDeclaration =
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
 
-  if not state.has() or state.curr() != CSS_IDENT_TOKEN:
+  if not state.has() or state.peek() != CSS_IDENT_TOKEN:
     raise newException(SyntaxError, "No ident token found!")
 
   let d = state.consumeDeclaration()
@@ -759,14 +776,14 @@ proc parseListOfDeclarations2*(inputStream: Stream): seq[CSSDeclaration] =
   return state.parseListOfDeclarations2()
 
 proc parseComponentValue(state: var CSSParseState): CSSComponentValue =
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
   if not state.has():
     raise newException(SyntaxError, "EOF reached!")
 
   result = state.consumeComponentValue()
 
-  while state.has() and state.curr() == CSS_WHITESPACE_TOKEN:
+  while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
     discard state.consume()
   if state.has():
     raise newException(SyntaxError, "EOF not reached!")
diff --git a/src/css/mediaquery.nim b/src/css/mediaquery.nim
index b589a355..2292a27c 100644
--- a/src/css/mediaquery.nim
+++ b/src/css/mediaquery.nim
@@ -1,5 +1,4 @@
 import tables
-import unicode
 
 import css/cssparser
 
@@ -103,7 +102,7 @@ template expect_bool(b: bool, sfalse: string, strue: string) =
   if not (cval of CSSToken): return nil
   let tok = CSSToken(cval)
   if tok.tokenType != CSS_IDENT_TOKEN: return nil
-  let s = $tok.value
+  let s = tok.value
   case s
   of strue: b = true
   of sfalse: b = false
@@ -155,7 +154,7 @@ proc parseMediaInParens(parser: var MediaQueryParser): MediaQuery =
     get_tok(tok)
     fparser.skipBlanks()
     if tok.tokenType == CSS_IDENT_TOKEN:
-      let tokval = $tok.value
+      let tokval = tok.value
       case tokval
       of "not":
         return fparser.parseMediaCondition(true)
@@ -187,7 +186,7 @@ proc parseMediaCondition(parser: var MediaQueryParser, non = false, noor = false
   if not non:
     let cval = parser.consume()
     if cval of CSSToken and CSSToken(cval).tokenType == CSS_IDENT_TOKEN:
-      if $CSSToken(cval).value == "not":
+      if CSSToken(cval).value == "not":
         non = true
     else:
       parser.reconsume()
@@ -211,7 +210,7 @@ proc parseMediaCondition(parser: var MediaQueryParser, non = false, noor = false
   var tok: CSSToken
   get_idtok(tok)
   parser.skipBlanks()
-  let tokval = $tok.value
+  let tokval = tok.value
   case tokval
   of "and":
     return parser.parseMediaAnd(result)
@@ -231,7 +230,7 @@ proc parseMediaQuery(parser: var MediaQueryParser): MediaQuery =
     if cval of CSSToken:
       let tok = CSSToken(cval)
       if tok.tokenType == CSS_IDENT_TOKEN: 
-        let tokval = $tok.value
+        let tokval = tok.value
         case tokval
         of "not":
           non = true
@@ -256,7 +255,7 @@ proc parseMediaQuery(parser: var MediaQueryParser): MediaQuery =
     if cval of CSSToken:
       let tok = CSSToken(cval)
       if tok.tokenType == CSS_IDENT_TOKEN: 
-        let tokval = $tok.value
+        let tokval = tok.value
         if result == nil:
           if tokval in MediaTypes:
             let mq = MediaQuery(t: CONDITION_MEDIA, media: MediaTypes[tokval])
@@ -287,7 +286,7 @@ proc parseMediaQuery(parser: var MediaQueryParser): MediaQuery =
     if cval of CSSToken:
       let tok = CSSToken(cval)
       if tok.tokenType == CSS_IDENT_TOKEN: 
-        let tokval = $tok.value
+        let tokval = tok.value
         if tokval != "and":
           return nil
       else:
diff --git a/src/css/selectorparser.nim b/src/css/selectorparser.nim
index 7c4834e6..012d6e68 100644
--- a/src/css/selectorparser.nim
+++ b/src/css/selectorparser.nim
@@ -191,11 +191,11 @@ proc parseSelectorToken(state: var SelectorParser, csstoken: CSSToken) =
   of CSS_IDENT_TOKEN:
     case state.query
     of QUERY_CLASS:
-      state.addSelector(Selector(t: CLASS_SELECTOR, class: $csstoken.value))
+      state.addSelector(Selector(t: CLASS_SELECTOR, class: csstoken.value))
     of QUERY_TYPE:
-      state.addSelector(Selector(t: TYPE_SELECTOR, tag: tagType($csstoken.value)))
+      state.addSelector(Selector(t: TYPE_SELECTOR, tag: tagType(csstoken.value)))
     of QUERY_PSEUDO:
-      case $csstoken.value
+      case csstoken.value
       of "before":
         state.addSelector(Selector(t: PSELEM_SELECTOR, elem: PSEUDO_BEFORE))
       of "after":
@@ -213,7 +213,7 @@ proc parseSelectorToken(state: var SelectorParser, csstoken: CSSToken) =
       of "checked":
         state.addSelector(Selector(t: PSEUDO_SELECTOR, pseudo: PSEUDO_CHECKED))
     of QUERY_PSELEM:
-      case $csstoken.value
+      case csstoken.value
       of "before":
         state.addSelector(Selector(t: PSELEM_SELECTOR, elem: PSEUDO_BEFORE))
       of "after":
@@ -238,7 +238,7 @@ proc parseSelectorToken(state: var SelectorParser, csstoken: CSSToken) =
       state.addSelector(Selector(t: UNIVERSAL_SELECTOR))
     else: discard
   of CSS_HASH_TOKEN:
-    state.addSelector(Selector(t: ID_SELECTOR, id: $csstoken.value))
+    state.addSelector(Selector(t: ID_SELECTOR, id: csstoken.value))
   of CSS_COMMA_TOKEN:
     if state.selectors[^1].len > 0:
       state.addSelectorList()
@@ -264,15 +264,15 @@ proc parseSelectorSimpleBlock(state: var SelectorParser, cssblock: CSSSimpleBloc
           case state.query
           of QUERY_ATTR:
             state.query = QUERY_DELIM
-            state.addSelector(Selector(t: ATTR_SELECTOR, attr: $csstoken.value, rel: ' '))
+            state.addSelector(Selector(t: ATTR_SELECTOR, attr: csstoken.value, rel: ' '))
           of QUERY_VALUE:
-            state.getLastSel().value = $csstoken.value
+            state.getLastSel().value = csstoken.value
             break
           else: discard
         of CSS_STRING_TOKEN:
           case state.query
           of QUERY_VALUE:
-            state.getLastSel().value = $csstoken.value
+            state.getLastSel().value = csstoken.value
             break
           else: discard
         of CSS_DELIM_TOKEN:
@@ -291,7 +291,7 @@ proc parseSelectorSimpleBlock(state: var SelectorParser, cssblock: CSSSimpleBloc
   else: discard
 
 proc parseSelectorFunction(state: var SelectorParser, cssfunction: CSSFunction) =
-  case $cssfunction.name
+  case cssfunction.name
   of "not", "is":
     if state.query != QUERY_PSEUDO:
       return
@@ -309,7 +309,7 @@ proc parseSelectorFunction(state: var SelectorParser, cssfunction: CSSFunction)
     state.query = QUERY_TYPE
     return
   else: return
-  var fun = Selector(t: FUNC_SELECTOR, name: $cssfunction.name)
+  var fun = Selector(t: FUNC_SELECTOR, name: cssfunction.name)
   state.addSelector(fun)
 
   let osels = state.selectors
diff --git a/src/css/sheet.nim b/src/css/sheet.nim
index be8ac80c..b3fbefa7 100644
--- a/src/css/sheet.nim
+++ b/src/css/sheet.nim
@@ -1,6 +1,5 @@
 import streams
 import tables
-import unicode
 
 import css/mediaquery
 import css/cssparser
@@ -172,7 +171,7 @@ proc addRule(stylesheet: var CSSStylesheet, rule: CSSQualifiedRule) =
     stylesheet.add(r)
 
 proc addAtRule(stylesheet: var CSSStylesheet, atrule: CSSAtRule) =
-  case $atrule.name
+  case atrule.name
   of "media":
     let query = parseMediaQueryList(atrule.prelude)
     let rules = atrule.oblock.value.parseListOfRules()
diff --git a/src/css/values.nim b/src/css/values.nim
index 9c1a93c9..3fd8e38b 100644
--- a/src/css/values.nim
+++ b/src/css/values.nim
@@ -1,4 +1,3 @@
-import unicode
 import tables
 import sugar
 import sequtils
@@ -221,13 +220,13 @@ func valueType(prop: CSSPropertyType): CSSValueType =
 macro `{}`*(vals: CSSComputedValues, s: string): untyped =
   let t = propertyType($s)
   let vs = $valueType(t)
-  let s = vs.split(Rune('_'))[1..^1].join("_").tolower()
+  let s = vs.split('_')[1..^1].join("_").tolower()
   result = newDotExpr(newTree(nnkBracketExpr, vals, newLit(t)), newIdentNode(s))
 
 macro `{}=`*(vals: CSSComputedValues, s: string, v: typed): untyped =
   let t = propertyType($s)
   let vs = $valueType(t)
-  let s = vs.split(Rune('_'))[1..^1].join("_").tolower()
+  let s = vs.split('_')[1..^1].join("_").tolower()
   let expr = newDotExpr(newTree(nnkBracketExpr, vals, newLit(t)), newIdentNode(s))
   result = quote do:
     `expr` = `v`
@@ -493,8 +492,8 @@ func cssColor(d: CSSDeclaration): CSSColor =
           raise newException(CSSValueError, "Invalid color")
       of CSS_IDENT_TOKEN:
         let s = tok.value
-        if $s in Colors:
-          return Colors[$s]
+        if s in Colors:
+          return Colors[s]
         else:
           raise newException(CSSValueError, "Invalid color")
       else:
@@ -502,7 +501,7 @@ func cssColor(d: CSSDeclaration): CSSColor =
     elif d.value[0] of CSSFunction:
       let f = CSSFunction(d.value[0])
       #TODO calc etc (cssnumber function or something)
-      case $f.name
+      case f.name
       of "rgb":
         if f.value.len != 3:
           raise newException(CSSValueError, "Invalid color")
@@ -540,9 +539,9 @@ func cssLength(d: CSSDeclaration): CSSLength =
     of CSS_PERCENTAGE_TOKEN:
       return cssLength(tok.nvalue, "%")
     of CSS_DIMENSION_TOKEN:
-      return cssLength(tok.nvalue, $tok.unit)
+      return cssLength(tok.nvalue, tok.unit)
     of CSS_IDENT_TOKEN:
-      if $tok.value == "auto":
+      if tok.value == "auto":
         return CSSLength(auto: true)
     else: discard
   raise newException(CSSValueError, "Invalid length")
@@ -552,9 +551,9 @@ func cssWordSpacing(d: CSSDeclaration): CSSLength =
     let tok = CSSToken(d.value[0])
     case tok.tokenType
     of CSS_DIMENSION_TOKEN:
-      return cssLength(tok.nvalue, $tok.unit)
+      return cssLength(tok.nvalue, tok.unit)
     of CSS_IDENT_TOKEN:
-      if $tok.value == "normal":
+      if tok.value == "normal":
         return CSSLength(auto: true)
     else: discard
   raise newException(CSSValueError, "Invalid word spacing")
@@ -565,7 +564,7 @@ func cssGlobal*(d: CSSDeclaration): CSSGlobalValueType =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "inherit": return VALUE_INHERIT
       of "initial": return VALUE_INITIAL
       of "unset": return VALUE_UNSET
@@ -577,14 +576,14 @@ func cssString(d: CSSDeclaration): string =
     let tok = getToken(d)
     case tok.tokenType
     of CSS_IDENT_TOKEN, CSS_STRING_TOKEN:
-      return $tok.value
+      return tok.value
     else: return
 
 func cssDisplay(d: CSSDeclaration): CSSDisplay =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "block": return DISPLAY_BLOCK
       of "inline": return DISPLAY_INLINE
       of "list-item": return DISPLAY_LIST_ITEM
@@ -605,7 +604,7 @@ func cssFontStyle(d: CSSDeclaration): CSSFontStyle =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "normal": return FONTSTYLE_NORMAL
       of "italic": return FONTSTYLE_ITALIC
       of "oblique": return FONTSTYLE_OBLIQUE
@@ -616,7 +615,7 @@ func cssWhiteSpace(d: CSSDeclaration): CSSWhitespace =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "normal": return WHITESPACE_NORMAL
       of "nowrap": return WHITESPACE_NOWRAP
       of "pre": return WHITESPACE_PRE
@@ -630,7 +629,7 @@ func cssFontWeight(d: CSSDeclaration): int =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "normal": return 400
       of "bold": return 700
       of "lighter": return 400
@@ -645,7 +644,7 @@ func cssTextDecoration(d: CSSDeclaration): CSSTextDecoration =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "none": return TEXT_DECORATION_NONE
       of "underline": return TEXT_DECORATION_UNDERLINE
       of "overline": return TEXT_DECORATION_OVERLINE
@@ -657,7 +656,7 @@ func cssWordBreak(d: CSSDeclaration): CSSWordBreak =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "normal": return WORD_BREAK_NORMAL
       of "break-all": return WORD_BREAK_BREAK_ALL
       of "keep-all": return WORD_BREAK_KEEP_ALL
@@ -667,7 +666,7 @@ func cssListStyleType(d: CSSDeclaration): CSSListStyleType =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "none": return LIST_STYLE_TYPE_NONE
       of "disc": return LIST_STYLE_TYPE_DISC
       of "circle": return LIST_STYLE_TYPE_CIRCLE
@@ -682,7 +681,7 @@ func cssVerticalAlign(d: CSSDeclaration): CSSVerticalAlign =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      case $tok.value
+      case tok.value
       of "baseline": result.keyword = VERTICAL_ALIGN_BASELINE
       of "sub": result.keyword = VERTICAL_ALIGN_SUB
       of "super": result.keyword = VERTICAL_ALIGN_SUPER
@@ -706,7 +705,7 @@ func cssLineHeight(d: CSSDeclaration): CSSLength =
     of CSS_NUMBER_TOKEN:
       return cssLength(tok.nvalue * 100, "%")
     of CSS_IDENT_TOKEN:
-      if $tok.value == "normal":
+      if tok.value == "normal":
         return CSSLength(auto: true)
     else:
       return cssLength(d)
@@ -716,7 +715,7 @@ func cssTextAlign(d: CSSDeclaration): CSSTextAlign =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      return case $tok.value
+      return case tok.value
       of "start": TEXT_ALIGN_START
       of "end": TEXT_ALIGN_END
       of "left": TEXT_ALIGN_LEFT
@@ -731,7 +730,7 @@ func cssListStylePosition(d: CSSDeclaration): CSSListStylePosition =
   if isToken(d):
     let tok = getToken(d)
     if tok.tokenType == CSS_IDENT_TOKEN:
-      return case $tok.value
+      return case tok.value
       of "outside": LIST_STYLE_POSITION_OUTSIDE
       of "inside": LIST_STYLE_POSITION_INSIDE
       else: raise newException(CSSValueError, "Invalid list style position")
@@ -806,7 +805,7 @@ func getDefault(t: CSSPropertyType): CSSComputedValue = {.cast(noSideEffect).}:
   return defaultTable[t]
 
 func getComputedValue(d: CSSDeclaration, parent: CSSComputedValues): tuple[a:CSSComputedValue,b:CSSGlobalValueType] =
-  let name = $d.name
+  let name = d.name
   let ptype = propertyType(name)
   let vtype = valueType(ptype)
 
diff --git a/src/utils/twtstr.nim b/src/utils/twtstr.nim
index f7d16618..63d9a730 100644
--- a/src/utils/twtstr.nim
+++ b/src/utils/twtstr.nim
@@ -33,7 +33,7 @@ const AsciiUpperAlpha* = {'A'..'Z'}
 const AsciiLowerAlpha* = {'a'..'z'}
 const AsciiAlpha* = (AsciiUpperAlpha + AsciiLowerAlpha)
 const AllChars = {chr(0x00)..chr(0xFF)}
-const NonAscii = (AllChars - Ascii)
+const NonAscii* = (AllChars - Ascii)
 const AsciiDigit* = {'0'..'9'}
 const AsciiHexDigit* = (AsciiDigit + {'a'..'f', 'A'..'F'})
 const AsciiWhitespace* = {' ', '\n', '\r', '\t', '\f'}