css: change type of rvalue to char (from Rune)

We did were not using it as a unicode value anywhere, it was just
making our life harder.

If we ever need unicode delimiter tokens, we can re-introduce them
as a different token type.

4 files changed, 62 insertions, 52 deletions

diff --git a/src/css/cssparser.nim b/src/css/cssparser.nim
index 2b286782..ef167fb2 100644
--- a/src/css/cssparser.nim
+++ b/src/css/cssparser.nim
@@ -41,7 +41,7 @@ type
       value*: string
       tflaga*: tflaga
     of CSS_DELIM_TOKEN:
-      rvalue*: Rune
+      cvalue*: char
     of CSS_NUMBER_TOKEN, CSS_PERCENTAGE_TOKEN, CSS_DIMENSION_TOKEN:
       nvalue*: float64
       tflagb*: tflagb
@@ -91,7 +91,10 @@ proc `$`*(c: CSSParsedItem): string =
     of CSS_STRING_TOKEN:
       result &= ("\"" & c.value & "\"")
     of CSS_DELIM_TOKEN:
-      result &= c.rvalue
+      if c.cvalue != char(128):
+        result &= c.cvalue
+      else:
+        result &= "<UNICODE>"
     of CSS_DIMENSION_TOKEN:
       case c.tflagb
       of TFLAGB_NUMBER:
@@ -145,7 +148,7 @@ proc `$`*(c: CSSParsedItem): string =
 func `==`*(a: CSSParsedItem, b: CSSTokenType): bool =
   return a of CSSToken and CSSToken(a).tokenType == b
 
-const IdentStart = AsciiAlpha + NonAscii + {'_'} 
+const IdentStart = AsciiAlpha + NonAscii + {'_'}
 const Ident = IdentStart + AsciiDigit + {'-'}
 
 proc consume(state: var CSSTokenizerState): char =
@@ -153,8 +156,12 @@ proc consume(state: var CSSTokenizerState): char =
   inc state.at
   return state.curr
 
-proc consumeRune(state: var CSSTokenizerState): Rune =
-  fastRuneAt(state.buf, state.at, result)
+proc consumeRChar(state: var CSSTokenizerState): char =
+  var r: Rune
+  fastRuneAt(state.buf, state.at, r)
+  if r.isAscii():
+    return cast[char](r)
+  return char(128)
 
 proc reconsume(state: var CSSTokenizerState) =
   dec state.at
@@ -416,7 +423,7 @@ proc consumeToken(state: var CSSTokenizerState): CSSToken =
       result.value = consumeIdentSequence(state)
     else:
       state.reconsume()
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: state.consumeRune())
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: state.consumeRChar())
   of '(': return CSSToken(tokenType: CSS_LPAREN_TOKEN)
   of ')': return CSSToken(tokenType: CSS_RPAREN_TOKEN)
   of '{': return CSSToken(tokenType: CSS_LBRACE_TOKEN)
@@ -426,7 +433,7 @@ proc consumeToken(state: var CSSTokenizerState): CSSToken =
       state.reconsume()
       return state.consumeNumericToken()
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: c)
   of ',': return CSSToken(tokenType: CSS_COMMA_TOKEN)
   of '-':
     if state.startsWithNumber():
@@ -441,13 +448,13 @@ proc consumeToken(state: var CSSTokenizerState): CSSToken =
         state.reconsume()
         return state.consumeIdentLikeToken()
       else:
-        return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+        return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: c)
   of '.':
     if state.startsWithNumber():
       state.reconsume()
       return state.consumeNumericToken()
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: c)
   of ':': return CSSToken(tokenType: CSS_COLON_TOKEN)
   of ';': return CSSToken(tokenType: CSS_SEMICOLON_TOKEN)
   of '<':
@@ -457,20 +464,20 @@ proc consumeToken(state: var CSSTokenizerState): CSSToken =
       discard state.consume()
       return CSSToken(tokenType: CSS_CDO_TOKEN)
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: c)
   of '@':
     if state.next3startsWithIdentSequence():
       let name = state.consumeIdentSequence()
       return CSSToken(tokenType: CSS_AT_KEYWORD_TOKEN, value: name)
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: c)
   of '[': return CSSToken(tokenType: CSS_LBRACKET_TOKEN)
   of '\\':
     if state.isValidEscape():
       state.reconsume()
       return state.consumeIdentLikeToken()
     else:
-      return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: Rune(c))
+      return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: c)
   of ']': return CSSToken(tokenType: CSS_RBRACKET_TOKEN)
   of AsciiDigit:
     state.reconsume()
@@ -480,7 +487,7 @@ proc consumeToken(state: var CSSTokenizerState): CSSToken =
     return state.consumeIdentLikeToken()
   else:
     state.reconsume()
-    return CSSToken(tokenType: CSS_DELIM_TOKEN, rvalue: state.consumeRune())
+    return CSSToken(tokenType: CSS_DELIM_TOKEN, cvalue: state.consumeRChar())
 
 proc tokenizeCSS*(inputStream: Stream): seq[CSSParsedItem] =
   var state: CSSTokenizerState
@@ -520,7 +527,7 @@ proc consumeSimpleBlock(state: var CSSParseState): CSSSimpleBlock =
   of CSS_LPAREN_TOKEN: ending = CSS_RPAREN_TOKEN
   of CSS_LBRACKET_TOKEN: ending = CSS_RBRACKET_TOKEN
   else: doAssert false
-  
+
   result = CSSSimpleBlock(token: t)
   while state.at < state.tokens.len:
     let t = state.consume()
@@ -613,7 +620,7 @@ proc consumeDeclaration(state: var CSSParseState): Option[CSSDeclaration] =
           inc k
           l = i
       elif k == 1 and decl.value[i] == CSS_DELIM_TOKEN:
-        if CSSToken(decl.value[i]).rvalue == Rune('!'):
+        if CSSToken(decl.value[i]).cvalue == '!':
           decl.important = true
           decl.value.delete(l)
           decl.value.delete(i)
@@ -841,7 +848,8 @@ proc parseAnB*(state: var CSSParseState): Option[CSSAnB] =
     while state.has() and state.peek() == CSS_WHITESPACE_TOKEN:
       discard state.consume()
   template get_plus: bool =
-    if state.peek() == CSS_DELIM_TOKEN and CSSToken(state.peek()).rvalue == Rune('+'):
+    let tok = state.peek()
+    if tok == CSS_DELIM_TOKEN and CSSToken(tok).cvalue == '+':
       discard state.consume()
       true
     else:
@@ -894,9 +902,9 @@ proc parseAnB*(state: var CSSParseState): Option[CSSAnB] =
         return some((1, 0))
       let tok2 = get_tok
       if tok2.tokenType == CSS_DELIM_TOKEN:
-        let sign = case tok2.rvalue
-        of Rune('+'): 1
-        of Rune('-'): -1
+        let sign = case tok2.cvalue
+        of '+': 1
+        of '-': -1
         else: return none(CSSAnB)
         let tok3 = get_tok
         fail_non_signless_integer tok3, some((1, 0))
@@ -910,9 +918,9 @@ proc parseAnB*(state: var CSSParseState): Option[CSSAnB] =
         return some((-1, 0))
       let tok2 = get_tok
       if tok2.tokenType == CSS_DELIM_TOKEN:
-        let sign = case tok2.rvalue
-        of Rune('+'): 1
-        of Rune('-'): -1
+        let sign = case tok2.cvalue
+        of '+': 1
+        of '-': -1
         else: return none(CSSAnB)
         let tok3 = get_tok
         fail_non_signless_integer tok3, some((-1, 0))
@@ -957,9 +965,9 @@ proc parseAnB*(state: var CSSParseState): Option[CSSAnB] =
         return some((int(tok.nvalue), 0))
       let tok2 = get_tok
       if tok2.tokenType == CSS_DELIM_TOKEN:
-        let sign = case tok2.rvalue
-        of Rune('+'): 1
-        of Rune('-'): -1
+        let sign = case tok2.cvalue
+        of '+': 1
+        of '-': -1
         else: return none(CSSAnB)
         let tok3 = get_tok
         fail_non_signless_integer tok3, some((int(tok.nvalue), 0))
diff --git a/src/css/mediaquery.nim b/src/css/mediaquery.nim
index 8e2b53e6..f92ecfb2 100644
--- a/src/css/mediaquery.nim
+++ b/src/css/mediaquery.nim
@@ -1,6 +1,5 @@
 import strutils
 import tables
-import unicode
 
 import css/cssparser
 import css/values
@@ -190,15 +189,14 @@ template expect_bool(b: bool, sfalse: string, strue: string) =
 template expect_comparison(comparison: var MediaQueryComparison) =
   let tok = consume_token()
   if tok != CSS_DELIM_TOKEN: return nil
-  if not tok.rvalue.isAscii(): return nil
-  let c = char(tok.rvalue)
+  let c = tok.cvalue
   if c notin {'=', '<', '>'}: return nil
   block parse:
-    case char(tok.rvalue)
+    case c
     of '<':
       if parser.has():
         let tok = skip_consume()
-        if tok == CSS_DELIM_TOKEN and tok.rvalue == Rune('='):
+        if tok == CSS_DELIM_TOKEN and tok.cvalue == '=':
           comparison = COMPARISON_LE
           break parse
         parser.reconsume()
@@ -206,7 +204,7 @@ template expect_comparison(comparison: var MediaQueryComparison) =
     of '>':
       if parser.has():
         let tok = skip_consume()
-        if tok == CSS_DELIM_TOKEN and tok.rvalue == Rune('='):
+        if tok == CSS_DELIM_TOKEN and tok.cvalue == '=':
           comparison = COMPARISON_GE
           break parse
         parser.reconsume()
@@ -440,7 +438,7 @@ proc parseMediaQuery(parser: var MediaQueryParser): MediaQuery =
     let cval = parser.consume()
     if cval of CSSToken:
       let tok = CSSToken(cval)
-      if tok.tokenType == CSS_IDENT_TOKEN: 
+      if tok.tokenType == CSS_IDENT_TOKEN:
         let tokval = tok.value
         case tokval
         of "not":
@@ -465,7 +463,7 @@ proc parseMediaQuery(parser: var MediaQueryParser): MediaQuery =
     let cval = parser.consume()
     if cval of CSSToken:
       let tok = CSSToken(cval)
-      if tok.tokenType == CSS_IDENT_TOKEN: 
+      if tok.tokenType == CSS_IDENT_TOKEN:
         let tokval = tok.value
         if result == nil:
           if tokval in MediaTypes:
@@ -496,7 +494,7 @@ proc parseMediaQuery(parser: var MediaQueryParser): MediaQuery =
     let cval = parser.consume()
     if cval of CSSToken:
       let tok = CSSToken(cval)
-      if tok.tokenType == CSS_IDENT_TOKEN: 
+      if tok.tokenType == CSS_IDENT_TOKEN:
         let tokval = tok.value
         if tokval != "and":
           return nil
diff --git a/src/css/selectorparser.nim b/src/css/selectorparser.nim
index 62c1033a..6720754a 100644
--- a/src/css/selectorparser.nim
+++ b/src/css/selectorparser.nim
@@ -1,7 +1,6 @@
 import options
 import streams
 import strutils
-import unicode
 
 import css/cssparser
 import utils/twtstr
@@ -334,18 +333,23 @@ proc parseAttributeSelector(state: var SelectorParser, cssblock: CSSSimpleBlock)
   if not state2.has(): return Selector(t: ATTR_SELECTOR, attr: attr.value, rel: ' ')
   let delim0 = get_tok state2.consume()
   if delim0.tokenType != CSS_DELIM_TOKEN: fail
-  case delim0.rvalue
-  of Rune('~'), Rune('|'), Rune('^'), Rune('$'), Rune('*'):
+  case delim0.cvalue
+  of '~', '|', '^', '$', '*':
     let delim1 = get_tok state2.consume()
     if delim1.tokenType != CSS_DELIM_TOKEN: fail
-  of Rune('='):
+  of '=':
     discard
   else: fail
   state2.skipWhitespace()
   if not state2.has(): fail
   let value = get_tok state2.consume()
   if value.tokenType notin {CSS_IDENT_TOKEN, CSS_STRING_TOKEN}: fail
-  return Selector(t: ATTR_SELECTOR, attr: attr.value, value: value.value, rel: cast[char](delim0.rvalue))
+  return Selector(
+    t: ATTR_SELECTOR,
+    attr: attr.value,
+    value: value.value,
+    rel: delim0.cvalue
+  )
 
 proc parseClassSelector(state: var SelectorParser): Selector =
   if not state.has(): fail
@@ -375,14 +379,14 @@ proc parseCompoundSelector(state: var SelectorParser): CompoundSelector =
         result.add(Selector(t: ID_SELECTOR, id: tok.value))
       of CSS_COMMA_TOKEN: break
       of CSS_DELIM_TOKEN:
-        case tok.rvalue
-        of Rune('.'):
+        case tok.cvalue
+        of '.':
           inc state.at
           result.add(state.parseClassSelector())
-        of Rune('*'):
+        of '*':
           inc state.at
           result.add(Selector(t: UNIVERSAL_SELECTOR))
-        of Rune('>'), Rune('+'), Rune('~'): break
+        of '>', '+', '~': break
         else: fail
       of CSS_WHITESPACE_TOKEN:
         # skip trailing whitespace
@@ -390,7 +394,7 @@ proc parseCompoundSelector(state: var SelectorParser): CompoundSelector =
           inc state.at
         elif state.peek(1) == CSS_DELIM_TOKEN:
           let tok = CSSToken(state.peek(1))
-          if tok.rvalue == Rune('>') or tok.rvalue == Rune('+') or tok.rvalue == Rune('~'):
+          if tok.cvalue in {'>', '+', '~'}:
             inc state.at
         break
       else: fail
@@ -411,10 +415,10 @@ proc parseComplexSelector(state: var SelectorParser): ComplexSelector =
     let tok = get_tok state.consume()
     case tok.tokenType
     of CSS_DELIM_TOKEN:
-      case tok.rvalue
-      of Rune('>'): result[^1].ct = CHILD_COMBINATOR
-      of Rune('+'): result[^1].ct = NEXT_SIBLING_COMBINATOR
-      of Rune('~'): result[^1].ct = SUBSEQ_SIBLING_COMBINATOR
+      case tok.cvalue
+      of '>': result[^1].ct = CHILD_COMBINATOR
+      of '+': result[^1].ct = NEXT_SIBLING_COMBINATOR
+      of '~': result[^1].ct = SUBSEQ_SIBLING_COMBINATOR
       else: fail
     of CSS_WHITESPACE_TOKEN:
       result[^1].ct = DESCENDANT_COMBINATOR
diff --git a/src/css/values.nim b/src/css/values.nim
index 1fe63ed4..8d377134 100644
--- a/src/css/values.nim
+++ b/src/css/values.nim
@@ -2,7 +2,6 @@ import tables
 import macros
 import options
 import strutils
-import unicode
 
 import css/cssparser
 import css/selectorparser
@@ -149,7 +148,7 @@ type
   CSSCounterReset* = object
     name*: string
     num*: int
-  
+
   CSSComputedValue* = ref object
     t*: CSSPropertyType
     case v*: CSSValueType
@@ -540,7 +539,8 @@ func cssColor*(val: CSSComponentValue): Result[RGBAColor, string] =
         elif commaMode:
           return err("Invalid color")
         elif slash:
-          if f.value[i] != CSS_DELIM_TOKEN or CSSToken(f.value[i]).rvalue != Rune('/'):
+          let tok = f.value[i]
+          if tok != CSS_DELIM_TOKEN or CSSToken(tok).cvalue != '/':
             return err("Invalid color")
           inc i
           f.value.skipWhitespace(i)
@@ -990,7 +990,7 @@ proc getValueFromDecl(val: CSSComputedValue, d: CSSDeclaration,
     vtype: CSSValueType, ptype: CSSPropertyType): Err[string] =
   var i = 0
   d.value.skipWhitespace(i)
-  if i >= d.value.len: 
+  if i >= d.value.len:
     return err("Empty value")
   let cval = d.value[i]
   inc i