about summary refs log tree commit diff stats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buffer/buffer.nim23
-rw-r--r--src/config/config.nim453
-rw-r--r--src/config/toml.nim46
-rw-r--r--src/display/client.nim25
-rw-r--r--src/display/pager.nim49
-rw-r--r--src/display/term.nim24
-rw-r--r--src/encoding/decoderstream.nim34
-rw-r--r--src/html/htmlparser.nim100
-rw-r--r--src/ips/editor.nim4
-rw-r--r--src/ips/forkserver.nim8
-rw-r--r--src/ips/serialize.nim2
-rw-r--r--src/main.nim37
-rw-r--r--src/render/rendertext.nim33
-rw-r--r--src/types/buffersource.nim2
-rw-r--r--src/utils/opt.nim34
-rw-r--r--src/utils/twtstr.nim6
16 files changed, 549 insertions, 331 deletions
diff --git a/src/buffer/buffer.nim b/src/buffer/buffer.nim
index 4a091091..b0dc0696 100644
--- a/src/buffer/buffer.nim
+++ b/src/buffer/buffer.nim
@@ -17,8 +17,8 @@ import css/cssparser
 import css/mediaquery
 import css/sheet
 import css/stylednode
-import data/charset
 import config/config
+import data/charset
 import html/dom
 import html/env
 import html/htmlparser
@@ -72,7 +72,6 @@ type
   Buffer* = ref object
     fd: int
     alive: bool
-    cs: Charset
     readbufsize: int
     contenttype: string
     lines: FlexibleGrid
@@ -239,6 +238,11 @@ macro task(fun: typed) =
   pfun.istask = true
   fun
 
+func charsets(buffer: Buffer): seq[Charset] =
+  if buffer.source.charset.isSome:
+    return @[buffer.source.charset.get]
+  return buffer.config.charsets
+
 func getTitleAttr(node: StyledNode): string =
   if node == nil:
     return ""
@@ -646,12 +650,9 @@ proc finishLoad(buffer: Buffer): EmptyPromise =
     buffer.available = 0
     if buffer.window == nil:
       buffer.window = newWindow(buffer.config.scripting)
-    let (doc, cs) = parseHTML(buffer.sstream, fallbackcs = buffer.cs, window = buffer.window, url = buffer.url)
+    let doc = parseHTML(buffer.sstream, charsets = buffer.charsets,
+      window = buffer.window, url = buffer.url)
     buffer.document = doc
-    if buffer.document == nil: # needsreinterpret
-      buffer.sstream.setPosition(0)
-      let (doc, _) = parseHTML(buffer.sstream, cs = some(cs), window = buffer.window, url = buffer.url)
-      buffer.document = doc
     buffer.state = LOADING_RESOURCES
     p = buffer.loadResources(buffer.document)
   else:
@@ -741,8 +742,9 @@ proc cancel*(buffer: Buffer): int {.proxy.} =
     buffer.available = 0
     if buffer.window == nil:
       buffer.window = newWindow(buffer.config.scripting)
-    let (doc, _) = parseHTML(buffer.sstream, cs = some(buffer.cs), window = buffer.window, url = buffer.url) # confidence: certain
-    buffer.document = doc
+    buffer.document = parseHTML(buffer.sstream,
+      charsets = buffer.charsets, window = buffer.window,
+      url = buffer.url, canReinterpret = false)
     buffer.do_reshape()
   return buffer.lines.len
 
@@ -1220,7 +1222,6 @@ proc launchBuffer*(config: BufferConfig, source: BufferSource,
                    mainproc: Pid) =
   let buffer = Buffer(
     alive: true,
-    cs: CHARSET_UTF_8,
     userstyle: parseStylesheet(config.userstyle),
     attrs: attrs,
     config: config,
@@ -1235,7 +1236,7 @@ proc launchBuffer*(config: BufferConfig, source: BufferSource,
   buffer.selector = newSelector[int]()
   loader.registerFun = proc(fd: int) = buffer.selector.registerHandle(fd, {Read}, 0)
   loader.unregisterFun = proc(fd: int) = buffer.selector.unregister(fd)
-  buffer.srenderer = newStreamRenderer(buffer.sstream)
+  buffer.srenderer = newStreamRenderer(buffer.sstream, buffer.charsets)
   if buffer.config.scripting:
     buffer.window = newWindow(buffer.config.scripting, some(buffer.loader))
   let socks = connectSocketStream(mainproc, false)
diff --git a/src/config/config.nim b/src/config/config.nim
index d3cad0b1..26c6d5ac 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -5,6 +5,7 @@ import streams
 
 import buffer/cell
 import config/toml
+import data/charset
 import io/request
 import io/urlfilter
 import js/javascript
@@ -13,6 +14,7 @@ import types/color
 import types/cookie
 import types/referer
 import types/url
+import utils/opt
 import utils/twtstr
 
 type
@@ -32,6 +34,7 @@ type
     sharecookiejar: Option[string]
     refererfrom*: Option[bool]
     scripting: Option[bool]
+    document_charset: seq[Charset]
 
   StaticOmniRule = object
     match: string
@@ -46,36 +49,60 @@ type
     sharecookiejar*: Option[string]
     refererfrom*: Option[bool]
     scripting*: Option[bool]
+    document_charset*: seq[Charset]
 
   OmniRule* = object
     match*: Regex
     subst*: (proc(s: string): Option[string])
 
-  Config* = ref ConfigObj
-  ConfigObj* = object
-    searchwrap* {.jsget, jsset.}: bool
-    maxredirect*: int
-    prependhttps* {.jsget, jsset.}: bool
-    termreload*: bool
-    nmap*: ActionMap
-    lemap*: ActionMap
-    stylesheet* {.jsget, jsset.}: string
-    startup*: string
-    ambiguous_double*: bool
-    hlcolor*: RGBAColor
+  StartConfig = object
+    visual_home*: string
+    startup_script*: string
     headless*: bool
-    colormode*: Option[ColorMode]
-    formatmode*: Option[FormatMode]
-    noformatmode*: FormatMode
-    altscreen*: Option[bool]
-    mincontrast*: int
-    editor*: string
+
+  CSSConfig = object
+    stylesheet*: string
+
+  SearchConfig = object
+    wrap*: bool
+
+  EncodingConfig = object
+    document_charset*: seq[Charset]
+
+  ExternalConfig = object
     tmpdir*: string
+    editor*: string
+
+  NetworkConfig = object
+    max_redirect*: int32
+    prepend_https*: bool
+
+  DisplayConfig = object
+    color_mode*: Option[ColorMode]
+    format_mode*: Option[FormatMode]
+    no_format_mode*: FormatMode
+    emulate_overline*: bool
+    alt_screen*: Option[bool]
+    highlight_color*: RGBAColor
+    double_width_ambiguous*: bool
+    minimum_contrast*: int32
+    force_clear*: bool
+
+  #TODO: add JS wrappers for objects
+  Config* = ref ConfigObj
+  ConfigObj* = object
+    includes: seq[string]
+    start*: StartConfig
+    search*: SearchConfig
+    css*: CSSConfig
+    encoding*: EncodingConfig
+    external*: ExternalConfig
+    network*: NetworkConfig
+    display*: DisplayConfig
     siteconf: seq[StaticSiteConfig]
     omnirules: seq[StaticOmniRule]
-    forceclear*: bool
-    emulateoverline*: bool
-    visualhome*: string
+    page*: ActionMap
+    line*: ActionMap
 
   BufferConfig* = object
     userstyle*: string
@@ -85,6 +112,7 @@ type
     refererfrom*: bool
     referrerpolicy*: ReferrerPolicy
     scripting*: bool
+    charsets*: seq[Charset]
 
   ForkServerConfig* = object
     tmpdir*: string
@@ -100,19 +128,21 @@ const DefaultHeaders* = {
 
 func getForkServerConfig*(config: Config): ForkServerConfig =
   return ForkServerConfig(
-    tmpdir: config.tmpdir,
-    ambiguous_double: config.ambiguous_double
+    tmpdir: config.external.tmpdir,
+    ambiguous_double: config.display.double_width_ambiguous
   )
 
 proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar = nil,
-                      headers: HeaderList = nil, refererfrom = false, scripting = false): BufferConfig =
+      headers: HeaderList = nil, refererfrom = false, scripting = false,
+      charsets = config.encoding.document_charset): BufferConfig =
   result = BufferConfig(
-    userstyle: config.stylesheet,
+    userstyle: config.css.stylesheet,
     filter: newURLFilter(scheme = some(location.scheme), default = true),
     cookiejar: cookiejar,
     headers: headers,
     refererfrom: refererfrom,
-    scripting: scripting
+    scripting: scripting,
+    charsets: charsets
   )
   new(result.headers)
   result.headers[] = DefaultHeaders
@@ -123,7 +153,8 @@ proc getSiteConfig*(config: Config, jsctx: JSContext): seq[SiteConfig] =
       cookie: sc.cookie,
       scripting: sc.scripting,
       sharecookiejar: sc.sharecookiejar,
-      refererfrom: sc.refererfrom
+      refererfrom: sc.refererfrom,
+      charsets: sc.document_charset
     )
     if sc.url.isSome:
       conf.url = compileRegex(sc.url.get, 0)
@@ -220,8 +251,8 @@ proc readUserStylesheet(dir, file: string): string =
       result = f.readAll()
       f.close()
 
-proc parseConfig(config: Config, dir: string, stream: Stream)
-proc parseConfig*(config: Config, dir: string, s: string)
+proc parseConfig(config: Config, dir: string, stream: Stream, name = "<input>")
+proc parseConfig*(config: Config, dir: string, s: string, name = "<input>")
 
 proc loadConfig*(config: Config, s: string) {.jsfunc.} =
   let s = if s.len > 0 and s[0] == '/':
@@ -233,167 +264,231 @@ proc loadConfig*(config: Config, s: string) {.jsfunc.} =
 
 proc bindPagerKey*(config: Config, key, action: string) {.jsfunc.} =
   let k = getRealKey(key)
-  config.nmap[k] = action
+  config.page[k] = action
   var teststr = ""
   for c in k:
     teststr &= c
-    if teststr notin config.nmap:
-      config.nmap[teststr] = "client.feedNext()"
+    if teststr notin config.page:
+      config.page[teststr] = "client.feedNext()"
 
 proc bindLineKey*(config: Config, key, action: string) {.jsfunc.} =
   let k = getRealKey(key)
-  config.lemap[k] = action
+  config.line[k] = action
   var teststr = ""
   for c in k:
     teststr &= c
-    if teststr notin config.nmap:
-      config.lemap[teststr] = "client.feedNext()"
+    if teststr notin config.line:
+      config.line[teststr] = "client.feedNext()"
+
+proc parseConfigValue(x: var object, v: TomlValue, k: string)
+proc parseConfigValue(x: var bool, v: TomlValue, k: string)
+proc parseConfigValue(x: var string, v: TomlValue, k: string)
+proc parseConfigValue(x: var seq[object], v: TomlValue, k: string)
+proc parseConfigValue[T](x: var seq[T], v: TomlValue, k: string)
+proc parseConfigValue(x: var Charset, v: TomlValue, k: string)
+proc parseConfigValue(x: var int32, v: TomlValue, k: string)
+proc parseConfigValue(x: var int64, v: TomlValue, k: string)
+proc parseConfigValue(x: var Option[ColorMode], v: TomlValue, k: string)
+proc parseConfigValue(x: var Option[FormatMode], v: TomlValue, k: string)
+proc parseConfigValue(x: var FormatMode, v: TomlValue, k: string)
+proc parseConfigValue(x: var RGBAColor, v: TomlValue, k: string)
+proc parseConfigValue(x: var Option[bool], v: TomlValue, k: string)
+proc parseConfigValue[T](x: var Option[T], v: TomlValue, k: string)
+proc parseConfigValue(x: var ActionMap, v: TomlValue, k: string)
+proc parseConfigValue(x: var CSSConfig, v: TomlValue, k: string)
+
+proc typeCheck(v: TomlValue, vt: ValueType, k: string) =
+  if v.vt != vt:
+    raise newException(ValueError, "invalid type for key " & k &
+      " (got " & $v.vt & ", expected " & $vt & ")")
+
+proc typeCheck(v: TomlValue, vt: set[ValueType], k: string) =
+  if v.vt notin vt:
+    raise newException(ValueError, "invalid type for key " & k &
+      " (got " & $v.vt & ", expected " & $vt & ")")
+
+proc parseConfigValue(x: var object, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_TABLE, k)
+  for fk, fv in x.fieldPairs:
+    let kebabk = snakeToKebabCase(fk)
+    if kebabk in v:
+      let kkk = if k != "":
+        k & "." & fk
+      else:
+        fk
+      parseConfigValue(fv, v[kebabk], kkk)
+
+proc parseConfigValue(x: var bool, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_BOOLEAN, k)
+  x = v.b
+
+proc parseConfigValue(x: var string, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_STRING, k)
+  x = v.s
+
+proc parseConfigValue(x: var seq[object], v: TomlValue, k: string) =
+  typeCheck(v, {VALUE_TABLE_ARRAY, VALUE_ARRAY}, k)
+  if v.vt == VALUE_ARRAY:
+    #TODO if array and size != 0
+    # actually, arrays and table arrays should be the same data type
+    assert v.a.len == 0
+    x.setLen(0)
+  else:
+    for i in 0 ..< v.ta.len:
+      var y: typeof(x[0])
+      let tab = TomlValue(vt: VALUE_TABLE, t: v.ta[i])
+      parseConfigValue(y, tab, k & "[" & $i & "]")
+      x.add(y)
+
+proc parseConfigValue[T](x: var seq[T], v: TomlValue, k: string) =
+  typeCheck(v, {VALUE_STRING, VALUE_ARRAY}, k)
+  if v.vt != VALUE_ARRAY:
+    var y: T
+    parseConfigValue(y, v, k)
+    x.add(y)
+  else:
+    for i in 0 ..< v.a.len:
+      var y: T
+      parseConfigValue(y, v.a[i], k & "[" & $i & "]")
+      x.add(y)
+
+proc parseConfigValue(x: var Charset, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_STRING, k)
+  x = getCharset(v.s)
+  if x == CHARSET_UNKNOWN:
+    raise newException(ValueError, "unknown charset '" & v.s & "' for key " &
+      k)
+
+proc parseConfigValue(x: var int32, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_INTEGER, k)
+  x = int32(v.i)
+
+proc parseConfigValue(x: var int64, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_INTEGER, k)
+  x = v.i
+
+proc parseConfigValue(x: var Option[ColorMode], v: TomlValue, k: string) =
+  typeCheck(v, VALUE_STRING, k)
+  case v.s
+  of "auto": x = none(ColorMode)
+  of "monochrome": x = some(MONOCHROME)
+  of "ansi": x = some(ANSI)
+  of "8bit": x = some(EIGHT_BIT)
+  of "24bit": x = some(TRUE_COLOR)
+  else:
+    raise newException(ValueError, "unknown color mode '" & v.s &
+      "' for key " & k)
 
-proc parseConfig(config: Config, dir: string, t: TomlValue) =
-  for k, v in t:
-    case k
+proc parseConfigValue(x: var Option[FormatMode], v: TomlValue, k: string) =
+  typeCheck(v, {VALUE_STRING, VALUE_ARRAY}, k)
+  if v.vt == VALUE_STRING and v.s == "auto":
+    x = none(FormatMode)
+  else:
+    var y: FormatMode
+    parseConfigValue(y, v, k)
+    x = some(y)
+
+proc parseConfigValue(x: var FormatMode, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_ARRAY, k)
+  for i in 0 ..< v.a.len:
+    let s = v.a[i].s
+    let kk = k & "[" & $i & "]"
+    case s
+    of "bold": x.incl(FLAG_BOLD)
+    of "italic": x.incl(FLAG_ITALIC)
+    of "underline": x.incl(FLAG_UNDERLINE)
+    of "reverse": x.incl(FLAG_REVERSE)
+    of "strike": x.incl(FLAG_STRIKE)
+    of "overline": x.incl(FLAG_OVERLINE)
+    of "blink": x.incl(FLAG_BLINK)
+    else:
+      raise newException(ValueError, "unknown format mode '" & s &
+        "' for key " & kk)
+
+proc parseConfigValue(x: var RGBAColor, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_STRING, k)
+  let c = parseRGBAColor(v.s)
+  if c.isNone:
+      raise newException(ValueError, "invalid color '" & v.s &
+        "' for key " & k)
+  x = c.get
+
+proc parseConfigValue(x: var Option[bool], v: TomlValue, k: string) =
+  typeCheck(v, {VALUE_STRING, VALUE_BOOLEAN}, k)
+  if v.vt == VALUE_STRING:
+    if v.s == "auto":
+      x = none(bool)
+    else:
+      raise newException(ValueError, "invalid value '" & v.s &
+        "' for key " & k)
+  else:
+    var y: bool
+    parseConfigValue(y, v, k)
+    x = some(y)
+
+proc parseConfigValue[T](x: var Option[T], v: TomlValue, k: string) =
+  var y: T
+  parseConfigValue(y, v, k)
+  x = some(y)
+
+proc parseConfigValue(x: var ActionMap, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_TABLE, k)
+  for kk, vv in v:
+    typeCheck(vv, VALUE_STRING, k & "[" & kk & "]")
+    x[getRealKey(kk)] = vv.s
+
+var gdir {.compileTime.}: string
+proc parseConfigValue(x: var CSSConfig, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_TABLE, k)
+  let dir = gdir
+  for kk, vv in v:
+    let kkk = if k != "":
+      k & "." & kk
+    else:
+      kk
+    case kk
     of "include":
-      if v.vt == VALUE_STRING:
-        when nimvm:
-          config.parseConfig(dir, staticRead(dir / v.s))
-        else:
-          config.parseConfig(dir, newFileStream(dir / v.s))
-      elif t.vt == VALUE_ARRAY:
-        for v in t.a:
-          when nimvm:
-            config.parseConfig(dir, staticRead(dir / v.s))
-          else:
-            config.parseConfig(dir, newFileStream(dir / v.s))
-    of "search":
-      for k, v in v:
-        case k
-        of "wrap":
-          config.searchwrap = v.b
-    of "start":
-      for k, v in v:
-        case k
-        of "visual-home":
-          config.visualhome = v.s
-        of "run-script":
-          config.startup = v.s
-        of "headless":
-          config.headless = v.b
-    of "network":
-      for k, v in v:
-        case k
-        of "max-redirect":
-          config.maxredirect = int(v.i)
-        of "prepend-https":
-          config.prependhttps = v.b
-    of "page":
-      for k, v in v:
-        config.nmap[getRealKey(k)] = v.s
-    of "line":
-      for k, v in v:
-        config.lemap[getRealKey(k)] = v.s
-    of "css":
-      for k, v in v:
-        case k
-        of "include":
-          case v.vt
-          of VALUE_STRING:
-            config.stylesheet &= readUserStylesheet(dir, v.s)
-          of VALUE_ARRAY:
-            for child in v.a:
-              config.stylesheet &= readUserStylesheet(dir, v.s)
-          else: discard
-        of "inline":
-          config.stylesheet &= v.s
-    of "display":
-      template get_format_mode(v: TomlValue): FormatMode =
-        var mode: FormatMode
-        for vv in v.a:
-          case vv.s
-          of "bold": mode.incl(FLAG_BOLD)
-          of "italic": mode.incl(FLAG_ITALIC)
-          of "underline": mode.incl(FLAG_UNDERLINE)
-          of "reverse": mode.incl(FLAG_REVERSE)
-          of "strike": mode.incl(FLAG_STRIKE)
-          of "overline": mode.incl(FLAG_OVERLINE)
-          of "blink": mode.incl(FLAG_BLINK)
-        mode
-      for k, v in v:
-        case k
-        of "alt-screen":
-          if v.vt == VALUE_BOOLEAN:
-            config.altscreen = some(v.b)
-          elif v.vt == VALUE_STRING and v.s == "auto":
-            config.altscreen = none(bool)
-        of "color-mode":
-          case v.s
-          of "auto": config.colormode = none(ColorMode)
-          of "monochrome": config.colormode = some(MONOCHROME)
-          of "ansi": config.colormode = some(ANSI)
-          of "8bit": config.colormode = some(EIGHT_BIT)
-          of "24bit": config.colormode = some(TRUE_COLOR)
-        of "format-mode":
-          if v.vt == VALUE_STRING and v.s == "auto":
-            config.formatmode = none(FormatMode)
-          elif v.vt == VALUE_ARRAY:
-            config.formatmode = some(get_format_mode v)
-        of "no-format-mode":
-          config.noformatmode = get_format_mode v
-        of "highlight-color":
-          config.hlcolor = parseRGBAColor(v.s).get
-        of "double-width-ambiguous":
-          config.ambiguous_double = v.b
-        of "minimum-contrast":
-          config.mincontrast = int(v.i)
-        of "force-clear": config.forceclear = v.b
-        of "emulate-overline": config.emulateoverline = v.b
-    of "external":
-      for k, v in v:
-        case k
-        of "editor": config.editor = v.s
-        of "tmpdir": config.tmpdir = v.s
-    of "siteconf":
-      for v in v:
-        var conf = StaticSiteConfig()
-        for k, v in v:
-          case k
-          of "url": conf.url = some(v.s)
-          of "host": conf.host = some(v.s)
-          of "rewrite-url": conf.subst = some(v.s)
-          of "referer-from": conf.refererfrom = some(v.b)
-          of "cookie": conf.cookie = some(v.b)
-          of "third-party-cookie":
-            if v.vt == VALUE_STRING:
-              conf.thirdpartycookie = @[v.s]
-            else:
-              for v in v.a:
-                conf.thirdpartycookie.add(v.s)
-          of "share-cookie-jar": conf.sharecookiejar = some(v.s)
-          of "scripting": conf.scripting = some(v.b)
-        assert conf.url.isSome != conf.host.isSome
-        config.siteconf.add(conf)
-    of "omnirule":
-      if v.vt == VALUE_ARRAY and v.a.len == 0:
-        config.omnirules.setLen(0)
+      typeCheck(vv, {VALUE_STRING, VALUE_ARRAY}, kkk)
+      case vv.vt
+      of VALUE_STRING:
+        x.stylesheet &= readUserStylesheet(dir, vv.s)
+      of VALUE_ARRAY:
+        for child in vv.a:
+          x.stylesheet &= readUserStylesheet(dir, vv.s)
+      else: discard
+    of "inline":
+      typeCheck(vv, VALUE_STRING, kkk)
+      x.stylesheet &= vv.s
+
+proc parseConfig(config: Config, dir: string, t: TomlValue) =
+  gdir = dir
+  parseConfigValue(config[], t, "")
+  while config.includes.len > 0:
+    #TODO: warn about recursive includes
+    let includes = config.includes
+    config.includes.setLen(0)
+    for s in includes:
+      when nimvm:
+        config.parseConfig(dir, staticRead(dir / s))
       else:
-        for v in v:
-          var rule = StaticOmniRule()
-          for k, v in v:
-            case k
-            of "match": rule.match = v.s
-            of "substitute-url": rule.subst = v.s
-          if rule.match != "":
-            assert rule.subst != "", "Unspecified substitution for rule " & rule.match
-            config.omnirules.add(rule)
-
-proc parseConfig(config: Config, dir: string, stream: Stream) =
-  config.parseConfig(dir, parseToml(stream))
-
-proc parseConfig*(config: Config, dir: string, s: string) =
-  config.parseConfig(dir, newStringStream(s))
+        config.parseConfig(dir, newFileStream(dir / s))
+  #TODO: for omnirules/siteconf, check if substitution rules are specified?
+
+proc parseConfig(config: Config, dir: string, stream: Stream, name = "<input>") =
+  let toml = parseToml(stream, dir / name)
+  if toml.isOk:
+    config.parseConfig(dir, toml.get)
+  else:
+    eprint("Fatal error: Failed to parse config\n")
+    eprint(toml.error & "\n")
+    quit(1)
+
+proc parseConfig*(config: Config, dir: string, s: string, name = "<input>") =
+  config.parseConfig(dir, newStringStream(s), name)
 
 proc staticReadConfig(): ConfigObj =
   var config = new(Config)
-  config.parseConfig("res", staticRead"res/config.toml")
+  config.parseConfig("res", staticRead"res/config.toml", "config.toml")
   return config[]
 
 const defaultConfig = staticReadConfig()
@@ -404,13 +499,13 @@ proc readConfig(config: Config, dir: string) =
     config.parseConfig(dir, fs)
 
 proc getNormalAction*(config: Config, s: string): string =
-  if config.nmap.hasKey(s):
-    return config.nmap[s]
+  if config.page.hasKey(s):
+    return config.page[s]
   return ""
 
 proc getLinedAction*(config: Config, s: string): string =
-  if config.lemap.hasKey(s):
-    return config.lemap[s]
+  if config.line.hasKey(s):
+    return config.line[s]
   return ""
 
 proc readConfig*(): Config =
diff --git a/src/config/toml.nim b/src/config/toml.nim
index 89277d24..c34e46a7 100644
--- a/src/config/toml.nim
+++ b/src/config/toml.nim
@@ -5,6 +5,7 @@ import strutils
 import strformat
 import unicode
 
+import utils/opt
 import utils/twtstr
 
 type
@@ -21,6 +22,7 @@ type
   SyntaxError = object of ValueError
 
   TomlParser = object
+    filename: string
     at: int
     line: int
     stream: Stream
@@ -89,10 +91,10 @@ func peek(state: TomlParser, i: int, len: int): string =
   return state.buf.substr(state.at + i, state.at + i + len)
 
 proc syntaxError(state: TomlParser, msg: string) =
-  raise newException(SyntaxError, fmt"on line {state.line}: {msg}")
+  raise newException(SyntaxError, fmt"{state.filename}({state.line}): {msg}")
 
 proc valueError(state: TomlParser, msg: string) =
-  raise newException(ValueError, fmt"on line {state.line}: {msg}")
+  raise newException(ValueError, fmt"{state.filename}({state.line}): {msg}")
 
 proc consume(state: var TomlParser): char =
   result = state.buf[state.at]
@@ -405,6 +407,7 @@ proc consumeArray(state: var TomlParser): TomlValue =
         result.ta.add(val.t)
       else:
         result.a.add(val)
+      val = nil
     else:
       if val != nil:
         state.syntaxError("missing comma")
@@ -479,27 +482,28 @@ proc consumeValue(state: var TomlParser): TomlValue =
     else:
       state.syntaxError(fmt"invalid character in value: {c}")
 
-proc parseToml*(inputStream: Stream): TomlValue =
+proc parseToml*(inputStream: Stream, filename = "<input>"): Result[TomlValue, string] =
   var state: TomlParser
   state.stream = inputStream
   state.line = 1
   state.root = TomlTable()
-
-  while state.has():
-    if state.consumeNoState():
-      let kvpair = TomlKVPair(state.node)
-      kvpair.value = state.consumeValue()
-
+  state.filename = filename
+  try:
     while state.has():
-      let c = state.consume()
-      case c
-      of '\n':
-        state.flushLine()
-        break
-      of '#':
-        state.consumeComment()
-      of '\t', ' ': discard
-      else: state.syntaxError(fmt"invalid character after value: {c}")
-  inputStream.close()
-
-  return TomlValue(vt: VALUE_TABLE, t: state.root)
+      if state.consumeNoState():
+        let kvpair = TomlKVPair(state.node)
+        kvpair.value = state.consumeValue()
+      while state.has():
+        let c = state.consume()
+        case c
+        of '\n':
+          state.flushLine()
+          break
+        of '#':
+          state.consumeComment()
+        of '\t', ' ': discard
+        else: state.syntaxError(fmt"invalid character after value: {c}")
+    inputStream.close()
+    return ok(TomlValue(vt: VALUE_TABLE, t: state.root))
+  except SyntaxError, ValueError:
+    return err(getCurrentExceptionMsg())
diff --git a/src/display/client.nim b/src/display/client.nim
index d0f618e1..6e9f6331 100644
--- a/src/display/client.nim
+++ b/src/display/client.nim
@@ -16,6 +16,7 @@ import bindings/quickjs
 import buffer/container
 import css/sheet
 import config/config
+import data/charset
 import display/pager
 import display/term
 import html/dom
@@ -450,7 +451,8 @@ proc newConsole(pager: Pager, tty: File): Console =
     if pipe(pipefd) == -1:
       raise newException(Defect, "Failed to open console pipe.")
     let url = newURL("javascript:console.show()")
-    result.container = pager.readPipe0(some("text/plain"), pipefd[0], option(url), "Browser console")
+    result.container = pager.readPipe0(some("text/plain"), none(Charset),
+      pipefd[0], option(url), "Browser console")
     var f: File
     if not open(f, pipefd[1], fmWrite):
       raise newException(Defect, "Failed to open file for console pipe.")
@@ -477,7 +479,8 @@ proc dumpBuffers(client: Client) =
       quit(1)
   stdout.close()
 
-proc launchClient*(client: Client, pages: seq[string], ctype: Option[string], dump: bool) =
+proc launchClient*(client: Client, pages: seq[string], ctype: Option[string],
+    cs: Option[Charset], dump: bool) =
   var tty: File
   var dump = dump
   if not dump:
@@ -499,26 +502,26 @@ proc launchClient*(client: Client, pages: seq[string], ctype: Option[string], du
   client.console = newConsole(client.pager, tty)
   client.alive = true
   addExitProc((proc() = client.quit()))
-  if client.config.startup != "":
-    let s = if fileExists(client.config.startup):
-      readFile(client.config.startup)
+  if client.config.start.startup_script != "":
+    let s = if fileExists(client.config.start.startup_script):
+      readFile(client.config.start.startup_script)
     else:
-      client.config.startup
-    client.command0(s, client.config.startup, silence = true)
-  client.userstyle = client.config.stylesheet.parseStylesheet()
+      client.config.start.startup_script
+    client.command0(s, client.config.start.startup_script, silence = true)
+  client.userstyle = client.config.css.stylesheet.parseStylesheet()
 
   if not stdin.isatty():
-    client.pager.readPipe(ctype, stdin.getFileHandle())
+    client.pager.readPipe(ctype, cs, stdin.getFileHandle())
 
   for page in pages:
-    client.pager.loadURL(page, ctype = ctype)
+    client.pager.loadURL(page, ctype = ctype, cs = cs)
   client.acceptBuffers()
   client.pager.refreshStatusMsg()
   if not dump:
     client.inputLoop()
   else:
     client.dumpBuffers()
-  if client.config.headless:
+  if client.config.start.headless:
     client.headlessLoop()
   client.quit()
 
diff --git a/src/display/pager.nim b/src/display/pager.nim
index 805f0f51..bd07d52e 100644
--- a/src/display/pager.nim
+++ b/src/display/pager.nim
@@ -12,6 +12,7 @@ when defined(posix):
 import buffer/cell
 import buffer/container
 import config/config
+import data/charset
 import display/term
 import io/lineedit
 import io/promise
@@ -120,16 +121,16 @@ proc getter(pager: Pager, s: string): Option[JSValue] {.jsgetprop.} =
 proc searchNext(pager: Pager) {.jsfunc.} =
   if pager.regex.issome:
     if not pager.reverseSearch:
-      pager.container.cursorNextMatch(pager.regex.get, pager.config.searchwrap)
+      pager.container.cursorNextMatch(pager.regex.get, pager.config.search.wrap)
     else:
-      pager.container.cursorPrevMatch(pager.regex.get, pager.config.searchwrap)
+      pager.container.cursorPrevMatch(pager.regex.get, pager.config.search.wrap)
 
 proc searchPrev(pager: Pager) {.jsfunc.} =
   if pager.regex.issome:
     if not pager.reverseSearch:
-      pager.container.cursorPrevMatch(pager.regex.get, pager.config.searchwrap)
+      pager.container.cursorPrevMatch(pager.regex.get, pager.config.search.wrap)
     else:
-      pager.container.cursorNextMatch(pager.regex.get, pager.config.searchwrap)
+      pager.container.cursorNextMatch(pager.regex.get, pager.config.search.wrap)
 
 proc getLineHist(pager: Pager, mode: LineMode): LineHistory =
   if pager.linehist[mode] == nil:
@@ -245,7 +246,7 @@ proc refreshDisplay(pager: Pager, container = pager.container) =
       let area = hl.colorArea(container.fromy + by, startw .. startw + aw)
       for i in area:
         var hlformat = pager.display[dls + i - startw].format
-        hlformat.bgcolor = pager.config.hlcolor.cellColor()
+        hlformat.bgcolor = pager.config.display.highlight_color.cellColor()
         pager.display[dls + i - startw].format = hlformat
     inc by
 
@@ -544,6 +545,7 @@ proc applySiteconf(pager: Pager, request: Request): BufferConfig =
   var cookiejar: CookieJar
   var headers: HeaderList
   var scripting: bool
+  var charsets = pager.config.encoding.document_charset
   for sc in pager.siteconf:
     if sc.url.isSome and not sc.url.get.match(url):
       continue
@@ -566,12 +568,15 @@ proc applySiteconf(pager: Pager, request: Request): BufferConfig =
       scripting = sc.scripting.get
     if sc.refererfrom.isSome:
       refererfrom = sc.refererfrom.get
-  return pager.config.getBufferConfig(request.url, cookiejar, headers, refererfrom, scripting)
+    if sc.document_charset.len > 0:
+      charsets = sc.document_charset
+  return pager.config.getBufferConfig(request.url, cookiejar, headers,
+    refererfrom, scripting, charsets)
 
 # Load request in a new buffer.
 proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
-              ctype = none(string), replace: Container = nil,
-              redirectdepth = 0, referrer: Container = nil) =
+    ctype = none(string), cs = none(Charset), replace: Container = nil,
+    redirectdepth = 0, referrer: Container = nil) =
   if referrer != nil and referrer.config.refererfrom:
     request.referer = referrer.source.location
   var bufferconfig = pager.applySiteconf(request)
@@ -587,6 +592,7 @@ proc gotoURL(pager: Pager, request: Request, prevurl = none(URL),
       t: LOAD_REQUEST,
       request: request,
       contenttype: ctype,
+      charset: cs,
       location: request.url
     )
     if referrer != nil:
@@ -615,7 +621,8 @@ proc omniRewrite(pager: Pager, s: string): string =
 # * file://$PWD/<file>
 # * https://<url>
 # So we attempt to load both, and see what works.
-proc loadURL*(pager: Pager, url: string, ctype = none(string)) =
+proc loadURL*(pager: Pager, url: string, ctype = none(string),
+    cs = none(Charset)) =
   let url0 = pager.omniRewrite(url)
   let url = if url[0] == '~': expandPath(url0) else: url0
   let firstparse = parseURL(url)
@@ -624,10 +631,10 @@ proc loadURL*(pager: Pager, url: string, ctype = none(string)) =
       some(pager.container.source.location)
     else:
       none(URL)
-    pager.gotoURL(newRequest(firstparse.get), prev, ctype)
+    pager.gotoURL(newRequest(firstparse.get), prev, ctype, cs)
     return
   var urls: seq[URL]
-  if pager.config.prependhttps and url[0] != '/':
+  if pager.config.network.prepend_https and url[0] != '/':
     let pageurl = parseURL("https://" & url)
     if pageurl.isSome: # attempt to load remote page
       urls.add(pageurl.get)
@@ -640,22 +647,25 @@ proc loadURL*(pager: Pager, url: string, ctype = none(string)) =
     pager.alert("Invalid URL " & url)
   else:
     let prevc = pager.container
-    pager.gotoURL(newRequest(urls.pop()), ctype = ctype)
+    pager.gotoURL(newRequest(urls.pop()), ctype = ctype, cs = cs)
     if pager.container != prevc:
       pager.container.retry = urls
 
-proc readPipe0*(pager: Pager, ctype: Option[string], fd: FileHandle, location: Option[URL], title: string): Container =
+proc readPipe0*(pager: Pager, ctype: Option[string], cs: Option[Charset],
+    fd: FileHandle, location: Option[URL], title: string): Container =
   let source = BufferSource(
     t: LOAD_PIPE,
     fd: fd,
     contenttype: some(ctype.get("text/plain")),
+    charset: cs,
     location: location.get(newURL("file://-"))
   )
   let bufferconfig = pager.config.getBufferConfig(source.location)
   return pager.dispatcher.newBuffer(bufferconfig, source, title = title)
 
-proc readPipe*(pager: Pager, ctype: Option[string], fd: FileHandle) =
-  let container = pager.readPipe0(ctype, fd, none(URL), "*pipe*")
+proc readPipe*(pager: Pager, ctype: Option[string], cs: Option[Charset],
+    fd: FileHandle) =
+  let container = pager.readPipe0(ctype, cs, fd, none(URL), "*pipe*")
   pager.addContainer(container)
 
 proc command(pager: Pager) {.jsfunc.} =
@@ -680,9 +690,9 @@ proc updateReadLineISearch(pager: Pager, linemode: LineMode) =
     if pager.iregex.isSome:
       pager.container.hlon = true
       if linemode == ISEARCH_F:
-        pager.container.cursorNextMatch(pager.iregex.get, pager.config.searchwrap)
+        pager.container.cursorNextMatch(pager.iregex.get, pager.config.search.wrap)
       else:
-        pager.container.cursorPrevMatch(pager.iregex.get, pager.config.searchwrap)
+        pager.container.cursorPrevMatch(pager.iregex.get, pager.config.search.wrap)
     pager.container.pushCursorPos()
   of FINISH:
     if pager.iregex.isSome:
@@ -754,7 +764,8 @@ proc load(pager: Pager, s = "") {.jsfunc.} =
 
 # Reload the page in a new buffer, then kill the previous buffer.
 proc reload(pager: Pager) {.jsfunc.} =
-  pager.gotoURL(newRequest(pager.container.source.location), none(URL), pager.container.contenttype, pager.container)
+  pager.gotoURL(newRequest(pager.container.source.location), none(URL),
+    pager.container.contenttype, replace = pager.container)
 
 proc authorize(pager: Pager) =
   pager.setLineEdit("Username: ", USERNAME)
@@ -799,7 +810,7 @@ proc handleEvent0(pager: Pager, container: Container, event: ContainerEvent): bo
     if pager.container == container:
       pager.authorize()
   of REDIRECT:
-    if container.redirectdepth < pager.config.maxredirect:
+    if container.redirectdepth < pager.config.network.max_redirect:
       pager.alert("Redirecting to " & $event.request.url)
       pager.gotoURL(event.request, some(container.source.location),
         replace = container, redirectdepth = container.redirectdepth + 1,
diff --git a/src/display/term.nim b/src/display/term.nim
index 239b1a53..ce518428 100644
--- a/src/display/term.nim
+++ b/src/display/term.nim
@@ -390,8 +390,8 @@ proc showCursor*(term: Terminal) =
   term.outfile.showCursor()
 
 func emulateOverline(term: Terminal): bool =
-  term.config.emulateoverline and FLAG_OVERLINE notin term.formatmode and
-  FLAG_UNDERLINE in term.formatmode
+  term.config.display.emulate_overline and
+    FLAG_OVERLINE notin term.formatmode and FLAG_UNDERLINE in term.formatmode
 
 proc writeGrid*(term: Terminal, grid: FixedGrid, x = 0, y = 0) =
   for ly in y ..< y + grid.height:
@@ -412,28 +412,28 @@ proc writeGrid*(term: Terminal, grid: FixedGrid, x = 0, y = 0) =
           j += cell[].width()
 
 proc applyConfig(term: Terminal) =
-  if term.config.colormode.isSome:
-    term.colormode = term.config.colormode.get
+  if term.config.display.color_mode.isSome:
+    term.colormode = term.config.display.color_mode.get
   elif term.isatty():
     term.colormode = ANSI
     let colorterm = getEnv("COLORTERM")
     case colorterm
     of "24bit", "truecolor": term.colormode = TRUE_COLOR
-  if term.config.formatmode.isSome:
-    term.formatmode = term.config.formatmode.get
+  if term.config.display.format_mode.isSome:
+    term.formatmode = term.config.display.format_mode.get
   for fm in FormatFlags:
-    if fm in term.config.noformatmode:
+    if fm in term.config.display.no_format_mode:
       term.formatmode.excl(fm)
-  if term.isatty() and term.config.altscreen.isSome:
-    term.smcup = term.config.altscreen.get
-  term.mincontrast = term.config.mincontrast
+  if term.isatty() and term.config.display.alt_screen.isSome:
+    term.smcup = term.config.display.alt_screen.get
+  term.mincontrast = term.config.display.minimum_contrast
 
 proc outputGrid*(term: Terminal) =
-  if term.config.termreload:
+  if term.config.display.force_clear:
     term.applyConfig()
   term.outfile.write(term.resetFormat())
   let samesize = term.canvas.width == term.pcanvas.width and term.canvas.height == term.pcanvas.height
-  if term.config.forceclear or not term.cleared or not samesize:
+  if term.config.display.force_clear or not term.cleared or not samesize:
     term.outfile.write(term.generateFullOutput(term.canvas))
     term.cleared = true
   else:
diff --git a/src/encoding/decoderstream.nim b/src/encoding/decoderstream.nim
index 362d9607..425f264f 100644
--- a/src/encoding/decoderstream.nim
+++ b/src/encoding/decoderstream.nim
@@ -505,26 +505,26 @@ proc decodeShiftJIS(stream: DecoderStream, iq: var seq[uint8],
   while i < ilen:
     let b = iq[i]
     if lead != 0:
-      let l = lead
-      lead = 0
+      var ptrisnull = true;
+      var p = 0u16
       let offset = if b < 0x7Fu8: 0x40u16 else: 0x41u16
-      let leadoffset = if l < 0xA0: 0x81u16 else: 0xC1u16
+      let leadoffset = if lead < 0xA0: 0x81u16 else: 0xC1u16
       if b in 0x40u8..0x7Eu8 or b in 0x80u8..0xFCu8:
-        let p = (uint16(l) - leadoffset) * 188 + uint16(b) - offset
-        if p in 8836u16..10715u16:
-          stream.append_codepoint 0xE000u16 - 8836 + p, oq, olen, n
-          inc i
-          continue
-        if p < Jis0208Decode.len and Jis0208Decode[p] != 0:
-          let c = Jis0208Decode[p]
-          stream.append_codepoint c, oq, olen, n
-          inc i
-          continue
-      if cast[char](b) in Ascii:
-        continue # prepend (no inc i)
+        p = (uint16(lead) - leadoffset) * 188 + uint16(b) - offset
+        ptrisnull = false
+      lead = 0
+      if not ptrisnull and p in 8836u16..10715u16:
+        stream.append_codepoint 0xE000u16 - 8836 + p, oq, olen, n
+        inc i
+        continue
+      elif not ptrisnull and p < Jis0208Decode.len and Jis0208Decode[p] != 0:
+        let c = Jis0208Decode[p]
+        stream.append_codepoint c, oq, olen, n
       else:
         stream.handleError(oq, olen, n)
         if stream.isend: break
+        if cast[char](b) in Ascii:
+          continue # prepend (no inc i)
     elif cast[char](b) in Ascii or b == 0x80:
       stream.append_codepoint b, oq, olen, n
     elif b in 0xA1u8..0xDFu8:
@@ -640,11 +640,11 @@ proc decodeSingleByte(stream: DecoderStream, iq: var seq[uint8],
     if c in Ascii:
       stream.append_codepoint c, oq, olen, n
     else:
-      let p = map[c]
+      let p = map[cast[char](iq[i] - 0x80)]
       if p == 0u16:
         stream.handleError(oq, olen, n)
       else:
-        stream.append_codepoint cast[uint32](oq), oq, olen, n
+        stream.append_codepoint cast[uint32](p), oq, olen, n
 
 proc decodeReplacement(stream: DecoderStream, oq: ptr UncheckedArray[uint32], olen: int, n: var int) =
   if not stream.replreported:
diff --git a/src/html/htmlparser.nim b/src/html/htmlparser.nim
index 8cd34cd9..d03e0d24 100644
--- a/src/html/htmlparser.nim
+++ b/src/html/htmlparser.nim
@@ -2162,8 +2162,7 @@ proc constructTree(parser: var HTML5Parser): Document =
     else:
       parser.processInForeignContent(token)
     if parser.needsreinterpret:
-      return nil
-
+      break
   return parser.document
 
 proc finishParsing(parser: var HTML5Parser) =
@@ -2175,47 +2174,68 @@ proc finishParsing(parser: var HTML5Parser) =
     script.execute()
   #TODO events
 
-proc parseHTML*(inputStream: Stream, cs = none(Charset), fallbackcs = CHARSET_UTF_8, window: Window = nil, url: URL = nil): (Document, Charset) =
-  var parser: HTML5Parser
-  var bom: string
-  if cs.isSome:
-    parser.charset = cs.get
-    parser.confidence = CONFIDENCE_CERTAIN
-  else:
-    # bom sniff
-    const u8bom = char(0xEF) & char(0xBB) & char(0xBF)
-    const bebom = char(0xFE) & char(0xFF)
-    const lebom = char(0xFF) & char(0xFE)
-    bom = inputStream.readStr(2)
-    if bom == bebom:
-      parser.charset = CHARSET_UTF_16_BE
-      parser.confidence = CONFIDENCE_CERTAIN
-      bom = ""
-    elif bom == lebom:
-      parser.charset = CHARSET_UTF_16_LE
-      parser.confidence = CONFIDENCE_CERTAIN
-      bom = ""
+proc parseHTML*(inputStream: Stream, charsets: seq[Charset] = @[],
+    fallbackcs = CHARSET_UTF_8, window: Window = nil,
+    url: URL = nil, canReinterpret = true): Document =
+  var charsetStack: seq[Charset]
+  for i in countdown(charsets.high, 0):
+    charsetStack.add(charsets[i])
+  var canReinterpret = canReinterpret
+  while true:
+    var parser: HTML5Parser
+    var bom: string
+    let islastcs = charsetStack.len == 0
+    if not islastcs:
+      parser.charset = charsetStack.pop()
+      if not canReinterpret:
+        parser.confidence = CONFIDENCE_CERTAIN
     else:
-      bom &= inputStream.readChar()
-      if bom == u8bom:
-        parser.charset = CHARSET_UTF_8
+      # bom sniff
+      const u8bom = char(0xEF) & char(0xBB) & char(0xBF)
+      const bebom = char(0xFE) & char(0xFF)
+      const lebom = char(0xFF) & char(0xFE)
+      bom = inputStream.readStr(2)
+      if bom == bebom:
+        parser.charset = CHARSET_UTF_16_BE
+        parser.confidence = CONFIDENCE_CERTAIN
+        bom = ""
+      elif bom == lebom:
+        parser.charset = CHARSET_UTF_16_LE
         parser.confidence = CONFIDENCE_CERTAIN
         bom = ""
       else:
-        parser.charset = fallbackcs
-  let decoder = newDecoderStream(inputStream, parser.charset)
-  for c in bom:
-    decoder.prepend(cast[uint32](c))
-  parser.document = newDocument()
-  parser.document.contentType = "text/html"
-  if window != nil:
-    parser.document.window = window
-    window.document = parser.document
-  parser.document.url = url
-  parser.tokenizer = newTokenizer(decoder)
-  let document = parser.constructTree()
-  parser.finishParsing()
-  return (document, parser.charset)
+        bom &= inputStream.readChar()
+        if bom == u8bom:
+          parser.charset = CHARSET_UTF_8
+          parser.confidence = CONFIDENCE_CERTAIN
+          bom = ""
+        else:
+          parser.charset = fallbackcs
+    let em = if islastcs or not canReinterpret:
+      DECODER_ERROR_MODE_REPLACEMENT
+    else:
+      DECODER_ERROR_MODE_FATAL
+    let decoder = newDecoderStream(inputStream, parser.charset, errormode = em)
+    for c in bom:
+      decoder.prepend(cast[uint32](c))
+    parser.document = newDocument()
+    parser.document.contentType = "text/html"
+    if window != nil:
+      parser.document.window = window
+      window.document = parser.document
+    parser.document.url = url
+    parser.tokenizer = newTokenizer(decoder)
+    let document = parser.constructTree()
+    if parser.needsreinterpret and canReinterpret:
+      inputStream.setPosition(0)
+      charsetStack.add(parser.charset)
+      canReinterpret = false
+      continue
+    if decoder.failed and canReinterpret:
+      inputStream.setPosition(0)
+      continue
+    parser.finishParsing()
+    return document
 
 proc newDOMParser*(): DOMParser {.jsctor.} =
   new(result)
@@ -2223,7 +2243,7 @@ proc newDOMParser*(): DOMParser {.jsctor.} =
 proc parseFromString(parser: DOMParser, str: string, t: string): Document {.jserr, jsfunc.} =
   case t
   of "text/html":
-    let (res, _) = parseHTML(newStringStream(str))
+    let res = parseHTML(newStringStream(str))
     return res
   of "text/xml", "application/xml", "application/xhtml+xml", "image/svg+xml":
     JS_ERR JS_InternalError, "XML parsing is not supported yet"
diff --git a/src/ips/editor.nim b/src/ips/editor.nim
index 4361c39f..0e2c91c8 100644
--- a/src/ips/editor.nim
+++ b/src/ips/editor.nim
@@ -30,7 +30,7 @@ func formatEditorName(editor, file: string, line: int): string =
     result &= file
 
 proc openEditor*(term: Terminal, config: Config, file: string, line = 1): bool =
-  var editor = config.editor
+  var editor = config.external.editor
   if editor == "":
     editor = getEnv("EDITOR")
     if editor == "":
@@ -43,7 +43,7 @@ proc openEditor*(term: Terminal, config: Config, file: string, line = 1): bool =
 var tmpf_seq: int
 proc openInEditor*(term: Terminal, config: Config, input: var string): bool =
   try:
-    let tmpdir = config.tmpdir
+    let tmpdir = config.external.tmpdir
     if not dirExists(tmpdir):
       createDir(tmpdir)
     var tmpf = tmpdir / "chatmp" & $tmpf_seq
diff --git a/src/ips/forkserver.nim b/src/ips/forkserver.nim
index 009c5491..e7ddf1c0 100644
--- a/src/ips/forkserver.nim
+++ b/src/ips/forkserver.nim
@@ -75,8 +75,8 @@ proc forkLoader(ctx: var ForkServerContext, config: LoaderConfig): Pid =
       let e = getCurrentException()
       # taken from system/excpt.nim
       let msg = e.getStackTrace() & "Error: unhandled exception: " & e.msg &
-        " [" & $e.name & "]"
-      eprint(msg)
+        " [" & $e.name & "]\n"
+      stderr.write(msg)
     doAssert false
   let readfd = pipefd[0] # get read
   discard close(pipefd[1]) # close write
@@ -121,8 +121,8 @@ proc forkBuffer(ctx: var ForkServerContext): Pid =
       let e = getCurrentException()
       # taken from system/excpt.nim
       let msg = e.getStackTrace() & "Error: unhandled exception: " & e.msg &
-        " [" & $e.name & "]"
-      eprint(msg)
+        " [" & $e.name & "]\n"
+      stderr.write(msg)
     doAssert false
   ctx.children.add((pid, loaderPid))
   return pid
diff --git a/src/ips/serialize.nim b/src/ips/serialize.nim
index b8ffab0a..0636e2e9 100644
--- a/src/ips/serialize.nim
+++ b/src/ips/serialize.nim
@@ -325,6 +325,7 @@ proc swrite*(stream: Stream, source: BufferSource) =
   of LOAD_PIPE: stream.swrite(source.fd)
   stream.swrite(source.location)
   stream.swrite(source.contenttype)
+  stream.swrite(source.charset)
 
 proc sread*(stream: Stream, source: var BufferSource) =
   var t: BufferSourceType
@@ -341,6 +342,7 @@ proc sread*(stream: Stream, source: var BufferSource) =
     stream.sread(source.fd)
   stream.sread(source.location)
   stream.sread(source.contenttype)
+  stream.sread(source.charset)
 
 func slen*(source: BufferSource): int =
   result += slen(source.t)
diff --git a/src/main.nim b/src/main.nim
index a0a6b064..0371c801 100644
--- a/src/main.nim
+++ b/src/main.nim
@@ -9,12 +9,13 @@ when defined(profile):
   import nimprof
 
 import config/config
+import data/charset
 import display/client
 import ips/forkserver
 import utils/twtstr
 
 let conf = readConfig()
-set_cjk_ambiguous(conf.ambiguous_double)
+set_cjk_ambiguous(conf.display.double_width_ambiguous)
 let params = commandLineParams()
 
 proc version(long: static bool = false): string =
@@ -34,7 +35,8 @@ Options:
     -c, --css <stylesheet>      Pass stylesheet (e.g. -c 'a{color: blue}')
     -o, --opt <config>          Pass config options (e.g. -o 'page.q="QUIT"')
     -T, --type <type>           Specify content mime type
-    -M, --monochrome            Alias of -o color-mode='monochrome'
+    -I, --input-charset <name>  Specify document charset
+    -M, --monochrome            Set color-mode to 'monochrome'
     -V, --visual                Visual startup mode
     -r, --run <script/file>     Run passed script or file
     -h, --help                  Print this usage message
@@ -47,6 +49,7 @@ Options:
 
 var i = 0
 var ctype = none(string)
+var cs = none(Charset)
 var pages: seq[string]
 var dump = false
 var visual = false
@@ -62,15 +65,25 @@ while i < params.len:
     echo version(true)
     quit(0)
   of "-M", "--monochrome":
-    conf.colormode = some(MONOCHROME)
+    conf.display.colormode = some(MONOCHROME)
   of "-V", "--visual":
     visual = true
-  of "-T":
+  of "-T", "--type":
     inc i
     if i < params.len:
       ctype = some(params[i])
     else:
       help(1)
+  of "-I", "--input-charset":
+    inc i
+    if i < params.len:
+      let c = getCharset(params[i])
+      if c == CHARSET_UNKNOWN:
+        stderr.write("Unknown charset " & params[i] & "\n")
+        quit(1)
+      cs = some(c)
+    else:
+      help(1)
   of "-":
     discard # emulate programs that accept - as stdin
   of "-d", "-dump", "--dump":
@@ -78,7 +91,7 @@ while i < params.len:
   of "-c", "--css":
     inc i
     if i < params.len:
-      conf.stylesheet &= params[i]
+      conf.css.stylesheet &= params[i]
     else:
       help(1)
   of "-o", "--opt":
@@ -92,8 +105,8 @@ while i < params.len:
   of "-r", "--run":
     inc i
     if i < params.len:
-      conf.startup = params[i]
-      conf.headless = true
+      conf.start.startup_script = params[i]
+      conf.start.headless = true
       dump = true
     else:
       help(1)
@@ -108,7 +121,7 @@ while i < params.len:
 
 if pages.len == 0 and stdin.isatty():
   if visual:
-    pages.add(conf.visualhome)
+    pages.add(conf.start.visual_home)
   else:
     let http = getEnv("HTTP_HOME")
     if http != "": pages.add(http)
@@ -116,17 +129,17 @@ if pages.len == 0 and stdin.isatty():
       let www = getEnv("WWW_HOME")
       if www != "": pages.add(www)
 
-if pages.len == 0 and not conf.headless:
+if pages.len == 0 and not conf.start.headless:
   if stdin.isatty:
     help(1)
 
-conf.nmap = constructActionTable(conf.nmap)
-conf.lemap = constructActionTable(conf.lemap)
+conf.page = constructActionTable(conf.page)
+conf.line = constructActionTable(conf.line)
 disp.forkserver.loadForkServerConfig(conf)
 
 let c = newClient(conf, disp)
 try:
-  c.launchClient(pages, ctype, dump)
+  c.launchClient(pages, ctype, cs, dump)
 except CatchableError:
   c.flushConsole()
   raise
diff --git a/src/render/rendertext.nim b/src/render/rendertext.nim
index e91c1e10..d0576c75 100644
--- a/src/render/rendertext.nim
+++ b/src/render/rendertext.nim
@@ -10,14 +10,35 @@ type StreamRenderer* = object
   ansiparser: AnsiCodeParser
   format: Format
   af: bool
+  stream: Stream
   decoder: DecoderStream
+  charsets: seq[Charset]
   newline: bool
   w: int
 
-proc newStreamRenderer*(stream: Stream): StreamRenderer =
+proc newStreamRenderer*(stream: Stream, charsets: seq[Charset]): StreamRenderer =
   result.format = newFormat()
   result.ansiparser.state = PARSE_DONE
-  result.decoder = newDecoderStream(stream, CHARSET_UTF_8)
+  for i in countdown(charsets.high, 0):
+    result.charsets.add(charsets[i])
+  let cs = result.charsets.pop()
+  let em = if charsets.len > 0:
+    DECODER_ERROR_MODE_FATAL
+  else:
+    DECODER_ERROR_MODE_REPLACEMENT
+  result.stream = stream
+  result.decoder = newDecoderStream(stream, cs, errormode = em)
+
+proc rewind(renderer: var StreamRenderer) =
+  renderer.stream.setPosition(0)
+  let cs = renderer.charsets.pop()
+  let em = if renderer.charsets.len > 0:
+    DECODER_ERROR_MODE_FATAL
+  else:
+    DECODER_ERROR_MODE_REPLACEMENT
+  renderer.decoder = newDecoderStream(renderer.stream, cs, errormode = em)
+  renderer.format = newFormat()
+  renderer.ansiparser.state = PARSE_DONE
 
 proc renderStream*(grid: var FlexibleGrid, renderer: var StreamRenderer, len: int) =
   if len == 0: return
@@ -28,7 +49,13 @@ proc renderStream*(grid: var FlexibleGrid, renderer: var StreamRenderer, len: in
 
   if grid.len == 0: grid.addLine()
   var buf = newSeq[Rune](len * 4)
-  let n = renderer.decoder.readData(addr buf[0], buf.len * sizeof(buf[0]))
+  var n: int
+  while true:
+    n = renderer.decoder.readData(addr buf[0], buf.len * sizeof(buf[0]))
+    if renderer.decoder.failed:
+      renderer.rewind()
+      continue
+    break
   for i in 0 ..< n div sizeof(buf[0]):
     if renderer.newline:
       # avoid newline at end of stream
diff --git a/src/types/buffersource.nim b/src/types/buffersource.nim
index dbf8fbbe..109e8361 100644
--- a/src/types/buffersource.nim
+++ b/src/types/buffersource.nim
@@ -3,6 +3,7 @@ import options
 when defined(posix):
   import posix
 
+import data/charset
 import io/request
 import types/url
 
@@ -13,6 +14,7 @@ type
   BufferSource* = object
     location*: URL
     contenttype*: Option[string] # override
+    charset*: Option[Charset] # override
     case t*: BufferSourceType
     of CLONE:
       clonepid*: Pid
diff --git a/src/utils/opt.nim b/src/utils/opt.nim
new file mode 100644
index 00000000..21d79af2
--- /dev/null
+++ b/src/utils/opt.nim
@@ -0,0 +1,34 @@
+# Inspired by nim-results.
+
+type
+  Result*[T, E] = object
+    val: T
+    has: bool
+    when not (E is void):
+      ex: E
+
+  Opt*[T] = Result[T, void]
+
+template ok*[T, E](t: type Result[T, E], x: T): Result[T, E] =
+  Result[T, E](val: x, has: true)
+
+template ok*[T](x: T): auto =
+  ok(typeof(result), x)
+
+template ok*[T, E](res: var Result[T, E], x: T): Result[T, E] =
+  res.val = x
+  res.has = true
+
+template err*[T, E](t: type Result[T, E], e: E): Result[T, E] =
+  Result[T, E](ex: e)
+
+template err*[E](e: E): auto =
+  err(typeof(result), e)
+
+template err*[T, E](res: var Result[T, E], e: E) =
+  res.ex = e
+
+template isOk*(res: Result): bool = res.has
+template isErr*(res: Result): bool = not res.has
+template get*[T, E](res: Result[T, E]): T = res.val
+template error*[T, E](res: Result[T, E]): E = res.ex
diff --git a/src/utils/twtstr.nim b/src/utils/twtstr.nim
index 027b26d1..d67e1ab7 100644
--- a/src/utils/twtstr.nim
+++ b/src/utils/twtstr.nim
@@ -116,6 +116,12 @@ func toScreamingSnakeCase*(str: string): string = # input is camel case
     else:
       result &= c.toUpperAscii()
 
+func snakeToKebabCase*(str: string): string =
+  result = str
+  for c in result.mitems:
+    if c == '_':
+      c = '-'
+
 func isAscii*(r: Rune): bool =
   return cast[uint32](r) < 128
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-01-11 00:15:38 +0000