config: better path handling; fix array parsing bug

* Paths are now parsed through an unified code path with some useful
  additions like environment variable substitution.
* Fix a bug in parseConfigValue where strings would be appended to
  existing arrays (and not override them).
* Fix beforeLast calling afterLast for some reason.
* Add a default CGI directory.

3 files changed, 320 insertions, 17 deletions

diff --git a/src/config/chapath.nim b/src/config/chapath.nim
new file mode 100644
index 00000000..caaf9435
--- /dev/null
+++ b/src/config/chapath.nim
@@ -0,0 +1,286 @@
+import std/options
+import std/os
+import std/posix
+
+import js/error
+import js/fromjs
+import js/javascript
+import js/tojs
+import types/opt
+import utils/twtstr
+
+type ChaPath* = distinct string
+
+func `$`*(p: ChaPath): string =
+  return string(p)
+
+type
+  UnquoteContext = object
+    state: UnquoteState
+    s: string
+    p: string
+    i: int
+    identStr: string
+    subChar: char
+    hasColon: bool
+    terminal: Option[char]
+
+  UnquoteState = enum
+    STATE_NORMAL, STATE_TILDE, STATE_DOLLAR, STATE_IDENT, STATE_BSLASH,
+    STATE_CURLY_START, STATE_CURLY, STATE_CURLY_HASH, STATE_CURLY_PERC,
+    STATE_CURLY_COLON, STATE_CURLY_EXPAND, STATE_DONE
+
+  ChaPathError = string
+
+  ChaPathResult[T] = Result[T, ChaPathError]
+
+proc unquote(p: string, starti: var int, terminal: Option[char]):
+    ChaPathResult[string]
+
+proc stateNormal(ctx: var UnquoteContext, c: char) =
+  case c
+  of '$': ctx.state = STATE_DOLLAR
+  of '\\': ctx.state = STATE_BSLASH
+  of '~':
+    if ctx.i == 0:
+      ctx.state = STATE_TILDE
+    else:
+      ctx.s &= c
+  elif ctx.terminal.isSome and ctx.terminal.get == c:
+    ctx.state = STATE_DONE
+  else:
+    ctx.s &= c
+
+proc flushTilde(ctx: var UnquoteContext) =
+  if ctx.identStr == "":
+    ctx.s &= getHomeDir()
+  else:
+    let p = getpwnam(cstring(ctx.identStr))
+    if p != nil:
+      ctx.s &= $p.pw_dir
+    ctx.identStr = ""
+  ctx.state = STATE_NORMAL
+
+proc stateTilde(ctx: var UnquoteContext, c: char) =
+  if c != '/':
+    ctx.identStr &= c
+  else:
+    ctx.flushTilde()
+
+# Kind of a hack. We special case `\$' (backslash-dollar) in TOML, so that
+# it produces itself in dquote strings.
+# Thus by applying stateBSlash we get '\$' -> "$", but also "\$" -> "$".
+proc stateBSlash(ctx: var UnquoteContext, c: char) =
+  if c != '$':
+    ctx.s &= '\\'
+  ctx.s &= c
+  ctx.state = STATE_NORMAL
+
+proc stateDollar(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # $
+  case c
+  of '$':
+    ctx.s &= $getCurrentProcessId()
+    ctx.state = STATE_NORMAL
+  of '0':
+    # Note: we intentionally use getAppFileName so that any symbolic links
+    # are resolved.
+    ctx.s &= getAppFileName()
+    ctx.state = STATE_NORMAL
+  of '1'..'9':
+    return err("Parameter substitution is not supported")
+  of AsciiAlpha:
+    ctx.identStr = $c
+    ctx.state = STATE_IDENT
+  of '{':
+    ctx.state = STATE_CURLY_START
+  else:
+    # > If an unquoted '$' is followed by a character that is not one of
+    # > the following: [...] the result is unspecified.
+    # just error out here to be safe
+    return err("Invalid dollar substitution")
+  ok()
+
+proc flushIdent(ctx: var UnquoteContext) =
+  ctx.s &= getEnv(ctx.identStr)
+  ctx.identStr = ""
+
+const BareChars = AsciiAlphaNumeric + {'_'}
+
+proc stateIdent(ctx: var UnquoteContext, c: char) =
+  # $ident
+  if c in BareChars:
+    ctx.identStr &= c
+  else:
+    ctx.flushIdent()
+    dec ctx.i
+    ctx.state = STATE_NORMAL
+
+proc stateCurlyStart(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # ${
+  case c
+  of '#':
+    ctx.state = STATE_CURLY_HASH
+  of '%':
+    ctx.state = STATE_CURLY_PERC
+  of BareChars:
+    ctx.state = STATE_CURLY
+    dec ctx.i
+  else:
+    return err("unexpected character in substitution: '" & c & "'")
+  return ok()
+
+proc stateCurly(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # ${ident
+  case c
+  of '}':
+    ctx.s &= $getEnv(ctx.identStr)
+    ctx.state = STATE_NORMAL
+    return ok()
+  of '$': # allow $ as first char only
+    if ctx.identStr.len > 0:
+      return err("unexpected dollar sign in substitution")
+    ctx.identStr &= c
+    return ok()
+  of ':', '-', '?', '+': # note: we don't support `=' (assign)
+    if ctx.identStr.len > 0:
+      return err("substitution without parameter name")
+    if c == ':':
+      ctx.state = STATE_CURLY_COLON
+    else:
+      ctx.subChar = c
+      ctx.state = STATE_CURLY_EXPAND
+    return ok()
+  of '1'..'9':
+    return err("Parameter substitution is not supported")
+  of BareChars - {'1'..'9'}:
+    ctx.identStr &= c
+    return ok()
+  else:
+    return err("unexpected character in substitution: '" & c & "'")
+
+proc stateCurlyHash(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # ${#ident
+  if c == '}':
+    let s = getEnv(ctx.identStr)
+    ctx.s &= $s.len
+    ctx.identStr = ""
+    ctx.state = STATE_NORMAL
+    return ok()
+  if c == '$': # allow $ as first char only
+    if ctx.identStr.len > 0:
+      return err("unexpected dollar sign in substitution")
+    # fall through
+  elif c notin BareChars:
+    return err("unexpected character in substitution: '" & c & "'")
+  ctx.identStr &= c
+  return ok()
+
+proc stateCurlyPerc(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # ${%ident
+  if c == '}':
+    if ctx.identStr == "CHA_BIN_DIR":
+      ctx.s &= getAppFileName().beforeLast('/')
+    else:
+      return err("Unknown internal variable " & ctx.identStr)
+    ctx.identStr = ""
+    ctx.state = STATE_NORMAL
+    return ok()
+  if c notin BareChars:
+    return err("unexpected character in substitution: '" & c & "'")
+  ctx.identStr &= c
+  return ok()
+
+proc stateCurlyColon(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # ${ident:
+  if c notin {'-', '?', '+'}: # Note: we don't support `=' (assign)
+    return err("unexpected character after colon: '" & c & "'")
+  ctx.hasColon = true
+  ctx.subChar = c
+  ctx.state = STATE_CURLY_EXPAND
+  return ok()
+
+proc flushCurlyExpand(ctx: var UnquoteContext, word: string):
+    ChaPathResult[void] =
+  case ctx.subChar
+  of '-':
+    if ctx.hasColon:
+      ctx.s &= getEnv(ctx.identStr, word)
+    else:
+      if existsEnv(ctx.identStr):
+        ctx.s &= getEnv(ctx.identStr)
+      else:
+        ctx.s &= word
+  of '?':
+    if ctx.hasColon:
+      let s = getEnv(ctx.identStr)
+      if s.len == 0:
+        return err(word)
+      ctx.s &= s
+    else:
+      if not existsEnv(ctx.identStr):
+        return err(word)
+      ctx.s &= getEnv(ctx.identStr)
+  of '+':
+    if ctx.hasColon:
+      if getEnv(ctx.identStr).len > 0:
+        ctx.s &= word
+    else:
+      if existsEnv(ctx.identStr):
+        ctx.s &= word
+  else: assert false
+  ctx.subChar = '\0'
+  ctx.hasColon = false
+
+proc stateCurlyExpand(ctx: var UnquoteContext, c: char): ChaPathResult[void] =
+  # ${ident:-[word], ${ident:=[word], ${ident:?[word], ${ident:+[word]
+  # word must be unquoted too.
+  let word = ?unquote(ctx.p, ctx.i, some('}'))
+  ctx.flushCurlyExpand(word)
+
+proc unquote(p: string, starti: var int, terminal: Option[char]):
+    ChaPathResult[string] =
+  var ctx = UnquoteContext(p: p, i: starti, terminal: terminal)
+  while ctx.i < p.len:
+    let c = p[ctx.i]
+    case ctx.state
+    of STATE_NORMAL: ctx.stateNormal(c)
+    of STATE_TILDE: ctx.stateTilde(c)
+    of STATE_BSLASH: ctx.stateBSlash(c)
+    of STATE_DOLLAR: ?ctx.stateDollar(c)
+    of STATE_IDENT: ctx.stateIdent(c)
+    of STATE_CURLY_START: ?ctx.stateCurlyStart(c)
+    of STATE_CURLY: ?ctx.stateCurly(c)
+    of STATE_CURLY_HASH: ?ctx.stateCurlyHash(c)
+    of STATE_CURLY_PERC: ?ctx.stateCurlyPerc(c)
+    of STATE_CURLY_COLON: ?ctx.stateCurlyColon(c)
+    of STATE_CURLY_EXPAND: ?ctx.stateCurlyExpand(c)
+    of STATE_DONE: break
+    inc ctx.i
+  case ctx.state
+  of STATE_NORMAL, STATE_DONE: discard
+  of STATE_TILDE: ctx.flushTilde()
+  of STATE_BSLASH: ctx.s &= '\\'
+  of STATE_DOLLAR: ctx.s &= '$'
+  of STATE_IDENT: ctx.flushIdent()
+  of STATE_CURLY_START, STATE_CURLY, STATE_CURLY_HASH, STATE_CURLY_PERC,
+      STATE_CURLY_COLON:
+    return err("} expected")
+  of STATE_CURLY_EXPAND:
+    ?ctx.flushCurlyExpand("")
+  starti = ctx.i
+  return ok(ctx.s)
+
+proc unquote(p: string): ChaPathResult[string] =
+  var dummy = 0
+  return unquote(p, dummy, none(char))
+
+proc toJS*(ctx: JSContext, p: ChaPath): JSValue =
+  toJS(ctx, $p)
+
+proc fromJS2*(ctx: JSContext, val: JSValue, o: var JSResult[ChaPath]) =
+  o = cast[JSResult[ChaPath]](fromJS[string](ctx, val))
+
+proc unquote*(p: ChaPath): ChaPathResult[string] =
+  let s = ?unquote(string(p))
+  return ok(normalizedPath(s))
diff --git a/src/config/config.nim b/src/config/config.nim
index 82b532c4..5f8e9c17 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -3,6 +3,7 @@ import options
 import os
 import streams
 
+import config/chapath
 import config/mailcap
 import config/mimetypes
 import config/toml
@@ -86,12 +87,12 @@ type
     document_charset* {.jsgetset.}: seq[Charset]
 
   ExternalConfig = object
-    tmpdir* {.jsgetset.}: string
+    tmpdir* {.jsgetset.}: ChaPath
     editor* {.jsgetset.}: string
-    mailcap* {.jsgetset.}: seq[string]
-    mime_types* {.jsgetset.}: seq[string]
-    cgi_dir* {.jsgetset.}: seq[string]
-    urimethodmap* {.jsgetset.}: seq[string]
+    mailcap* {.jsgetset.}: seq[ChaPath]
+    mime_types* {.jsgetset.}: seq[ChaPath]
+    cgi_dir* {.jsgetset.}: seq[ChaPath]
+    urimethodmap* {.jsgetset.}: seq[ChaPath]
     w3m_cgi_compat* {.jsgetset.}: bool
 
   InputConfig = object
@@ -121,7 +122,7 @@ type
   Config* = ref ConfigObj
   ConfigObj* = object
     configdir {.jsget.}: string
-    `include` {.jsget.}: seq[string]
+    `include` {.jsget.}: seq[ChaPath]
     start* {.jsget.}: StartConfig
     search* {.jsget.}: SearchConfig
     css* {.jsget.}: CSSConfig
@@ -224,7 +225,7 @@ func getDefaultHeaders*(config: Config): Headers =
 proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar,
     headers: Headers, referer_from, scripting: bool, charsets: seq[Charset],
     images: bool, userstyle: string, proxy: URL, mimeTypes: MimeTypes,
-    urimethodmap: URIMethodMap): BufferConfig =
+    urimethodmap: URIMethodMap, cgiDir: seq[string]): BufferConfig =
   let filter = newURLFilter(
     scheme = some(location.scheme),
     allowschemes = @["data"],
@@ -242,7 +243,7 @@ proc getBufferConfig*(config: Config, location: URL, cookiejar: CookieJar,
       filter: filter,
       cookiejar: cookiejar,
       proxy: proxy,
-      cgiDir: config.external.cgi_dir,
+      cgiDir: cgiDir,
       urimethodmap: urimethodmap,
       w3mCGICompat: config.external.w3m_cgi_compat
     )
@@ -327,16 +328,20 @@ func getRealKey(key: string): string =
     realk &= '\\'
   return realk
 
-proc openFileExpand(dir, file: string): FileStream =
+proc openFileExpand(dir: string, file: ChaPath): FileStream =
+  let file0 = file.unquote()
+  if file0.isNone:
+    raise newException(ValueError, file0.error)
+  let file = file0.get
   if file.len == 0:
     return nil
-  if file[0] == '~' or file[0] == '/':
-    return newFileStream(expandPath(file))
+  if file[0] == '/':
+    return newFileStream(file)
   else:
-    return newFileStream(expandPath(dir / file))
+    return newFileStream(dir / file)
 
 proc readUserStylesheet(dir, file: string): string =
-  let s = openFileExpand(dir, file)
+  let s = openFileExpand(dir, ChaPath(file))
   if s != nil:
     result = s.readAll()
     s.close()
@@ -390,8 +395,11 @@ proc getURIMethodMap*(config: Config): URIMethodMap =
   return urimethodmap
 
 proc getForkServerConfig*(config: Config): ForkServerConfig =
+  let tmpdir0 = config.external.tmpdir.unquote()
+  if tmpdir0.isNone:
+    raise newException(ValueError, tmpdir0.error)
   return ForkServerConfig(
-    tmpdir: config.external.tmpdir,
+    tmpdir: tmpdir0.get,
     ambiguous_double: config.display.double_width_ambiguous
   )
 
@@ -411,6 +419,7 @@ proc loadConfig*(config: Config, s: string) {.jsfunc.} =
 proc parseConfigValue(x: var object, v: TomlValue, k: string)
 proc parseConfigValue(x: var bool, v: TomlValue, k: string)
 proc parseConfigValue(x: var string, v: TomlValue, k: string)
+proc parseConfigValue(x: var ChaPath, v: TomlValue, k: string)
 proc parseConfigValue[T](x: var seq[T], v: TomlValue, k: string)
 proc parseConfigValue(x: var Charset, v: TomlValue, k: string)
 proc parseConfigValue(x: var int32, v: TomlValue, k: string)
@@ -463,12 +472,16 @@ proc parseConfigValue(x: var string, v: TomlValue, k: string) =
   typeCheck(v, VALUE_STRING, k)
   x = v.s
 
+proc parseConfigValue(x: var ChaPath, v: TomlValue, k: string) =
+  typeCheck(v, VALUE_STRING, k)
+  x = ChaPath(v.s)
+
 proc parseConfigValue[T](x: var seq[T], v: TomlValue, k: string) =
   typeCheck(v, {VALUE_STRING, VALUE_ARRAY}, k)
   if v.vt != VALUE_ARRAY:
     var y: T
     parseConfigValue(y, v, k)
-    x.add(y)
+    x = @[y]
   else:
     if not v.ad:
       x.setLen(0)
@@ -597,7 +610,7 @@ proc parseConfig(config: Config, dir: string, t: TomlValue) =
     config.`include`.setLen(0)
     for s in includes:
       when nimvm:
-        config.parseConfig(dir, staticRead(dir / s))
+        config.parseConfig(dir, staticRead(dir / string(s)))
       else:
         config.parseConfig(dir, openFileExpand(dir, s))
   config.configdir = dir
@@ -629,7 +642,10 @@ proc staticReadConfig(): ConfigObj =
 const defaultConfig = staticReadConfig()
 
 proc readConfig(config: Config, dir, name: string) =
-  let fs = openFileExpand(dir, name)
+  let fs = if name.len > 0 and name[0] == '/':
+    newFileStream(name)
+  else:
+    newFileStream(dir / name)
   if fs != nil:
     config.parseConfig(dir, fs)
 
diff --git a/src/config/toml.nim b/src/config/toml.nim
index 4c195871..9932cc83 100644
--- a/src/config/toml.nim
+++ b/src/config/toml.nim
@@ -236,6 +236,7 @@ proc consumeString(state: var TomlParser, first: char):
       of '\\': res &= '\\'
       of 'u', 'U': res &= ?state.consumeEscape(c)
       of '\n': ml_trim = true
+      of '$': res &= "\\$" # special case for substitution in paths
       else: return state.err("invalid escape sequence \\" & c)
       escape = false
     elif ml_trim: