local CGI improvements, move data: to cgi-bin

error codes are WIP, not final yet...

8 files changed, 163 insertions, 61 deletions

diff --git a/Makefile b/Makefile
index 8441fc19..ea0512af 100644
--- a/Makefile
+++ b/Makefile
@@ -41,7 +41,7 @@ endif
 FLAGS += --nimcache:"$(OBJDIR)/$(TARGET)"
 
 .PHONY: all
-all: $(OUTDIR_BIN)/cha $(OUTDIR_LIBEXEC)/gopher2html $(OUTDIR_CGI_BIN)/gmifetch $(OUTDIR_LIBEXEC)/gmi2html $(OUTDIR_CGI_BIN)/cha-finger $(OUTDIR_CGI_BIN)/about
+all: $(OUTDIR_BIN)/cha $(OUTDIR_LIBEXEC)/gopher2html $(OUTDIR_CGI_BIN)/gmifetch $(OUTDIR_LIBEXEC)/gmi2html $(OUTDIR_CGI_BIN)/cha-finger $(OUTDIR_CGI_BIN)/about $(OUTDIR_CGI_BIN)/data
 
 $(OUTDIR_BIN)/cha: lib/libquickjs.a src/*.nim src/**/*.nim res/* res/**/*
 	@mkdir -p "$(OUTDIR)/$(TARGET)/bin"
@@ -68,6 +68,9 @@ $(OUTDIR_CGI_BIN)/cha-finger: adapter/finger/cha-finger
 $(OUTDIR_CGI_BIN)/about: adapter/about/about.nim
 	$(NIMC) $(FLAGS) -o:"$(OUTDIR_CGI_BIN)/about" adapter/about/about.nim
 
+$(OUTDIR_CGI_BIN)/data: adapter/data/data.nim
+	$(NIMC) $(FLAGS) -o:"$(OUTDIR_CGI_BIN)/data" adapter/data/data.nim
+
 CFLAGS = -g -Wall -O2 -DCONFIG_VERSION=\"$(shell cat lib/quickjs/VERSION)\"
 QJSOBJ = $(OBJDIR)/quickjs
 
@@ -144,6 +147,7 @@ uninstall:
 	rm -f $(LIBEXECDIR_CHAWAN)/gopher2html
 	rm -f $(LIBEXECDIR_CHAWAN)/gmi2html
 	rm -f $(LIBEXECDIR_CHAWAN)/cgi-bin/about
+	rm -f $(LIBEXECDIR_CHAWAN)/cgi-bin/data
 	rm -f $(LIBEXECDIR_CHAWAN)/cgi-bin/gmifetch
 	rm -f $(LIBEXECDIR_CHAWAN)/cgi-bin/cha-finger
 	rmdir $(LIBEXECDIR_CHAWAN)/cgi-bin && rmdir $(LIBEXECDIR_CHAWAN) || true
diff --git a/adapter/data/data.nim b/adapter/data/data.nim
new file mode 100644
index 00000000..7b022976
--- /dev/null
+++ b/adapter/data/data.nim
@@ -0,0 +1,29 @@
+import std/envvars
+import std/base64
+import std/strutils
+
+import utils/twtstr
+
+proc main() =
+  let str = getEnv("MAPPED_URI_PATH")
+  const si = "data:".len # start index
+  var ct = str.until(',', si)
+  for c in ct:
+    if c notin AsciiAlphaNumeric and c != '/':
+      stdout.write("Cha-Control: ConnectionError -7 invalid data URL")
+      return
+  let sd = si + ct.len + 1 # data start
+  let body = percentDecode(str, sd)
+  if ct.endsWith(";base64"):
+    try:
+      let d = base64.decode(body) # decode from ct end + 1
+      ct.setLen(ct.len - ";base64".len) # remove base64 indicator
+      stdout.write("Content-Type: " & ct & "\n\n")
+      stdout.write(d)
+    except ValueError:
+      stdout.write("Cha-Control: ConnectionError -7 invalid data URL")
+  else:
+    stdout.write("Content-Type: " & ct & "\n\n")
+    stdout.write(body)
+
+main()
diff --git a/doc/localcgi.md b/doc/localcgi.md
index 9943fbda..56927634 100644
--- a/doc/localcgi.md
+++ b/doc/localcgi.md
@@ -34,8 +34,47 @@ use a custom scheme for local CGI instead of interpreting all requests to
 a designated path as a CGI request. (This incompatibility is bridged over when
 `external.cgi-dir` is true.)
 
-Also, for now Chawan has no equivalent to the W3m-control headers (but this
-may change in the future).
+## Headers
+
+Local CGI scripts may send some headers that Chawan will interpret
+specially (and thus will not pass forward to e.g. the fetch API, etc):
+
+* `Status`: interpreted as the HTTP status code.
+* `Cha-Control`: special header, see below.
+
+Note that these headers MUST be sent before any regular headers. Headers
+received after a regular header or a `Cha-Control: ControlDone` header will be
+treated as regular headers.
+
+The `Cha-Control` header's value is parsed as follows:
+
+```
+Cha-Control-Value = Command *Parameter
+Command = ALPHA *ALPHA
+Parameter = *SPACE *CHAR
+```
+
+In other words, it is `Command [Param1] [Param2] ...`.
+
+Currently available commands are:
+
+* `Connected`: Takes no parameters. Must be the first reported header;
+  it means that connection to the server has been successfully established,
+  but no data has been received yet. When any other header is sent first,
+  Chawan will act as if a `Cha-Control: Connected` header had been implicitly
+  sent before that.
+* `ConnectionError`: Must be the first reported header. Parameter 1 is the
+  error code, see below. If any following parameters are given, they are
+  concatenated to form a custom error message. TODO implement this
+* `ControlDone`: Signals that no more special headers will be sent; this
+  means that `Cha-Control` and `Status` headers sent after this must be
+  interpreted as regular headers (and thus e.g. will be available for JS
+  code calling the script using the fetch API).  
+  WARNING: this header must be sent before any non-hardcoded headers that
+  take external input. For example, a HTTP client would have to send
+  `Cha-Control: ControlDone` before returning the retrieved headers.
+
+TODO insert list of public error codes here
 
 ## Environment variables
 
diff --git a/src/config/config.nim b/src/config/config.nim
index db4af966..3a14377f 100644
--- a/src/config/config.nim
+++ b/src/config/config.nim
@@ -411,6 +411,7 @@ const DefaultURIMethodMap = parseURIMethodMap("""
 finger:		cgi-bin:cha-finger?%s
 gemini:		cgi-bin:gmifetch?%s
 about:		cgi-bin:about
+data:		cgi-bin:data
 """)
 
 proc getURIMethodMap*(config: Config): URIMethodMap =
diff --git a/src/loader/cgi.nim b/src/loader/cgi.nim
index 592111c9..659047da 100644
--- a/src/loader/cgi.nim
+++ b/src/loader/cgi.nim
@@ -58,6 +58,66 @@ proc setupEnv(cmd, scriptName, pathInfo, requestURI: string, request: Request,
       putEnv("HTTPS_proxy", s)
     putEnv("ALL_PROXY", s)
 
+type ControlResult = enum
+  RESULT_CONTROL_DONE, RESULT_CONTROL_CONTINUE, RESULT_ERROR
+
+proc handleFirstLine(handle: LoaderHandle, line: string, headers: Headers,
+    status: var int): ControlResult =
+  let k = line.until(':')
+  if k.len == line.len:
+    # invalid
+    discard handle.sendResult(ERROR_CGI_MALFORMED_HEADER)
+    return RESULT_ERROR
+  let v = line.substr(k.len + 1).strip()
+  if k.equalsIgnoreCase("Status"):
+    status = parseInt32(v).get(0)
+    return RESULT_CONTROL_CONTINUE
+  if k.equalsIgnoreCase("Cha-Control"):
+    if v.startsWithIgnoreCase("Connected"):
+      discard handle.sendResult(0) # success
+      return RESULT_CONTROL_CONTINUE
+    elif v.startsWithIgnoreCase("ConnectionError"):
+      let errs = v.substr("ConnectionError".len + 1).split(' ')
+      if errs.len == 0:
+        discard handle.sendResult(ERROR_CGI_INVALID_CHA_CONTROL)
+      else:
+        let fb = int32(ERROR_CGI_INVALID_CHA_CONTROL)
+        let code = int(parseInt32(errs[0]).get(fb))
+        discard handle.sendResult(code)
+      return RESULT_ERROR
+    elif v.startsWithIgnoreCase("ControlDone"):
+      return RESULT_CONTROL_DONE
+    discard handle.sendResult(ERROR_CGI_INVALID_CHA_CONTROL)
+    return RESULT_ERROR
+  headers.add(k, v)
+  return RESULT_CONTROL_DONE
+
+proc handleControlLine(handle: LoaderHandle, line: string, headers: Headers,
+    status: var int): ControlResult =
+  let k = line.until(':')
+  if k.len == line.len:
+    # invalid
+    return RESULT_ERROR
+  let v = line.substr(k.len + 1).strip()
+  if k.equalsIgnoreCase("Status"):
+    status = parseInt32(v).get(0)
+    return RESULT_CONTROL_CONTINUE
+  if k.equalsIgnoreCase("Cha-Control"):
+    if v.startsWithIgnoreCase("ControlDone"):
+      return RESULT_CONTROL_DONE
+    return RESULT_ERROR
+  headers.add(k, v)
+  return RESULT_CONTROL_DONE
+
+# returns false if transfer was interrupted
+proc handleLine(handle: LoaderHandle, line: string, headers: Headers) =
+  let k = line.until(':')
+  if k.len == line.len:
+    # invalid
+    return
+  let v = line.substr(k.len + 1).strip()
+  headers.add(k, v)
+
 proc loadCGI*(handle: LoaderHandle, request: Request, cgiDir: seq[string],
     prevURL: URL) =
   template t(body: untyped) =
@@ -159,20 +219,28 @@ proc loadCGI*(handle: LoaderHandle, request: Request, cgiDir: seq[string],
     let ps = newPosixStream(pipefd[0])
     let headers = newHeaders()
     var status = 200
-    while not ps.atEnd:
-      let line = ps.readLine()
-      if line == "": #\r\n
-        break
-      let k = line.until(':')
-      if k == line:
-        # invalid?
-        discard
-      else:
-        let v = line.substr(k.len + 1).strip()
-        if k.equalsIgnoreCase("Status"):
-          status = parseInt32(v).get(0)
-        else:
-          headers.add(k, v)
+    if ps.atEnd:
+      # no data?
+      discard handle.sendResult(ERROR_CGI_NO_DATA)
+      return
+    let line = ps.readLine()
+    if line == "": #\r\n
+      # no headers, body comes immediately
+      t handle.sendResult(0) # success
+    else:
+      var res = handle.handleFirstLine(line, headers, status)
+      if res == RESULT_ERROR:
+        return
+      while not ps.atEnd and res == RESULT_CONTROL_CONTINUE:
+        let line = ps.readLine()
+        res = handle.handleControlLine(line, headers, status)
+        if res == RESULT_ERROR:
+          return
+      while not ps.atEnd:
+        let line = ps.readLine()
+        if line == "": #\r\n
+          break
+        handle.handleLine(line, headers)
     t handle.sendStatus(status)
     t handle.sendHeaders(headers)
     var buffer: array[4096, uint8]
diff --git a/src/loader/connecterror.nim b/src/loader/connecterror.nim
index 8f2f95d2..913e007f 100644
--- a/src/loader/connecterror.nim
+++ b/src/loader/connecterror.nim
@@ -1,6 +1,9 @@
 import bindings/curl
 
 type ConnectErrorCode* = enum
+  ERROR_CGI_NO_DATA = (-17, "CGI script returned no data")
+  ERROR_CGI_MALFORMED_HEADER = (-16, "CGI script returned a malformed header")
+  ERROR_CGI_INVALID_CHA_CONTROL = (-15, "CGI got invalid Cha-Control header")
   ERROR_TOO_MANY_REWRITES = (-14, "too many URI method map rewrites")
   ERROR_INVALID_URI_METHOD_ENTRY = (-13, "invalid URI method entry")
   ERROR_CGI_FILE_NOT_FOUND = (-12, "CGI file not found")
@@ -8,8 +11,8 @@ type ConnectErrorCode* = enum
   ERROR_FAIL_SETUP_CGI = (-10, "failed to set up CGI script")
   ERROR_NO_CGI_DIR = (-9, "no local-CGI directory configured")
   ERROR_INVALID_METHOD = (-8, "invalid method")
-  ERROR_INVALID_DATA_URL = (-7, "invalid data URL")
-  ERROR_ABOUT_PAGE_NOT_FOUND = (-6, "about page not found")
+  ERROR_INVALID_URL = (-7, "invalid URL")
+  ERROR_CONNECTION_REFUSED = (-6, "connection refused")
   ERROR_FILE_NOT_FOUND = (-5, "file not found")
   ERROR_SOURCE_NOT_FOUND = (-4, "clone source could not be found")
   ERROR_LOADER_KILLED = (-3, "loader killed during transfer")
diff --git a/src/loader/data.nim b/src/loader/data.nim
deleted file mode 100644
index 832bb9b9..00000000
--- a/src/loader/data.nim
+++ /dev/null
@@ -1,38 +0,0 @@
-import base64
-import strutils
-
-import loader/connecterror
-import loader/headers
-import loader/loaderhandle
-import loader/request
-import types/url
-import utils/twtstr
-
-proc loadData*(handle: LoaderHandle, request: Request) =
-  template t(body: untyped) =
-    if not body:
-      return
-  var str = $request.url
-  let si = "data:".len # start index
-  var ct = ""
-  for i in si ..< str.len:
-    if str[i] == ',':
-      break
-    ct &= str[i]
-  let sd = si + ct.len + 1 # data start
-  let s = percentDecode(str, sd)
-  if ct.endsWith(";base64"):
-    try:
-      let d = base64.decode(s) # decode from ct end + 1
-      t handle.sendResult(0)
-      t handle.sendStatus(200)
-      ct.setLen(ct.len - ";base64".len) # remove base64 indicator
-      t handle.sendHeaders(newHeaders({"Content-Type": ct}))
-      t handle.sendData(d)
-    except ValueError:
-      discard handle.sendResult(ERROR_INVALID_DATA_URL)
-  else:
-    t handle.sendResult(0)
-    t handle.sendStatus(200)
-    t handle.sendHeaders(newHeaders({"Content-Type": ct}))
-    t handle.sendData(s)
diff --git a/src/loader/loader.nim b/src/loader/loader.nim
index 5fb49c07..b6250098 100644
--- a/src/loader/loader.nim
+++ b/src/loader/loader.nim
@@ -31,7 +31,6 @@ import js/javascript
 import loader/cgi
 import loader/connecterror
 import loader/curlhandle
-import loader/data
 import loader/file
 import loader/ftp
 import loader/gopher
@@ -147,9 +146,6 @@ proc loadResource(ctx: LoaderContext, request: Request, handle: LoaderHandle) =
       let handleData = handle.loadHttp(ctx.curlm, request)
       if handleData != nil:
         ctx.handleList.add(handleData)
-    of "data":
-      handle.loadData(request)
-      handle.close()
     of "ftp", "ftps", "sftp":
       let handleData = handle.loadFtp(ctx.curlm, request)
       if handleData != nil: