start emitting indent tokens

5 files changed, 227 insertions, 31 deletions

diff --git a/mu-init.subx b/mu-init.subx
index 4ba24788..4bd7abc6 100644
--- a/mu-init.subx
+++ b/mu-init.subx
@@ -14,8 +14,9 @@ Entry:
   bd/copy-to-ebp 0/imm32
   #
 #?   (main 0 0 Primary-bus-secondary-drive)
-#?   (test-tokenize-backquote)
-#?   (test-tokenize-stream-literal)
+#?   (test-tokenize-indent)
+#?   (test-run-integer)
+#?   (test-run-expand-trace)
   # always first run tests
   (run-tests)
   (num-test-failures)  # => eax
diff --git a/shell/parenthesize.mu b/shell/parenthesize.mu
new file mode 100644
index 00000000..67b50854
--- /dev/null
+++ b/shell/parenthesize.mu
@@ -0,0 +1,21 @@
+# TODO: not really implemented yet
+fn parenthesize in: (addr stream token), out: (addr stream token), trace: (addr trace) {
+  trace-text trace, "parenthesize", "insert parens"
+  trace-lower trace
+  rewind-stream in
+  {
+    var done?/eax: boolean <- stream-empty? in
+    compare done?, 0/false
+    break-if-!=
+    #
+    var token-storage: token
+    var token/edx: (addr token) <- address token-storage
+    read-from-stream in, token
+    var is-indent?/eax: boolean <- indent-token? token
+    compare is-indent?, 0/false
+    loop-if-!=
+    write-to-stream out, token  # shallow copy
+    loop
+  }
+  trace-higher trace
+}
diff --git a/shell/read.mu b/shell/read.mu
index 3804c9e4..e5eea010 100644
--- a/shell/read.mu
+++ b/shell/read.mu
@@ -1,8 +1,7 @@
 fn read-cell in: (addr gap-buffer), out: (addr handle cell), trace: (addr trace) {
-  # TODO: we may be able to generate tokens lazily and drop this stream.
-  # Depends on how we implement indent-sensitivity and infix.
+  # eagerly tokenize everything so that the phases are easier to see in the trace
   var tokens-storage: (stream token 0x400)
-  var tokens/ecx: (addr stream token) <- address tokens-storage
+  var tokens/edx: (addr stream token) <- address tokens-storage
   tokenize in, tokens, trace
   var error?/eax: boolean <- has-errors? trace
   compare error?, 0/false
@@ -10,7 +9,16 @@ fn read-cell in: (addr gap-buffer), out: (addr handle cell), trace: (addr trace)
     break-if-=
     return
   }
-  # TODO: insert parens
+  # insert more parens based on indentation
+  var parenthesized-tokens-storage: (stream token 0x400)
+  var parenthesized-tokens/ecx: (addr stream token) <- address parenthesized-tokens-storage
+  parenthesize tokens, parenthesized-tokens, trace
+  var error?/eax: boolean <- has-errors? trace
+  compare error?, 0/false
+  {
+    break-if-=
+    return
+  }
   # TODO: transform infix
-  parse-input tokens, out, trace
+  parse-input parenthesized-tokens, out, trace
 }
diff --git a/shell/sandbox.mu b/shell/sandbox.mu
index 2fa49e30..39173f99 100644
--- a/shell/sandbox.mu
+++ b/shell/sandbox.mu
@@ -959,8 +959,8 @@ fn test-run-expand-trace {
   check-background-color-in-screen-row screen, 7/bg=cursor, 2/y, " ||||||", "F - test-run-expand-trace/expand-1/cursor"
   check-screen-row screen,                                  3/y, " ...   ", "F - test-run-expand-trace/expand-2"
   check-background-color-in-screen-row screen, 7/bg=cursor, 3/y, "       ", "F - test-run-expand-trace/expand-2/cursor"
-  check-screen-row screen,                                  4/y, " 1 pars", "F - test-run-expand-trace/expand-2"
-  check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "       ", "F - test-run-expand-trace/expand-2/cursor"
+  check-screen-row screen,                                  4/y, " 1 inse", "F - test-run-expand-trace/expand-3"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "       ", "F - test-run-expand-trace/expand-3/cursor"
 }
 
 fn test-run-can-rerun-when-expanding-trace {
@@ -1006,7 +1006,7 @@ fn test-run-can-rerun-when-expanding-trace {
   check-background-color-in-screen-row screen, 7/bg=cursor, 2/y, " ||||||", "F - test-run-can-rerun-when-expanding-trace/pre2-1/cursor"
   check-screen-row screen,                                  3/y, " ...   ", "F - test-run-can-rerun-when-expanding-trace/pre2-2"
   check-background-color-in-screen-row screen, 7/bg=cursor, 3/y, "       ", "F - test-run-can-rerun-when-expanding-trace/pre2-2/cursor"
-  check-screen-row screen,                                  4/y, " 1 pars", "F - test-run-can-rerun-when-expanding-trace/pre2-2"
+  check-screen-row screen,                                  4/y, " 1 inse", "F - test-run-can-rerun-when-expanding-trace/pre2-2"
   check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "       ", "F - test-run-can-rerun-when-expanding-trace/pre2-2/cursor"
   # move cursor down and expand
   edit-sandbox sandbox, 0x6a/j, 0/no-globals, 0/no-disk
@@ -1024,8 +1024,12 @@ fn test-run-can-rerun-when-expanding-trace {
   check-background-color-in-screen-row screen, 7/bg=cursor, 3/y, " ||||||", "F - test-run-can-rerun-when-expanding-trace/expand-2/cursor"
   check-screen-row screen,                                  4/y, " ...   ", "F - test-run-can-rerun-when-expanding-trace/expand-3"
   check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "       ", "F - test-run-can-rerun-when-expanding-trace/expand-3/cursor"
-  check-screen-row screen,                                  5/y, " 2 => 1", "F - test-run-can-rerun-when-expanding-trace/expand-4"
+  check-screen-row screen,                                  5/y, " 2 next", "F - test-run-can-rerun-when-expanding-trace/expand-4"
   check-background-color-in-screen-row screen, 7/bg=cursor, 5/y, "       ", "F - test-run-can-rerun-when-expanding-trace/expand-4/cursor"
+  check-screen-row screen,                                  6/y, " ...   ", "F - test-run-can-rerun-when-expanding-trace/expand-5"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 6/y, "       ", "F - test-run-can-rerun-when-expanding-trace/expand-5/cursor"
+  check-screen-row screen,                                  7/y, " 2 => 1", "F - test-run-can-rerun-when-expanding-trace/expand-6"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 7/y, "       ", "F - test-run-can-rerun-when-expanding-trace/expand-6/cursor"
 }
 
 fn test-run-preserves-trace-view-on-rerun {
@@ -1069,16 +1073,18 @@ fn test-run-preserves-trace-view-on-rerun {
   check-background-color-in-screen-row screen, 7/bg=cursor, 2/y, " ||||||||||            ", "F - test-run-preserves-trace-view-on-rerun/pre2-1/cursor"
   check-screen-row screen,                                  3/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre2-2"
   check-background-color-in-screen-row screen, 7/bg=cursor, 3/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-2/cursor"
-  check-screen-row screen,                                  4/y, " 1 parse               ", "F - test-run-preserves-trace-view-on-rerun/pre2-3"
+  check-screen-row screen,                                  4/y, " 1 insert parens       ", "F - test-run-preserves-trace-view-on-rerun/pre2-3"
   check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-3/cursor"
-  check-screen-row screen,                                  5/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre2-4"
+  check-screen-row screen,                                  5/y, " 1 parse               ", "F - test-run-preserves-trace-view-on-rerun/pre2-4"
   check-background-color-in-screen-row screen, 7/bg=cursor, 5/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-4/cursor"
-  check-screen-row screen,                                  6/y, " 1 macroexpand 7       ", "F - test-run-preserves-trace-view-on-rerun/pre2-5"
+  check-screen-row screen,                                  6/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre2-5"
   check-background-color-in-screen-row screen, 7/bg=cursor, 6/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-5/cursor"
-  check-screen-row screen,                                  7/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre2-6"
+  check-screen-row screen,                                  7/y, " 1 macroexpand 7       ", "F - test-run-preserves-trace-view-on-rerun/pre2-6"
   check-background-color-in-screen-row screen, 7/bg=cursor, 7/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-6/cursor"
-  check-screen-row screen,                                  8/y, " 1 => 7                ", "F - test-run-preserves-trace-view-on-rerun/pre2-7"
+  check-screen-row screen,                                  8/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre2-7"
   check-background-color-in-screen-row screen, 7/bg=cursor, 8/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-7/cursor"
+  check-screen-row screen,                                  9/y, " 1 => 7                ", "F - test-run-preserves-trace-view-on-rerun/pre2-8"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 9/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre2-8/cursor"
   # move cursor down below the macroexpand line and expand
   edit-sandbox sandbox, 0x6a/j, 0/no-globals, 0/no-disk
   render-sandbox screen, sandbox, 0/x, 0/y, 0x80/width, 0x10/height, 1/show-cursor
@@ -1090,6 +1096,8 @@ fn test-run-preserves-trace-view-on-rerun {
   render-sandbox screen, sandbox, 0/x, 0/y, 0x80/width, 0x10/height, 1/show-cursor
   edit-sandbox sandbox, 0x6a/j, 0/no-globals, 0/no-disk
   render-sandbox screen, sandbox, 0/x, 0/y, 0x80/width, 0x10/height, 1/show-cursor
+  edit-sandbox sandbox, 0x6a/j, 0/no-globals, 0/no-disk
+  render-sandbox screen, sandbox, 0/x, 0/y, 0x80/width, 0x10/height, 1/show-cursor
   #
   check-screen-row screen,                                  1/y, " 7                     ", "F - test-run-preserves-trace-view-on-rerun/pre3-0"
   check-background-color-in-screen-row screen, 7/bg=cursor, 1/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-0/cursor"
@@ -1097,16 +1105,18 @@ fn test-run-preserves-trace-view-on-rerun {
   check-background-color-in-screen-row screen, 7/bg=cursor, 2/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-1/cursor"
   check-screen-row screen,                                  3/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-2"
   check-background-color-in-screen-row screen, 7/bg=cursor, 3/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-2/cursor"
-  check-screen-row screen,                                  4/y, " 1 parse               ", "F - test-run-preserves-trace-view-on-rerun/pre3-3"
+  check-screen-row screen,                                  4/y, " 1 insert parens       ", "F - test-run-preserves-trace-view-on-rerun/pre3-3"
   check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-3/cursor"
-  check-screen-row screen,                                  5/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-4"
+  check-screen-row screen,                                  5/y, " 1 parse               ", "F - test-run-preserves-trace-view-on-rerun/pre3-4"
   check-background-color-in-screen-row screen, 7/bg=cursor, 5/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-4/cursor"
-  check-screen-row screen,                                  6/y, " 1 macroexpand 7       ", "F - test-run-preserves-trace-view-on-rerun/pre3-5"
+  check-screen-row screen,                                  6/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-5"
   check-background-color-in-screen-row screen, 7/bg=cursor, 6/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-5/cursor"
-  check-screen-row screen,                                  7/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-6"
-  check-background-color-in-screen-row screen, 7/bg=cursor, 7/y, " |||                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-6/cursor"
-  check-screen-row screen,                                  8/y, " 1 => 7                ", "F - test-run-preserves-trace-view-on-rerun/pre3-7"
-  check-background-color-in-screen-row screen, 7/bg=cursor, 8/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-7/cursor"
+  check-screen-row screen,                                  7/y, " 1 macroexpand 7       ", "F - test-run-preserves-trace-view-on-rerun/pre3-6"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 7/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-6/cursor"
+  check-screen-row screen,                                  8/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-7"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 8/y, " |||                   ", "F - test-run-preserves-trace-view-on-rerun/pre3-7/cursor"
+  check-screen-row screen,                                  9/y, " 1 => 7                ", "F - test-run-preserves-trace-view-on-rerun/pre3-8"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 9/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/pre3-8/cursor"
   # expand
   edit-sandbox sandbox, 0xa/newline, 0/no-globals, 0/no-disk
   clear-screen screen
@@ -1118,14 +1128,16 @@ fn test-run-preserves-trace-view-on-rerun {
   check-background-color-in-screen-row screen, 7/bg=cursor, 2/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-1/cursor"
   check-screen-row screen,                                  3/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/expand-2"
   check-background-color-in-screen-row screen, 7/bg=cursor, 3/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-2/cursor"
-  check-screen-row screen,                                  4/y, " 1 parse               ", "F - test-run-preserves-trace-view-on-rerun/expand-3"
+  check-screen-row screen,                                  4/y, " 1 insert parens       ", "F - test-run-preserves-trace-view-on-rerun/expand-3"
   check-background-color-in-screen-row screen, 7/bg=cursor, 4/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-3/cursor"
-  check-screen-row screen,                                  5/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/expand-4"
+  check-screen-row screen,                                  5/y, " 1 parse               ", "F - test-run-preserves-trace-view-on-rerun/expand-4"
   check-background-color-in-screen-row screen, 7/bg=cursor, 5/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-4/cursor"
-  check-screen-row screen,                                  6/y, " 1 macroexpand 7       ", "F - test-run-preserves-trace-view-on-rerun/expand-5"
+  check-screen-row screen,                                  6/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/expand-5"
   check-background-color-in-screen-row screen, 7/bg=cursor, 6/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-5/cursor"
-  check-screen-row screen,                                  7/y, " 2 macroexpand-iter 7  ", "F - test-run-preserves-trace-view-on-rerun/expand-6"
-  check-background-color-in-screen-row screen, 7/bg=cursor, 7/y, " ||||||||||||||||||||  ", "F - test-run-preserves-trace-view-on-rerun/expand-6/cursor"
-  check-screen-row screen,                                  8/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/expand-7"
-  check-background-color-in-screen-row screen, 7/bg=cursor, 8/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-7/cursor"
+  check-screen-row screen,                                  7/y, " 1 macroexpand 7       ", "F - test-run-preserves-trace-view-on-rerun/expand-6"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 7/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-6/cursor"
+  check-screen-row screen,                                  8/y, " 2 macroexpand-iter 7  ", "F - test-run-preserves-trace-view-on-rerun/expand-7"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 8/y, " ||||||||||||||||||||  ", "F - test-run-preserves-trace-view-on-rerun/expand-7/cursor"
+  check-screen-row screen,                                  9/y, " ...                   ", "F - test-run-preserves-trace-view-on-rerun/expand-8"
+  check-background-color-in-screen-row screen, 7/bg=cursor, 9/y, "                       ", "F - test-run-preserves-trace-view-on-rerun/expand-8/cursor"
 }
diff --git a/shell/tokenize.mu b/shell/tokenize.mu
index 97696cd3..6b1cbffb 100644
--- a/shell/tokenize.mu
+++ b/shell/tokenize.mu
@@ -1,10 +1,13 @@
-# tokens are like cells, but not recursive
+# The language is indent-sensitive.
+# Each line consists of an initial indent token followed by other tokens.
 type token {
   type: int
   # type 0: default
   # type 1: stream
   text-data: (handle stream byte)
   # type 2: skip (end of line or end of file)
+  # type 3: indent
+  number-data: int
 }
 
 fn tokenize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
@@ -54,6 +57,11 @@ fn test-tokenize-number {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-number/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-number/before-indent"
+  read-from-stream stream, curr-token
   var number?/eax: boolean <- number-token? curr-token
   check number?, "F - test-tokenize-number"
   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
@@ -77,6 +85,11 @@ fn test-tokenize-negative-number {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-negative-number/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-negative-number/before-indent"
+  read-from-stream stream, curr-token
   var number?/eax: boolean <- number-token? curr-token
   check number?, "F - test-tokenize-negative-number"
   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
@@ -100,6 +113,11 @@ fn test-tokenize-number-followed-by-hyphen {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-number-followed-by-hyphen/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-number-followed-by-hyphen/before-indent"
+  read-from-stream stream, curr-token
   var number?/eax: boolean <- number-token? curr-token
   check number?, "F - test-tokenize-number-followed-by-hyphen"
   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
@@ -123,6 +141,11 @@ fn test-tokenize-quote {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-quote/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-quote/before-indent"
+  read-from-stream stream, curr-token
   var quote?/eax: boolean <- quote-token? curr-token
   check quote?, "F - test-tokenize-quote: quote"
   read-from-stream stream, curr-token
@@ -150,6 +173,11 @@ fn test-tokenize-backquote {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-backquote/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-backquote/before-indent"
+  read-from-stream stream, curr-token
   var backquote?/eax: boolean <- backquote-token? curr-token
   check backquote?, "F - test-tokenize-backquote: backquote"
   read-from-stream stream, curr-token
@@ -177,6 +205,11 @@ fn test-tokenize-unquote {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-unquote/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-unquote/before-indent"
+  read-from-stream stream, curr-token
   var unquote?/eax: boolean <- unquote-token? curr-token
   check unquote?, "F - test-tokenize-unquote: unquote"
   read-from-stream stream, curr-token
@@ -204,6 +237,11 @@ fn test-tokenize-unquote-splice {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-unquote-splice/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-unquote-splice/before-indent"
+  read-from-stream stream, curr-token
   var unquote-splice?/eax: boolean <- unquote-splice-token? curr-token
   check unquote-splice?, "F - test-tokenize-unquote-splice: unquote-splice"
 }
@@ -224,6 +262,11 @@ fn test-tokenize-dotted-list {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-dotted-list/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-dotted-list/before-indent"
+  read-from-stream stream, curr-token
   var open-paren?/eax: boolean <- open-paren-token? curr-token
   check open-paren?, "F - test-tokenize-dotted-list: open paren"
   read-from-stream stream, curr-token  # skip a
@@ -252,6 +295,11 @@ fn test-tokenize-stream-literal {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-stream-literal/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-stream-literal/before-indent"
+  read-from-stream stream, curr-token
   var stream?/eax: boolean <- stream-token? curr-token
   check stream?, "F - test-tokenize-stream-literal: type"
   var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
@@ -278,6 +326,11 @@ fn test-tokenize-stream-literal-in-tree {
   var curr-token-storage: token
   var curr-token/ebx: (addr token) <- address curr-token-storage
   read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-stream-literal-in-tree/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-stream-literal-in-tree/before-indent"
+  read-from-stream stream, curr-token
   var bracket?/eax: boolean <- bracket-token? curr-token
   check bracket?, "F - test-tokenize-stream-literal-in-tree: open paren"
   read-from-stream stream, curr-token
@@ -294,11 +347,55 @@ fn test-tokenize-stream-literal-in-tree {
   check empty?, "F - test-tokenize-stream-literal-in-tree: empty?"
 }
 
+fn test-tokenize-indent {
+  var in-storage: gap-buffer
+  var in/esi: (addr gap-buffer) <- address in-storage
+  initialize-gap-buffer-with in, "abc\n  def"
+  #
+  var stream-storage: (stream token 0x10)
+  var stream/edi: (addr stream token) <- address stream-storage
+  #
+  var trace-storage: trace
+  var trace/edx: (addr trace) <- address trace-storage
+  initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
+  tokenize in, stream, trace
+  #
+  var curr-token-storage: token
+  var curr-token/ebx: (addr token) <- address curr-token-storage
+  read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-indent/before-indent-type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 0/spaces, "F - test-tokenize-indent/before-indent"
+  read-from-stream stream, curr-token
+  var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
+  var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
+  check-stream-equal curr-token-data, "abc", "F - test-tokenize-indent/before"
+  #
+  read-from-stream stream, curr-token
+  var curr-token-type/eax: (addr int) <- get curr-token, type
+  check-ints-equal *curr-token-type, 3/indent, "F - test-tokenize-indent/type"
+  var curr-token-data/eax: (addr int) <- get curr-token, number-data
+  check-ints-equal *curr-token-data, 2/spaces, "F - test-tokenize-indent"
+  #
+  read-from-stream stream, curr-token
+  var curr-token-data-ah/eax: (addr handle stream byte) <- get curr-token, text-data
+  var curr-token-data/eax: (addr stream byte) <- lookup *curr-token-data-ah
+  check-stream-equal curr-token-data, "def", "F - test-tokenize-indent/after"
+}
+
 # caller is responsible for threading start-of-line? between calls to next-token
 # 'in' may contain whitespace if start-of-line?
 fn next-token in: (addr gap-buffer), out: (addr token), start-of-line?: boolean, trace: (addr trace) -> _/edi: boolean {
   trace-text trace, "tokenize", "next-token"
   trace-lower trace
+  {
+    compare start-of-line?, 0/false
+    break-if-=
+    next-indent-token in, out, trace
+    trace-higher trace
+    return 0/not-at-start-of-line
+  }
   skip-spaces-from-gap-buffer in
   {
     var g/eax: grapheme <- peek-from-gap-buffer in
@@ -701,6 +798,52 @@ fn rest-of-line in: (addr gap-buffer), _out: (addr token), trace: (addr trace) {
   }
 }
 
+fn next-indent-token in: (addr gap-buffer), _out: (addr token), trace: (addr trace) {
+  trace-text trace, "tokenize", "indent"
+  trace-lower trace
+  var out/edi: (addr token) <- copy _out
+  var out-type/eax: (addr int) <- get out, type
+  copy-to *out-type, 3/indent
+  var dest/edi: (addr int) <- get out, number-data
+  copy-to *dest, 0
+  {
+    var done?/eax: boolean <- gap-buffer-scan-done? in
+    compare done?, 0/false
+    break-if-!=
+    var g/eax: grapheme <- peek-from-gap-buffer in
+    {
+      {
+        var should-trace?/eax: boolean <- should-trace? trace
+        compare should-trace?, 0/false
+      }
+      break-if-=
+      var stream-storage: (stream byte 0x40)
+      var stream/esi: (addr stream byte) <- address stream-storage
+      write stream, "next: "
+      var gval/eax: int <- copy g
+      write-int32-hex stream, gval
+      trace trace, "tokenize", stream
+    }
+    # if non-space, break
+    compare g, 0x20/space
+    break-if-!=
+    g <- read-from-gap-buffer in
+    increment *dest
+    loop
+  }
+  trace-higher trace
+  {
+    var should-trace?/eax: boolean <- should-trace? trace
+    compare should-trace?, 0/false
+    break-if-=
+    var stream-storage: (stream byte 0x40)
+    var stream/esi: (addr stream byte) <- address stream-storage
+    write stream, "=> indent "
+    write-int32-hex stream, *dest
+    trace trace, "tokenize", stream
+  }
+}
+
 fn symbol-grapheme? g: grapheme -> _/eax: boolean {
   ## whitespace
   compare g, 9/tab
@@ -1178,6 +1321,17 @@ fn skip-token? _self: (addr token) -> _/eax: boolean {
   return 1/true
 }
 
+fn indent-token? _self: (addr token) -> _/eax: boolean {
+  var self/eax: (addr token) <- copy _self
+  var in-type/eax: (addr int) <- get self, type
+  compare *in-type, 3/indent
+  {
+    break-if-=
+    return 0/false
+  }
+  return 1/true
+}
+
 fn allocate-token _self-ah: (addr handle token) {
   var self-ah/eax: (addr handle token) <- copy _self-ah
   allocate self-ah