diff options
| -rw-r--r-- | shell/tokenize.mu | 17 |
1 files changed, 12 insertions, 5 deletions
diff --git a/shell/tokenize.mu b/shell/tokenize.mu index 775f7df7..39922a83 100644 --- a/shell/tokenize.mu +++ b/shell/tokenize.mu @@ -11,6 +11,7 @@ fn tokenize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) trace-text trace, "tokenize", "tokenize" trace-lower trace rewind-gap-buffer in + var at-start-of-line?/edi: boolean <- copy 1/true { var done?/eax: boolean <- gap-buffer-scan-done? in compare done?, 0/false @@ -18,7 +19,7 @@ fn tokenize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) # var token-storage: token var token/edx: (addr token) <- address token-storage - next-token in, token, trace + at-start-of-line? <- next-token in, token, at-start-of-line?, trace var error?/eax: boolean <- has-errors? trace compare error?, 0/false { @@ -293,7 +294,9 @@ fn test-tokenize-stream-literal-in-tree { check empty?, "F - test-tokenize-stream-literal-in-tree: empty?" } -fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trace) { +# caller is responsible for threading start-of-line? between calls to next-token +# 'in' may contain whitespace if start-of-line? +fn next-token in: (addr gap-buffer), _out-token: (addr token), start-of-line?: boolean, trace: (addr trace) -> _/edi: boolean { trace-text trace, "tokenize", "next-token" trace-lower trace skip-spaces-from-gap-buffer in @@ -301,20 +304,22 @@ fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trac var g/eax: grapheme <- peek-from-gap-buffer in compare g, 0xa/newline break-if-!= + trace-text trace, "tokenize", "newline" g <- read-from-gap-buffer in var out-token/eax: (addr token) <- copy _out-token var out-token-type/eax: (addr int) <- get out-token, type copy-to *out-token-type, 2/skip - return + return 1/at-start-of-line } { var done?/eax: boolean <- gap-buffer-scan-done? in compare done?, 0/false break-if-= + trace-text trace, "tokenize", "end" var out-token/eax: (addr token) <- copy _out-token var out-token-type/eax: (addr int) <- get out-token, type copy-to *out-token-type, 2/skip - return + return 1/at-start-of-line } var _g/eax: grapheme <- peek-from-gap-buffer in var g/ecx: grapheme <- copy _g @@ -363,6 +368,7 @@ fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trac compare g, 0x23/comment break-if-!= rest-of-line in, out, trace + copy-to start-of-line?, 1/true break $next-token:case } # special-case: '-' @@ -399,7 +405,7 @@ fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trac compare g, 0x5d/close-square-bracket break-if-!= error trace, "unbalanced ']'" - return + return start-of-line? } # other brackets are always single-char tokens { @@ -464,6 +470,7 @@ fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trac write-stream stream, out trace trace, "tokenize", stream } + return start-of-line? } fn next-symbol-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) { |