diff options
| author | Kartik K. Agaram <vc@akkartik.com> | 2021-06-18 19:54:01 -0700 |
|---|---|---|
| committer | Kartik K. Agaram <vc@akkartik.com> | 2021-06-18 19:54:01 -0700 |
| commit | 24275c3828ce2bcda8984762c8bcfcdd0a79ef00 (patch) | |
| tree | 737ae29d1c625f99376304984ea991820088c4c7 | |
| parent | dd133c2a095228e8424a388ca575a8086ee34690 (diff) | |
| download | mu-24275c3828ce2bcda8984762c8bcfcdd0a79ef00.tar.gz | |
newlines are now a token
| -rw-r--r-- | shell/gap-buffer.mu | 4 | ||||
| -rw-r--r-- | shell/tokenize.mu | 12 |
2 files changed, 12 insertions, 4 deletions
diff --git a/shell/gap-buffer.mu b/shell/gap-buffer.mu index 7acaeb42..0077d6fd 100644 --- a/shell/gap-buffer.mu +++ b/shell/gap-buffer.mu @@ -1187,7 +1187,7 @@ fn test-read-from-gap-buffer { check-ints-equal x, 0/nul, "F - test-read-from-gap-buffer/right-4" } -fn skip-whitespace-from-gap-buffer self: (addr gap-buffer) { +fn skip-spaces-from-gap-buffer self: (addr gap-buffer) { var done?/eax: boolean <- gap-buffer-scan-done? self compare done?, 0/false break-if-!= @@ -1195,8 +1195,6 @@ fn skip-whitespace-from-gap-buffer self: (addr gap-buffer) { { compare g, 0x20/space break-if-= - compare g, 0xa/newline - break-if-= return } g <- read-from-gap-buffer self diff --git a/shell/tokenize.mu b/shell/tokenize.mu index 8c7583a9..775f7df7 100644 --- a/shell/tokenize.mu +++ b/shell/tokenize.mu @@ -296,7 +296,17 @@ fn test-tokenize-stream-literal-in-tree { fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trace) { trace-text trace, "tokenize", "next-token" trace-lower trace - skip-whitespace-from-gap-buffer in + skip-spaces-from-gap-buffer in + { + var g/eax: grapheme <- peek-from-gap-buffer in + compare g, 0xa/newline + break-if-!= + g <- read-from-gap-buffer in + var out-token/eax: (addr token) <- copy _out-token + var out-token-type/eax: (addr int) <- get out-token, type + copy-to *out-token-type, 2/skip + return + } { var done?/eax: boolean <- gap-buffer-scan-done? in compare done?, 0/false |