diff options
| -rw-r--r-- | shell/tokenize.mu | 58 |
1 files changed, 58 insertions, 0 deletions
diff --git a/shell/tokenize.mu b/shell/tokenize.mu index d591646c..6734e13f 100644 --- a/shell/tokenize.mu +++ b/shell/tokenize.mu @@ -30,6 +30,39 @@ fn tokenize in: (addr gap-buffer), out: (addr stream cell), trace: (addr trace) trace-higher trace } +fn test-tokenize-dotted-list { + # in: "(a . b)" + var in-storage: gap-buffer + var in/esi: (addr gap-buffer) <- address in-storage + initialize-gap-buffer in, 0x10 + add-code-point-at-gap in, 0x28/open-paren + add-code-point-at-gap in, 0x61/a + add-code-point-at-gap in, 0x20/space + add-code-point-at-gap in, 0x2e/dot + add-code-point-at-gap in, 0x20/space + add-code-point-at-gap in, 0x62/b + add-code-point-at-gap in, 0x29/close-paren + # + var stream-storage: (stream cell 0x10) + var stream/edi: (addr stream cell) <- address stream-storage + # + tokenize in, stream, 0/no-trace + # + var curr-token-storage: cell + var curr-token/ebx: (addr cell) <- address curr-token-storage + read-from-stream stream, curr-token + var open-paren?/eax: boolean <- open-paren-token? curr-token + check open-paren?, "F - test-tokenize-dotted-list: open paren" + read-from-stream stream, curr-token # skip a + read-from-stream stream, curr-token + var dot?/eax: boolean <- dot-token? curr-token + check dot?, "F - test-tokenize-dotted-list: dot" + read-from-stream stream, curr-token # skip b + read-from-stream stream, curr-token + var close-paren?/eax: boolean <- close-paren-token? curr-token + check close-paren?, "F - test-tokenize-dotted-list: close paren" +} + fn next-token in: (addr gap-buffer), _out-cell: (addr cell), trace: (addr trace) { trace-text trace, "read", "next-token" trace-lower trace @@ -585,3 +618,28 @@ fn close-paren-token? _in: (addr cell) -> _/eax: boolean { } return 0/false } + +fn dot-token? _in: (addr cell) -> _/eax: boolean { + var in/eax: (addr cell) <- copy _in + var in-data-ah/eax: (addr handle stream byte) <- get in, text-data + var _in-data/eax: (addr stream byte) <- lookup *in-data-ah + var in-data/ecx: (addr stream byte) <- copy _in-data + rewind-stream in-data + var g/eax: grapheme <- read-grapheme in-data + compare g, 0x2e/dot + { + break-if-!= + var result/eax: boolean <- stream-empty? in-data + return result + } + return 0/false +} + +fn test-dot-token { + var tmp-storage: (handle cell) + var tmp-ah/eax: (addr handle cell) <- address tmp-storage + new-symbol tmp-ah, "." + var tmp/eax: (addr cell) <- lookup *tmp-ah + var result/eax: boolean <- dot-token? tmp + check result, "F - test-dot-token" +} |