# out is not allocated
fn read-cell in: (addr gap-buffer), out: (addr handle cell), trace: (addr trace) {
trace-text trace, "read", ""
trace-lower trace
trace-text trace, "read", "tokenize"
rewind-gap-buffer in
var token-storage: (stream byte 0x1000) # strings can be large
var token/ecx: (addr stream byte) <- address token-storage
{
var done?/eax: boolean <- gap-buffer-scan-done? in
compare done?, 0/false
break-if-!=
next-token in, token, trace
var error?/eax: boolean <- has-errors? trace
compare error?, 0/false
break-if-!=
read-symbol token, out
loop
}
trace-higher trace # tokenize
# TODO:
# insert parens
# transform infix
# token tree
# syntax tree
trace-higher trace # read
}
fn next-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
trace-text trace, "read", "next-token"
trace-lower trace
$next-token:body: {
clear-stream out
skip-whitespace-from-gap-buffer in
var g/eax: grapheme <- peek-from-gap-buffer in
{
var digit?/eax: boolean <- is-decimal-digit? g
compare digit?, 0/false
break-if-=
next-number-token in, out, trace
break $next-token:body
}
next-symbol-token in, out, trace
}
trace-higher trace
var stream-storage: (stream byte 0x40)
var stream/eax: (addr stream byte) <- address stream-storage
write stream, "next-token: result"
trace trace, "read", stream
}
fn next-symbol-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
{
var done?/eax: boolean <- gap-buffer-scan-done? in
compare done?, 0/false
break-if-!=
var g/eax: grapheme <- peek-from-gap-buffer in
# if non-symbol, return
var symbol-grapheme?/eax: boolean <- is-symbol-grapheme? g
{
compare symbol-grapheme?, 0/false
break-if-!=
return
}
var g/eax: grapheme <- read-from-gap-buffer in
write-grapheme out, g
loop
}
}
fn is-symbol-grapheme? g: grapheme -> _/eax: boolean {
compare g, 0x20/space
{
break-if-!=
return 0/false
}
compare g, 0xa/newline
{
break-if-!=
return 0/false
}
compare g, 0x22/double-quote
{
break-if-!=
return 0/false
}
compare g, 0x28/open-paren
{
break-if-!=
return 0/false
}
compare g, 0x29/close-paren
{
break-if-!=
return 0/false
}
compare g, 0x5b/open-square-bracket
{
break-if-!=
return 0/false
}
compare g, 0x5d/close-square-bracket
{
break-if-!=
return 0/false
}
compare g, 0x7b/open-curly-bracket
{
break-if-!=
return 0/false
}
compare g, 0x7d/close-curly-bracket
{
break-if-!=
return 0/false
}
return 1/true
}
fn next-number-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
var done?/eax: boolean <- gap-buffer-scan-done? in
compare done?, 0/false
break-if-!=
var g/eax: grapheme <- peek-from-gap-buffer in
# if not symbol grapheme, return
{
var symbol-grapheme?/eax: boolean <- is-symbol-grapheme? g
compare symbol-grapheme?, 0/false
break-if-!=
return
}
# if not digit grapheme, abort
{
var digit?/eax: boolean <- is-decimal-digit? g
compare digit?, 0/false
break-if-!=
error trace, "invalid number"
return
}
var g/eax: grapheme <- read-from-gap-buffer in
write-grapheme out, g
loop
}
fn read-symbol in: (addr stream byte), _out: (addr handle cell) {
var out/eax: (addr handle cell) <- copy _out
new-symbol out
var out-a/eax: (addr cell) <- lookup *out
var out-data-ah/eax: (addr handle stream byte) <- get out-a, text-data
var _out-data/eax: (addr stream byte) <- lookup *out-data-ah
var out-data/edi: (addr stream byte) <- copy _out-data
{
var done?/eax: boolean <- stream-empty? in
compare done?, 0/false
break-if-!=
var g/eax: grapheme <- read-grapheme in
write-grapheme out-data, g
loop
}
}