7779

2 files changed, 25 insertions, 157 deletions

diff --git a/baremetal/124next-token.subx b/baremetal/124next-token.subx
index a4490e6d..6ad00907 100644
--- a/baremetal/124next-token.subx
+++ b/baremetal/124next-token.subx
@@ -7,159 +7,6 @@
 
 # extract the next run of characters that are different from a given 'delimiter' (skipping multiple delimiters if necessary)
 # on reaching end of file, return an empty interval
-next-token:  # in: (addr stream byte), delimiter: byte, out: (addr slice)
-    # . prologue
-    55/push-ebp
-    89/copy                         3/mod/direct    5/rm32/ebp    .           .             .           4/r32/esp   .               .                 # copy esp to ebp
-    # . save registers
-    50/push-eax
-    51/push-ecx
-    56/push-esi
-    57/push-edi
-    # esi = in
-    8b/copy                         1/mod/*+disp8   5/rm32/ebp    .           .             .           6/r32/esi   8/disp8         .                 # copy *(ebp+8) to esi
-    # edi = out
-    8b/copy                         1/mod/*+disp8   5/rm32/ebp    .           .             .           7/r32/edi   0x10/disp8      .                 # copy *(ebp+16) to edi
-    # skip-chars-matching(in, delimiter)
-    # . . push args
-    ff          6/subop/push        1/mod/*+disp8   5/rm32/ebp    .           .             .           .           0xc/disp8       .                 # push *(ebp+12)
-    56/push-esi
-    # . . call
-    e8/call  skip-chars-matching/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
-    # out->start = &in->data[in->read]
-    8b/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           1/r32/ecx   4/disp8         .                 # copy *(esi+4) to ecx
-    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  1/index/ecx   .           0/r32/eax   0xc/disp8       .                 # copy esi+ecx+12 to eax
-    89/copy                         0/mod/indirect  7/rm32/edi    .           .             .           0/r32/eax   .               .                 # copy eax to *edi
-    # skip-chars-not-matching(in, delimiter)
-    # . . push args
-    ff          6/subop/push        1/mod/*+disp8   5/rm32/ebp    .           .             .           .           0xc/disp8       .                 # push *(ebp+12)
-    56/push-esi
-    # . . call
-    e8/call  skip-chars-not-matching/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
-    # out->end = &in->data[in->read]
-    8b/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           1/r32/ecx   4/disp8         .                 # copy *(esi+4) to ecx
-    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  1/index/ecx   .           0/r32/eax   0xc/disp8       .                 # copy esi+ecx+12 to eax
-    89/copy                         1/mod/*+disp8   7/rm32/edi    .           .             .           0/r32/eax   4/disp8         .                 # copy eax to *(edi+4)
-    # . restore registers
-    5f/pop-to-edi
-    5e/pop-to-esi
-    59/pop-to-ecx
-    58/pop-to-eax
-    # . epilogue
-    89/copy                         3/mod/direct    4/rm32/esp    .           .             .           5/r32/ebp   .               .                 # copy ebp to esp
-    5d/pop-to-ebp
-    c3/return
-
-test-next-token:
-    # . prologue
-    55/push-ebp
-    89/copy                         3/mod/direct    5/rm32/ebp    .           .             .           4/r32/esp   .               .                 # copy esp to ebp
-    # setup
-    # . clear-stream(_test-stream)
-    # . . push args
-    68/push  _test-stream/imm32
-    # . . call
-    e8/call  clear-stream/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
-    # var slice/ecx: slice
-    68/push  0/imm32/end
-    68/push  0/imm32/start
-    89/copy                         3/mod/direct    1/rm32/ecx    .           .             .           4/r32/esp   .               .                 # copy esp to ecx
-    # write(_test-stream, "  ab")
-    # . . push args
-    68/push  "  ab"/imm32
-    68/push  _test-stream/imm32
-    # . . call
-    e8/call  write/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
-    # next-token(_test-stream, 0x20/space, slice)
-    # . . push args
-    51/push-ecx
-    68/push  0x20/imm32
-    68/push  _test-stream/imm32
-    # . . call
-    e8/call  next-token/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
-    # check-ints-equal(slice->start - _test-stream->data, 2, msg)
-    # . check-ints-equal(slice->start - _test-stream, 14, msg)
-    # . . push args
-    68/push  "F - test-next-token: start"/imm32
-    68/push  0xe/imm32
-    # . . push slice->start - _test-stream
-    8b/copy                         0/mod/indirect  1/rm32/ecx    .           .             .           0/r32/eax   .               .                 # copy *ecx to eax
-    81          5/subop/subtract    3/mod/direct    0/rm32/eax    .           .             .           .           .               _test-stream/imm32 # subtract from eax
-    50/push-eax
-    # . . call
-    e8/call  check-ints-equal/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
-    # check-ints-equal(slice->end - _test-stream->data, 4, msg)
-    # . check-ints-equal(slice->end - _test-stream, 16, msg)
-    # . . push args
-    68/push  "F - test-next-token: end"/imm32
-    68/push  0x10/imm32
-    # . . push slice->end - _test-stream
-    8b/copy                         1/mod/*+disp8   1/rm32/ecx    .           .             .           0/r32/eax   4/disp8         .                 # copy *(ecx+4) to eax
-    81          5/subop/subtract    3/mod/direct    0/rm32/eax    .           .             .           .           .               _test-stream/imm32 # subtract from eax
-    50/push-eax
-    # . . call
-    e8/call  check-ints-equal/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
-    # . epilogue
-    89/copy                         3/mod/direct    4/rm32/esp    .           .             .           5/r32/ebp   .               .                 # copy ebp to esp
-    5d/pop-to-ebp
-    c3/return
-
-test-next-token-Eof:
-    # . prologue
-    55/push-ebp
-    89/copy                         3/mod/direct    5/rm32/ebp    .           .             .           4/r32/esp   .               .                 # copy esp to ebp
-    # setup
-    # . clear-stream(_test-stream)
-    # . . push args
-    68/push  _test-stream/imm32
-    # . . call
-    e8/call  clear-stream/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
-    # var slice/ecx: slice
-    68/push  0/imm32/end
-    68/push  0/imm32/start
-    89/copy                         3/mod/direct    1/rm32/ecx    .           .             .           4/r32/esp   .               .                 # copy esp to ecx
-    # write nothing to _test-stream
-    # next-token(_test-stream, 0x20/space, slice)
-    # . . push args
-    51/push-ecx
-    68/push  0x20/imm32
-    68/push  _test-stream/imm32
-    # . . call
-    e8/call  next-token/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
-    # check-ints-equal(slice->end, slice->start, msg)
-    # . . push args
-    68/push  "F - test-next-token-Eof"/imm32
-    ff          6/subop/push        1/mod/*+disp8   1/rm32/ecx    .           .             .           .           4/disp8         .                 # push *(ecx+4)
-    ff          6/subop/push        0/mod/indirect  1/rm32/ecx    .           .             .           .           .               .                 # push *ecx
-    # . . call
-    e8/call  check-ints-equal/disp32
-    # . . discard args
-    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
-    # . epilogue
-    89/copy                         3/mod/direct    4/rm32/esp    .           .             .           5/r32/ebp   .               .                 # copy ebp to esp
-    5d/pop-to-ebp
-    c3/return
-
-# extract the next run of characters that are different from a given 'delimiter' (skipping multiple delimiters if necessary)
-# on reaching end of file, return an empty interval
 next-token-from-slice:  # start: (addr byte), end: (addr byte), delimiter: byte, out: (addr slice)
     # . prologue
     55/push-ebp
diff --git a/baremetal/shell/read.mu b/baremetal/shell/read.mu
index ced559b1..b6446b55 100644
--- a/baremetal/shell/read.mu
+++ b/baremetal/shell/read.mu
@@ -7,10 +7,31 @@ fn read-cell in: (addr gap-buffer), out: (addr handle cell) {
   #   token tree
   #   syntax tree
   rewind-gap-buffer in
-  read-symbol in, out
+  var token-storage: (stream byte 0x1000)  # strings can be large
+  var token/ecx: (addr stream byte) <- address token-storage
+  {
+    var done?/eax: boolean <- gap-buffer-scan-done? in
+    compare done?, 0/false
+    break-if-!=
+    next-token in, token
+    read-symbol token, out
+    loop
+  }
 }
 
-fn read-symbol in: (addr gap-buffer), _out: (addr handle cell) {
+fn next-token in: (addr gap-buffer), out: (addr stream byte) {
+  clear-stream out
+  {
+    var done?/eax: boolean <- gap-buffer-scan-done? in
+    compare done?, 0/false
+    break-if-!=
+    var g/eax: grapheme <- read-from-gap-buffer in
+    write-grapheme out, g
+    loop
+  }
+}
+
+fn read-symbol in: (addr stream byte), _out: (addr handle cell) {
   var out/eax: (addr handle cell) <- copy _out
   new-symbol out
   var out-a/eax: (addr cell) <- lookup *out
@@ -18,10 +39,10 @@ fn read-symbol in: (addr gap-buffer), _out: (addr handle cell) {
   var _out-data/eax: (addr stream byte) <- lookup *out-data-ah
   var out-data/edi: (addr stream byte) <- copy _out-data
   {
-    var done?/eax: boolean <- gap-buffer-scan-done? in
+    var done?/eax: boolean <- stream-empty? in
     compare done?, 0/false
     break-if-!=
-    var g/eax: grapheme <- read-from-gap-buffer in
+    var g/eax: grapheme <- read-grapheme in
     write-grapheme out-data, g
     loop
   }