7802

Better tracing of the tokenization phase.

3 files changed, 235 insertions, 24 deletions

diff --git a/baremetal/113write-stream.subx b/baremetal/113write-stream.subx
new file mode 100644
index 00000000..7dd93fba
--- /dev/null
+++ b/baremetal/113write-stream.subx
@@ -0,0 +1,173 @@
+# write-stream: like write, but write streams rather than strings
+
+== code
+#   instruction                     effective address                                                   register    displacement    immediate
+# . op          subop               mod             rm32          base        index         scale       r32
+# . 1-3 bytes   3 bits              2 bits          3 bits        3 bits      3 bits        2 bits      2 bits      0/1/2/4 bytes   0/1/2/4 bytes
+
+write-stream:  # f: (addr stream byte), s: (addr stream byte)
+    # . prologue
+    55/push-ebp
+    89/copy                         3/mod/direct    5/rm32/ebp    .           .             .           4/r32/esp   .               .                 # copy esp to ebp
+    # . save registers
+    50/push-eax
+    56/push-esi
+    57/push-edi
+    # edi = f
+    8b/copy                         1/mod/*+disp8   5/rm32/ebp    .           .                         7/r32/edi   8/disp8         .                 # copy *(ebp+8) to edi
+    # esi = s
+    8b/copy                         1/mod/*+disp8   5/rm32/ebp    .           .                         6/r32/esi   0xc/disp8       .                 # copy *(ebp+12) to esi
+    # eax = _append-4(&f->data[f->write], &f->data[f->size], &s->data[s->read], &s->data[s->write])
+    # . . push &s->data[s->write]
+    8b/copy                         0/mod/indirect  6/rm32/esi    .           .             .           0/r32/eax   .               .                 # copy *esi to eax
+    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  0/index/eax   .           0/r32/eax   0xc/disp8       .                 # copy esi+eax+12 to eax
+    50/push-eax
+    # . . push &s->data[s->read]
+    8b/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           0/r32/eax   4/disp8         .                 # copy *(esi+4) to eax
+    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  0/index/eax   .           0/r32/eax   0xc/disp8       .                 # copy esi+eax+12 to eax
+    50/push-eax
+    # . . push &f->data[f->size]
+    8b/copy                         1/mod/*+disp8   7/rm32/edi    .           .             .           0/r32/eax   8/disp8         .                 # copy *(edi+8) to eax
+    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    7/base/edi  0/index/eax   .           0/r32/eax   0xc/disp8       .                 # copy edi+eax+12 to eax
+    50/push-eax
+    # . . push &f->data[f->write]
+    8b/copy                         0/mod/indirect  7/rm32/edi    .           .             .           0/r32/eax   .               .                 # copy *edi to eax
+    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    7/base/edi  0/index/eax   .           0/r32/eax   0xc/disp8       .                 # copy edi+eax+12 to eax
+    50/push-eax
+    # . . call
+    e8/call  _append-4/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0x10/imm32        # add to esp
+    # f->write += eax
+    01/add                          0/mod/indirect  7/rm32/edi    .           .             .           0/r32/eax   .               .                 # add eax to *edi
+    # s->read += eax
+    01/add                          1/mod/*+disp8   6/rm32/esi    .           .             .           0/r32/eax   4/disp8         .                 # add eax to *(esi+4)
+$write-stream:end:
+    # . restore registers
+    5f/pop-to-edi
+    5e/pop-to-esi
+    58/pop-to-eax
+    # . epilogue
+    89/copy                         3/mod/direct    4/rm32/esp    .           .             .           5/r32/ebp   .               .                 # copy ebp to esp
+    5d/pop-to-ebp
+    c3/return
+
+test-write-stream-single:
+    # setup
+    # . clear-stream(_test-stream)
+    # . . push args
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  clear-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
+    # . clear-stream(_test-stream2)
+    # . . push args
+    68/push  _test-stream2/imm32
+    # . . call
+    e8/call  clear-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
+    # . write(_test-stream2, "Ab")
+    # . . push args
+    68/push  "Ab"/imm32
+    68/push  _test-stream2/imm32
+    # . . call
+    e8/call  write/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
+    # write-stream(_test-stream, _test-stream2)
+    # . . push args
+    68/push  _test-stream2/imm32
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  write-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
+    # check-stream-equal(_test-stream, "Ab", msg)
+    # . . push args
+    68/push  "F - test-write-stream-single"/imm32
+    68/push  "Ab"/imm32
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  check-stream-equal/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
+    # . end
+    c3/return
+
+test-write-stream-appends:
+    # setup
+    # . clear-stream(_test-stream)
+    # . . push args
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  clear-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
+    # . clear-stream(_test-stream2)
+    # . . push args
+    68/push  _test-stream2/imm32
+    # . . call
+    e8/call  clear-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
+    # . write(_test-stream2, "C")
+    # . . push args
+    68/push  "C"/imm32
+    68/push  _test-stream2/imm32
+    # . . call
+    e8/call  write/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
+    # first write
+    # . write-stream(_test-stream, _test-stream2)
+    # . . push args
+    68/push  _test-stream2/imm32
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  write-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
+    # second write
+    # . write(_test-stream2, "D")
+    # . . push args
+    68/push  "D"/imm32
+    68/push  _test-stream2/imm32
+    # . . call
+    e8/call  write/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
+    # . write-stream(_test-stream, _test-stream2)
+    # . . push args
+    68/push  _test-stream2/imm32
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  write-stream/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               8/imm32           # add to esp
+    # check-stream-equal(_test-stream, "CD", msg)
+    # . . push args
+    68/push  "F - test-write-stream-appends"/imm32
+    68/push  "CD"/imm32
+    68/push  _test-stream/imm32
+    # . . call
+    e8/call  check-stream-equal/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               0xc/imm32         # add to esp
+    # . end
+    c3/return
+
+== data
+
+_test-stream2:  # (stream byte)
+    # current write index
+    4/imm32
+    # current read index
+    1/imm32
+    # size
+    8/imm32
+    # data
+    41/A 42/B 43/C 44/D 00 00 00 00  # 8 bytes
+
+# . . vim:nowrap:textwidth=0
diff --git a/baremetal/400.mu b/baremetal/400.mu
index 51730e0c..9d0c095f 100644
--- a/baremetal/400.mu
+++ b/baremetal/400.mu
@@ -24,6 +24,7 @@ sig check-stream-equal f: (addr stream byte), s: (addr array byte), msg: (addr a
 sig next-stream-line-equal? f: (addr stream byte), s: (addr array byte) -> _/eax: boolean
 sig check-next-stream-line-equal f: (addr stream byte), s: (addr array byte), msg: (addr array byte)
 sig write f: (addr stream byte), s: (addr array byte)
+sig write-stream f: (addr stream byte), s: (addr stream byte)
 sig read-byte s: (addr stream byte) -> _/eax: byte
 sig append-byte f: (addr stream byte), n: int
 #sig to-hex-char in/eax: int -> out/eax: int
diff --git a/baremetal/shell/read.mu b/baremetal/shell/read.mu
index dcb5b070..6b1a0a82 100644
--- a/baremetal/shell/read.mu
+++ b/baremetal/shell/read.mu
@@ -32,6 +32,14 @@ fn next-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace
     skip-whitespace-from-gap-buffer in
     var g/eax: grapheme <- peek-from-gap-buffer in
     {
+      var stream-storage: (stream byte 0x40)
+      var stream/esi: (addr stream byte) <- address stream-storage
+      write stream, "next: "
+      var gval/eax: int <- copy g
+      write-int32-hex stream, gval
+      trace trace, "read", stream
+    }
+    {
       var digit?/eax: boolean <- is-decimal-digit? g
       compare digit?, 0/false
       break-if-=
@@ -43,27 +51,40 @@ fn next-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace
   trace-higher trace
   var stream-storage: (stream byte 0x40)
   var stream/eax: (addr stream byte) <- address stream-storage
-  write stream, "next-token: result"
+  write stream, "=> "
+  write-stream stream, out
   trace trace, "read", stream
 }
 
 fn next-symbol-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
-  {
+  trace-text trace, "read", "looking for a symbol"
+  trace-lower trace
+  $next-symbol-token:loop: {
     var done?/eax: boolean <- gap-buffer-scan-done? in
     compare done?, 0/false
     break-if-!=
     var g/eax: grapheme <- peek-from-gap-buffer in
+    {
+      var stream-storage: (stream byte 0x40)
+      var stream/esi: (addr stream byte) <- address stream-storage
+      write stream, "next: "
+      var gval/eax: int <- copy g
+      write-int32-hex stream, gval
+      trace trace, "read", stream
+    }
     # if non-symbol, return
-    var symbol-grapheme?/eax: boolean <- is-symbol-grapheme? g
     {
+      var symbol-grapheme?/eax: boolean <- is-symbol-grapheme? g
       compare symbol-grapheme?, 0/false
       break-if-!=
-      return
+      trace-text trace, "read", "stop"
+      break $next-symbol-token:loop
     }
     var g/eax: grapheme <- read-from-gap-buffer in
     write-grapheme out, g
     loop
   }
+  trace-higher trace
 }
 
 fn is-symbol-grapheme? g: grapheme -> _/eax: boolean {
@@ -116,31 +137,47 @@ fn is-symbol-grapheme? g: grapheme -> _/eax: boolean {
 }
 
 fn next-number-token in: (addr gap-buffer), out: (addr stream byte), trace: (addr trace) {
-  var done?/eax: boolean <- gap-buffer-scan-done? in
-  compare done?, 0/false
-  break-if-!=
-  var g/eax: grapheme <- peek-from-gap-buffer in
-  # if not symbol grapheme, return
-  {
-    var symbol-grapheme?/eax: boolean <- is-symbol-grapheme? g
-    compare symbol-grapheme?, 0/false
-    break-if-!=
-    return
-  }
-  # if not digit grapheme, abort
-  {
-    var digit?/eax: boolean <- is-decimal-digit? g
-    compare digit?, 0/false
+  trace-text trace, "read", "looking for a number"
+  trace-lower trace
+  $next-number-token:loop: {
+    var done?/eax: boolean <- gap-buffer-scan-done? in
+    compare done?, 0/false
     break-if-!=
-    error trace, "invalid number"
-    return
+    var g/eax: grapheme <- peek-from-gap-buffer in
+    {
+      var stream-storage: (stream byte 0x40)
+      var stream/esi: (addr stream byte) <- address stream-storage
+      write stream, "next: "
+      var gval/eax: int <- copy g
+      write-int32-hex stream, gval
+      trace trace, "read", stream
+    }
+    # if not symbol grapheme, return
+    {
+      var symbol-grapheme?/eax: boolean <- is-symbol-grapheme? g
+      compare symbol-grapheme?, 0/false
+      break-if-!=
+      trace-text trace, "read", "stop"
+      break $next-number-token:loop
+    }
+    # if not digit grapheme, abort
+    {
+      var digit?/eax: boolean <- is-decimal-digit? g
+      compare digit?, 0/false
+      break-if-!=
+      error trace, "invalid number"
+      return
+    }
+    trace-text trace, "read", "append"
+    var g/eax: grapheme <- read-from-gap-buffer in
+    write-grapheme out, g
+    loop
   }
-  var g/eax: grapheme <- read-from-gap-buffer in
-  write-grapheme out, g
-  loop
+  trace-higher trace
 }
 
 fn read-symbol in: (addr stream byte), _out: (addr handle cell) {
+  rewind-stream in
   var out/eax: (addr handle cell) <- copy _out
   new-symbol out
   var out-a/eax: (addr cell) <- lookup *out