7329 - snapshot: advent day 4 part 2

I've found two bugs in SubX libraries:

1. next-word had an out-of-bounds read
2. next-word was skipping comments, because that's what I need during bootstrapping.
I've created a new variant called next-raw-word that doesn't skip comments.
These really need better names.

We're now at the point where 4b.mu has the right structure and returns
identical result to 4a.mu.

16 files changed, 147 insertions, 2 deletions

diff --git a/127next-word.subx b/127next-word.subx
index 6705437f..5af326d4 100644
--- a/127next-word.subx
+++ b/127next-word.subx
@@ -5,7 +5,8 @@
 # . op          subop               mod             rm32          base        index         scale       r32
 # . 1-3 bytes   3 bits              2 bits          3 bits        3 bits      3 bits        2 bits      2 bits      0/1/2/4 bytes   0/1/2/4 bytes
 
-# (re)compute the bounds of the next word in the line
+# (re)compute the bounds of the next word in the line (surrounded by whitespace,
+# treating '#' comments as a single word)
 # return empty string on reaching end of file
 next-word:  # line: (addr stream byte), out: (addr slice)
     # . prologue
@@ -56,6 +57,7 @@ $next-word:comment:
     8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  0/index/eax   .           0/r32/eax   0xc/disp8       .                 # copy esi+eax+12 to eax
     89/copy                         1/mod/*+disp8   7/rm32/edi    .           .             .           0/r32/eax   4/disp8         .                 # copy eax to *(edi+4)
     # . line->read = line->write
+    8b/copy                         0/mod/indirect  6/rm32/esi    .           .             .           0/r32/eax   .               .                 # copy *esi to eax
     89/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           0/r32/eax   4/disp8         .                 # copy eax to *(esi+4)
     # . return
     eb/jump  $next-word:end/disp8
@@ -298,3 +300,63 @@ test-next-word-returns-empty-string-on-newline:
     89/copy                         3/mod/direct    4/rm32/esp    .           .             .           5/r32/ebp   .               .                 # copy ebp to esp
     5d/pop-to-ebp
     c3/return
+
+# (re)compute the bounds of the next word in the line (separated by whitespace)
+# return empty string on reaching end of file
+next-raw-word:  # line: (addr stream byte), out: (addr slice)
+    # . prologue
+    55/push-ebp
+    89/copy                         3/mod/direct    5/rm32/ebp    .           .             .           4/r32/esp   .               .                 # copy esp to ebp
+    # . save registers
+    50/push-eax
+    51/push-ecx
+    56/push-esi
+    57/push-edi
+    # esi = line
+    8b/copy                         1/mod/*+disp8   5/rm32/ebp    .           .             .           6/r32/esi   8/disp8         .                 # copy *(ebp+8) to esi
+    # edi = out
+    8b/copy                         1/mod/*+disp8   5/rm32/ebp    .           .             .           7/r32/edi   0xc/disp8       .                 # copy *(ebp+12) to edi
+    # skip-chars-matching-whitespace(line)
+    # . . push args
+    ff          6/subop/push        1/mod/*+disp8   5/rm32/ebp    .           .             .           .           8/disp8         .                 # push *(ebp+8)
+    # . . call
+    e8/call  skip-chars-matching-whitespace/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
+$next-raw-word:check0:
+    # if (line->read >= line->write) clear out and return
+    # . eax = line->read
+    8b/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           0/r32/eax   4/disp8         .                 # copy *(esi+4) to eax
+    # . if (eax < line->write) goto next check
+    3b/compare                      0/mod/indirect  6/rm32/esi    .           .             .           0/r32/eax   .               .                 # compare eax with *esi
+    7c/jump-if-<  $next-raw-word:word-exists/disp8
+    # . return out
+    c7          0/subop/copy        0/mod/direct    7/rm32/edi    .           .             .           .           .               0/imm32           # copy to *edi
+    c7          0/subop/copy        1/mod/*+disp8   7/rm32/edi    .           .             .           .           4/disp8         0/imm32           # copy to *(edi+4)
+    eb/jump  $next-raw-word:end/disp8
+$next-raw-word:word-exists:
+    # out->start = &line->data[line->read]
+    8b/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           1/r32/ecx   4/disp8         .                 # copy *(esi+4) to ecx
+    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  1/index/ecx   .           0/r32/eax   0xc/disp8       .                 # copy esi+ecx+12 to eax
+    89/copy                         0/mod/indirect  7/rm32/edi    .           .             .           0/r32/eax   .               .                 # copy eax to *edi
+    # skip-chars-not-matching-whitespace(line)  # including trailing newline
+    # . . push args
+    ff          6/subop/push        1/mod/*+disp8   5/rm32/ebp    .           .             .           .           8/disp8         .                 # push *(ebp+8)
+    # . . call
+    e8/call  skip-chars-not-matching-whitespace/disp32
+    # . . discard args
+    81          0/subop/add         3/mod/direct    4/rm32/esp    .           .             .           .           .               4/imm32           # add to esp
+    # out->end = &line->data[line->read]
+    8b/copy                         1/mod/*+disp8   6/rm32/esi    .           .             .           1/r32/ecx   4/disp8         .                 # copy *(esi+4) to ecx
+    8d/copy-address                 1/mod/*+disp8   4/rm32/sib    6/base/esi  1/index/ecx   .           0/r32/eax   0xc/disp8       .                 # copy esi+ecx+12 to eax
+    89/copy                         1/mod/*+disp8   7/rm32/edi    .           .             .           0/r32/eax   4/disp8         .                 # copy eax to *(edi+4)
+$next-raw-word:end:
+    # . restore registers
+    5f/pop-to-edi
+    5e/pop-to-esi
+    59/pop-to-ecx
+    58/pop-to-eax
+    # . epilogue
+    89/copy                         3/mod/direct    4/rm32/esp    .           .             .           5/r32/ebp   .               .                 # copy ebp to esp
+    5d/pop-to-ebp
+    c3/return
diff --git a/400.mu b/400.mu
index d3ff0e9b..23e3ce2d 100644
--- a/400.mu
+++ b/400.mu
@@ -113,7 +113,8 @@ sig write-stream-data f: (addr buffered-file), s: (addr stream byte)
 sig write-int32-decimal out: (addr stream byte), n: int
 sig is-decimal-digit? c: grapheme -> _/eax: boolean
 sig to-decimal-digit in: grapheme -> _/eax: int
-sig next-word line: (addr stream byte), out: (addr slice)
+sig next-word line: (addr stream byte), out: (addr slice)  # skips '#' comments
+sig next-raw-word line: (addr stream byte), out: (addr slice)  # does not skip '#' comments
 sig has-metadata? word: (addr slice), s: (addr string) -> _/eax: boolean
 sig is-valid-name? in: (addr slice) -> _/eax: boolean
 sig is-label? word: (addr slice) -> _/eax: boolean
diff --git a/apps/advent2020/4b.mu b/apps/advent2020/4b.mu
new file mode 100644
index 00000000..acc3b89d
--- /dev/null
+++ b/apps/advent2020/4b.mu
@@ -0,0 +1,82 @@
+# https://adventofcode.com/2020/day/4
+#
+# To run (on Linux):
+#   $ git clone https://github.com/akkartik/mu
+#   $ cd mu
+#   $ ./translate_mu apps/advent2020/4b.mu
+#   $ ./a.elf < input
+#
+# You'll need to register to download the 'input' file for yourself.
+
+fn main -> _/ebx: int {
+  var curr-passport-field-count/esi: int <- copy 0
+  var valid-passport-count/edi: int <- copy 0
+  var line-storage: (stream byte 0x100)  # 256 bytes
+  var line/ecx: (addr stream byte) <- address line-storage
+  var key-slice-storage: slice
+  var key-slice/edx: (addr slice) <- address key-slice-storage
+  var val-slice-storage: slice
+  var val-slice/ebx: (addr slice) <- address val-slice-storage
+  $main:line-loop: {
+    # read line from stdin
+    clear-stream line
+    read-line-from-real-keyboard line
+    # if line is empty (not even a newline), quit
+    var done?/eax: boolean <- stream-empty? line
+    compare done?, 0  # false
+    break-if-!=
+    print-stream-to-real-screen line
+    # if line has just a newline, process passport
+    skip-chars-matching-whitespace line
+    var new-passport?/eax: boolean <- stream-empty? line
+    {
+      compare new-passport?, 0  # false
+      break-if-=
+      compare curr-passport-field-count, 7
+      {
+        break-if-!=
+        valid-passport-count <- increment
+        print-string 0, "=> "
+        print-int32-decimal 0, valid-passport-count
+        print-string 0, "\n"
+      }
+      curr-passport-field-count <- copy 0
+      loop $main:line-loop
+    }
+    $main:word-loop: {
+      skip-chars-matching-whitespace line
+      var done?/eax: boolean <- stream-empty? line
+      compare done?, 0  # false
+      break-if-!=
+      next-token line, 0x3a, key-slice  # ':'
+      var dummy/eax: byte <- read-byte line  # skip ':'
+      next-raw-word line, val-slice
+      print-slice-to-real-screen key-slice
+      print-string 0, " : "
+      print-slice-to-real-screen val-slice
+      print-string 0, "\n"
+      # treat cid as optional
+      var optional?/eax: boolean <- slice-equal? key-slice, "cid"
+      compare optional?, 0  # false
+      {
+        break-if-!=
+        # otherwise assume there are no invalid fields and no duplicate fields
+        curr-passport-field-count <- increment
+        print-string 0, "-> "
+        print-int32-decimal 0, curr-passport-field-count
+        print-string 0, "\n"
+      }
+      loop
+    }
+    loop
+  }
+  # process final passport
+  compare curr-passport-field-count, 7
+  {
+    break-if-!=
+    valid-passport-count <- increment
+  }
+  print-int32-decimal 0, valid-passport-count
+  print-string 0, "\n"
+  return 0
+}
diff --git a/apps/assort b/apps/assort
index ee69e7e4..edf90788 100755
--- a/apps/assort
+++ b/apps/assort
Binary files differdiff --git a/apps/braces b/apps/braces
index cd098220..610431a5 100755
--- a/apps/braces
+++ b/apps/braces
Binary files differdiff --git a/apps/calls b/apps/calls
index 440185ca..19dfedb4 100755
--- a/apps/calls
+++ b/apps/calls
Binary files differdiff --git a/apps/crenshaw2-1 b/apps/crenshaw2-1
index 5a88adb7..d0ac4e5e 100755
--- a/apps/crenshaw2-1
+++ b/apps/crenshaw2-1
Binary files differdiff --git a/apps/crenshaw2-1b b/apps/crenshaw2-1b
index 95447807..c285a509 100755
--- a/apps/crenshaw2-1b
+++ b/apps/crenshaw2-1b
Binary files differdiff --git a/apps/dquotes b/apps/dquotes
index 69183207..d4bef5ad 100755
--- a/apps/dquotes
+++ b/apps/dquotes
Binary files differdiff --git a/apps/factorial b/apps/factorial
index 7040ea3e..305695b6 100755
--- a/apps/factorial
+++ b/apps/factorial
Binary files differdiff --git a/apps/hex b/apps/hex
index 22db822a..c8f3d5aa 100755
--- a/apps/hex
+++ b/apps/hex
Binary files differdiff --git a/apps/mu b/apps/mu
index 913cf1b1..e7183ec5 100755
--- a/apps/mu
+++ b/apps/mu
Binary files differdiff --git a/apps/pack b/apps/pack
index 29e59e58..0665c0b9 100755
--- a/apps/pack
+++ b/apps/pack
Binary files differdiff --git a/apps/sigils b/apps/sigils
index 3300a975..b14ff723 100755
--- a/apps/sigils
+++ b/apps/sigils
Binary files differdiff --git a/apps/survey b/apps/survey
index 1adeef7c..67808f49 100755
--- a/apps/survey
+++ b/apps/survey
Binary files differdiff --git a/apps/tests b/apps/tests
index b606df1d..b673925e 100755
--- a/apps/tests
+++ b/apps/tests
Binary files differ