strip spaces when tokenizing

Thanks Max Bernstein for reporting this.

2 files changed, 24 insertions, 0 deletions

diff --git a/shell/sandbox.mu b/shell/sandbox.mu
index 7d85ed59..37e94c12 100644
--- a/shell/sandbox.mu
+++ b/shell/sandbox.mu
@@ -210,6 +210,29 @@ fn test-run-integer {
   check-screen-row screen, 2/y, "=> 1 ", "F - test-run-integer/2"
 }
 
+fn test-run-with-spaces {
+  var sandbox-storage: sandbox
+  var sandbox/esi: (addr sandbox) <- address sandbox-storage
+  initialize-sandbox sandbox
+  # type "1"
+  edit-sandbox sandbox, 0x20/space
+  edit-sandbox sandbox, 0x31/1
+  edit-sandbox sandbox, 0x20/space
+  edit-sandbox sandbox, 0xa/newline
+  # eval
+  edit-sandbox sandbox, 0x13/ctrl-s
+  # setup: screen
+  var screen-on-stack: screen
+  var screen/edi: (addr screen) <- address screen-on-stack
+  initialize-screen screen, 0x80/width, 0x10/height
+  #
+  render-sandbox screen, sandbox, 0/x, 0/y, 0x80/width, 0x10/height
+  check-screen-row screen, 0/y, " 1   ", "F - test-run-with-spaces/0"
+  check-screen-row screen, 1/y, "     ", "F - test-run-with-spaces/1"
+  check-screen-row screen, 2/y, "...  ", "F - test-run-with-spaces/2"
+  check-screen-row screen, 3/y, "=> 1 ", "F - test-run-with-spaces/3"
+}
+
 fn test-run-error-invalid-integer {
   var sandbox-storage: sandbox
   var sandbox/esi: (addr sandbox) <- address sandbox-storage
diff --git a/shell/tokenize.mu b/shell/tokenize.mu
index d28f59dc..8391ae7e 100644
--- a/shell/tokenize.mu
+++ b/shell/tokenize.mu
@@ -9,6 +9,7 @@ fn tokenize in: (addr gap-buffer), out: (addr stream cell), trace: (addr trace)
   var token-storage: cell
   var token/edx: (addr cell) <- address token-storage
   {
+    skip-whitespace-from-gap-buffer in
     var done?/eax: boolean <- gap-buffer-scan-done? in
     compare done?, 0/false
     break-if-!=