about summary refs log tree commit diff stats
diff options
context:
space:
mode:
authorKartik K. Agaram <vc@akkartik.com>2021-06-18 19:40:35 -0700
committerKartik K. Agaram <vc@akkartik.com>2021-06-18 19:40:35 -0700
commitb5306cabd38da962842aea7d536d44e7edca50bc (patch)
tree0fcb1f4470c3b087f7a45218ac9bdbff2260f353
parent2acc00f9b90111e64a3671c59cdeba02c9874f99 (diff)
downloadmu-b5306cabd38da962842aea7d536d44e7edca50bc.tar.gz
.
-rw-r--r--shell/tokenize.mu8
1 files changed, 2 insertions, 6 deletions
diff --git a/shell/tokenize.mu b/shell/tokenize.mu
index 2db1e98f..e019a3b4 100644
--- a/shell/tokenize.mu
+++ b/shell/tokenize.mu
@@ -10,14 +10,14 @@ fn tokenize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace)
   trace-text trace, "tokenize", "tokenize"
   trace-lower trace
   rewind-gap-buffer in
-  var token-storage: token
-  var token/edx: (addr token) <- address token-storage
   {
     skip-whitespace-from-gap-buffer in
     var done?/eax: boolean <- gap-buffer-scan-done? in
     compare done?, 0/false
     break-if-!=
     #
+    var token-storage: token
+    var token/edx: (addr token) <- address token-storage
     next-token in, token, trace
     var error?/eax: boolean <- has-errors? trace
     compare error?, 0/false
@@ -307,10 +307,6 @@ fn next-token in: (addr gap-buffer), _out-token: (addr token), trace: (addr trac
     trace trace, "tokenize", stream
   }
   var out-token/eax: (addr token) <- copy _out-token
-  {
-    var out-token-type/eax: (addr int) <- get out-token, type
-    copy-to *out-token-type, 0/default
-  }
   var out-ah/edi: (addr handle stream byte) <- get out-token, text-data
   $next-token:allocate: {
     # Allocate a large buffer if it's a stream.