diff options
author | Kartik K. Agaram <vc@akkartik.com> | 2021-05-29 16:07:39 -0700 |
---|---|---|
committer | Kartik K. Agaram <vc@akkartik.com> | 2021-05-29 16:07:39 -0700 |
commit | 7b84872380e409fe6bb636b6a6df31a25c8c109d (patch) | |
tree | 09c372d360ae49cb24bb5ee9d2e87fa199caa3f8 /shell | |
parent | 0bba6ccfc25d77d5148a2a24f0185aefa1481994 (diff) | |
download | mu-7b84872380e409fe6bb636b6a6df31a25c8c109d.tar.gz |
shell: non-stream tokens are now small
Diffstat (limited to 'shell')
-rw-r--r-- | shell/tokenize.mu | 14 |
1 files changed, 11 insertions, 3 deletions
diff --git a/shell/tokenize.mu b/shell/tokenize.mu index 214c0d9b..7c006f93 100644 --- a/shell/tokenize.mu +++ b/shell/tokenize.mu @@ -236,9 +236,17 @@ fn next-token in: (addr gap-buffer), _out-cell: (addr cell), trace: (addr trace) copy-to *out-cell-type, 0/uninitialized } var out-ah/edi: (addr handle stream byte) <- get out-cell, text-data - # TODO: obscenely pessimally sized - # I'm allocating 1KB for every. single. token. Just because a whole definition needs to fit in a string sometimes. Absolutely bonkers. - populate-stream out-ah, 0x400/max-definition-size + $next-token:allocate: { + # Allocate a large buffer if it's a stream. + # Sometimes a whole function definition will need to fit in it. + compare g, 0x5b/open-square-bracket + { + break-if-!= + populate-stream out-ah, 0x400/max-definition-size=1KB + break $next-token:allocate + } + populate-stream out-ah, 0x40 + } var _out/eax: (addr stream byte) <- lookup *out-ah var out/edi: (addr stream byte) <- copy _out clear-stream out |