1 ## insert explicit parens based on indentation
  2 
  3 # Design goals:
  4 #  keywords in other languages should look different from functions: def, if, while, etc.
  5 #  fully-parenthesized expressions should not be messed with
  6 #    ignore indent when lines start with parens
  7 #    ignore indent inside parens
  8 #    no modes to disable this pass
  9 #  introduce no new operators
 10 #    the language doesn't use nested lists like Scheme's `cond`
 11 #    lines with one word are never wrapped in parens
 12 #  encourage macros to explicitly insert all parens
 13 #    ignore indent inside backquote
 14 
 15 fn parenthesize in: (addr stream token), out: (addr stream token), trace: (addr trace) {
 16   trace-text trace, "parenthesize", "insert parens"
 17   trace-lower trace
 18   var buffer-storage: (stream token 0x40)
 19   var buffer/edi: (addr stream token) <- address buffer-storage
 20   var curr-line-indent: int
 21   var num-words-in-line: int
 22   var paren-at-start-of-line?: boolean
 23   var explicit-open-parens-storage: int
 24   var explicit-open-parens/ebx: (addr int) <- address explicit-open-parens-storage
 25   var implicit-open-parens-storage: int-stack
 26   var implicit-open-parens/esi: (addr int-stack) <- address implicit-open-parens-storage
 27   initialize-int-stack implicit-open-parens, 0x10  # potentially a major memory leak
 28   rewind-stream in
 29   {
 30     var done?/eax: boolean <- stream-empty? in
 31     compare done?, 0/false
 32     break-if-!=
 33     #
 34     var curr-token-storage: token
 35     var curr-token/ecx: (addr token) <- address curr-token-storage
 36     read-from-stream in, curr-token
 37 #?     dump-token-from-cursor curr-token
 38     # update state
 39     {
 40       var is-indent?/eax: boolean <- indent-token? curr-token
 41       compare is-indent?, 0/false
 42       break-if-=
 43       copy-to num-words-in-line, 0
 44       copy-to paren-at-start-of-line?, 0/false
 45       var tmp/eax: int <- indent-level curr-token
 46       copy-to curr-line-indent, tmp
 47     }
 48     {
 49       var is-word?/eax: boolean <- word-token? curr-token
 50       compare is-word?, 0/false
 51       break-if-=
 52       increment num-words-in-line
 53     }
 54     {
 55       compare num-words-in-line, 0
 56       break-if-!=
 57       var is-open?/eax: boolean <- open-paren-token? curr-token
 58       compare is-open?, 0/false
 59       break-if-=
 60       copy-to paren-at-start-of-line?, 1/true
 61     }
 62     #
 63     $parenthesize:emit: {
 64       {
 65         compare paren-at-start-of-line?, 0/false
 66         break-if-=
 67 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "A", 7/fg 0/bg
 68         emit-all buffer, curr-token, out, explicit-open-parens
 69         break $parenthesize:emit
 70       }
 71       {
 72         var is-indent?/eax: boolean <- indent-token? curr-token
 73         compare is-indent?, 0/false
 74         break-if-=
 75 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "B", 7/fg 0/bg
 76         emit-all buffer, curr-token, out, explicit-open-parens
 77         break $parenthesize:emit
 78       }
 79       {
 80         compare num-words-in-line, 2
 81         break-if->=
 82 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "C", 7/fg 0/bg
 83         write-to-stream buffer, curr-token
 84         break $parenthesize:emit
 85       }
 86       {
 87         compare num-words-in-line, 2
 88         break-if-!=
 89         var is-word?/eax: boolean <- word-token? curr-token
 90         compare is-word?, 0/false
 91         break-if-=
 92         compare *explicit-open-parens, 0
 93         break-if-!=
 94 #?         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen "(\n", 7/fg 0/bg
 95         var paren-storage: token
 96         var paren-token/eax: (addr token) <- address paren-storage
 97         initialize-token paren-token, "("
 98         write-to-stream out, paren-token
 99         push-int-stack implicit-open-parens, curr-line-indent
100       }
101       emit-all buffer, curr-token, out, explicit-open-parens
102     }
103     {
104       var is-indent?/eax: boolean <- indent-token? curr-token
105       compare is-indent?, 0/false
106       break-if-=
107       {
108         # . loop check
109         var done?/eax: boolean <- int-stack-empty? implicit-open-parens
110         compare done?, 0/false
111         break-if-!=
112         var top-indent/eax: int <- int-stack-top implicit-open-parens
113         compare top-indent, curr-line-indent
114         break-if-<
115         # . loop body
116         var paren-storage: token
117         var paren-token/eax: (addr token) <- address paren-storage
118         initialize-token paren-token, ")"
119         write-to-stream out, paren-token
120         # . update
121         var dummy/eax: int <- pop-int-stack implicit-open-parens
122         loop
123       }
124     }
125     loop
126   }
127   emit-all buffer, 0/no-curr-token, out, explicit-open-parens
128   {
129     # . loop check
130     var done?/eax: boolean <- int-stack-empty? implicit-open-parens
131     compare done?, 0/false
132     break-if-!=
133     # . loop body
134     var paren-storage: token
135     var paren-token/eax: (addr token) <- address paren-storage
136     initialize-token paren-token, ")"
137     write-to-stream out, paren-token
138     # . update
139     var dummy/eax: int <- pop-int-stack implicit-open-parens
140     loop
141   }
142   trace-higher trace
143 }
144 
145 fn indent-level _in: (addr token) -> _/eax: int {
146   var in/eax: (addr token) <- copy _in
147   var result/eax: (addr int) <- get in, number-data
148   return *result
149 }
150 
151 fn word-token? in: (addr token) -> _/eax: boolean {
152   {
153     var is-indent?/eax: boolean <- indent-token? in
154     compare is-indent?, 0/false
155     break-if-!=
156     var is-bracket?/eax: boolean <- bracket-token? in  # overzealously checks for [], but shouldn't ever encounter it
157     compare is-bracket?, 0/false
158     break-if-!=
159     var is-quote?/eax: boolean <- quote-token? in
160     compare is-quote?, 0/false
161     break-if-!=
162     var is-backquote?/eax: boolean <- backquote-token? in
163     compare is-backquote?, 0/false
164     break-if-!=
165     var is-unquote?/eax: boolean <- unquote-token? in
166     compare is-unquote?, 0/false
167     break-if-!=
168     var is-unquote-splice?/eax: boolean <- unquote-splice-token? in
169     compare is-unquote-splice?, 0/false
170     break-if-!=
171     return 1/true
172   }
173   return 0/false
174 }
175 
176 fn emit-all first: (addr stream token), second: (addr token), out: (addr stream token), explicit-open-parens: (addr int) {
177   rewind-stream first
178   {
179     var done?/eax: boolean <- stream-empty? first
180     compare done?, 0/false
181     break-if-!=
182     var curr-token-storage: token
183     var curr-token/eax: (addr token) <- address curr-token-storage
184     read-from-stream first, curr-token
185     emit curr-token, out, explicit-open-parens
186     loop
187   }
188   clear-stream first
189   {
190     compare second, 0
191     break-if-=
192     emit second, out, explicit-open-parens
193   }
194 }
195 
196 fn emit t: (addr token), out: (addr stream token), explicit-open-parens: (addr int) {
197   {
198     var is-indent?/eax: boolean <- indent-token? t
199     compare is-indent?, 0/false
200     break-if-=
201     return
202   }
203   write-to-stream out, t
204   var explicit-open-parens/edi: (addr int) <- copy explicit-open-parens
205   {
206     var is-open?/eax: boolean <- open-paren-token? t
207     compare is-open?, 0/false
208     break-if-=
209     increment *explicit-open-parens
210   }
211   {
212     var is-close?/eax: boolean <- close-paren-token? t
213     compare is-close?, 0/false
214     break-if-=
215     decrement *explicit-open-parens
216     compare *explicit-open-parens, 0
217     break-if->=
218     abort "emit: extra ')'"
219   }
220 }
221 
222 # helper for checking parenthesize
223 fn emit-salient-tokens in: (addr stream token), out: (addr stream token) {
224   rewind-stream in
225   {
226     var done?/eax: boolean <- stream-empty? in
227     compare done?, 0/false
228     break-if-!=
229     var token-storage: token
230     var token/edx: (addr token) <- address token-storage
231     read-from-stream in, token
232     # skip tokens should be skipped
233     var is-skip?/eax: boolean <- skip-token? token
234     compare is-skip?, 0/false
235     loop-if-!=
236     # indent tokens should be skipped
237     var is-indent?/eax: boolean <- indent-token? token
238     compare is-indent?, 0/false
239     loop-if-!=
240     #
241     write-to-stream out, token  # shallow copy
242     loop
243   }
244 }
245 
246 fn test-parenthesize {
247   check-parenthesize "a b c  ", "(a b c)", "F - test-parenthesize/1"
248   check-parenthesize "a (b)", "(a (b))", "F - test-parenthesize/2"
249   check-parenthesize "a (b c)", "(a (b c))", "F - test-parenthesize/3"
250   check-parenthesize "a (b c) d", "(a (b c) d)", "F - test-parenthesize/4"
251   check-parenthesize "a b c\nd ef", "(a b c) (d ef)", "F - test-parenthesize/5-multiple-lines"
252   check-parenthesize "a b c\n  d ef", "(a b c (d ef))", "F - test-parenthesize/6-indented"
253   check-parenthesize "a b c\n  (d ef)", "(a b c (d ef))", "F - test-parenthesize/7-indented"
254   check-parenthesize "a b c\n  (d ef)\n  g", "(a b c (d ef) g)", "F - test-parenthesize/8-indented"
255   check-parenthesize "a b c\n  d e\n    f\ny", "(a b c (d e f)) y", "F - test-parenthesize/9-indented"
256   check-parenthesize "#a\na b", "(a b)", "F - test-parenthesize/10-initial-comment"
257 #? a b c
258 #?     d ef
259 #? 
260 #?   g
261 #?   check-parenthesize "a b c\n    d ef\n\n  g", "(a b c (d ef) g)", "F - test-parenthesize/11-comments"
262 #?   check-parenthesize "a b c\n    d ef\n\n  g #abc", "(a b c (d ef)) g", "F - test-parenthesize/11-comments"
263   check-parenthesize "a b c\n    d ef\n\n  g #abc", "(a b c (d ef) g)", "F - test-parenthesize/11-comments"
264 #? a b c
265 #?   '(d ef)
266 #? 
267 #?   g #abc
268 #?   check-parenthesize "a b c\n  '(d ef)\n  g #abc", "(a b c '(d ef) g)", "F - test-parenthesize/12-quotes-and-comments"
269   check-parenthesize "a b c\n  '(d ef)\n\n  g #abc", "(a b c '(d ef) g)", "F - test-parenthesize/12-quotes-and-comments"
270   check-parenthesize "  a b c", "(a b c)", "F - test-parenthesize/13-initial-indent"
271   check-parenthesize "    a b c\n  34", "(a b c) 34", "F - test-parenthesize/14-initial-indent"
272   check-parenthesize "def foo\n    a b c\n  d e\nnewdef", "(def foo (a b c) (d e)) newdef", "F - test-parenthesize/14"
273   check-parenthesize "  a a\n    a\ny", "(a a a) y", "F - test-parenthesize/15-group-before-too-much-outdent"
274   check-parenthesize "a `(b c)", "(a `(b c))", "F - test-parenthesize/16-backquote"
275   check-parenthesize "'a b c", "('a b c)", "F - test-parenthesize/17-quote"
276   check-parenthesize ",a b c", "(,a b c)", "F - test-parenthesize/18-unquote"
277   check-parenthesize ",@a b c", "(,@a b c)", "F - test-parenthesize/19-unquote-splice"
278   check-parenthesize "a b\n  'c\n  ,d\n  e", "(a b 'c ,d e)", "F - test-parenthesize/20-quotes-are-not-words"
279   check-parenthesize "def foo\n#a b c\n  d e\nnew", "(def foo (d e)) new", "F - test-parenthesize/21-group-across-comments"
280 }
281 
282 fn test-parenthesize-skips-lines-with-initial-parens {
283   check-parenthesize "(a b c)", "(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/1"
284   check-parenthesize "(a (b c))", "(a (b c))", "F - test-parenthesize-skips-lines-with-initial-parens/2"
285   check-parenthesize "(a () b)", "(a () b)", "F - test-parenthesize-skips-lines-with-initial-parens/3"
286   check-parenthesize "  (a b c)", "(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/initial-indent"
287   check-parenthesize "(a b c\n  bc\n    def\n  gh)", "(a b c bc def gh)", "F - test-parenthesize-skips-lines-with-initial-parens/outdent"
288   check-parenthesize "(a b c\n  (def gh)\n    (i j k)\n  lm\n\n\n    (no p))", "(a b c (def gh) (i j k) lm (no p))", "F - test-parenthesize-skips-lines-with-initial-parens/fully-parenthesized"
289   check-parenthesize ",(a b c)", ",(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-unquote"
290   check-parenthesize ",@(a b c)", ",@(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-unquote-splice"
291   check-parenthesize ",,(a b c)", ",,(a b c)", "F - test-parenthesize-skips-lines-with-initial-parens/after-nested-unquote"
292   check-parenthesize "(def foo\n    #a b c\n  d e)\nnew", "(def foo d e) new", "F - test-parenthesize-skips-lines-with-initial-parens/across-comment"
293   check-parenthesize "`(def foo\n    #a b c\n  d e)\nnew", "`(def foo d e) new", "F - test-parenthesize-skips-lines-with-initial-parens/across-comment-after-backquote"
294   check-parenthesize "  (a b c\n    d e)", "(a b c d e)", "F - test-parenthesize-skips-lines-with-initial-parens/with-indent"
295   check-parenthesize "def foo(a (b)\n    c d)\n  d e\nnew", "(def foo (a (b) c d) (d e)) new", "F - test-parenthesize-skips-lines-with-initial-parens/inside-arg-lists"
296 }
297 
298 fn test-parenthesize-skips-single-word-lines {
299   # lines usually get grouped with later indented lines
300   check-parenthesize "a b\n  c", "(a b c)", "F - test-parenthesize-skips-single-word-lines/0"
301   # but single-word lines don't
302   check-parenthesize "a\n  c", "a c", "F - test-parenthesize-skips-single-word-lines/1"
303   check-parenthesize "a", "a", "F - test-parenthesize-skips-single-word-lines/2"
304   check-parenthesize "a  \nb\nc", "a b c", "F - test-parenthesize-skips-single-word-lines/3"
305 }
306 
307 fn check-parenthesize actual: (addr array byte), expected: (addr array byte), message: (addr array byte) {
308   var trace-storage: trace
309   var trace/edx: (addr trace) <- address trace-storage
310   initialize-trace trace, 1/only-errors, 0x10/capacity, 0/visible
311   #
312   var actual-buffer-storage: gap-buffer
313   var actual-buffer/eax: (addr gap-buffer) <- address actual-buffer-storage
314   initialize-gap-buffer-with actual-buffer, actual
315   var actual-tokens-storage: (stream token 0x40)
316   var actual-tokens/esi: (addr stream token) <- address actual-tokens-storage
317   tokenize-and-parenthesize actual-buffer, actual-tokens, trace
318   #
319   var expected-buffer-storage: gap-buffer
320   var expected-buffer/eax: (addr gap-buffer) <- address expected-buffer-storage
321   initialize-gap-buffer-with expected-buffer, expected
322   var expected-tokens-storage: (stream token 0x40)
323   var expected-tokens/edi: (addr stream token) <- address expected-tokens-storage
324   tokenize-salient expected-buffer, expected-tokens, trace
325   #
326   rewind-stream actual-tokens
327   check-token-streams-data-equal actual-tokens, expected-tokens, message
328 }
329 
330 fn check-token-streams-data-equal actual: (addr stream token), expected: (addr stream token), message: (addr array byte) {
331   rewind-stream actual
332   rewind-stream expected
333   {
334     # loop termination checks
335     var actual-done?/eax: boolean <- stream-empty? actual
336     {
337       compare actual-done?, 0/false
338       break-if-=
339       var expected-done?/eax: boolean <- stream-empty? expected
340       compare expected-done?, 0/false
341       {
342         break-if-!=
343         # actual empty, but expected not empty
344         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
345         draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ": too short\n", 3/fg=cyan 0/bg
346         count-test-failure
347         return
348       }
349       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ".", 3/fg/cyan, 0/bg
350       return
351     }
352     var expected-done?/eax: boolean <- stream-empty? expected
353     compare expected-done?, 0/false
354     {
355       break-if-=
356       # actual not empty, but expected empty
357       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
358       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, ": too long\n", 3/fg=cyan 0/bg
359       count-test-failure
360       return
361     }
362     # loop body
363     var curr-token-storage: token
364     var curr-token/ecx: (addr token) <- address curr-token-storage
365     read-from-stream actual, curr-token
366 #?     dump-token-from-cursor curr-token
367     var expected-token-storage: token
368     var expected-token/edx: (addr token) <- address expected-token-storage
369     read-from-stream expected, expected-token
370 #?     dump-token-from-cursor expected-token
371     var match?/eax: boolean <- tokens-equal? curr-token, expected-token
372     compare match?, 0/false
373     {
374       break-if-!=
375       draw-text-wrapping-right-then-down-from-cursor-over-full-screen 0/screen, message, 3/fg=cyan 0/bg
376       count-test-failure
377       return
378     }
379     loop
380   }
381 }
382 
383 fn tokenize-and-parenthesize in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
384   var tokens-storage: (stream token 0x400)
385   var tokens/edx: (addr stream token) <- address tokens-storage
386   tokenize in, tokens, trace
387   var error?/eax: boolean <- has-errors? trace
388   compare error?, 0/false
389   {
390     break-if-=
391     return
392   }
393   parenthesize tokens, out, trace
394 }
395 
396 fn tokenize-salient in: (addr gap-buffer), out: (addr stream token), trace: (addr trace) {
397   var tokens-storage: (stream token 0x400)
398   var tokens/edx: (addr stream token) <- address tokens-storage
399   tokenize in, tokens, trace
400   var error?/eax: boolean <- has-errors? trace
401   compare error?, 0/false
402   {
403     break-if-=
404     return
405   }
406   emit-salient-tokens tokens, out
407 }