# Some tokenization primitives. == code # instruction effective address register displacement immediate # . op subop mod rm32 base index scale r32 # . 1-3 bytes 3 bits 2 bits 3 bits 3 bits 3 bits 2 bits 2 bits 0/1/2/4 bytes 0/1/2/4 bytes # extract the next run of characters that are different from a given 'delimiter' (skipping multiple delimiters if necessary) # on reaching end of file, return an empty interval next-token: # in: (addr stream byte), delimiter: byte, out: (addr slice) # . prologue 55/push-ebp 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp # . save registers 50/push-eax 51/push-ecx 56/push-esi 57/push-edi # esi = in 8b/copy 1/mod/*+disp8 5/rm32/ebp . . . 6/r32/esi 8/disp8 . # copy *(ebp+8) to esi # edi = out 8b/copy 1/mod/*+disp8 5/rm32/ebp . . . 7/r32/edi 0x10/disp8 . # copy *(ebp+16) to edi # skip-chars-matching(in, delimiter) # . . push args ff 6/subop/push 1/mod/*+disp8 5/rm32/ebp . . . . 0xc/disp8 . # push *(ebp+12) 56/push-esi # . . call e8/call skip-chars-matching/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp # out->start = &in->data[in->read] 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 1/r32/ecx 4/disp8 . # copy *(esi+4) to ecx 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/eax 0xc/disp8 . # copy esi+ecx+12 to eax 89/copy 0/mod/indirect 7/rm32/edi . . . 0/r32/eax . . # copy eax to *edi # skip-chars-not-matching(in, delimiter) # . . push args ff 6/subop/push 1/mod/*+disp8 5/rm32/ebp . . . . 0xc/disp8 . # push *(ebp+12) 56/push-esi # . . call e8/call skip-chars-not-matching/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp # out->end = &in->data[in->read] 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 1/r32/ecx 4/disp8 . # copy *(esi+4) to ecx 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/eax 0xc/disp8 . # copy esi+ecx+12 to eax 89/copy 1/mod/*+disp8 7/rm32/edi . . . 0/r32/eax 4/disp8 . # copy eax to *(edi+4) # . restore registers 5f/pop-to-edi 5e/pop-to-esi 59/pop-to-ecx 58/pop-to-eax # . epilogue 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp 5d/pop-to-ebp c3/return test-next-token: # . prologue 55/push-ebp 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp # setup # . clear-stream(_test-stream) # . . push args 68/push _test-stream/imm32 # . . call e8/call clear-stream/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp # var slice/ecx: slice 68/push 0/imm32/end 68/push 0/imm32/start 89/copy 3/mod/direct 1/rm32/ecx . . . 4/r32/esp . . # copy esp to ecx # write(_test-stream, " ab") # . . push args 68/push " ab"/imm32 68/push _test-stream/imm32 # . . call e8/call write/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp # next-token(_test-stream, 0x20/space, slice) # . . push args 51/push-ecx 68/push 0x20/imm32 68/push _test-stream/imm32 # . . call e8/call next-token/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp # check-ints-equal(slice->start - _test-stream->data, 2, msg) # . check-ints-equal(slice->start - _test-stream, 14, msg) # . . push args 68/push "F - test-next-token: start"/imm32 68/push 0xe/imm32 # . . push slice->start - _test-stream 8b/copy 0/mod/indirect 1/rm32/ecx . . . 0/r32/eax . . # copy *ecx to eax 81 5/subop/subtract 3/mod/direct 0/rm32/eax . . . . . _test-stream/imm32 # subtract from eax 50/push-eax # . . call e8/call check-ints-equal/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp # check-ints-equal(slice->end - _test-stream->data, 4, msg) # . check-ints-equal(slice->end - _test-stream, 16, msg) # . . push args 68/push "F - test-next-token: end"/imm32 68/push 0x10/imm32 # . . push slice->end - _test-stream 8b/copy 1/mod/*+disp8 1/rm32/ecx . . . 0/r32/eax 4/disp8 . # copy *(ecx+4) to eax 81 5/subop/subtract 3/mod/direct 0/rm32/eax . . . . . _test-stream/imm32 # subtract from eax 50/push-eax # . . call e8/call check-ints-equal/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp # . epilogue 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp 5d/pop-to-ebp c3/return test-next-token-Eof: # . prologue 55/push-ebp 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp # setup # . clear-stream(_test-stream) # . . push args 68/push _test-stream/imm32 # . . call e8/call clear-stream/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp # var slice/ecx: slice 68/push 0/imm32/end 68/push 0/imm32/start 89/copy 3/mod/direct 1/rm32/ecx . . . 4/r32/esp . . # copy esp to ecx # write nothing to _test-stream # next-token(_test-stream, 0x20/space, slice) # . . push args 51/push-ecx 68/push 0x20/imm32 68/push _test-stream/imm32 # . . call e8/call next-token/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp # check-ints-equal(slice->end, slice->start, msg) # . . push args 68/push "F - test-next-token-Eof"/imm32 ff 6/subop/push 1/mod/*+disp8 1/rm32/ecx . . . . 4/disp8 . # push *(ecx+4) ff 6/subop/push 0/mod/indirect 1/rm32/ecx . . . . . . # push *ecx # . . call e8/call check-ints-equal/disp32 # . . discard args 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp # . epilogue 89/copy 3/mod/direct 4/rm32/esp . . .
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>Mu - 062rewrite_stash.cc</title>
<meta name="Generator" content="Vim/7.4">
<meta name="plugin-version" content="vim7.4_v2">
<meta name="syntax" content="cpp">
<meta name="settings" content="use_css,pre_wrap,no_foldcolumn,expand_tabs,prevent_copy=">
<meta name="colorscheme" content="minimal">
<style type="text/css">
<!--
pre { white-space: pre-wrap; font-family: monospace; color: #eeeeee; background-color: #080808; }
body { font-size: 12pt; font-family: monospace; color: #eeeeee; background-color: #080808; }
* { font-size: 12pt; font-size: 1em; }
.Constant { color: #00a0a0; }
.traceContains { color: #008000; }
.Comment { color: #9090ff; }
.Delimiter { color: #800080; }
.Special { color: #c00000; }
.Identifier { color: #fcb165; }
.Normal { color: #eeeeee; background-color: #080808; padding-bottom: 1px; }
-->
</style>
<script type='text/javascript'>
<!--
-->
</script>
</head>
<body>
<pre id='vimCodeElement'>
<span class="Comment">//: when encountering other types, try to convert them to strings using</span>
<span class="Comment">//: 'to-text'</span>
<span class="Delimiter">:(scenarios transform)</span>
<span class="Delimiter">:(scenario rewrite_stashes_to_text)</span>
recipe main [
local-scope
<span class="Normal">n</span>:number<span class="Special"> <- </span>copy <span class="Constant">34</span>
stash n
]
<span class="traceContains">+transform: {stash_2_0: ("address" "array" "character")} <- to-text-line {n: "number"}</span>
<span class="traceContains">+transform: stash {stash_2_0: ("address" "array" "character")}</span>
<span class="Delimiter">:(scenario rewrite_traces_to_text)</span>
recipe main [
local-scope
<span class="Normal">n</span>:number<span class="Special"> <- </span>copy <span class="Constant">34</span>
trace <span class="Constant">2</span><span class="Delimiter">,</span> [app]<span class="Delimiter">,</span> n
]
<span class="traceContains">+transform: {trace_2_2: ("address" "array" "character")} <- to-text-line {n: "number"}</span>
<span class="traceContains">+transform: trace {2: "literal"}, {"app": "literal-string"}, {trace_2_2: ("address" "array" "character")}</span>
<span class="Comment">//: special case: rewrite attempts to stash contents of most arrays to avoid</span>
<span class="Comment">//: passing addresses around</span>
<span class="Delimiter">:(scenario rewrite_stashes_of_arrays)</span>
recipe main [
local-scope
<span class="Normal">n</span>:address:array:number<span class="Special"> <- </span><span class="Normal">new</span> number:type<span class="Delimiter">,</span> <span class="Constant">3</span>
stash *n
]
<span class="traceContains">+transform: {stash_2_0: ("address" "array" "character")} <- array-to-text-line {n: ("address" "array" "number")}</span>
<span class="traceContains">+transform: stash {stash_2_0: ("address" "array" "character")}</span>
<span class="Delimiter">:(scenario ignore_stashes_of_static_arrays)</span>
recipe main [
local-scope
<span class="Normal">n</span>:array:number:<span class="Constant">3</span><span class="Special"> <- </span>create-array
stash n
]
<span class="traceContains">+transform: stash {n: ("array" "number" "3")}</span>
<span class="Delimiter">:(scenario rewrite_stashes_of_recipe_header_products)</span>
container foo [
<span class="Normal">x</span>:number
]
recipe bar <span class="Delimiter">-></span> x:foo [
local-scope
load-ingredients
x<span class="Special"> <- </span>merge <span class="Constant">34</span>
stash x
]
<span class="traceContains">+transform: stash {stash_2_0: ("address" "array" "character")}</span>
<span class="Comment">//: misplaced; should be in instruction inserting/deleting transforms, but has</span>
<span class="Comment">//: prerequisites: deduce_types_from_header and check_or_set_types_by_name</span>
<span class="Delimiter">:(after "Transform.push_back(deduce_types_from_header)")</span>
Transform<span class="Delimiter">.</span>push_back<span class="Delimiter">(</span>rewrite_stashes_to_text<span class="Delimiter">);</span>
<span class="Delimiter">:(code)</span>
<span class="Normal">void</span> rewrite_stashes_to_text<span class="Delimiter">(</span>recipe_ordinal r<span class="Delimiter">)</span> <span class="Delimiter">{</span>
recipe& caller = get<span class="Delimiter">(</span>Recipe<span class="Delimiter">,</span> r<span class="Delimiter">);</span>
trace<span class="Delimiter">(</span><span class="Constant">9991</span><span class="Delimiter">,</span> <span class="Constant">"transform"</span><span class="Delimiter">)</span> << <span class="Constant">"--- rewrite 'stash' instructions in recipe "</span> << caller<span class="Delimiter">.</span>name << end<span class="Delimiter">();</span>
<span class="Comment">// in recipes without named locations, 'stash' is still not extensible</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>contains_numeric_locations<span class="Delimiter">(</span>caller<span class="Delimiter">))</span> <span class="Identifier">return</span><span class="Delimiter">;</span>
rewrite_stashes_to_text<span class="Delimiter">(</span>caller<span class="Delimiter">);</span>
<span class="Delimiter">}</span>
<span class="Normal">void</span> rewrite_stashes_to_text<span class="Delimiter">(</span>recipe& caller<span class="Delimiter">)</span> <span class="Delimiter">{</span>
vector<instruction> new_instructions<span class="Delimiter">;</span>
<span class="Normal">for</span> <span class="Delimiter">(</span><span class="Normal">int</span> i = <span class="Constant">0</span><span class="Delimiter">;</span> i < SIZE<span class="Delimiter">(</span>caller<span class="Delimiter">.</span>steps<span class="Delimiter">);</span> ++i<span class="Delimiter">)</span> <span class="Delimiter">{</span>
instruction& inst = caller<span class="Delimiter">.</span>steps<span class="Delimiter">.</span>at<span class="Delimiter">(</span>i<span class="Delimiter">);</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>inst<span class="Delimiter">.</span>name == <span class="Constant">"stash"</span><span class="Delimiter">)</span> <span class="Delimiter">{</span>
<span class="Normal">for</span> <span class="Delimiter">(</span><span class="Normal">int</span> j = <span class="Constant">0</span><span class="Delimiter">;</span> j < SIZE<span class="Delimiter">(</span>inst<span class="Delimiter">.</span>ingredients<span class="Delimiter">);</span> ++j<span class="Delimiter">)</span> <span class="Delimiter">{</span>
ostringstream ingredient_name<span class="Delimiter">;</span>
ingredient_name << <span class="Constant">"stash_"</span> << i << <span class="Constant">'_'</span> << j << <span class="Constant">":address:array:character"</span><span class="Delimiter">;</span>
rewrite_stash_to_text<span class="Delimiter">(</span>inst<span class="Delimiter">.</span>ingredients<span class="Delimiter">.</span>at<span class="Delimiter">(</span>j<span class="Delimiter">),</span> new_instructions<span class="Delimiter">,</span> ingredient_name<span class="Delimiter">.</span>str<span class="Delimiter">());</span>
<span class="Delimiter">}</span>
<span class="Delimiter">}</span>
<span class="Normal">else</span> <span class="Normal">if</span> <span class="Delimiter">(</span>inst<span class="Delimiter">.</span>name == <span class="Constant">"trace"</span><span class="Delimiter">)</span> <span class="Delimiter">{</span>
<span class="Normal">for</span> <span class="Delimiter">(</span><span class="Normal">int</span> j = <span class="Comment">/*</span><span class="Comment">skip</span><span class="Comment">*/</span><span class="Constant">2</span><span class="Delimiter">;</span> j < SIZE<span class="Delimiter">(</span>inst<span class="Delimiter">.</span>ingredients<span class="Delimiter">);</span> ++j<span class="Delimiter">)</span> <span class="Delimiter">{</span>
ostringstream ingredient_name<span class="Delimiter">;</span>
ingredient_name << <span class="Constant">"trace_"</span> << i << <span class="Constant">'_'</span> << j << <span class="Constant">":address:array:character"</span><span class="Delimiter">;</span>
rewrite_stash_to_text<span class="Delimiter">(</span>inst<span class="Delimiter">.</span>ingredients<span class="Delimiter">.</span>at<span class="Delimiter">(</span>j<span class="Delimiter">),</span> new_instructions<span class="Delimiter">,</span> ingredient_name<span class="Delimiter">.</span>str<span class="Delimiter">());</span>
<span class="Delimiter">}</span>
<span class="Delimiter">}</span>
trace<span class="Delimiter">(</span><span class="Constant">9993</span><span class="Delimiter">,</span> <span class="Constant">"transform"</span><span class="Delimiter">)</span> << to_string<span class="Delimiter">(</span>inst<span class="Delimiter">)</span> << end<span class="Delimiter">();</span>
new_instructions<span class="Delimiter">.</span>push_back<span class="Delimiter">(</span>inst<span class="Delimiter">);</span>
<span class="Delimiter">}</span>
caller<span class="Delimiter">.</span>steps<span class="Delimiter">.</span>swap<span class="Delimiter">(</span>new_instructions<span class="Delimiter">);</span>
<span class="Delimiter">}</span>
<span class="Comment">// add an instruction to convert reagent 'r' to text in list 'out', then</span>
<span class="Comment">// replace r with converted text</span>
<span class="Normal">void</span> rewrite_stash_to_text<span class="Delimiter">(</span>reagent& r<span class="Delimiter">,</span> vector<instruction>& out<span class="Delimiter">,</span> <span class="Normal">const</span> string& tmp_var<span class="Delimiter">)</span> <span class="Delimiter">{</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>!r<span class="Delimiter">.</span>type<span class="Delimiter">)</span> <span class="Identifier">return</span><span class="Delimiter">;</span> <span class="Comment">// error; will be handled elsewhere</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>is_literal<span class="Delimiter">(</span>r<span class="Delimiter">))</span> <span class="Identifier">return</span><span class="Delimiter">;</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>is_mu_string<span class="Delimiter">(</span>r<span class="Delimiter">))</span> <span class="Identifier">return</span><span class="Delimiter">;</span>
<span class="Comment">// don't try to extend static arrays</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>is_static_array<span class="Delimiter">(</span>r<span class="Delimiter">))</span> <span class="Identifier">return</span><span class="Delimiter">;</span>
instruction def<span class="Delimiter">;</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>is_lookup_of_address_of_array<span class="Delimiter">(</span>r<span class="Delimiter">))</span> <span class="Delimiter">{</span>
def<span class="Delimiter">.</span>name = <span class="Constant">"array-to-text-line"</span><span class="Delimiter">;</span>
reagent<span class="Comment">/*</span><span class="Comment">copy</span><span class="Comment">*/</span> tmp = r<span class="Delimiter">;</span>
drop_one_lookup<span class="Delimiter">(</span>tmp<span class="Delimiter">);</span>
def<span class="Delimiter">.</span>ingredients<span class="Delimiter">.</span>push_back<span class="Delimiter">(</span>tmp<span class="Delimiter">);</span>
<span class="Delimiter">}</span>
<span class="Normal">else</span> <span class="Delimiter">{</span>
def<span class="Delimiter">.</span>name = <span class="Constant">"to-text-line"</span><span class="Delimiter">;</span>
def<span class="Delimiter">.</span>ingredients<span class="Delimiter">.</span>push_back<span class="Delimiter">(</span>r<span class="Delimiter">);</span>
<span class="Delimiter">}</span>
def<span class="Delimiter">.</span>products<span class="Delimiter">.</span>push_back<span class="Delimiter">(</span>reagent<span class="Delimiter">(</span>tmp_var<span class="Delimiter">));</span>
trace<span class="Delimiter">(</span><span class="Constant">9993</span><span class="Delimiter">,</span> <span class="Constant">"transform"</span><span class="Delimiter">)</span> << to_string<span class="Delimiter">(</span>def<span class="Delimiter">)</span> << end<span class="Delimiter">();</span>
out<span class="Delimiter">.</span>push_back<span class="Delimiter">(</span>def<span class="Delimiter">);</span>
r<span class="Delimiter">.</span>clear<span class="Delimiter">();</span> <span class="Comment">// reclaim old memory</span>
r = reagent<span class="Delimiter">(</span>tmp_var<span class="Delimiter">);</span>
<span class="Delimiter">}</span>
<span class="Normal">bool</span> is_lookup_of_address_of_array<span class="Delimiter">(</span>reagent<span class="Comment">/*</span><span class="Comment">copy</span><span class="Comment">*/</span> x<span class="Delimiter">)</span> <span class="Delimiter">{</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>x<span class="Delimiter">.</span>type<span class="Delimiter">-></span>atom<span class="Delimiter">)</span> <span class="Identifier">return</span> <span class="Constant">false</span><span class="Delimiter">;</span>
<span class="Normal">if</span> <span class="Delimiter">(</span>x<span class="Delimiter">.</span>type<span class="Delimiter">-></span>left<span class="Delimiter">-></span>name != <span class="Constant">"address"</span><span class="Delimiter">)</span> <span class="Identifier">return</span> <span class="Constant">false</span><span class="Delimiter">;</span>
<span class="Normal">if</span> <span