From 71e4f3812982dba2efb471283d310224e8db363e Mon Sep 17 00:00:00 2001 From: "Kartik K. Agaram" Date: Wed, 3 Mar 2021 22:09:50 -0800 Subject: 7842 - new directory organization Baremetal is now the default build target and therefore has its sources at the top-level. Baremetal programs build using the phase-2 Mu toolchain that requires a Linux kernel. This phase-2 codebase which used to be at the top-level is now under the linux/ directory. Finally, the phase-2 toolchain, while self-hosting, has a way to bootstrap from a C implementation, which is now stored in linux/bootstrap. The bootstrap C implementation uses some literate programming tools that are now in linux/bootstrap/tools. So the whole thing has gotten inverted. Each directory should build one artifact and include the main sources (along with standard library). Tools used for building it are relegated to sub-directories, even though those tools are often useful in their own right, and have had lots of interesting programs written using them. A couple of things have gotten dropped in this process: - I had old ways to run on just a Linux kernel, or with a Soso kernel. No more. - I had some old tooling for running a single test at the cursor. I haven't used that lately. Maybe I'll bring it back one day. The reorg isn't done yet. Still to do: - redo documentation everywhere. All the README files, all other markdown, particularly vocabulary.md. - clean up how-to-run comments at the start of programs everywhere - rethink what to do with the html/ directory. Do we even want to keep supporting it? In spite of these shortcomings, all the scripts at the top-level, linux/ and linux/bootstrap are working. The names of the scripts also feel reasonable. This is a good milestone to take stock at. --- linux/127next-word.subx | 362 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 linux/127next-word.subx (limited to 'linux/127next-word.subx') diff --git a/linux/127next-word.subx b/linux/127next-word.subx new file mode 100644 index 00000000..5af326d4 --- /dev/null +++ b/linux/127next-word.subx @@ -0,0 +1,362 @@ +# Tokenize by whitespace. + +== code +# instruction effective address register displacement immediate +# . op subop mod rm32 base index scale r32 +# . 1-3 bytes 3 bits 2 bits 3 bits 3 bits 3 bits 2 bits 2 bits 0/1/2/4 bytes 0/1/2/4 bytes + +# (re)compute the bounds of the next word in the line (surrounded by whitespace, +# treating '#' comments as a single word) +# return empty string on reaching end of file +next-word: # line: (addr stream byte), out: (addr slice) + # . prologue + 55/push-ebp + 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp + # . save registers + 50/push-eax + 51/push-ecx + 56/push-esi + 57/push-edi + # esi = line + 8b/copy 1/mod/*+disp8 5/rm32/ebp . . . 6/r32/esi 8/disp8 . # copy *(ebp+8) to esi + # edi = out + 8b/copy 1/mod/*+disp8 5/rm32/ebp . . . 7/r32/edi 0xc/disp8 . # copy *(ebp+12) to edi + # skip-chars-matching-whitespace(line) + # . . push args + ff 6/subop/push 1/mod/*+disp8 5/rm32/ebp . . . . 8/disp8 . # push *(ebp+8) + # . . call + e8/call skip-chars-matching-whitespace/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp +$next-word:check0: + # if (line->read >= line->write) clear out and return + # . eax = line->read + 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 0/r32/eax 4/disp8 . # copy *(esi+4) to eax + # . if (eax < line->write) goto next check + 3b/compare 0/mod/indirect 6/rm32/esi . . . 0/r32/eax . . # compare eax with *esi + 7c/jump-if-< $next-word:check-for-comment/disp8 + # . return out + c7 0/subop/copy 0/mod/direct 7/rm32/edi . . . . . 0/imm32 # copy to *edi + c7 0/subop/copy 1/mod/*+disp8 7/rm32/edi . . . . 4/disp8 0/imm32 # copy to *(edi+4) + eb/jump $next-word:end/disp8 +$next-word:check-for-comment: + # out->start = &line->data[line->read] + 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 1/r32/ecx 4/disp8 . # copy *(esi+4) to ecx + 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/eax 0xc/disp8 . # copy esi+ecx+12 to eax + 89/copy 0/mod/indirect 7/rm32/edi . . . 0/r32/eax . . # copy eax to *edi + # if (line->data[line->read] == '#') out->end = &line->data[line->write]), skip rest of stream and return + # . eax = line->data[line->read] + 31/xor 3/mod/direct 0/rm32/eax . . . 0/r32/eax . . # clear eax + 8a/copy-byte 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/AL 0xc/disp8 . # copy byte at *(esi+ecx+12) to AL + # . compare + 3d/compare-eax-and 0x23/imm32/pound + 75/jump-if-!= $next-word:regular-word/disp8 +$next-word:comment: + # . out->end = &line->data[line->write] + 8b/copy 0/mod/indirect 6/rm32/esi . . . 0/r32/eax . . # copy *esi to eax + 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 0/index/eax . 0/r32/eax 0xc/disp8 . # copy esi+eax+12 to eax + 89/copy 1/mod/*+disp8 7/rm32/edi . . . 0/r32/eax 4/disp8 . # copy eax to *(edi+4) + # . line->read = line->write + 8b/copy 0/mod/indirect 6/rm32/esi . . . 0/r32/eax . . # copy *esi to eax + 89/copy 1/mod/*+disp8 6/rm32/esi . . . 0/r32/eax 4/disp8 . # copy eax to *(esi+4) + # . return + eb/jump $next-word:end/disp8 +$next-word:regular-word: + # otherwise skip-chars-not-matching-whitespace(line) # including trailing newline + # . . push args + ff 6/subop/push 1/mod/*+disp8 5/rm32/ebp . . . . 8/disp8 . # push *(ebp+8) + # . . call + e8/call skip-chars-not-matching-whitespace/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp + # out->end = &line->data[line->read] + 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 1/r32/ecx 4/disp8 . # copy *(esi+4) to ecx + 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/eax 0xc/disp8 . # copy esi+ecx+12 to eax + 89/copy 1/mod/*+disp8 7/rm32/edi . . . 0/r32/eax 4/disp8 . # copy eax to *(edi+4) +$next-word:end: + # . restore registers + 5f/pop-to-edi + 5e/pop-to-esi + 59/pop-to-ecx + 58/pop-to-eax + # . epilogue + 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp + 5d/pop-to-ebp + c3/return + +test-next-word: + # . prologue + 55/push-ebp + 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp + # setup + # . clear-stream(_test-stream) + # . . push args + 68/push _test-stream/imm32 + # . . call + e8/call clear-stream/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp + # var slice/ecx: slice + 68/push 0/imm32/end + 68/push 0/imm32/start + 89/copy 3/mod/direct 1/rm32/ecx . . . 4/r32/esp . . # copy esp to ecx + # write(_test-stream, " ab") + # . . push args + 68/push " ab"/imm32 + 68/push _test-stream/imm32 + # . . call + e8/call write/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # next-word(_test-stream, slice) + # . . push args + 51/push-ecx + 68/push _test-stream/imm32 + # . . call + e8/call next-word/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # check-ints-equal(slice->start - _test-stream->data, 2, msg) + # . check-ints-equal(slice->start - _test-stream, 14, msg) + # . . push args + 68/push "F - test-next-word: start"/imm32 + 68/push 0xe/imm32 + # . . push slice->start - _test-stream + 8b/copy 0/mod/indirect 1/rm32/ecx . . . 0/r32/eax . . # copy *ecx to eax + 81 5/subop/subtract 3/mod/direct 0/rm32/eax . . . . . _test-stream/imm32 # subtract from eax + 50/push-eax + # . . call + e8/call check-ints-equal/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp + # check-ints-equal(slice->end - _test-stream->data, 4, msg) + # . check-ints-equal(slice->end - _test-stream, 16, msg) + # . . push args + 68/push "F - test-next-word: end"/imm32 + 68/push 0x10/imm32 + # . . push slice->end - _test-stream + 8b/copy 1/mod/*+disp8 1/rm32/ecx . . . 0/r32/eax 4/disp8 . # copy *(ecx+4) to eax + 81 5/subop/subtract 3/mod/direct 0/rm32/eax . . . . . _test-stream/imm32 # subtract from eax + 50/push-eax + # . . call + e8/call check-ints-equal/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp + # . epilogue + 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp + 5d/pop-to-ebp + c3/return + +test-next-word-returns-whole-comment: + # . prologue + 55/push-ebp + 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp + # setup + # . clear-stream(_test-stream) + # . . push args + 68/push _test-stream/imm32 + # . . call + e8/call clear-stream/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp + # var slice/ecx: slice + 68/push 0/imm32/end + 68/push 0/imm32/start + 89/copy 3/mod/direct 1/rm32/ecx . . . 4/r32/esp . . # copy esp to ecx + # write(_test-stream, " # a") + # . . push args + 68/push " # a"/imm32 + 68/push _test-stream/imm32 + # . . call + e8/call write/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # next-word(_test-stream, slice) + # . . push args + 51/push-ecx + 68/push _test-stream/imm32 + # . . call + e8/call next-word/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # check-ints-equal(slice->start - _test-stream->data, 2, msg) + # . check-ints-equal(slice->start - _test-stream, 14, msg) + # . . push args + 68/push "F - test-next-word-returns-whole-comment: start"/imm32 + 68/push 0xe/imm32 + # . . push slice->start - _test-stream + 8b/copy 0/mod/indirect 1/rm32/ecx . . . 0/r32/eax . . # copy *ecx to eax + 81 5/subop/subtract 3/mod/direct 0/rm32/eax . . . . . _test-stream/imm32 # subtract from eax + 50/push-eax + # . . call + e8/call check-ints-equal/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp + # check-ints-equal(slice->end - _test-stream->data, 5, msg) + # . check-ints-equal(slice->end - _test-stream, 17, msg) + # . . push args + 68/push "F - test-next-word-returns-whole-comment: end"/imm32 + 68/push 0x11/imm32 + # . . push slice->end - _test-stream + 8b/copy 1/mod/*+disp8 1/rm32/ecx . . . 0/r32/eax 4/disp8 . # copy *(ecx+4) to eax + 81 5/subop/subtract 3/mod/direct 0/rm32/eax . . . . . _test-stream/imm32 # subtract from eax + 50/push-eax + # . . call + e8/call check-ints-equal/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp + # . epilogue + 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp + 5d/pop-to-ebp + c3/return + +test-next-word-returns-empty-string-on-eof: + # . prologue + 55/push-ebp + 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp + # setup + # . clear-stream(_test-stream) + # . . push args + 68/push _test-stream/imm32 + # . . call + e8/call clear-stream/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp + # var slice/ecx: slice + 68/push 0/imm32/end + 68/push 0/imm32/start + 89/copy 3/mod/direct 1/rm32/ecx . . . 4/r32/esp . . # copy esp to ecx + # write nothing to _test-stream + # next-word(_test-stream, slice) + # . . push args + 51/push-ecx + 68/push _test-stream/imm32 + # . . call + e8/call next-word/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # check-ints-equal(slice->end - slice->start, 0, msg) + # . . push args + 68/push "F - test-next-word-returns-empty-string-on-eof"/imm32 + 68/push 0/imm32 + # . . push slice->end - slice->start + 8b/copy 1/mod/*+disp8 1/rm32/ecx . . . 0/r32/eax 4/disp8 . # copy *(ecx+4) to eax + 2b/subtract 0/mod/indirect 1/rm32/ecx . . . 0/r32/eax . . # subtract *ecx from eax + 50/push-eax + # . . call + e8/call check-ints-equal/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp + # . epilogue + 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp + 5d/pop-to-ebp + c3/return + +test-next-word-returns-empty-string-on-newline: + # . prologue + 55/push-ebp + 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp + # setup + # . clear-stream(_test-stream) + # . . push args + 68/push _test-stream/imm32 + # . . call + e8/call clear-stream/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp + # var slice/ecx: slice + 68/push 0/imm32/end + 68/push 0/imm32/start + 89/copy 3/mod/direct 1/rm32/ecx . . . 4/r32/esp . . # copy esp to ecx + # write some whitespace and a newline + # . . push args + 68/push " \n"/imm32 + 68/push _test-stream/imm32 + # . . call + e8/call write/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # next-word(_test-stream, slice) + # . . push args + 51/push-ecx + 68/push _test-stream/imm32 + # . . call + e8/call next-word/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 8/imm32 # add to esp + # check-ints-equal(slice->end - slice->start, 0, msg) + # . . push args + 68/push "F - test-next-word-returns-empty-string-on-newline"/imm32 + 68/push 0/imm32 + # . . push slice->end - slice->start + 8b/copy 1/mod/*+disp8 1/rm32/ecx . . . 0/r32/eax 4/disp8 . # copy *(ecx+4) to eax + 2b/subtract 0/mod/indirect 1/rm32/ecx . . . 0/r32/eax . . # subtract *ecx from eax + 50/push-eax + # . . call + e8/call check-ints-equal/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 0xc/imm32 # add to esp + # . epilogue + 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp + 5d/pop-to-ebp + c3/return + +# (re)compute the bounds of the next word in the line (separated by whitespace) +# return empty string on reaching end of file +next-raw-word: # line: (addr stream byte), out: (addr slice) + # . prologue + 55/push-ebp + 89/copy 3/mod/direct 5/rm32/ebp . . . 4/r32/esp . . # copy esp to ebp + # . save registers + 50/push-eax + 51/push-ecx + 56/push-esi + 57/push-edi + # esi = line + 8b/copy 1/mod/*+disp8 5/rm32/ebp . . . 6/r32/esi 8/disp8 . # copy *(ebp+8) to esi + # edi = out + 8b/copy 1/mod/*+disp8 5/rm32/ebp . . . 7/r32/edi 0xc/disp8 . # copy *(ebp+12) to edi + # skip-chars-matching-whitespace(line) + # . . push args + ff 6/subop/push 1/mod/*+disp8 5/rm32/ebp . . . . 8/disp8 . # push *(ebp+8) + # . . call + e8/call skip-chars-matching-whitespace/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp +$next-raw-word:check0: + # if (line->read >= line->write) clear out and return + # . eax = line->read + 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 0/r32/eax 4/disp8 . # copy *(esi+4) to eax + # . if (eax < line->write) goto next check + 3b/compare 0/mod/indirect 6/rm32/esi . . . 0/r32/eax . . # compare eax with *esi + 7c/jump-if-< $next-raw-word:word-exists/disp8 + # . return out + c7 0/subop/copy 0/mod/direct 7/rm32/edi . . . . . 0/imm32 # copy to *edi + c7 0/subop/copy 1/mod/*+disp8 7/rm32/edi . . . . 4/disp8 0/imm32 # copy to *(edi+4) + eb/jump $next-raw-word:end/disp8 +$next-raw-word:word-exists: + # out->start = &line->data[line->read] + 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 1/r32/ecx 4/disp8 . # copy *(esi+4) to ecx + 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/eax 0xc/disp8 . # copy esi+ecx+12 to eax + 89/copy 0/mod/indirect 7/rm32/edi . . . 0/r32/eax . . # copy eax to *edi + # skip-chars-not-matching-whitespace(line) # including trailing newline + # . . push args + ff 6/subop/push 1/mod/*+disp8 5/rm32/ebp . . . . 8/disp8 . # push *(ebp+8) + # . . call + e8/call skip-chars-not-matching-whitespace/disp32 + # . . discard args + 81 0/subop/add 3/mod/direct 4/rm32/esp . . . . . 4/imm32 # add to esp + # out->end = &line->data[line->read] + 8b/copy 1/mod/*+disp8 6/rm32/esi . . . 1/r32/ecx 4/disp8 . # copy *(esi+4) to ecx + 8d/copy-address 1/mod/*+disp8 4/rm32/sib 6/base/esi 1/index/ecx . 0/r32/eax 0xc/disp8 . # copy esi+ecx+12 to eax + 89/copy 1/mod/*+disp8 7/rm32/edi . . . 0/r32/eax 4/disp8 . # copy eax to *(edi+4) +$next-raw-word:end: + # . restore registers + 5f/pop-to-edi + 5e/pop-to-esi + 59/pop-to-ecx + 58/pop-to-eax + # . epilogue + 89/copy 3/mod/direct 4/rm32/esp . . . 5/r32/ebp . . # copy ebp to esp + 5d/pop-to-ebp + c3/return -- cgit 1.4.1-2-gfad0