7842 - new directory organization

Baremetal is now the default build target and therefore has its sources
at the top-level. Baremetal programs build using the phase-2 Mu toolchain
that requires a Linux kernel. This phase-2 codebase which used to be at
the top-level is now under the linux/ directory. Finally, the phase-2 toolchain,
while self-hosting, has a way to bootstrap from a C implementation, which
is now stored in linux/bootstrap. The bootstrap C implementation uses some
literate programming tools that are now in linux/bootstrap/tools.

So the whole thing has gotten inverted. Each directory should build one
artifact and include the main sources (along with standard library). Tools
used for building it are relegated to sub-directories, even though those
tools are often useful in their own right, and have had lots of interesting
programs written using them.

A couple of things have gotten dropped in this process:
  - I had old ways to run on just a Linux kernel, or with a Soso kernel.
    No more.
  - I had some old tooling for running a single test at the cursor. I haven't
    used that lately. Maybe I'll bring it back one day.

The reorg isn't done yet. Still to do:
  - redo documentation everywhere. All the README files, all other markdown,
    particularly vocabulary.md.
  - clean up how-to-run comments at the start of programs everywhere
  - rethink what to do with the html/ directory. Do we even want to keep
    supporting it?

In spite of these shortcomings, all the scripts at the top-level, linux/
and linux/bootstrap are working. The names of the scripts also feel reasonable.
This is a good milestone to take stock at.

1 files changed, 359 insertions, 0 deletions

diff --git a/linux/braces.subx b/linux/braces.subx
new file mode 100644
index 00000000..d6b8f0f4
--- /dev/null
+++ b/linux/braces.subx
@@ -0,0 +1,359 @@
+# Structured control flow using break/loop rather than jump.
+#
+# To run (on Linux):
+#   $ ./translate_subx init.linux [012]*.subx apps/subx-params.subx apps/braces.subx
+#   $ mv a.elf apps/braces
+#
+# Example 1:
+#   $ cat x.subx
+#   {
+#     7c/jump-if-< break/disp8
+#     74/jump-if-= loop/disp8
+#   }
+#   $ cat x.subx |apps/braces
+#   _loop1:
+#     7c/jump-if-< _break1/disp8
+#     74/jump-if-= _loop1/disp8
+#   _break1:
+#
+# Example 2:
+#   $ cat x.subx
+#   {
+#     7c/jump-if-< break/disp8
+#   }
+#   {
+#     74/jump-if-= loop/disp8
+#   }
+#   $ cat x.subx |apps/braces
+#   _loop1:
+#     7c/jump-if-< _break1/disp8
+#   _break1:
+#   _loop2:
+#     74/jump-if-= _loop2/disp8
+#   _break2:
+#
+# Example 3:
+#   $ cat x.subx
+#   {
+#     {
+#       74/jump-if-= loop/disp8
+#     }
+#     7c/jump-if-< loop/disp8
+#   }
+#   $ cat x.subx |apps/braces
+#   _loop1:
+#     _loop2:
+#       74/jump-if-= _loop2/disp8
+#     _break2:
+#     7c/jump-if-< _loop1/disp8
+#   _break1:
+
+== code
+
+Entry:  # run tests if necessary, a REPL if not
+    # . prologue
+    89/<- %ebp 4/r32/esp
+    # initialize heap
+    (new-segment *Heap-size Heap)
+    # if (argc <= 1) goto interactive
+    81 7/subop/compare *ebp 1/imm32
+    7e/jump-if-<= $subx-braces-main:interactive/disp8
+    # if (argv[1] != "test")) goto interactive
+    (kernel-string-equal? *(ebp+8) "test")  # => eax
+    3d/compare-eax-and 0/imm32/false
+    74/jump-if-= $subx-braces-main:interactive/disp8
+    #
+    (run-tests)
+    # syscall(exit, *Num-test-failures)
+    8b/-> *Num-test-failures 3/r32/ebx
+    eb/jump $subx-braces-main:end/disp8
+$subx-braces-main:interactive:
+    (subx-braces Stdin Stdout)
+    # syscall(exit, 0)
+    bb/copy-to-ebx 0/imm32
+$subx-braces-main:end:
+    e8/call syscall_exit/disp32
+
+subx-braces:  # in: (addr buffered-file), out: (addr buffered-file)
+    # pseudocode:
+    #   var line: (stream byte 512)
+    #   var label-stack: (stack int 32)  # at most 32 levels of nesting
+    #   var next-label-id: int = 1
+    #   while true
+    #     clear-stream(line)
+    #     read-line-buffered(in, line)
+    #     if (line->write == 0) break                           # end of file
+    #     skip-chars-matching-whitespace(line)
+    #     if line->data[line->read] == '{'
+    #       print(out, "_loop" next-label-id ":\n")
+    #       push(label-stack, next-label-id)
+    #       ++next-label-id
+    #       continue
+    #     if line->data[line->read] == '}'
+    #       var top = pop(label-stack)
+    #       print(out, "_break" top ":\n")
+    #       continue
+    #     while true
+    #       var word-slice: (addr slice) = next-word-or-string(line)
+    #       if slice-empty?(word-slice)                         # end of line
+    #         break
+    #       if slice-starts-with?(word-slice, "#")              # comment
+    #         continue
+    #       if slice-starts-with?(word-slice, "break/")
+    #         var top = top(label-stack)
+    #         print(out, "_break" top)
+    #         word-slice->start += len("break")
+    #       else if slice-starts-with?(word-slice, "loop/")
+    #         var top = top(label-stack)
+    #         print(out, "_loop" top)
+    #         word-slice->start += len("loop")
+    #       print(out, word-slice " ")
+    #     print(out, "\n")
+    #   flush(out)
+    # . prologue
+    55/push-ebp
+    89/<- %ebp 4/r32/esp
+    # . save registers
+    50/push-eax
+    51/push-ecx
+    52/push-edx
+    53/push-ebx
+    56/push-esi
+    57/push-edi
+    # esi = in
+    8b/-> *(ebp+8) 6/r32/esi
+    # var line/ecx: (stream byte 512)
+    81 5/subop/subtract %esp 0x200/imm32
+    68/push 0x200/imm32/length
+    68/push 0/imm32/read
+    68/push 0/imm32/write
+    89/<- %ecx 4/r32/esp
+    # var label-stack/edx: (stack int 32)
+    81 5/subop/subtract %esp 0x80/imm32
+    68/push 0x80/imm32/length
+    68/push 0/imm32/top
+    89/<- %edx 4/r32/esp
+    # var next-label-id/ebx: int = 1
+    c7 0/subop/copy %ebx 1/imm32
+    # var word-slice/edi: slice
+    68/push 0/imm32/end
+    68/push 0/imm32/start
+    89/<- %edi 4/r32/esp
+$subx-braces:line-loop:
+    (clear-stream %ecx)
+    (read-line-buffered %esi %ecx)
+$subx-braces:check0:
+    # if (line->write == 0) break
+    81 7/subop/compare *ecx 0/imm32
+    0f 84/jump-if-=  $subx-braces:break/disp32
+    (skip-chars-matching-whitespace %ecx)
+$subx-braces:check-for-curly-open:
+    # if (line->data[line->read] != '{') goto next check
+    # . eax = line->data[line->read]
+    8b/-> *(ecx+4) 0/r32/eax
+    8a/copy-byte *(ecx+eax+0xc) 0/r32/AL
+    81 4/subop/and %eax 0xff/imm32
+    # . if (eax != '{') continue
+    3d/compare-eax-and 0x7b/imm32/open-curly
+    0f 85/jump-if-!= $subx-braces:check-for-curly-closed/disp32
+$subx-braces:emit-curly-open:
+    # print(out, "_loop" next-label-id ":")
+    (write-buffered *(ebp+0xc) "_loop")
+    (write-int32-hex-buffered *(ebp+0xc) %ebx)
+    (write-buffered *(ebp+0xc) ":")
+    # push(label-stack, next-label-id)
+    (push %edx %ebx)
+    # ++next-label-id
+    ff 0/subop/increment %ebx
+    # continue
+    e9/jump  $subx-braces:next-line/disp32
+$subx-braces:check-for-curly-closed:
+    # if (line->data[line->read] != '}') goto next check
+    3d/compare-eax-and 0x7d/imm32/close-curly
+    0f 85/jump-if-= $subx-braces:word-loop/disp32
+$subx-braces:emit-curly-closed:
+    # eax = pop(label-stack)
+    (pop %edx)
+    # print(out, "_break" eax ":")
+    (write-buffered *(ebp+0xc) "_break")
+    (write-int32-hex-buffered *(ebp+0xc) %eax)
+    (write-buffered *(ebp+0xc) ":")
+    # continue
+    e9/jump  $subx-braces:next-line/disp32
+$subx-braces:word-loop:
+    (next-word-or-string %ecx %edi)
+$subx-braces:check1:
+    # if (slice-empty?(word-slice)) break
+    (slice-empty? %edi)
+    3d/compare-eax-and 0/imm32/false
+    0f 85/jump-if-!= $subx-braces:next-line/disp32
+$subx-braces:check-for-comment:
+    # if (slice-starts-with?(word-slice, "#")) continue
+    # . eax = *word-slice->start
+    8b/-> *edi 0/r32/eax
+    8a/copy-byte *eax 0/r32/AL
+    81 4/subop/and %eax 0xff/imm32
+    # . if (eax == '#') continue
+    3d/compare-eax-and 0x23/imm32/hash
+    74/jump-if-= $subx-braces:word-loop/disp8
+$subx-braces:check-for-break:
+    # if (!slice-starts-with?(word-slice, "break/")) goto next check
+    # . eax = slice-starts-with?(word-slice, "break/")
+    (slice-starts-with? %edi "break/")
+    # . if (eax == false) goto next check
+    3d/compare-eax-and 0/imm32/false
+    74/jump-if-= $subx-braces:check-for-loop/disp8
+$subx-braces:emit-break:
+    (top %edx)
+    # print(out, "_break" eax)
+    (write-buffered *(ebp+0xc) "_break")
+    (write-int32-hex-buffered *(ebp+0xc) %eax)
+    # word-slice->start += len("break")
+    81 0/subop/add *edi 5/imm32/strlen
+    # emit rest of word as usual
+    eb/jump $subx-braces:emit-word-slice/disp8
+$subx-braces:check-for-loop:
+    # if (!slice-starts-with?(word-slice, "loop/")) emit word
+    # . eax = slice-starts-with?(word-slice, "loop/")
+    (slice-starts-with? %edi "loop/")
+    # . if (eax == false) goto next check
+    3d/compare-eax-and 0/imm32/false
+    74/jump-if-= $subx-braces:emit-word-slice/disp8
+$subx-braces:emit-loop:
+    (top %edx)
+    # print(out, "_loop" eax)
+    (write-buffered *(ebp+0xc) "_loop")
+    (write-int32-hex-buffered *(ebp+0xc) %eax)
+    # word-slice->start += len("loop")
+    81 0/subop/add *edi 4/imm32/strlen
+    # fall through
+$subx-braces:emit-word-slice:
+    # print(out, word-slice " ")
+    (write-slice-buffered *(ebp+0xc) %edi)
+    (write-buffered *(ebp+0xc) Space)
+    # loop to next word
+    e9/jump $subx-braces:word-loop/disp32
+$subx-braces:next-line:
+    # print(out, "\n")
+    (write-buffered *(ebp+0xc) Newline)
+    # loop to next line
+    e9/jump $subx-braces:line-loop/disp32
+$subx-braces:break:
+    (flush *(ebp+0xc))
+$subx-braces:end:
+    # . reclaim locals
+    81 0/subop/add %esp 0x29c/imm32
+    # . restore registers
+    5f/pop-to-edi
+    5e/pop-to-esi
+    5b/pop-to-ebx
+    5a/pop-to-edx
+    59/pop-to-ecx
+    58/pop-to-eax
+    # . epilogue
+    89/<- %esp 5/r32/ebp
+    5d/pop-to-ebp
+    c3/return
+
+test-subx-braces-passes-most-words-through:
+    # . prologue
+    55/push-ebp
+    89/<- %ebp 4/r32/esp
+    # setup
+    (clear-stream _test-input-stream)
+    (clear-stream _test-output-stream)
+    (clear-stream $_test-input-buffered-file->buffer)
+    (clear-stream $_test-output-buffered-file->buffer)
+    # test
+    (write _test-input-stream "== abcd 0x1")
+    (subx-braces _test-input-buffered-file _test-output-buffered-file)
+    # check that the line just passed through
+    (flush _test-output-buffered-file)
+#?     # dump _test-output-stream {{{
+#?     (write 2 "^")
+#?     (write-stream 2 _test-output-stream)
+#?     (write 2 "$\n")
+#?     # }}}
+    (check-stream-equal _test-output-stream "== abcd 0x1 \n" "F - test-subx-braces-passes-most-words-through")
+    # . epilogue
+    89/<- %esp 5/r32/ebp
+    5d/pop-to-ebp
+    c3/return
+
+test-subx-braces-1:
+    # input:
+    #   {
+    #   ab break/imm32
+    #   cd loop/imm32
+    #   }
+    #
+    # output:
+    #   _loop1:
+    #   ab _break1/imm32
+    #   cd _loop1/imm32
+    #   _break1:
+    #
+    # . prologue
+    55/push-ebp
+    89/<- %ebp 4/r32/esp
+    # setup
+    (clear-stream _test-input-stream)
+    (clear-stream _test-output-stream)
+    (clear-stream $_test-input-buffered-file->buffer)
+    (clear-stream $_test-output-buffered-file->buffer)
+    # test
+    (write _test-input-stream "{\nab break/imm32\ncd loop/imm32\n}")
+    (subx-braces _test-input-buffered-file _test-output-buffered-file)
+    # check that the line just passed through
+    (flush _test-output-buffered-file)
+#?     # dump _test-output-stream {{{
+#?     (write 2 "^")
+#?     (write-stream 2 _test-output-stream)
+#?     (write 2 "$\n")
+#?     # }}}
+    (check-stream-equal _test-output-stream "_loop0x00000001:\nab _break0x00000001/imm32 \ncd _loop0x00000001/imm32 \n_break0x00000001:\n" "F - test-subx-braces-1")
+    # . epilogue
+    89/<- %esp 5/r32/ebp
+    5d/pop-to-ebp
+    c3/return
+
+test-subx-braces-2:
+    # input:
+    #   {
+    #   {
+    #   ab break/imm32
+    #   }
+    #   cd loop/imm32
+    #   }
+    #
+    # output:
+    #   _loop1:
+    #   _loop2:
+    #   ab _break2/imm32
+    #   _break2:
+    #   cd _loop1/imm32
+    #   _break1:
+    #
+    # . prologue
+    55/push-ebp
+    89/<- %ebp 4/r32/esp
+    # setup
+    (clear-stream _test-input-stream)
+    (clear-stream _test-output-stream)
+    (clear-stream $_test-input-buffered-file->buffer)
+    (clear-stream $_test-output-buffered-file->buffer)
+    # test
+    (write _test-input-stream "{\n{\nab break/imm32\n}\ncd loop/imm32\n}")
+    (subx-braces _test-input-buffered-file _test-output-buffered-file)
+    # check that the line just passed through
+    (flush _test-output-buffered-file)
+#?     # dump _test-output-stream {{{
+#?     (write 2 "^")
+#?     (write-stream 2 _test-output-stream)
+#?     (write 2 "$\n")
+#?     # }}}
+    (check-stream-equal _test-output-stream "_loop0x00000001:\n_loop0x00000002:\nab _break0x00000002/imm32 \n_break0x00000002:\ncd _loop0x00000001/imm32 \n_break0x00000001:\n" "F - test-subx-braces-2")
+    # . epilogue
+    89/<- %esp 5/r32/ebp
+    5d/pop-to-ebp
+    c3/return