#!/bin/sh # Experimental alternative to build2 that can run steps in parallel if their # dependencies are met. Caveats: # # 1. We rely on the OS to schedule steps, so thousands of independent tasks # will likely be counter-productive. # 2. Can run out of virtual memory if you spawn too many say $CC processes. # 3. Compilation errors can cause the script to hang. We tag the most common # suspects with '|| quit', but can't eliminate the possibility entirely. # 4. Ugly as heck! This version really benefits from comparisons with its # 'upstream', build2. And even then, diff gets confused. # 5. There's a mechanical difficulty: we use mktemp to reliably create # temporary filenames, which has the side effect of also creating the # files. So zero-size files are considered equivalent to non-existent # files. When some commands have trouble with this (e.g. ar) we need to # delete the empty file, which can expose us to a race condition wrt # mktemp. set -e # stop immediately on error # Some environment variables that can be passed in. For example, to turn off # optimization: # $ CFLAGS=-g ./build4 test "$CXX" || export CXX=c++ test "$CC" || export CC=cc test "$CFLAGS" || export CFLAGS="-g -O2" export CFLAGS="$CFLAGS -Wall -Wextra -ftrapv -fno-strict-aliasing" export CXXFLAGS="-std=c++98 $CFLAGS" # CI has an ancient version; don't expect recent dialects # Outline: # [0-9]*.cc -> mu.cc -> .build/*.cc -> .build/*.o -> .build/mu_bin # (layers) | | | | # tangle cleave $CXX $CXX # can also be called with a layer to only build until # $ ./build4 --until 050 UNTIL_LAYER=${2:-zzz} # there's two mechanisms for fast builds here: # - if a command is quick to run, always run it but update the result only on any change # - otherwise run it only if the output is 'older_than' the inputs # # avoid combining both mechanisms for a single file # otherwise you'll see spurious messages about files being updated # risk: a file may unnecessarily update without changes, causing unnecessary work downstream # return 1 if $1 is older than _any_ of the remaining args older_than() { local target=$1 shift if [ ! -s $target ] then #? echo "$target has size zero" #? echo "updating $target" >&2 return 0 # success fi local f for f in $* do if [ $f -nt $target ] then #? echo "updating $target" >&2 return 0 # success fi done return 1 # failure } # redirect to $1, unless it's already identical update() { if [ ! -e $1 ] then cat > $1 else cat > $1.tmp diff -q $1 $1.tmp >/dev/null && rm $1.tmp || mv $1.tmp $1 fi } update_cp() { if [ ! -e $2/$1 ] then cp $1 $2 elif [ $1 -nt $2/$1 ] then cp $1 $2 fi } noisy_cd() { cd $1 echo "-- `pwd`" >&2 } mv_if_exists() { test -e $1 && mv $1 $2 return 0 } # wait for all the given filenames to exist # exit immediately if a special file called '.quit' exists QUITFILE=`pwd`/.quit rm -f $QUITFILE wait_for_all() { # could use inotify on Linux while ! all_exist "$@" do #? echo waiting: $* test -e $QUITFILE && return 1 # some step had an error; stop all waiting steps using errexit sleep 1 done return 0 } quit() { touch $QUITFILE exit 1 } all_exist() { for f in "$@" do test -e $f || return 1 done return 0 } TMP=`mktemp` mv_if_exists ../../tools/enumerate $TMP ( wait_for_all ../../tools/enumerate.cc older_than $TMP ../../tools/enumerate.cc && { echo "building enumerate" $CXX $CXXFLAGS ../../tools/enumerate.cc -o $TMP || quit echo "done building enumerate" } mv $TMP ../../tools/enumerate ) & TMP=`mktemp` mv_if_exists ../../tools/tangle $TMP ( wait_for_all ../../tools/tangle.cc older_than $TMP ../../tools/tangle.cc && { echo "building tangle" cd ../../tools # auto-generate various lists (ending in '_list' by convention) { # list of function declarations, so I can define them in any order grep -h "^[^ #].*) {" tangle.cc |sed 's/ {.*/;/' |update tangle.function_list # list of tests to run grep -h "^[[:space:]]*void test_" tangle.cc |sed 's/^\s*void \(.*\)() {$/\1,/' |update tangle.test_list grep -h "^\s*void test_" tangle.cc |sed 's/^\s*void \(.*\)() {.*/"\1",/' |update tangle.test_name_list # } # Now that we have all the _lists, compile 'tangle' $CXX $CXXFLAGS tangle.cc -o $TMP || quit $TMP test cd ../archive/1.vm echo "done building tangle" } mv $TMP ../../tools/tangle ) & wait_for_all ../../tools/enumerate echo "enumerating layers" LAYERS=$(../../tools/enumerate --until $UNTIL_LAYER |grep '\.cc$') echo "done enumerating layers" TMP=`mktemp` mv_if_exists mu.cc $TMP ( wait_for_all $LAYERS ../../tools/enumerate ../../tools/tangle older_than $TMP $LAYERS ../../tools/enumerate ../../tools/tangle && { echo "running tangle" # no update here; rely on 'update' calls downstream ../../tools/tangle $LAYERS >$TMP || quit echo "done running tangle" } mv $TMP mu.cc ) & TMP=`mktemp` mv_if_exists cleave/cleave $TMP ( wait_for_all cleave/cleave.cc older_than $TMP cleave/cleave.cc && { echo "building cleave" $CXX $CXXFLAGS cleave/cleave.cc -o $TMP || quit rm -rf .build echo "done building cleave" } mv $TMP cleave/cleave ) & wait_for_all mu.cc cleave/cleave # cleave/cleave just for the .build cleanup mkdir -p .build # auto-generate function declarations, so I can define them in any order # functions start out unindented, have all args on the same line, and end in ') {' # # \/ ignore struct/class methods grep -h "^[^[:space:]#].*) {$" mu.cc |grep -v ":.*(" |sed 's/ {.*/;/' |update .build/function_list # auto-generate list of tests to run grep -h "^\s*void test_" mu.cc |sed 's/^\s*void \(.*\)() {.*/\1,/' |update .build/test_list grep -h "^\s*void test_" mu.cc |sed 's/^\s*void \(.*\)() {.*/"\1",/' |update .build/test_name_list mkdir -p .build/termbox update_cp termbox/termbox.h .build/termbox TMP=`mktemp` mv_if_exists mu_bin $TMP ( wait_for_all mu.cc cleave/cleave termbox/*.c termbox/*.h termbox/*.inl older_than $TMP mu.cc *_list cleave/cleave termbox/* && { echo "building mu_bin" cleave/cleave mu.cc .build || quit cd .build # create the list of global variable declarations from the corresponding definitions grep ';' global_definitions_list |sed 's/[=(].*/;/' |sed 's/^[^\/# ]/extern &/' |sed 's/^extern extern /extern /' |update global_declarations_list for f in mu_*.cc do OBJ=`echo $f |sed 's/\.cc$/.o/'` TMP=`mktemp` mv_if_exists $OBJ $TMP