From a800400c360c302a06c4127a34023b92244bcbf6 Mon Sep 17 00:00:00 2001 From: "Kartik K. Agaram" Date: Mon, 29 Aug 2016 21:42:41 -0700 Subject: 3281 - faster incremental builds for layers Before: layers -> tangle -> g++ All changes to (C++) layers triggered recompilation of everything, taking 35s on my laptop, and over 4 minutes on a puny server with just 512MB of RAM. After: layers -> tangle -> cleave -> g++ Now a tiny edit takes just 5s to recompile on my laptop. My initial approach was to turn each function into a separate compilation unit under the .build/ directory. That blew up the time for a full/initial compilation to almost 6 minutes on my laptop. Trial and error showed 4 compilation units to be close to the sweet spot. Full compilation is still slightly slower (43s) but not by much. I could speed things up further by building multiple of the compilation units in parallel (the recursive invocation in 'makefile'). But that would put more pressure on a puny server, so I'm going to avoid getting too aggressive. --- Other considerations I spent some time manually testing the dependency structure to the makefile, making sure that files aren't unnecessarily written to disk, modifying their timestamp and triggering dependent work; that changes to layers don't unnecessarily modify the common headers or list of globals; that changes to the cleave/ tool itself rebuild the entire project; that the old auto-generated '_list' files plug in at the right stage in the pipeline; that changes to common headers trigger recompilation of everything; etc. Too bad it's not easy to write some tests for all this. I spent some time trying to make sure the makefile was not too opaque to a newcomer. The targets mostly flow from top to bottom. There's a little diagram at the top that is hopefully illuminating. When I had 700 compilation units for 700 functions I stopped printing each one of those compilation commands, but when I backed off to just 4 compilation units I decided to err on the side of making the build steps easy to see. --- makefile | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) (limited to 'makefile') diff --git a/makefile b/makefile index a4c97916..3938cbb2 100644 --- a/makefile +++ b/makefile @@ -1,27 +1,51 @@ +# [0-9]*.cc -> mu.cc -> .build/*.cc -> .build/*.o -> .build/mu_bin +# (layers) | | | | +# tangle cleave $CXX $CXX +# +# [0-9]*.mu -> core.mu + all: mu_bin core.mu CXX ?= c++ CXXFLAGS ?= -g -O3 -# reduce memory usage for small servers -CXXFLAGS := ${CXXFLAGS} --param ggc-min-expand=1 --param ggc-min-heapsize=32768 +CXXFLAGS := ${CXXFLAGS} -Wall -Wextra -ftrapv -fno-strict-aliasing + +core.mu: [0-9]*.mu mu.cc makefile + cat $$(./enumerate/enumerate --until zzz |grep '.mu$$') > core.mu + +mu_bin: mu.cc makefile function_list test_list cleave/cleave + @mkdir -p .build + @cp function_list test_list .build + @mkdir -p .build/termbox + @cp termbox/termbox.h .build/termbox + ./cleave/cleave mu.cc .build + @# recursive (potentially parallel) make to pick up BUILD_SRC after cleave + @make .build/mu_bin + cp .build/mu_bin . + +BUILD_SRC=$(wildcard .build/*.cc) +.build/mu_bin: $(BUILD_SRC:.cc=.o) termbox/libtermbox.a + ${CXX} .build/*.o termbox/libtermbox.a -o .build/mu_bin -mu_bin: makefile mu.cc termbox/libtermbox.a function_list test_list - ${CXX} ${CXXFLAGS} -Wall -Wextra -ftrapv -fno-strict-aliasing mu.cc termbox/libtermbox.a -o mu_bin +.build/%.o: .build/%.cc .build/header .build/global_declarations_list + @# explicitly state default rule since we added dependencies + ${CXX} ${CXXFLAGS} -c $< -o $@ # To see what the program looks like after all layers have been applied, read # mu.cc mu.cc: [0-9]*.cc enumerate/enumerate tangle/tangle ./tangle/tangle $$(./enumerate/enumerate --until zzz |grep -v '.mu$$') > mu.cc -core.mu: [0-9]*.mu mu.cc - cat $$(./enumerate/enumerate --until zzz |grep '.mu$$') > core.mu - enumerate/enumerate: cd enumerate && make tangle/tangle: cd tangle && make && ./tangle test +cleave/cleave: cleave/cleave.cc + cd cleave && make + rm -rf .build + termbox/libtermbox.a: termbox/*.c termbox/*.h termbox/*.inl cd termbox && make @@ -43,12 +67,17 @@ test_list: mu.cc @grep -h "^\s*void test_" mu.cc |perl -pwe 's/^\s*void (.*)\(\) \{.*/$$1,/' > test_list @grep -h "^\s*TEST(" mu.cc |perl -pwe 's/^\s*TEST\((.*)\)$$/test_$$1,/' >> test_list +.build/global_declarations_list: .build/global_definitions_list + grep ';' .build/global_definitions_list |perl -pwe 's/[=(].*/;/' |perl -pwe 's/^[^\/# ]/extern $$&/' |perl -pwe 's/^extern extern /extern /' > .build/global_declarations_list + .PHONY: all clean clena clena: clean clean: cd enumerate && make clean cd tangle && make clean + cd cleave && make clean cd termbox && make clean -rm mu.cc core.mu mu_bin *_list -rm -rf mu_bin.* + -rm -rf .build -- cgit 1.4.1-2-gfad0