diff options
Diffstat (limited to 'koch.py')
| -rwxr-xr-x | koch.py | 658 |
1 files changed, 0 insertions, 658 deletions
diff --git a/koch.py b/koch.py deleted file mode 100755 index 6430e4a6e..000000000 --- a/koch.py +++ /dev/null @@ -1,658 +0,0 @@ -#! /usr/bin/env python - -########################################################################## -## ## -## Build script of the Nimrod Compiler ## -## (c) 2009 Andreas Rumpf ## -## ## -########################################################################## - -import sys, os, os.path, re, shutil, time, getopt, glob, zlib, pickle -from pycompab import * - -# --------------------- constants ---------------------------------------- - -NIMROD_VERSION = '0.8.1' -# This string contains Nimrod's version. It is the only place -# where the version needs to be updated. The rest is done by -# the build process automatically. It is replaced **everywhere**! -# Format is: Major.Minor.Patch -# Major part: plan is to use number 1 for the first version that is stable; -# higher versions may be incompatible with previous versions -# Minor part: incremented if new features are added; unfortunately often -# not backwards-compatible -# Patch level: is increased for every patch; should be completely -# backwards-compatible - -EXPLAIN = true -force = false - -GENERATE_DIFF = false -# if set, a diff.log file is generated when bootstrapping - -USE_FPC = true - -BOOTCMD = "$1 cc --compile:build/platdef.c $2 rod/nimrod.nim" -# the command used for bootstrapping - -# ---------------------- compiler detection -------------------------------- - -def detect(cmd, lookFor="version"): - try: - pipe = os.popen4(cmd)[1] - except AttributeError: - pipe = os.popen(cmd) - result = None - for line in pipe.readlines(): - if find(lower(line), lookFor) >= 0: - result = line[:-1] - break - pipe.close() - if not result: - # don't give up yet; it may have written to stderr - if os.system(cmd) == 0: - result = cmd - return result - -def detectNimrod(): - if detect("nimrod"): - return "nimrod" - elif detect("bin/nimrod"): - return "bin/nimrod" - else: - Error("could not find a working nimrod executable") - -def detectNim(): - if USE_FPC and detect("fpc -h"): - # workaround a bug in the macosx version of nim: - if getHost() == "macosx": return "nim" - else: return "bin/nim" - return detectNimrod() - -def detectAndCompileNim(): - result = detectNim() - if result == "bin/nim" or result == "nim": - cmd_nim() - return result - -# -------------------------------------------------------------------------- - -def Error(msg): sys.exit("[Koch] *** ERROR: " + msg) -def Warn(msg): print("[Koch] *** WARNING: " + msg) -def Echo(msg): print("[Koch] " + msg) -def _Info(msg): print("[Koch] " + msg) - -_FINGERPRINTS_FILE = "koch.dat" - # in this file all the fingerprints are kept to allow recognizing when a file - # has changed. This works reliably, which cannot be said from just taking - # filetime-stamps. - -def SameFileContent(filenameA, filenameB): - SIZE = 4096*2 - result = true - a = open(filenameA, "rb") - b = open(filenameB, "rb") - while true: - x = a.read(SIZE) - y = b.read(SIZE) - if x != y: - result = false - break - elif len(x) < SIZE: # EOF? - break - a.close() - b.close() - return result - -def SplitArg(s): - if ':' in s: c = ':' - elif '=' in s: c = '=' - else: return (s, '') - i = find(s, c) - return (s[:i], s[i+1:]) - -_baseDir = os.getcwd() -BaseDir = _baseDir - -def Path(a): - # Gets a UNIX like path and converts it to a path on this platform. - # With UNIX like, I mean: slashes, not backslashes, only relative - # paths ('../etc') can be used - result = a - if os.sep != "/": result = replace(result, "/", os.sep) - if os.pardir != "..": result = replace(result, "..", os.pardir) - return result - -def Join(*args): - result = [] - for a in args[:-1]: - result.append(a) - if result[-1] != "/": result.append("/") - result.append(args[-1]) - return replace(join(result, ""), "//", "/") - -def Exec(command): - c = Path(command) - Echo(c) - result = os.system(c) - if result != 0: Error("execution of an external program failed") - return result - -def TryExec(command): - c = Path(command) - Echo(c) - result = os.system(c) - return result - -def RawExec(command): - Echo(command) - result = os.system(command) - if result != 0: Error("execution of an external program failed") - return result - -def Remove(f): - try: - os.remove(Path(f)) - except OSError: - Warn("could not remove: " + f) - -def Move(src, dest): - try: - m = shutil.move - except AttributeError: - def f(src, dest): - shutil.copy(src, dest) - Remove(src) - m = f - s = Path(src) - d = Path(dest) - try: - m(s, d) - except IOError: - Warn(Subs("could not move $1 to $2", s, d)) - except OSError: - Warn(Subs("could not move $1 to $2", s, d)) - -def Copy(src, dest): - s = Path(src) - d = Path(dest) - try: - shutil.copyfile(s, d) - except IOError: - Warn(Subs("could not copy $1 to $2", s, d)) - except OSError: - Warn(Subs("could not copy $1 to $2", s, d)) - -def RemoveDir(f): - try: - shutil.rmtree(Path(f)) - except OSError: - Warn("could not remove: " + f) - -def Exists(f): return os.path.exists(Path(f)) - -def Chdir(dest): - d = Path(dest) - try: - os.chdir(d) - except OSError: - Warn("could not switch to directory: " + d) - -def Mkdir(dest): - d = Path(dest) - try: - os.mkdir(d) - except OSError: - Warn("could not create directory: " + d) - -def Glob(pattern): # needed because glob.glob() is buggy on Windows 95: - # things like tests/t*.nim won't work - global _baseDir - (head, tail) = os.path.split(Path(pattern)) - result = [] - if os.path.exists(head): - try: - os.chdir(os.path.join(_baseDir, head)) - try: - for f in glob.glob(tail): result.append(os.path.join(head, f)) - except OSError: - result = [] - finally: - os.chdir(_baseDir) - return result - -def FilenameNoExt(f): - return os.path.splitext(os.path.basename(f))[0] - -def _Ext(trunc, posixFormat, winFormat): - (head, tail) = os.path.split(Path(trunc)) - if os.name == "posix": frmt = posixFormat - else: frmt = winFormat - return os.path.join(head, Subs(frmt, trunc=tail)) - -def DynExt(trunc): - return _Ext(trunc, 'lib${trunc}.so', '${trunc}.dll') - -def LibExt(trunc): - return _Ext(trunc, '${trunc}.a', '${trunc}.lib') - -def ScriptExt(trunc): - return _Ext(trunc, '${trunc}.sh', '${trunc}.bat') - -def ExeExt(trunc): - return _Ext(trunc, '${trunc}', '${trunc}.exe') - -def MakeExecutable(file): - os.chmod(file, 493) - -class Changed: - """ Returns a Changed object. check() returns true iff one of the - given files has changed. You have to call the object's success() - method if the build has been a success. - - Example: - - c = Changed("unique_name", "file1.pas file2.pas file3.pas") - if c.check(): - Exec("fpc file1.pas") - # Exec raises an exception if it fails, thus if we reach the - # next statement, it was a success: - c.success() - """ - def __init__(self, id, files, explain=false, - fingerprintsfile=_FINGERPRINTS_FILE): - # load the fingerprints file: - # fingerprints is a dict[target, files] where files is a dict[filename, hash] - self.fingers = {} # default value - if Exists(fingerprintsfile): - try: - self.fingers = pickle.load(open(fingerprintsfile, "rb")) - except OSError: - Error("Cannot read from " + fingerprintsfile) - self.filename = fingerprintsfile - self.id = id - self.files = files - self._hashStr = zlib.adler32 # our hash function - self.explain = explain - - def _hashFile(self, f): - x = open(f, "rb") - result = self._hashStr(x.read()) - x.close() # for other Python implementations - return result - - def check(self): - if type(self.files) == type(""): - self.files = split(self.files) - result = false - target = self.id - if not has_key(self.fingers, target): - self.fingers[target] = {} - if self.explain: _Info(Subs("no entries for target '$1'", target)) - result = true - for d in self.files: - if Exists(d): - n = self._hashFile(d) - if not has_key(self.fingers[target], d) or n != self.fingers[target][d]: - result = true - if self.explain: _Info(Subs("'$1' modified since last build", d)) - self.fingers[target][d] = n - else: - Warn(Subs("'$1' does not exist!", d)) - result = true - return result - - def update(self, filename): - self.fingers[self.id][filename] = self._hashFile(filename) - - def success(self): - pickle.dump(self.fingers, open(self.filename, "wb+")) - - -# -------------------------------------------------------------------------- - -def CogRule(name, filename, dependson): - def processCog(filename): - from cogapp import Cog - ret = Cog().main([sys.argv[0], "-r", Path(filename)]) - return ret - - c = Changed(name, filename + " " + dependson, EXPLAIN) - if c.check() or force: - if processCog(filename) == 0: - c.update(filename) - c.success() - else: - Error("Cog failed") - -_nim_exe = os.path.join(os.getcwd(), "bin", ExeExt("nim")) -_output_obj = os.path.join(os.getcwd(), "obj") -FPC_CMD = Subs(r"fpc -Cs16777216 -gl -bl -Crtoi -Sgidh -vw -Se1 -o$1 " - r"-FU$2 $3", _nim_exe, _output_obj, - os.path.join(os.getcwd(), "nim", "nimrod.pas")) - -def buildRod(options): - Exec(Subs("nim compile --compile:build/platdef.c $1 rod/nimrod", options)) - Move(ExeExt("rod/nimrod"), ExeExt("bin/nimrod")) - -def cmd_nim(): - CogRule("nversion", "nim/nversion.pas", "koch.py") - CogRule("msgs", "nim/msgs.pas", "data/messages.yml") - CogRule("ast", "nim/ast.pas", "koch.py data/magic.yml data/ast.yml") - CogRule("scanner", "nim/scanner.pas", "data/keywords.txt") - CogRule("paslex", "nim/paslex.pas", "data/pas_keyw.yml") - CogRule("wordrecg", "nim/wordrecg.pas", "data/keywords.txt") - CogRule("commands", "nim/commands.pas", - "data/basicopt.txt data/advopt.txt") - CogRule("macros", "lib/pure/macros.nim", "koch.py data/ast.yml") - c = Changed("nim", Glob("nim/*.pas"), EXPLAIN) - if c.check() or force: - Exec(FPC_CMD) - if Exists(ExeExt("bin/nim")): - c.success() - return true - return false - -def cmd_rod(options): - prereqs = Glob("lib/*.nim") + Glob("rod/*.nim") + [ - "lib/nimbase.h", "config/nimrod.cfg"] - c = Changed("rod", prereqs, EXPLAIN) - if c.check() or cmd_nim() or force: - buildRod(options) - if Exists(ExeExt("bin/nimrod")): - c.success() - -# ------------------- constants ----------------------------------------------- - -HELP = Subs("""\ -+-----------------------------------------------------------------+ -| Maintenance script for Nimrod | -| Version $1| -| (c) 2009 Andreas Rumpf | -+-----------------------------------------------------------------+ -Your Python version: $2 - -Usage: - koch.py [options] command [options for command] -Options: - --force, -f, -B, -b forces rebuild - --diff generates a diff.log file when bootstrapping - --help, -h shows this help and quits - --no_fpc bootstrap without FPC -Possible Commands: - nim builds the Pascal version of Nimrod - rod [options] builds the Nimrod version of Nimrod (with options) - clean cleans Nimrod project; removes generated files - boot [options] bootstraps with given command line options - rodsrc generates Nimrod version from Pascal version - web generates the website - profile profiles the Nimrod compiler - csource [options] builds the C sources for installation - zip builds the installation ZIP package - inno builds the Inno Setup installer -""", NIMROD_VERSION + ' ' * (44-len(NIMROD_VERSION)), sys.version) - -def main(args): - if len(args) == 0: - print(HELP) - else: - i = 0 - while args[i][:1] == "-": - a = args[i] - if a in ("--force", "-f", "-B", "-b"): - global force - force = true - elif a in ("-h", "--help", "-?"): - print(HELP) - return - elif a == "--diff": - global GENERATE_DIFF - GENERATE_DIFF = true - elif a == "--no_fpc": - global USE_FPC - USE_FPC = false - else: - Error("illegal option: " + a) - i = i + 1 - cmd = args[i] - if cmd == "rod": cmd_rod(join(args[i+1:])) - elif cmd == "nim": cmd_nim() - elif cmd == "clean": cmd_clean() - elif cmd == "boot": cmd_boot(join(args[i+1:])) - elif cmd == "rodsrc": cmd_rodsrc() - elif cmd == "web": cmd_web() - elif cmd == "profile": cmd_profile() - elif cmd == "zip": cmd_zip() - elif cmd == "inno": cmd_inno() - elif cmd == "csource": cmd_csource(join(args[i+1:])) - elif cmd == "install": cmd_install() # for backwards compability - else: Error("illegal command: " + cmd) - -def cmd_csource(args): - Exec(Subs( - "nimrod cc $2 -r tools/niminst --var:version=$1 csource rod/nimrod $2", - NIMROD_VERSION, args)) - -def cmd_zip(): - Exec(Subs("nimrod cc -r tools/niminst --var:version=$1 zip rod/nimrod", - NIMROD_VERSION)) - -def cmd_inno(): - Exec(Subs("nimrod cc -r tools/niminst --var:version=$1 inno rod/nimrod", - NIMROD_VERSION)) - -def cmd_install(): - Exec("sh ./build.sh") - -# -------------------------- bootstrap ---------------------------------------- - -def readCFiles(): - result = {} - if GENERATE_DIFF: - for f in Glob("rod/nimcache/rod/*.c") + Glob("rod/nimcache/lib/*.c"): - x = os.path.split(f)[1] - result[x] = open(f).readlines()[1:] - return result - -def genBootDiff(genA, genB): - def interestingDiff(a, b): - #a = re.sub(r"([a-zA-Z_]+)([0-9]+)", r"\1____", a) - #b = re.sub(r"([a-zA-Z_]+)([0-9]+)", r"\1____", b) - return a != b - - BOOTLOG = "bootdiff.log" - result = false - for f in Glob("diff/*.c"): Remove(f) - if Exists(BOOTLOG): Remove(BOOTLOG) - if GENERATE_DIFF: - lines = [] # lines of the generated logfile - if len(genA) != len(genB): Warn("number of generated files differ!") - for filename, acontent in genA.items(): - bcontent = genB[filename] - if bcontent != acontent: - lines.append("------------------------------------------------------") - lines.append(filename + " differs") - # write the interesting lines to the log file: - for i in range(min(len(acontent), len(bcontent))): - la = acontent[i][:-1] # without newline! - lb = bcontent[i][:-1] - if interestingDiff(la, lb): - lines.append(Subs("$1 - $2", i, la)) - lines.append(Subs("$1 + $2", i, lb)) - if len(acontent) > len(bcontent): - cont = acontent - marker = "-" - else: - cont = bcontent - marker = "+" - for i in range(min(len(acontent), len(bcontent)), len(cont)): - lines.append(Subs("$1 $2 $3", i, marker, cont[i])) - open(os.path.join("diff", "a_"+filename), "w+").write(join(acontent, "")) - open(os.path.join("diff", "b_"+filename), "w+").write(join(bcontent, "")) - if lines: result = true - open(BOOTLOG, "w+").write(join(lines, "\n")) - return result - -def cmd_rodsrc(): - "converts the src/*.pas files into Nimrod syntax" - PAS_FILES_BLACKLIST = split("""nsystem nmath nos osproc ntime strutils""") - compiler = detectAndCompileNim() - CMD = "$1 boot --skip_proj_cfg -o:rod/$2.nim nim/$3" - result = false - for fi in Glob("nim/*.pas"): - f = FilenameNoExt(fi) - if f in PAS_FILES_BLACKLIST: continue - c = Changed(f+"__rodsrc", fi, EXPLAIN) - if c.check() or force: - Exec(Subs(CMD, compiler, f, f+".pas")) - Exec(Subs("$1 parse rod/$2.nim", compiler, f)) - c.success() - result = true - return result - -def moveExes(): - Move(ExeExt("rod/nimrod"), ExeExt("bin/nimrod")) - -def cmd_boot(args): - def myExec(compiler, args=args): - Exec(Subs(BOOTCMD, compiler, args)) - # some C compilers (PellesC) output the executable to the - # wrong directory. We work around this bug here: - if Exists(ExeExt("rod/nimcache/nimrod")): - Move(ExeExt("rod/nimcache/nimrod"), ExeExt("rod/nimrod")) - - writePlatdefC(getNimrodPath()) - compiler = detectAndCompileNim() - cmd_rodsrc() # regenerate nimrod version of the files - - # move the new executable to bin directory (is done by cmd_rod()) - # use the new executable to compile the files in the bootstrap directory: - myExec(compiler) - genA = readCFiles() # first generation of generated C files - # move the new executable to bin directory: - moveExes() - # compile again and compare: - myExec("bin/nimrod") # here we always use the new executable - genB = readCFiles() # second generation of generated C files - diff = genBootDiff(genA, genB) - if diff: - Warn("generated C files are not equal: cycle once again...") - # check if the executables are the same (they should!): - if SameFileContent(Path(ExeExt("rod/nimrod")), - Path(ExeExt("bin/nimrod"))): - Echo("executables are equal: SUCCESS!") - else: - Echo("executables are not equal: cycle once again...") - diff = true - if diff: - # move the new executable to bin directory: - moveExes() - # use the new executable to compile Nimrod: - myExec("bin/nimrod") - if SameFileContent(Path(ExeExt("rod/nimrod")), - Path(ExeExt("bin/nimrod"))): - Echo("executables are equal: SUCCESS!") - else: - Warn("executables are still not equal") - -# ------------------ profile -------------------------------------------------- -def cmd_profile(): - Exec(Subs(BOOTCMD, "nimrod", "-d:release --profiler:on")) - moveExes() - Exec(Subs(BOOTCMD, "nimrod", "--compile_only")) - -# ------------------ web ------------------------------------------------------ - -def cmd_web(): - Exec(Subs("nimrod cc -r tools/nimweb.nim web/nimrod " - "--putenv:nimrodversion=$1", NIMROD_VERSION)) - -# ----------------------------------------------------------------------------- - -def getVersion(): - return NIMROD_VERSION - -# ------------------------------ clean ---------------------------------------- - -CLEAN_EXT = "ppu o obj dcu ~pas ~inc ~dsk ~dpr map tds err bak pyc exe rod" - -def cmd_clean(dir = "."): - L = [] - for x in split(CLEAN_EXT): - L.append(r".*\."+ x +"$") - extRegEx = re.compile(join(L, "|")) - if Exists("koch.dat"): Remove("koch.dat") - for f in Glob("*.pdb"): Remove(f) - for f in Glob("*.idb"): Remove(f) - for f in Glob("web/*.html"): Remove(f) - for f in Glob("doc/*.html"): Remove(f) - for f in Glob("rod/*.nim"): Remove(f) # remove generated source code - def visit(extRegEx, dirname, names): - if os.path.split(dirname)[1] == "nimcache": - shutil.rmtree(path=dirname, ignore_errors=true) - del names - else: - for name in names: - x = os.path.join(dirname, name) - if os.path.isdir(x): continue - if (extRegEx.match(name) - or (os.path.split(dirname)[1] == "tests" and ('.' not in name))): - if find(x, "/dist/") < 0 and find(x, "\\dist\\") < 0: - Echo("removing: " + x) - Remove(x) - os.path.walk(dir, visit, extRegEx) - -def getHost(): - # incomplete list that sys.platform may return: - # win32 aix3 aix4 atheos beos5 darwin freebsd2 freebsd3 freebsd4 freebsd5 - # freebsd6 freebsd7 generic irix5 irix6 linux2 mac netbsd1 next3 os2emx - # riscos sunos5 unixware7 - x = replace(lower(re.sub(r"[0-9]+$", r"", sys.platform)), "-", "") - if x == "win": return "windows" - elif x == "darwin": return "macosx" # probably Mac OS X - elif x == "sunos": return "solaris" - else: return x - -def mydirwalker(dir, L): - for name in os.listdir(dir): - path = os.path.join(dir, name) - if os.path.isdir(path): - mydirwalker(path, L) - else: - L.append(path) - -# --------------- install target ---------------------------------------------- - -def getOSandProcessor(): - host = getHost() - if host == "windows": processor = "i386" # BUGFIX - else: processor = os.uname()[4] - if lower(processor) in ("i686", "i586", "i468", "i386"): - processor = "i386" - if lower(processor) in ("x86_64", "x86-64", "amd64"): - processor = "amd64" - if find(lower(processor), "sparc") >= 0: - processor = "sparc" - return (host, processor) - -def writePlatdefC(nimrodpath): - host, processor = getOSandProcessor() - f = open(os.path.join(nimrodpath, "build/platdef.c"), "w+") - f.write(Subs('/* Generated by koch.py */\n' - 'char* nimOS(void) { return "$1"; }\n' - 'char* nimCPU(void) { return "$2"; }\n' - '\n', host, processor)) - f.close() - -def getNimrodPath(): - if os.name == "posix": - # Does not work 100% reliably. It is the best solution though. - p = replace(sys.argv[0], "./", "") - return os.path.split(os.path.join(os.getcwd(), p))[0] - else: # Windows - return os.path.split(sys.argv[0])[0] - -# ------------------- main ---------------------------------------------------- - -if __name__ == "__main__": - main(sys.argv[1:]) |