diff options
Diffstat (limited to 'koch.nim')
| -rw-r--r-- | koch.nim | 278 |
1 files changed, 174 insertions, 104 deletions
diff --git a/koch.nim b/koch.nim index bf7fb1e62..77bc2299f 100644 --- a/koch.nim +++ b/koch.nim @@ -1,17 +1,22 @@ # # # Maintenance program for Nim -# (c) Copyright 2017 Andreas Rumpf +# (c) Copyright 2024 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# See doc/koch.txt for documentation. +# See doc/koch.md for documentation. # const - NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master - # examples of possible values: #head, #ea82b54, 1.2.3 + # examples of possible values for repos: Head, ea82b54 + NimbleStableCommit = "4fb6f8e6c33963f6f510fe82d09ad2a61b5e4265" # 0.16.1 + AtlasStableCommit = "5faec3e9a33afe99a7d22377dd1b45a5391f5504" + ChecksumsStableCommit = "bd9bf4eaea124bf8d01e08f92ac1b14c6879d8d3" + SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" + + # examples of possible values for fusion: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" when not defined(windows): @@ -34,6 +39,9 @@ import std/[os, strutils, parseopt, osproc] # If this fails with: `Error: cannot open file: std/os`, see # https://github.com/nim-lang/Nim/pull/14291 for explanation + how to fix. +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + import tools / kochdocs import tools / deps @@ -44,7 +52,7 @@ const +-----------------------------------------------------------------+ | Maintenance program for Nim | | Version $1| -| (c) 2017 Andreas Rumpf | +| (c) 2024 Andreas Rumpf | +-----------------------------------------------------------------+ Build time: $2, $3 @@ -59,6 +67,7 @@ Options: --nim:path use specified path for nim binary --localdocs[:path] only build local documentations. If a path is not specified (or empty), the default is used. + --skipIntegrityCheck skips integrity check when booting the compiler Possible Commands: boot [options] bootstraps with given command line options distrohelper [bindir] helper for distro packagers @@ -67,6 +76,8 @@ Possible Commands: e.g. nimble) doesn't require network connectivity nimble builds the Nimble tool + atlas builds the Atlas tool + checksums installs the checksums dependency fusion installs fusion via Nimble Boot options: @@ -76,9 +87,12 @@ Boot options: -d:leanCompiler produce a compiler without JS codegen or documentation generator in order to use less RAM for bootstrapping + -d:nimHasLibFFI adds FFI support for allowing compile-time VM to + interface with native functions (experimental, + requires prior `koch installdeps libffi`) Commands for core developers: - runCI runs continuous integration (CI), e.g. from travis + runCI runs continuous integration (CI), e.g. from Github Actions docs [options] generates the full documentation csource -d:danger builds the C sources for installation pdf builds the PDF documentation @@ -89,10 +103,6 @@ Commands for core developers: tests [options] run the testsuite (run a subset of tests by specifying a category, e.g. `tests cat async`) temp options creates a temporary compiler for testing - pushcsource push generated C sources to its repo -Web options: - --googleAnalytics:UA-... add the given google analytics code to the docs. To - build the official docs, use UA-48159761-1 """ let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed e.g.: `nim c -o:koch_debug koch.nim` @@ -149,13 +159,27 @@ proc bundleNimbleExe(latest: bool, args: string) = let commit = if latest: "HEAD" else: NimbleStableCommit cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", commit = commit, allowBundled = true) + cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/checksums.git", + commit = ChecksumsStableCommit, allowBundled = true) # or copy it from dist? + cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/sat.git", + commit = SatStableCommit, allowBundled = true) # installer.ini expects it under $nim/bin nimCompile("dist/nimble/src/nimble.nim", - options = "-d:release --noNimblePath " & args) + options = "-d:release -d:nimNimbleBootstrap --noNimblePath " & args) + +proc bundleAtlasExe(latest: bool, args: string) = + let commit = if latest: "HEAD" else: AtlasStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/atlas.git", + commit = commit, allowBundled = true) + cloneDependency(distDir / "atlas" / distDir, "https://github.com/nim-lang/sat.git", + commit = SatStableCommit, allowBundled = true) + # installer.ini expects it under $nim/bin + nimCompile("dist/atlas/src/atlas.nim", + options = "-d:release --noNimblePath -d:nimAtlasBootstrap " & args) proc bundleNimsuggest(args: string) = nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", - options = "-d:release -d:danger " & args) + options = "-d:danger " & args) proc buildVccTool(args: string) = let input = "tools/vccexe/vccexe.nim" @@ -182,8 +206,14 @@ proc bundleWinTools(args: string) = nimCompile(r"tools\downloader.nim", options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args) +proc bundleChecksums(latest: bool) = + let commit = if latest: "HEAD" else: ChecksumsStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/checksums.git", commit, allowBundled = true) + proc zip(latest: bool; args: string) = + bundleChecksums(latest) bundleNimbleExe(latest, args) + bundleAtlasExe(latest, args) bundleNimsuggest(args) bundleNimpretty(args) bundleWinTools(args) @@ -216,13 +246,28 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile nimfind", "tools/nimfind.nim", - options = "-d:release " & args) - nimCompileFold("Compile testament", "testament/testament.nim", + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) + + # pre-packages a debug version of nim which can help in many cases investigate issuses + # withouth having to rebuild compiler. + # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler + # `-d:debug` should be changed to a flag that doesn't require re-compiling nim + # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = + "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, + outputName = "nim_dbg") + +proc testTools(args: string = "") = + nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", options = "-d:release " & args) + when defined(windows): buildVccTool(args) + bundleNimpretty(args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) proc nsis(latest: bool; args: string) = + bundleChecksums(latest) bundleNimbleExe(latest, args) + bundleAtlasExe(latest, args) bundleNimsuggest(args) bundleWinTools(args) # make sure we have generated the niminst executables: @@ -242,19 +287,21 @@ proc install(args: string) = geninstall() exec("sh ./install.sh $#" % args) -when false: - proc web(args: string) = - nimexec("js tools/dochack/dochack.nim") - nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" % - [args, VersionAsString]) - - proc website(args: string) = - nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" % - [args, VersionAsString]) - - proc pdf(args="") = - exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" % - [findNim().quoteShell(), args, VersionAsString], additionalPATH=findNim().splitFile.dir) +proc installDeps(dep: string, commit = "") = + # the hashes/urls are version controlled here, so can be changed seamlessly + # and tied to a nim release (mimicking git submodules) + var commit = commit + case dep + of "tinyc": + if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3" + cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit) + of "libffi": + # technically a nimble package, however to play nicely with --noNimblePath, + # let's just clone it wholesale: + if commit.len == 0: commit = "bb2bdaf1a29a4bff6fbd8ae4695877cbb3ec783e" + cloneDependency(distDir, "https://github.com/Araq/libffi", commit) + else: doAssert false, "unsupported: " & dep + # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206 # -------------- boot --------------------------------------------------------- @@ -283,9 +330,9 @@ proc thVersion(i: int): string = template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true" -proc boot(args: string) = +proc boot(args: string, skipIntegrityCheck: bool) = ## bootstrapping is a process that involves 3 steps: - ## 1. use csources to produce nim1.exe. This nim1.exe is buggy but + ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but ## rock solid for building a Nim compiler. It shouldn't be used for anything else. ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need. ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps. @@ -298,39 +345,41 @@ proc boot(args: string) = let smartNimcache = (if "release" in args or "danger" in args: "nimcache/r_" else: "nimcache/d_") & hostOS & "_" & hostCPU + bundleChecksums(false) + + let usingLibFFI = "nimHasLibFFI" in args + if usingLibFFI and not dirExists("dist/libffi"): + installDeps("libffi") + let nimStart = findStartNim().quoteShell() - for i in 0..2: - # Nim versions < (1, 1) expect Nim's exception type to have a 'raiseId' field for - # C++ interop. Later Nim versions do this differently and removed the 'raiseId' field. - # Thus we always bootstrap the first iteration with "c" and not with "cpp" as - # a workaround. - let defaultCommand = if useCpp and i > 0: "cpp" else: "c" + let times = 2 - ord(skipIntegrityCheck) + for i in 0..times: + let defaultCommand = if useCpp: "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" echo "iteration: ", i+1 var extraOption = "" var nimi = i.thVersion if i == 0: nimi = nimStart - extraOption.add " --skipUserCfg --skipParentCfg" + extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap" + + # --noNimblePath precludes nimble packages as dependencies to the compiler, + # so libffi is not "installed as a nimble package" + if usingLibFFI: extraOption.add " --path:./dist" # The configs are skipped for bootstrap # (1st iteration) to prevent newer flags from breaking bootstrap phase. let ret = execCmdEx(nimStart & " --version") doAssert ret.exitCode == 0 let version = ret.output.splitLines[0] - # remove these when csources get updated - template addLib() = + if version.startsWith "Nim Compiler Version 0.20.0": extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291 - if version.startsWith "Nim Compiler Version 0.19.0": - extraOption.add " -d:nimBoostrapCsources0_19_0" - addLib() - elif version.startsWith "Nim Compiler Version 0.20.0": addLib() # in order to use less memory, we split the build into two steps: # --compileOnly produces a $project.json file and does not run GCC/Clang. # jsonbuild then uses the $project.json file to build the Nim binary. - exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" % + exec "$# $# $# --nimcache:$# $# --noNimblePath --compileOnly compiler" / "nim.nim" % [nimi, bootOptions, extraOption, smartNimcache, args] - exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" % + exec "$# jsonscript --noNimblePath --nimcache:$# $# compiler" / "nim.nim" % [nimi, smartNimcache, args] if sameFileContent(output, i.thVersion): @@ -339,7 +388,9 @@ proc boot(args: string) = return copyExe(output, (i+1).thVersion) copyExe(output, finalDest) - when not defined(windows): echo "[Warning] executables are still not equal" + when not defined(windows): + if not skipIntegrityCheck: + echo "[Warning] executables are still not equal" # -------------- clean -------------------------------------------------------- @@ -457,9 +508,11 @@ proc temp(args: string) = result[1].add " " & quoteShell(args[i]) inc i + bundleChecksums(false) + let d = getAppDir() - var output = d / "compiler" / "nim".exe - var finalDest = d / "bin" / "nim_temp".exe + let output = d / "compiler" / "nim".exe + let finalDest = d / "bin" / "nim_temp".exe # 125 is the magic number to tell git bisect to skip the current commit. var (bootArgs, programArgs) = splitArgs(args) if "doc" notin programArgs and @@ -483,6 +536,22 @@ proc xtemp(cmd: string) = finally: copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe) +proc icTest(args: string) = + temp("") + let inp = os.parseCmdLine(args)[0] + let content = readFile(inp) + let nimExe = getAppDir() / "bin" / "nim_temp".exe + var i = 0 + for fragment in content.split("#!EDIT!#"): + let file = inp.replace(".nim", "_temp.nim") + writeFile(file, fragment) + var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd " + if i == 0: + cmd.add "-f " + cmd.add quoteShell(file) + exec(cmd) + inc i + proc buildDrNim(args: string) = if not dirExists("dist/nimz3"): exec("git clone https://github.com/zevv/nimz3.git dist/nimz3") @@ -510,32 +579,30 @@ proc hostInfo(): string = "hostOS: $1, hostCPU: $2, int: $3, float: $4, cpuEndian: $5, cwd: $6" % [hostOS, hostCPU, $int.sizeof, $float.sizeof, $cpuEndian, getCurrentDir()] -proc installDeps(dep: string, commit = "") = - # the hashes/urls are version controlled here, so can be changed seamlessly - # and tied to a nim release (mimicking git submodules) - var commit = commit - case dep - of "tinyc": - if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3" - cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit) - else: doAssert false, "unsupported: " & dep - # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206 - proc runCI(cmd: string) = doAssert cmd.len == 0, cmd # avoid silently ignoring echo "runCI: ", cmd echo hostInfo() # boot without -d:nimHasLibFFI to make sure this still works - kochExecFold("Boot in release mode", "boot -d:release -d:nimStrictMode") + # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in + # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd). + kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib") + + when false: # debugging: when you need to run only 1 test in CI, use something like this: + execFold("debugging test", "nim r tests/stdlib/tosproc.nim") + doAssert false, "debugging only" ## build nimble early on to enable remainder to depend on it if needed kochExecFold("Build Nimble", "nimble") + execFold("Install smtp", "nimble install smtp -y") + let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_") if getEnv("NIM_TEST_PACKAGES", "0") == "1": - execFold("Test selected Nimble packages", "nim r testament/testament $# pcat nimble-packages" % batchParam) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release") + execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam) else: - buildTools() + testTools() for a in "zip opengl sdl1 jester@#head".split: let buildDeps = "build"/"deps" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 @@ -546,51 +613,35 @@ proc runCI(cmd: string) = ## run tests execFold("Test nimscript", "nim e tests/test_nimscript.nims") when defined(windows): - # note: will be over-written below - execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") + execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") # main bottleneck here # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`. # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343 - execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam) + execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam) - block CT_FFI: + block: # nimHasLibFFI: when defined(posix): # windows can be handled in future PR's - execFold("nimble install -y libffi", "nimble install -y libffi") - const nimFFI = "./bin/nim.ctffi" + installDeps("libffi") + const nimFFI = "bin/nim.ctffi" # no need to bootstrap with koch boot (would be slower) let backend = if doUseCpp(): "cpp" else: "c" - execFold("build with -d:nimHasLibFFI", "nim $1 -d:release -d:nimHasLibFFI -o:$2 compiler/nim.nim" % [backend, nimFFI]) + execFold("build with -d:nimHasLibFFI", "nim $1 -d:release --noNimblePath -d:nimHasLibFFI --path:./dist -o:$2 compiler/nim.nim" % [backend, nimFFI]) execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend]) - execFold("Run nimdoc tests", "nim c -r nimdoc/tester") - execFold("Run rst2html tests", "nim c -r nimdoc/rsttester") - execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim") + execFold("Run nimdoc tests", "nim r nimdoc/tester") + execFold("Run rst2html tests", "nim r nimdoc/rsttester") + execFold("Run nimpretty tests", "nim r nimpretty/tester.nim") when defined(posix): - execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester") + # refs #18385, build with -d:release instead of -d:danger for testing + # We could also skip building nimsuggest in buildTools, or build it with -d:release + # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage + # of rebuilding is this won't affect bin/nimsuggest when running runCI locally + execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest") + execFold("Run nimsuggest tests", "nim r nimsuggest/tester") + + kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib") -proc pushCsources() = - if not dirExists("../csources/.git"): - quit "[Error] no csources git repository found" - csource("-d:danger") - let cwd = getCurrentDir() - try: - copyDir("build/c_code", "../csources/c_code") - copyFile("build/build.sh", "../csources/build.sh") - copyFile("build/build.bat", "../csources/build.bat") - copyFile("build/build64.bat", "../csources/build64.bat") - copyFile("build/makefile", "../csources/makefile") - - setCurrentDir("../csources") - for kind, path in walkDir("c_code"): - if kind == pcDir: - exec("git add " & path / "*.c") - exec("git commit -am \"updated csources to version " & NimVersion & "\"") - exec("git push origin master") - exec("git tag -am \"Version $1\" v$1" % NimVersion) - exec("git push origin v$1" % NimVersion) - finally: - setCurrentDir(cwd) proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) @@ -616,7 +667,7 @@ proc testUnixInstall(cmdLineRest: string) = execCleanPath("./koch tools") # check the tests work: putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH") - execCleanPath("./koch tests --nim:./bin/nim cat megatest", destDir / "bin") + execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin") else: echo "Version check: failure" finally: @@ -644,9 +695,16 @@ proc valgrind(cmd: string) = let supp = getAppDir() / "tools" / "nimgrind.supp" exec("valgrind --suppressions=" & supp & valcmd) -proc showHelp() = +proc showHelp(success: bool) = quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)), - CompileDate, CompileTime], QuitSuccess) + CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure) + +proc branchDone() = + let thisBranch = execProcess("git symbolic-ref --short HEAD").strip() + if thisBranch != "devel" and thisBranch != "": + exec("git checkout devel") + exec("git branch -D " & thisBranch) + exec("git pull --rebase") when isMainModule: var op = initOptParser() @@ -654,11 +712,13 @@ when isMainModule: latest = false localDocsOnly = false localDocsOut = "" + skipIntegrityCheck = false while true: op.next() case op.kind of cmdLongOption, cmdShortOption: case normalize(op.key) + of "help", "h": showHelp(success = true) of "latest": latest = true of "stable": latest = false of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir @@ -666,12 +726,14 @@ when isMainModule: localDocsOnly = true if op.val.len > 0: localDocsOut = op.val.absolutePath - else: showHelp() + of "skipintegritycheck": + skipIntegrityCheck = true + else: showHelp(success = false) of cmdArgument: case normalize(op.key) - of "boot": boot(op.cmdLineRest) + of "boot": boot(op.cmdLineRest, skipIntegrityCheck) of "clean": clean(op.cmdLineRest) - of "doc", "docs": buildDocs(op.cmdLineRest, localDocsOnly, localDocsOut) + of "doc", "docs": buildDocs(op.cmdLineRest & " --d:nimPreviewSlimSystem " & paCode, localDocsOnly, localDocsOut) of "doc0", "docs0": # undocumented command for Araq-the-merciful: buildDocs(op.cmdLineRest & gaCode) @@ -691,6 +753,7 @@ when isMainModule: of "xtemp": xtemp(op.cmdLineRest) of "wintools": bundleWinTools(op.cmdLineRest) of "nimble": bundleNimbleExe(latest, op.cmdLineRest) + of "atlas": bundleAtlasExe(latest, op.cmdLineRest) of "nimsuggest": bundleNimsuggest(op.cmdLineRest) # toolsNoNimble is kept for backward compatibility with build scripts of "toolsnonimble", "toolsnoexternal": @@ -698,13 +761,20 @@ when isMainModule: of "tools": buildTools(op.cmdLineRest) bundleNimbleExe(latest, op.cmdLineRest) - of "pushcsource", "pushcsources": pushCsources() + bundleAtlasExe(latest, op.cmdLineRest) + of "checksums": + bundleChecksums(latest) + of "pushcsource": + quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim" of "valgrind": valgrind(op.cmdLineRest) of "c2nim": bundleC2nim(op.cmdLineRest) of "drnim": buildDrNim(op.cmdLineRest) of "fusion": let suffix = if latest: HeadHash else: FusionStableHash exec("nimble install -y fusion@$#" % suffix) - else: showHelp() + of "ic": icTest(op.cmdLineRest) + of "branchdone": branchDone() + else: showHelp(success = false) break - of cmdEnd: break + of cmdEnd: + showHelp(success = false) |