diff options
Diffstat (limited to 'koch.nim')
| -rw-r--r-- | koch.nim | 465 |
1 files changed, 300 insertions, 165 deletions
diff --git a/koch.nim b/koch.nim index 7d7e252bf..77bc2299f 100644 --- a/koch.nim +++ b/koch.nim @@ -1,16 +1,27 @@ # # # Maintenance program for Nim -# (c) Copyright 2017 Andreas Rumpf +# (c) Copyright 2024 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# See doc/koch.txt for documentation. +# See doc/koch.md for documentation. # const - NimbleStableCommit = "4007b2a778429a978e12307bf13a038029b4c4d9" # master + # examples of possible values for repos: Head, ea82b54 + NimbleStableCommit = "4fb6f8e6c33963f6f510fe82d09ad2a61b5e4265" # 0.16.1 + AtlasStableCommit = "5faec3e9a33afe99a7d22377dd1b45a5391f5504" + ChecksumsStableCommit = "bd9bf4eaea124bf8d01e08f92ac1b14c6879d8d3" + SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" + + # examples of possible values for fusion: #head, #ea82b54, 1.2.3 + FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" + HeadHash = "#head" +when not defined(windows): + const + Z3StableCommit = "65de3f748a6812eecd7db7c478d5fc54424d368b" # the version of Z3 that DrNim uses when defined(gcc) and defined(windows): when defined(x86): @@ -23,10 +34,16 @@ when defined(amd64) and defined(windows) and defined(vcc): when defined(i386) and defined(windows) and defined(vcc): {.link: "icons/koch-i386-windows-vcc.res".} -import - os, strutils, parseopt, osproc +import std/[os, strutils, parseopt, osproc] + # Using `std/os` instead of `os` to fail early if config isn't set up properly. + # If this fails with: `Error: cannot open file: std/os`, see + # https://github.com/nim-lang/Nim/pull/14291 for explanation + how to fix. + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] import tools / kochdocs +import tools / deps const VersionAsString = system.NimVersion @@ -35,7 +52,7 @@ const +-----------------------------------------------------------------+ | Maintenance program for Nim | | Version $1| -| (c) 2017 Andreas Rumpf | +| (c) 2024 Andreas Rumpf | +-----------------------------------------------------------------+ Build time: $2, $3 @@ -43,42 +60,52 @@ Usage: koch [options] command [options for command] Options: --help, -h shows this help and quits - --latest bundle the installers with a bleeding edge Nimble - --stable bundle the installers with a stable Nimble (default) + --latest bundle the installers with bleeding edge versions of + external components. + --stable bundle the installers with stable versions of + external components (default). --nim:path use specified path for nim binary + --localdocs[:path] only build local documentations. If a path is not + specified (or empty), the default is used. + --skipIntegrityCheck skips integrity check when booting the compiler Possible Commands: boot [options] bootstraps with given command line options distrohelper [bindir] helper for distro packagers tools builds Nim related tools - toolsNoNimble builds Nim related tools (except nimble) + toolsNoExternal builds Nim related tools (except external tools, + e.g. nimble) doesn't require network connectivity nimble builds the Nimble tool + atlas builds the Atlas tool + checksums installs the checksums dependency + fusion installs fusion via Nimble + Boot options: -d:release produce a release version of the compiler - -d:useLinenoise use the linenoise library for interactive mode - (not needed on Windows) + -d:nimUseLinenoise use the linenoise library for interactive mode + `nim secret` (not needed on Windows) -d:leanCompiler produce a compiler without JS codegen or documentation generator in order to use less RAM for bootstrapping + -d:nimHasLibFFI adds FFI support for allowing compile-time VM to + interface with native functions (experimental, + requires prior `koch installdeps libffi`) Commands for core developers: - runCI runs continuous integration (CI), eg from travis + runCI runs continuous integration (CI), e.g. from Github Actions docs [options] generates the full documentation - csource -d:release builds the C sources for installation + csource -d:danger builds the C sources for installation pdf builds the PDF documentation zip builds the installation zip package xz builds the installation tar.xz package testinstall test tar.xz package; Unix only! + installdeps [options] installs external dependency (e.g. tinyc) to dist/ tests [options] run the testsuite (run a subset of tests by specifying a category, e.g. `tests cat async`) temp options creates a temporary compiler for testing - pushcsource push generated C sources to its repo -Web options: - --googleAnalytics:UA-... add the given google analytics code to the docs. To - build the official docs, use UA-48159761-1 """ -let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed eg: `nim c -o:koch_debug koch.nim` +let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed e.g.: `nim c -o:koch_debug koch.nim` else: getAppDir() / "koch".exe # works for winrelease proc kochExec*(cmd: string) = @@ -103,7 +130,7 @@ proc tryExec(cmd: string): bool = result = execShellCmd(cmd) == 0 proc safeRemove(filename: string) = - if existsFile(filename): removeFile(filename) + if fileExists(filename): removeFile(filename) proc overwriteFile(source, dest: string) = safeRemove(dest) @@ -116,6 +143,7 @@ proc copyExe(source, dest: string) = const compileNimInst = "tools/niminst/niminst" + distDir = "dist" proc csource(args: string) = nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " & @@ -123,52 +151,44 @@ proc csource(args: string) = [args, VersionAsString, compileNimInst]) proc bundleC2nim(args: string) = - if not dirExists("dist/c2nim/.git"): - exec("git clone https://github.com/nim-lang/c2nim.git dist/c2nim") + cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git") nimCompile("dist/c2nim/c2nim", options = "--noNimblePath --path:. " & args) proc bundleNimbleExe(latest: bool, args: string) = - if not dirExists("dist/nimble/.git"): - exec("git clone https://github.com/nim-lang/nimble.git dist/nimble") - if not latest: - withDir("dist/nimble"): - exec("git fetch") - exec("git checkout " & NimbleStableCommit) + let commit = if latest: "HEAD" else: NimbleStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", + commit = commit, allowBundled = true) + cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/checksums.git", + commit = ChecksumsStableCommit, allowBundled = true) # or copy it from dist? + cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/sat.git", + commit = SatStableCommit, allowBundled = true) # installer.ini expects it under $nim/bin nimCompile("dist/nimble/src/nimble.nim", - options = "-d:release --nilseqs:on " & args) - -proc buildNimble(latest: bool, args: string) = - # if koch is used for a tar.xz, build the dist/nimble we shipped - # with the tarball: - var installDir = "dist/nimble" - if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"): - discard "don't do the git dance" - else: - if not dirExists("dist/nimble/.git"): - if dirExists(installDir): - var id = 0 - while dirExists("dist/nimble" & $id): - inc id - installDir = "dist/nimble" & $id - exec("git clone https://github.com/nim-lang/nimble.git " & installDir) - withDir(installDir): - if latest: - exec("git checkout -f master") - exec("git pull") - else: - exec("git fetch") - exec("git checkout " & NimbleStableCommit) - nimCompile(installDir / "src/nimble.nim", - options = "--noNimblePath --nilseqs:on -d:release " & args) + options = "-d:release -d:nimNimbleBootstrap --noNimblePath " & args) + +proc bundleAtlasExe(latest: bool, args: string) = + let commit = if latest: "HEAD" else: AtlasStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/atlas.git", + commit = commit, allowBundled = true) + cloneDependency(distDir / "atlas" / distDir, "https://github.com/nim-lang/sat.git", + commit = SatStableCommit, allowBundled = true) + # installer.ini expects it under $nim/bin + nimCompile("dist/atlas/src/atlas.nim", + options = "-d:release --noNimblePath -d:nimAtlasBootstrap " & args) proc bundleNimsuggest(args: string) = nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", - options = "-d:release -d:danger " & args) + options = "-d:danger " & args) proc buildVccTool(args: string) = - nimCompileFold("Compile Vcc", "tools/vccexe/vccexe.nim ", options = args) + let input = "tools/vccexe/vccexe.nim" + if contains(args, "--cc:vcc"): + nimCompileFold("Compile Vcc", input, "build", options = args) + let fileName = input.splitFile.name + moveFile(exe("build" / fileName), exe("bin" / fileName)) + else: + nimCompileFold("Compile Vcc", input, options = args) proc bundleNimpretty(args: string) = nimCompileFold("Compile nimpretty", "nimpretty/nimpretty.nim", @@ -180,15 +200,20 @@ proc bundleWinTools(args: string) = buildVccTool(args) nimCompile("tools/nimgrab.nim", options = "-d:ssl " & args) nimCompile("tools/nimgrep.nim", options = args) - bundleC2nim(args) nimCompile("testament/testament.nim", options = args) when false: # not yet a tool worth including nimCompile(r"tools\downloader.nim", options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args) +proc bundleChecksums(latest: bool) = + let commit = if latest: "HEAD" else: ChecksumsStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/checksums.git", commit, allowBundled = true) + proc zip(latest: bool; args: string) = + bundleChecksums(latest) bundleNimbleExe(latest, args) + bundleAtlasExe(latest, args) bundleNimsuggest(args) bundleNimpretty(args) bundleWinTools(args) @@ -221,13 +246,28 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile nimfind", "tools/nimfind.nim", - options = "-d:release " & args) - nimCompileFold("Compile testament", "testament/testament.nim", + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) + + # pre-packages a debug version of nim which can help in many cases investigate issuses + # withouth having to rebuild compiler. + # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler + # `-d:debug` should be changed to a flag that doesn't require re-compiling nim + # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = + "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, + outputName = "nim_dbg") + +proc testTools(args: string = "") = + nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", options = "-d:release " & args) + when defined(windows): buildVccTool(args) + bundleNimpretty(args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) proc nsis(latest: bool; args: string) = + bundleChecksums(latest) bundleNimbleExe(latest, args) + bundleAtlasExe(latest, args) bundleNimsuggest(args) bundleWinTools(args) # make sure we have generated the niminst executables: @@ -247,40 +287,39 @@ proc install(args: string) = geninstall() exec("sh ./install.sh $#" % args) -when false: - proc web(args: string) = - nimexec("js tools/dochack/dochack.nim") - nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" % - [args, VersionAsString]) - - proc website(args: string) = - nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" % - [args, VersionAsString]) - - proc pdf(args="") = - exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" % - [findNim(), args, VersionAsString], additionalPATH=findNim().splitFile.dir) +proc installDeps(dep: string, commit = "") = + # the hashes/urls are version controlled here, so can be changed seamlessly + # and tied to a nim release (mimicking git submodules) + var commit = commit + case dep + of "tinyc": + if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3" + cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit) + of "libffi": + # technically a nimble package, however to play nicely with --noNimblePath, + # let's just clone it wholesale: + if commit.len == 0: commit = "bb2bdaf1a29a4bff6fbd8ae4695877cbb3ec783e" + cloneDependency(distDir, "https://github.com/Araq/libffi", commit) + else: doAssert false, "unsupported: " & dep + # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206 # -------------- boot --------------------------------------------------------- proc findStartNim: string = # we try several things before giving up: + # * nimExe # * bin/nim # * $PATH/nim # If these fail, we try to build nim with the "build.(sh|bat)" script. - var nim = "nim".exe - result = "bin" / nim - if existsFile(result): return - for dir in split(getEnv("PATH"), PathSep): - if existsFile(dir / nim): return dir / nim - - when defined(Posix): + let (nim, ok) = findNimImpl() + if ok: return nim + when defined(posix): const buildScript = "build.sh" - if existsFile(buildScript): + if fileExists(buildScript): if tryExec("./" & buildScript): return "bin" / nim else: const buildScript = "build.bat" - if existsFile(buildScript): + if fileExists(buildScript): if tryExec(buildScript): return "bin" / nim echo("Found no nim compiler and every attempt to build one failed!") @@ -289,16 +328,32 @@ proc findStartNim: string = proc thVersion(i: int): string = result = ("compiler" / "nim" & $i).exe -proc boot(args: string) = +template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true" + +proc boot(args: string, skipIntegrityCheck: bool) = + ## bootstrapping is a process that involves 3 steps: + ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but + ## rock solid for building a Nim compiler. It shouldn't be used for anything else. + ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need. + ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps. + ## This step ensures a minimum amount of quality. We know that nim2.exe can be used + ## for Nim compiler development. var output = "compiler" / "nim".exe var finalDest = "bin" / "nim".exe # default to use the 'c' command: - let useCpp = getEnv("NIM_COMPILE_TO_CPP", "false") == "true" + let useCpp = doUseCpp() let smartNimcache = (if "release" in args or "danger" in args: "nimcache/r_" else: "nimcache/d_") & hostOS & "_" & hostCPU - let nimStart = findStartNim() - for i in 0..2: + bundleChecksums(false) + + let usingLibFFI = "nimHasLibFFI" in args + if usingLibFFI and not dirExists("dist/libffi"): + installDeps("libffi") + + let nimStart = findStartNim().quoteShell() + let times = 2 - ord(skipIntegrityCheck) + for i in 0..times: let defaultCommand = if useCpp: "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" echo "iteration: ", i+1 @@ -306,23 +361,26 @@ proc boot(args: string) = var nimi = i.thVersion if i == 0: nimi = nimStart - extraOption.add " --skipUserCfg --skipParentCfg" + extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap" + + # --noNimblePath precludes nimble packages as dependencies to the compiler, + # so libffi is not "installed as a nimble package" + if usingLibFFI: extraOption.add " --path:./dist" # The configs are skipped for bootstrap # (1st iteration) to prevent newer flags from breaking bootstrap phase. let ret = execCmdEx(nimStart & " --version") doAssert ret.exitCode == 0 let version = ret.output.splitLines[0] - if version.startsWith "Nim Compiler Version 0.19.0": - extraOption.add " -d:nimBoostrapCsources0_19_0" - # remove this when csources get updated + if version.startsWith "Nim Compiler Version 0.20.0": + extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291 # in order to use less memory, we split the build into two steps: # --compileOnly produces a $project.json file and does not run GCC/Clang. # jsonbuild then uses the $project.json file to build the Nim binary. - exec "$# $# $# $# --nimcache:$# --compileOnly compiler" / "nim.nim" % - [nimi, bootOptions, extraOption, args, smartNimcache] - exec "$# jsonscript $# --nimcache:$# compiler" / "nim.nim" % - [nimi, args, smartNimcache] + exec "$# $# $# --nimcache:$# $# --noNimblePath --compileOnly compiler" / "nim.nim" % + [nimi, bootOptions, extraOption, smartNimcache, args] + exec "$# jsonscript --noNimblePath --nimcache:$# $# compiler" / "nim.nim" % + [nimi, smartNimcache, args] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) @@ -330,7 +388,9 @@ proc boot(args: string) = return copyExe(output, (i+1).thVersion) copyExe(output, finalDest) - when not defined(windows): echo "[Warning] executables are still not equal" + when not defined(windows): + if not skipIntegrityCheck: + echo "[Warning] executables are still not equal" # -------------- clean -------------------------------------------------------- @@ -414,7 +474,7 @@ proc winRelease*() = "web/upload/download/docs-$1.zip" % VersionAsString when true: inFold "winrelease csource": - csource("-d:release") + csource("-d:danger") when sizeof(pointer) == 4: winReleaseArch "32" when sizeof(pointer) == 8: @@ -426,8 +486,10 @@ template `|`(a, b): string = (if a.len > 0: a else: b) proc tests(args: string) = nimexec "cc --opt:speed testament/testament" - let tester = quoteShell(getCurrentDir() / "testament/testament".exe) - let success = tryExec tester & " " & (args|"all") + var testCmd = quoteShell(getCurrentDir() / "testament/testament".exe) + testCmd.add " " & quoteShell("--nim:" & findNim()) + testCmd.add " " & (args|"all") + let success = tryExec testCmd if not success: quit("tests failed", QuitFailure) @@ -446,16 +508,18 @@ proc temp(args: string) = result[1].add " " & quoteShell(args[i]) inc i + bundleChecksums(false) + let d = getAppDir() - var output = d / "compiler" / "nim".exe - var finalDest = d / "bin" / "nim_temp".exe + let output = d / "compiler" / "nim".exe + let finalDest = d / "bin" / "nim_temp".exe # 125 is the magic number to tell git bisect to skip the current commit. var (bootArgs, programArgs) = splitArgs(args) if "doc" notin programArgs and "threads" notin programArgs and - "js" notin programArgs: - bootArgs.add " -d:leanCompiler" - let nimexec = findNim() + "js" notin programArgs and "rst2html" notin programArgs: + bootArgs = " -d:leanCompiler" & bootArgs + let nimexec = findNim().quoteShell() exec(nimexec & " c -d:debug --debugger:native -d:nimBetterRun " & bootArgs & " " & (d / "compiler" / "nim"), 125) copyExe(output, finalDest) setCurrentDir(origDir) @@ -472,76 +536,115 @@ proc xtemp(cmd: string) = finally: copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe) +proc icTest(args: string) = + temp("") + let inp = os.parseCmdLine(args)[0] + let content = readFile(inp) + let nimExe = getAppDir() / "bin" / "nim_temp".exe + var i = 0 + for fragment in content.split("#!EDIT!#"): + let file = inp.replace(".nim", "_temp.nim") + writeFile(file, fragment) + var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd " + if i == 0: + cmd.add "-f " + cmd.add quoteShell(file) + exec(cmd) + inc i + +proc buildDrNim(args: string) = + if not dirExists("dist/nimz3"): + exec("git clone https://github.com/zevv/nimz3.git dist/nimz3") + when defined(windows): + if not dirExists("dist/dlls"): + exec("git clone -q https://github.com/nim-lang/dlls.git dist/dlls") + copyExe("dist/dlls/libz3.dll", "bin/libz3.dll") + execFold("build drnim", "nim c -o:$1 $2 drnim/drnim" % ["bin/drnim".exe, args]) + else: + if not dirExists("dist/z3"): + exec("git clone -q https://github.com/Z3Prover/z3.git dist/z3") + withDir("dist/z3"): + exec("git fetch") + exec("git checkout " & Z3StableCommit) + createDir("build") + withDir("build"): + exec("""cmake -DZ3_BUILD_LIBZ3_SHARED=FALSE -G "Unix Makefiles" ../""") + exec("make -j4") + execFold("build drnim", "nim cpp --dynlibOverride=libz3 -o:$1 $2 drnim/drnim" % ["bin/drnim".exe, args]) + # always run the tests for now: + exec("testament/testament".exe & " --nim:" & "drnim".exe & " pat drnim/tests") + + +proc hostInfo(): string = + "hostOS: $1, hostCPU: $2, int: $3, float: $4, cpuEndian: $5, cwd: $6" % + [hostOS, hostCPU, $int.sizeof, $float.sizeof, $cpuEndian, getCurrentDir()] + proc runCI(cmd: string) = doAssert cmd.len == 0, cmd # avoid silently ignoring - echo "runCI:", cmd - # note(@araq): Do not replace these commands with direct calls (eg boot()) - # as that would weaken our testing efforts. - when defined(posix): # appveyor (on windows) didn't run this - kochExecFold("Boot", "boot") + echo "runCI: ", cmd + echo hostInfo() # boot without -d:nimHasLibFFI to make sure this still works - kochExecFold("Boot in release mode", "boot -d:release -d:danger") + # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in + # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd). + kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib") + + when false: # debugging: when you need to run only 1 test in CI, use something like this: + execFold("debugging test", "nim r tests/stdlib/tosproc.nim") + doAssert false, "debugging only" ## build nimble early on to enable remainder to depend on it if needed kochExecFold("Build Nimble", "nimble") - when false: - execFold("nimble install -y libffi", "nimble install -y libffi") - kochExecFold("boot -d:release -d:nimHasLibFFI", "boot -d:release -d:nimHasLibFFI") + execFold("Install smtp", "nimble install smtp -y") - if getEnv("NIM_TEST_PACKAGES", "false") == "true": - execFold("Test selected Nimble packages", "nim c -r testament/testament cat nimble-packages") + let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_") + if getEnv("NIM_TEST_PACKAGES", "0") == "1": + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release") + execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam) else: - buildTools() # altenatively, kochExec "tools --toolsNoNimble" + testTools() + + for a in "zip opengl sdl1 jester@#head".split: + let buildDeps = "build"/"deps" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + # if this gives `Additional info: "build/deps" [OSError]`, make sure nimble is >= v0.12.0, + # otherwise `absolutePath` is needed, refs https://github.com/nim-lang/nimble/issues/901 + execFold("", "nimble install -y --nimbleDir:$# $#" % [buildDeps.quoteShell, a]) ## run tests execFold("Test nimscript", "nim e tests/test_nimscript.nims") when defined(windows): - # note: will be over-written below - execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") + execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") # main bottleneck here - execFold("Run tester", "nim c -r -d:nimCoroutines testament/testament --pedantic all -d:nimCoroutines") - - execFold("Run nimdoc tests", "nim c -r nimdoc/tester") - execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim") + # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`. + # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343 + execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam) + + block: # nimHasLibFFI: + when defined(posix): # windows can be handled in future PR's + installDeps("libffi") + const nimFFI = "bin/nim.ctffi" + # no need to bootstrap with koch boot (would be slower) + let backend = if doUseCpp(): "cpp" else: "c" + execFold("build with -d:nimHasLibFFI", "nim $1 -d:release --noNimblePath -d:nimHasLibFFI --path:./dist -o:$2 compiler/nim.nim" % [backend, nimFFI]) + execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend]) + + execFold("Run nimdoc tests", "nim r nimdoc/tester") + execFold("Run rst2html tests", "nim r nimdoc/rsttester") + execFold("Run nimpretty tests", "nim r nimpretty/tester.nim") when defined(posix): - execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester") + # refs #18385, build with -d:release instead of -d:danger for testing + # We could also skip building nimsuggest in buildTools, or build it with -d:release + # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage + # of rebuilding is this won't affect bin/nimsuggest when running runCI locally + execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest") + execFold("Run nimsuggest tests", "nim r nimsuggest/tester") + + kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib") - ## remaining actions - when defined(posix): - kochExecFold("Docs", "docs --git.commit:devel") - kochExecFold("C sources", "csource") - elif defined(windows): - when false: - kochExec "csource" - kochExec "zip" - -proc pushCsources() = - if not dirExists("../csources/.git"): - quit "[Error] no csources git repository found" - csource("-d:release") - let cwd = getCurrentDir() - try: - copyDir("build/c_code", "../csources/c_code") - copyFile("build/build.sh", "../csources/build.sh") - copyFile("build/build.bat", "../csources/build.bat") - copyFile("build/build64.bat", "../csources/build64.bat") - copyFile("build/makefile", "../csources/makefile") - - setCurrentDir("../csources") - for kind, path in walkDir("c_code"): - if kind == pcDir: - exec("git add " & path / "*.c") - exec("git commit -am \"updated csources to version " & NimVersion & "\"") - exec("git push origin master") - exec("git tag -am \"Version $1\" v$1" % NimVersion) - exec("git push origin v$1" % NimVersion) - finally: - setCurrentDir(cwd) proc testUnixInstall(cmdLineRest: string) = - csource("-d:release " & cmdLineRest) + csource("-d:danger" & cmdLineRest) xz(false, cmdLineRest) let oldCurrentDir = getCurrentDir() try: @@ -564,7 +667,7 @@ proc testUnixInstall(cmdLineRest: string) = execCleanPath("./koch tools") # check the tests work: putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH") - execCleanPath("./koch tests --nim:./bin/nim cat megatest", destDir / "bin") + execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin") else: echo "Version check: failure" finally: @@ -592,27 +695,45 @@ proc valgrind(cmd: string) = let supp = getAppDir() / "tools" / "nimgrind.supp" exec("valgrind --suppressions=" & supp & valcmd) -proc showHelp() = +proc showHelp(success: bool) = quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)), - CompileDate, CompileTime], QuitSuccess) + CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure) + +proc branchDone() = + let thisBranch = execProcess("git symbolic-ref --short HEAD").strip() + if thisBranch != "devel" and thisBranch != "": + exec("git checkout devel") + exec("git branch -D " & thisBranch) + exec("git pull --rebase") when isMainModule: var op = initOptParser() - var latest = false + var + latest = false + localDocsOnly = false + localDocsOut = "" + skipIntegrityCheck = false while true: op.next() case op.kind of cmdLongOption, cmdShortOption: case normalize(op.key) + of "help", "h": showHelp(success = true) of "latest": latest = true of "stable": latest = false of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir - else: showHelp() + of "localdocs": + localDocsOnly = true + if op.val.len > 0: + localDocsOut = op.val.absolutePath + of "skipintegritycheck": + skipIntegrityCheck = true + else: showHelp(success = false) of cmdArgument: case normalize(op.key) - of "boot": boot(op.cmdLineRest) + of "boot": boot(op.cmdLineRest, skipIntegrityCheck) of "clean": clean(op.cmdLineRest) - of "doc", "docs": buildDocs(op.cmdLineRest) + of "doc", "docs": buildDocs(op.cmdLineRest & " --d:nimPreviewSlimSystem " & paCode, localDocsOnly, localDocsOut) of "doc0", "docs0": # undocumented command for Araq-the-merciful: buildDocs(op.cmdLineRest & gaCode) @@ -625,21 +746,35 @@ when isMainModule: of "distrohelper": geninstall() of "install": install(op.cmdLineRest) of "testinstall": testUnixInstall(op.cmdLineRest) + of "installdeps": installDeps(op.cmdLineRest) of "runci": runCI(op.cmdLineRest) of "test", "tests": tests(op.cmdLineRest) of "temp": temp(op.cmdLineRest) of "xtemp": xtemp(op.cmdLineRest) of "wintools": bundleWinTools(op.cmdLineRest) - of "nimble": buildNimble(latest, op.cmdLineRest) + of "nimble": bundleNimbleExe(latest, op.cmdLineRest) + of "atlas": bundleAtlasExe(latest, op.cmdLineRest) of "nimsuggest": bundleNimsuggest(op.cmdLineRest) - of "toolsnonimble": + # toolsNoNimble is kept for backward compatibility with build scripts + of "toolsnonimble", "toolsnoexternal": buildTools(op.cmdLineRest) of "tools": buildTools(op.cmdLineRest) - buildNimble(latest, op.cmdLineRest) - of "pushcsource", "pushcsources": pushCsources() + bundleNimbleExe(latest, op.cmdLineRest) + bundleAtlasExe(latest, op.cmdLineRest) + of "checksums": + bundleChecksums(latest) + of "pushcsource": + quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim" of "valgrind": valgrind(op.cmdLineRest) of "c2nim": bundleC2nim(op.cmdLineRest) - else: showHelp() + of "drnim": buildDrNim(op.cmdLineRest) + of "fusion": + let suffix = if latest: HeadHash else: FusionStableHash + exec("nimble install -y fusion@$#" % suffix) + of "ic": icTest(op.cmdLineRest) + of "branchdone": branchDone() + else: showHelp(success = false) break - of cmdEnd: break + of cmdEnd: + showHelp(success = false) |