diff options
Diffstat (limited to 'koch.nim')
| -rw-r--r-- | koch.nim | 775 |
1 files changed, 491 insertions, 284 deletions
diff --git a/koch.nim b/koch.nim index 97e1da776..77bc2299f 100644 --- a/koch.nim +++ b/koch.nim @@ -1,14 +1,28 @@ # # # Maintenance program for Nim -# (c) Copyright 2017 Andreas Rumpf +# (c) Copyright 2024 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# See doc/koch.txt for documentation. +# See doc/koch.md for documentation. # +const + # examples of possible values for repos: Head, ea82b54 + NimbleStableCommit = "4fb6f8e6c33963f6f510fe82d09ad2a61b5e4265" # 0.16.1 + AtlasStableCommit = "5faec3e9a33afe99a7d22377dd1b45a5391f5504" + ChecksumsStableCommit = "bd9bf4eaea124bf8d01e08f92ac1b14c6879d8d3" + SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" + + # examples of possible values for fusion: #head, #ea82b54, 1.2.3 + FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" + HeadHash = "#head" +when not defined(windows): + const + Z3StableCommit = "65de3f748a6812eecd7db7c478d5fc54424d368b" # the version of Z3 that DrNim uses + when defined(gcc) and defined(windows): when defined(x86): {.link: "icons/koch.res".} @@ -16,12 +30,20 @@ when defined(gcc) and defined(windows): {.link: "icons/koch_icon.o".} when defined(amd64) and defined(windows) and defined(vcc): - {.link: "icons/koch-amd64-windows-vcc.res" .} + {.link: "icons/koch-amd64-windows-vcc.res".} when defined(i386) and defined(windows) and defined(vcc): - {.link: "icons/koch-i386-windows-vcc.res" .} + {.link: "icons/koch-i386-windows-vcc.res".} + +import std/[os, strutils, parseopt, osproc] + # Using `std/os` instead of `os` to fail early if config isn't set up properly. + # If this fails with: `Error: cannot open file: std/os`, see + # https://github.com/nim-lang/Nim/pull/14291 for explanation + how to fix. -import - os, strutils, parseopt, osproc, streams +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + +import tools / kochdocs +import tools / deps const VersionAsString = system.NimVersion @@ -30,7 +52,7 @@ const +-----------------------------------------------------------------+ | Maintenance program for Nim | | Version $1| -| (c) 2017 Andreas Rumpf | +| (c) 2024 Andreas Rumpf | +-----------------------------------------------------------------+ Build time: $2, $3 @@ -38,39 +60,59 @@ Usage: koch [options] command [options for command] Options: --help, -h shows this help and quits + --latest bundle the installers with bleeding edge versions of + external components. + --stable bundle the installers with stable versions of + external components (default). + --nim:path use specified path for nim binary + --localdocs[:path] only build local documentations. If a path is not + specified (or empty), the default is used. + --skipIntegrityCheck skips integrity check when booting the compiler Possible Commands: boot [options] bootstraps with given command line options distrohelper [bindir] helper for distro packagers tools builds Nim related tools + toolsNoExternal builds Nim related tools (except external tools, + e.g. nimble) + doesn't require network connectivity nimble builds the Nimble tool + atlas builds the Atlas tool + checksums installs the checksums dependency + fusion installs fusion via Nimble + Boot options: -d:release produce a release version of the compiler - -d:useLinenoise use the linenoise library for interactive mode - (not needed on Windows) + -d:nimUseLinenoise use the linenoise library for interactive mode + `nim secret` (not needed on Windows) + -d:leanCompiler produce a compiler without JS codegen or + documentation generator in order to use less RAM + for bootstrapping + -d:nimHasLibFFI adds FFI support for allowing compile-time VM to + interface with native functions (experimental, + requires prior `koch installdeps libffi`) Commands for core developers: - web [options] generates the website and the full documentation - website [options] generates only the website - csource -d:release builds the C sources for installation + runCI runs continuous integration (CI), e.g. from Github Actions + docs [options] generates the full documentation + csource -d:danger builds the C sources for installation pdf builds the PDF documentation zip builds the installation zip package xz builds the installation tar.xz package testinstall test tar.xz package; Unix only! - tests [options] run the testsuite + installdeps [options] installs external dependency (e.g. tinyc) to dist/ + tests [options] run the testsuite (run a subset of tests by + specifying a category, e.g. `tests cat async`) temp options creates a temporary compiler for testing - winrelease creates a Windows release - pushcsource push generated C sources to its repo -Web options: - --googleAnalytics:UA-... add the given google analytics code to the docs. To - build the official docs, use UA-48159761-1 """ -const gaCode = " --googleAnalytics:UA-48159761-1" +let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed e.g.: `nim c -o:koch_debug koch.nim` + else: getAppDir() / "koch".exe # works for winrelease -proc exe(f: string): string = - result = addFileExt(f, ExeExt) - when defined(windows): - result = result.replace('/','\\') +proc kochExec*(cmd: string) = + exec kochExe.quoteShell & " " & cmd + +proc kochExecFold*(desc, cmd: string) = + execFold(desc, kochExe.quoteShell & " " & cmd) template withDir(dir, body) = let old = getCurrentDir() @@ -78,78 +120,17 @@ template withDir(dir, body) = setCurrentDir(dir) body finally: - setCurrentdir(old) - -proc findNim(): string = - var nim = "nim".exe - result = "bin" / nim - if existsFile(result): return - for dir in split(getEnv("PATH"), PathSep): - if existsFile(dir / nim): return dir / nim - # assume there is a symlink to the exe or something: - return nim - -proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - let prevPath = getEnv("PATH") - if additionalPath.len > 0: - var absolute = additionalPATH - if not absolute.isAbsolute: - absolute = getCurrentDir() / absolute - echo("Adding to $PATH: ", absolute) - putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) - -proc nimexec(cmd: string) = - exec findNim() & " " & cmd + setCurrentDir(old) -proc execCleanPath(cmd: string, - additionalPath = ""; errorcode: int = QuitFailure) = - # simulate a poor man's virtual environment - let prevPath = getEnv("PATH") - when defined(windows): - let CleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT" - else: - const CleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin" - putEnv("PATH", CleanPath & PathSep & additionalPath) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) - -proc testUnixInstall() = - let oldCurrentDir = getCurrentDir() - try: - let destDir = getTempDir() - copyFile("build/nim-$1.tar.xz" % VersionAsString, - destDir / "nim-$1.tar.xz" % VersionAsString) - setCurrentDir(destDir) - execCleanPath("tar -xJf nim-$1.tar.xz" % VersionAsString) - setCurrentDir("nim-$1" % VersionAsString) - execCleanPath("sh build.sh") - # first test: try if './bin/nim --version' outputs something sane: - let output = execProcess("./bin/nim --version").splitLines - if output.len > 0 and output[0].contains(VersionAsString): - echo "Version check: success" - execCleanPath("./bin/nim c koch.nim") - execCleanPath("./koch boot -d:release", destDir / "bin") - # check the docs build: - execCleanPath("./koch web", destDir / "bin") - # check nimble builds: - execCleanPath("./koch tools") - # check the tests work: - execCleanPath("./koch tests", destDir / "bin") - else: - echo "Version check: failure" - finally: - setCurrentDir oldCurrentDir +let origDir = getCurrentDir() +setCurrentDir(getAppDir()) proc tryExec(cmd: string): bool = echo(cmd) result = execShellCmd(cmd) == 0 proc safeRemove(filename: string) = - if existsFile(filename): removeFile(filename) + if fileExists(filename): removeFile(filename) proc overwriteFile(source, dest: string) = safeRemove(dest) @@ -158,92 +139,98 @@ proc overwriteFile(source, dest: string) = proc copyExe(source, dest: string) = safeRemove(dest) copyFile(dest=dest, source=source) - inclFilePermissions(dest, {fpUserExec}) + inclFilePermissions(dest, {fpUserExec, fpGroupExec, fpOthersExec}) const compileNimInst = "tools/niminst/niminst" + distDir = "dist" proc csource(args: string) = nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " & "--main:compiler/nim.nim compiler/installer.ini $1") % [args, VersionAsString, compileNimInst]) -proc bundleNimbleSrc() = - ## bunldeNimbleSrc() bundles a specific Nimble commit with the tarball. We - ## always bundle the latest official release. - if not dirExists("dist/nimble/.git"): - exec("git clone https://github.com/nim-lang/nimble.git dist/nimble") - withDir("dist/nimble"): - exec("git checkout -f stable") - exec("git pull") - -proc bundleNimbleExe() = - bundleNimbleSrc() - # now compile Nimble and copy it to $nim/bin for the installer.ini - # to pick it up: - nimexec("c -d:release dist/nimble/src/nimble.nim") - copyExe("dist/nimble/src/nimble".exe, "bin/nimble".exe) - -proc buildNimble(latest: bool) = - # old installations created nim/nimblepkg/*.nim files. We remove these - # here so that it cannot cause problems (nimble bug #306): - if dirExists("bin/nimblepkg"): - removeDir("bin/nimblepkg") - # if koch is used for a tar.xz, build the dist/nimble we shipped - # with the tarball: - var installDir = "dist/nimble" - if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"): - discard "don't do the git dance" +proc bundleC2nim(args: string) = + cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git") + nimCompile("dist/c2nim/c2nim", + options = "--noNimblePath --path:. " & args) + +proc bundleNimbleExe(latest: bool, args: string) = + let commit = if latest: "HEAD" else: NimbleStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", + commit = commit, allowBundled = true) + cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/checksums.git", + commit = ChecksumsStableCommit, allowBundled = true) # or copy it from dist? + cloneDependency(distDir / "nimble" / distDir, "https://github.com/nim-lang/sat.git", + commit = SatStableCommit, allowBundled = true) + # installer.ini expects it under $nim/bin + nimCompile("dist/nimble/src/nimble.nim", + options = "-d:release -d:nimNimbleBootstrap --noNimblePath " & args) + +proc bundleAtlasExe(latest: bool, args: string) = + let commit = if latest: "HEAD" else: AtlasStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/atlas.git", + commit = commit, allowBundled = true) + cloneDependency(distDir / "atlas" / distDir, "https://github.com/nim-lang/sat.git", + commit = SatStableCommit, allowBundled = true) + # installer.ini expects it under $nim/bin + nimCompile("dist/atlas/src/atlas.nim", + options = "-d:release --noNimblePath -d:nimAtlasBootstrap " & args) + +proc bundleNimsuggest(args: string) = + nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", + options = "-d:danger " & args) + +proc buildVccTool(args: string) = + let input = "tools/vccexe/vccexe.nim" + if contains(args, "--cc:vcc"): + nimCompileFold("Compile Vcc", input, "build", options = args) + let fileName = input.splitFile.name + moveFile(exe("build" / fileName), exe("bin" / fileName)) else: - if not dirExists("dist/nimble/.git"): - if dirExists(installDir): - var id = 0 - while dirExists("dist/nimble" & $id): - inc id - installDir = "dist/nimble" & $id - exec("git clone https://github.com/nim-lang/nimble.git " & installDir) - withDir(installDir): - if latest: - exec("git checkout -f master") - else: - exec("git checkout -f stable") - exec("git pull") - nimexec("c --noNimblePath -p:compiler -d:release " & installDir / "src/nimble.nim") - copyExe(installDir / "src/nimble".exe, "bin/nimble".exe) - -proc bundleNimsuggest(buildExe: bool) = - if buildExe: - nimexec("c --noNimblePath -d:release -p:compiler nimsuggest/nimsuggest.nim") - copyExe("nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) - removeFile("nimsuggest/nimsuggest".exe) - -proc buildVccTool() = - nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe") - -proc bundleWinTools() = - nimexec("c tools/finish.nim") - copyExe("tools/finish".exe, "finish".exe) - removeFile("tools/finish".exe) - buildVccTool() - nimexec("c -o:bin/nimgrab.exe -d:ssl tools/nimgrab.nim") - nimexec("c -o:bin/nimgrep.exe tools/nimgrep.nim") + nimCompileFold("Compile Vcc", input, options = args) + +proc bundleNimpretty(args: string) = + nimCompileFold("Compile nimpretty", "nimpretty/nimpretty.nim", + options = "-d:release " & args) + +proc bundleWinTools(args: string) = + nimCompile("tools/finish.nim", outputDir = "", options = args) + + buildVccTool(args) + nimCompile("tools/nimgrab.nim", options = "-d:ssl " & args) + nimCompile("tools/nimgrep.nim", options = args) + nimCompile("testament/testament.nim", options = args) when false: # not yet a tool worth including - nimexec(r"c --cc:vcc --app:gui -o:bin\downloader.exe -d:ssl --noNimblePath " & - r"--path:..\ui tools\downloader.nim") - -proc zip(args: string) = - bundleNimbleExe() - bundleNimsuggest(true) - bundleWinTools() + nimCompile(r"tools\downloader.nim", + options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args) + +proc bundleChecksums(latest: bool) = + let commit = if latest: "HEAD" else: ChecksumsStableCommit + cloneDependency(distDir, "https://github.com/nim-lang/checksums.git", commit, allowBundled = true) + +proc zip(latest: bool; args: string) = + bundleChecksums(latest) + bundleNimbleExe(latest, args) + bundleAtlasExe(latest, args) + bundleNimsuggest(args) + bundleNimpretty(args) + bundleWinTools(args) nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" % ["tools/niminst/niminst".exe, VersionAsString]) -proc xz(args: string) = - bundleNimbleSrc() - bundleNimsuggest(false) +proc ensureCleanGit() = + let (outp, status) = osproc.execCmdEx("git diff") + if outp.len != 0: + quit "Not a clean git repository; 'git diff' not empty!" + if status != 0: + quit "Not a clean git repository; 'git diff' returned non-zero!" + +proc xz(latest: bool; args: string) = + ensureCleanGit() nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" % @@ -253,21 +240,36 @@ proc buildTool(toolname, args: string) = nimexec("cc $# $#" % [args, toolname]) copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe) -proc buildTools(latest: bool) = - nimexec "c --noNimblePath -p:compiler -d:release -o:" & ("bin/nimsuggest".exe) & - " nimsuggest/nimsuggest.nim" - - nimexec "c -d:release -o:" & ("bin/nimgrep".exe) & " tools/nimgrep.nim" - when defined(windows): buildVccTool() - - nimexec "c -o:" & ("bin/nimpretty".exe) & " nimpretty/nimpretty.nim" - - buildNimble(latest) - -proc nsis(args: string) = - bundleNimbleExe() - bundleNimsuggest(true) - bundleWinTools() +proc buildTools(args: string = "") = + bundleNimsuggest(args) + nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", + options = "-d:release " & args) + when defined(windows): buildVccTool(args) + bundleNimpretty(args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) + + # pre-packages a debug version of nim which can help in many cases investigate issuses + # withouth having to rebuild compiler. + # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler + # `-d:debug` should be changed to a flag that doesn't require re-compiling nim + # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = + "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, + outputName = "nim_dbg") + +proc testTools(args: string = "") = + nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", + options = "-d:release " & args) + when defined(windows): buildVccTool(args) + bundleNimpretty(args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) + +proc nsis(latest: bool; args: string) = + bundleChecksums(latest) + bundleNimbleExe(latest, args) + bundleAtlasExe(latest, args) + bundleNimsuggest(args) + bundleWinTools(args) # make sure we have generated the niminst executables: buildTool("tools/niminst/niminst", args) #buildTool("tools/nimgrep", args) @@ -285,39 +287,39 @@ proc install(args: string) = geninstall() exec("sh ./install.sh $#" % args) -proc web(args: string) = - nimexec("js tools/dochack/dochack.nim") - nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" % - [args, VersionAsString]) - -proc website(args: string) = - nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" % - [args, VersionAsString]) - -proc pdf(args="") = - exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" % - [findNim(), args, VersionAsString], additionalPATH=findNim().splitFile.dir) +proc installDeps(dep: string, commit = "") = + # the hashes/urls are version controlled here, so can be changed seamlessly + # and tied to a nim release (mimicking git submodules) + var commit = commit + case dep + of "tinyc": + if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3" + cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit) + of "libffi": + # technically a nimble package, however to play nicely with --noNimblePath, + # let's just clone it wholesale: + if commit.len == 0: commit = "bb2bdaf1a29a4bff6fbd8ae4695877cbb3ec783e" + cloneDependency(distDir, "https://github.com/Araq/libffi", commit) + else: doAssert false, "unsupported: " & dep + # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206 # -------------- boot --------------------------------------------------------- proc findStartNim: string = # we try several things before giving up: + # * nimExe # * bin/nim # * $PATH/nim # If these fail, we try to build nim with the "build.(sh|bat)" script. - var nim = "nim".exe - result = "bin" / nim - if existsFile(result): return - for dir in split(getEnv("PATH"), PathSep): - if existsFile(dir / nim): return dir / nim - - when defined(Posix): + let (nim, ok) = findNimImpl() + if ok: return nim + when defined(posix): const buildScript = "build.sh" - if existsFile(buildScript): + if fileExists(buildScript): if tryExec("./" & buildScript): return "bin" / nim else: const buildScript = "build.bat" - if existsFile(buildScript): + if fileExists(buildScript): if tryExec(buildScript): return "bin" / nim echo("Found no nim compiler and every attempt to build one failed!") @@ -326,26 +328,69 @@ proc findStartNim: string = proc thVersion(i: int): string = result = ("compiler" / "nim" & $i).exe -proc boot(args: string) = +template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true" + +proc boot(args: string, skipIntegrityCheck: bool) = + ## bootstrapping is a process that involves 3 steps: + ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but + ## rock solid for building a Nim compiler. It shouldn't be used for anything else. + ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need. + ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps. + ## This step ensures a minimum amount of quality. We know that nim2.exe can be used + ## for Nim compiler development. var output = "compiler" / "nim".exe var finalDest = "bin" / "nim".exe # default to use the 'c' command: - let bootOptions = if args.len == 0 or args.startsWith("-"): "c" else: "" - let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") & - hostOs & "_" & hostCpu + let useCpp = doUseCpp() + let smartNimcache = (if "release" in args or "danger" in args: "nimcache/r_" else: "nimcache/d_") & + hostOS & "_" & hostCPU + + bundleChecksums(false) + + let usingLibFFI = "nimHasLibFFI" in args + if usingLibFFI and not dirExists("dist/libffi"): + installDeps("libffi") - copyExe(findStartNim(), 0.thVersion) - for i in 0..2: + let nimStart = findStartNim().quoteShell() + let times = 2 - ord(skipIntegrityCheck) + for i in 0..times: + let defaultCommand = if useCpp: "cpp" else: "c" + let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" echo "iteration: ", i+1 - exec i.thVersion & " $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, args, - smartNimcache] + var extraOption = "" + var nimi = i.thVersion + if i == 0: + nimi = nimStart + extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap" + + # --noNimblePath precludes nimble packages as dependencies to the compiler, + # so libffi is not "installed as a nimble package" + if usingLibFFI: extraOption.add " --path:./dist" + # The configs are skipped for bootstrap + # (1st iteration) to prevent newer flags from breaking bootstrap phase. + let ret = execCmdEx(nimStart & " --version") + doAssert ret.exitCode == 0 + let version = ret.output.splitLines[0] + if version.startsWith "Nim Compiler Version 0.20.0": + extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291 + + # in order to use less memory, we split the build into two steps: + # --compileOnly produces a $project.json file and does not run GCC/Clang. + # jsonbuild then uses the $project.json file to build the Nim binary. + exec "$# $# $# --nimcache:$# $# --noNimblePath --compileOnly compiler" / "nim.nim" % + [nimi, bootOptions, extraOption, smartNimcache, args] + exec "$# jsonscript --noNimblePath --nimcache:$# $# compiler" / "nim.nim" % + [nimi, smartNimcache, args] + if sameFileContent(output, i.thVersion): copyExe(output, finalDest) echo "executables are equal: SUCCESS!" return copyExe(output, (i+1).thVersion) copyExe(output, finalDest) - when not defined(windows): echo "[Warning] executables are still not equal" + when not defined(windows): + if not skipIntegrityCheck: + echo "[Warning] executables are still not equal" # -------------- clean -------------------------------------------------------- @@ -408,9 +453,10 @@ proc winReleaseArch(arch: string) = withMingw r"..\mingw" & arch & r"\bin": # Rebuilding koch is necessary because it uses its pointer size to # determine which mingw link to put in the NSIS installer. - nimexec "c --cpu:$# koch" % cpu - exec "koch boot -d:release --cpu:$#" % cpu - exec "koch zip -d:release" + inFold "winrelease koch": + nimexec "c --cpu:$# koch" % cpu + kochExecFold("winrelease boot", "boot -d:release --cpu:$#" % cpu) + kochExecFold("winrelease zip", "zip -d:release") overwriteFile r"build\nim-$#.zip" % VersionAsString, r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch] @@ -419,16 +465,19 @@ proc winRelease*() = # anymore! # Build -docs file: when true: - web(gaCode) + inFold "winrelease buildDocs": + buildDocs(gaCode) withDir "web/upload/" & VersionAsString: - exec "7z a -tzip docs-$#.zip *.html" % VersionAsString + inFold "winrelease zipdocs": + exec "7z a -tzip docs-$#.zip *.html" % VersionAsString overwriteFile "web/upload/$1/docs-$1.zip" % VersionAsString, "web/upload/download/docs-$1.zip" % VersionAsString when true: - csource("-d:release") - when true: + inFold "winrelease csource": + csource("-d:danger") + when sizeof(pointer) == 4: winReleaseArch "32" - when true: + when sizeof(pointer) == 8: winReleaseArch "64" # -------------- tests -------------------------------------------------------- @@ -436,16 +485,11 @@ proc winRelease*() = template `|`(a, b): string = (if a.len > 0: a else: b) proc tests(args: string) = - # we compile the tester with taintMode:on to have a basic - # taint mode test :-) - nimexec "cc --taintMode:on --opt:speed tests/testament/tester" - # Since tests take a long time (on my machine), and we want to defy Murhpys - # law - lets make sure the compiler really is freshly compiled! - nimexec "c --lib:lib -d:release --opt:speed compiler/nim.nim" - let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe) - let success = tryExec tester & " " & (args|"all") - if not existsEnv("TRAVIS") and not existsEnv("APPVEYOR"): - exec tester & " html" + nimexec "cc --opt:speed testament/testament" + var testCmd = quoteShell(getCurrentDir() / "testament/testament".exe) + testCmd.add " " & quoteShell("--nim:" & findNim()) + testCmd.add " " & (args|"all") + let success = tryExec testCmd if not success: quit("tests failed", QuitFailure) @@ -464,14 +508,21 @@ proc temp(args: string) = result[1].add " " & quoteShell(args[i]) inc i - var output = "compiler" / "nim".exe - var finalDest = "bin" / "nim_temp".exe - # 125 is the magic number to tell git bisect to skip the current - # commit. - let (bootArgs, programArgs) = splitArgs(args) - let nimexec = findNim() - exec(nimexec & " c -d:debug " & bootArgs & " compiler" / "nim", 125) + bundleChecksums(false) + + let d = getAppDir() + let output = d / "compiler" / "nim".exe + let finalDest = d / "bin" / "nim_temp".exe + # 125 is the magic number to tell git bisect to skip the current commit. + var (bootArgs, programArgs) = splitArgs(args) + if "doc" notin programArgs and + "threads" notin programArgs and + "js" notin programArgs and "rst2html" notin programArgs: + bootArgs = " -d:leanCompiler" & bootArgs + let nimexec = findNim().quoteShell() + exec(nimexec & " c -d:debug --debugger:native -d:nimBetterRun " & bootArgs & " " & (d / "compiler" / "nim"), 125) copyExe(output, finalDest) + setCurrentDir(origDir) if programArgs.len > 0: exec(finalDest & " " & programArgs) proc xtemp(cmd: string) = @@ -485,28 +536,142 @@ proc xtemp(cmd: string) = finally: copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe) -proc pushCsources() = - if not dirExists("../csources/.git"): - quit "[Error] no csources git repository found" - csource("-d:release") - let cwd = getCurrentDir() +proc icTest(args: string) = + temp("") + let inp = os.parseCmdLine(args)[0] + let content = readFile(inp) + let nimExe = getAppDir() / "bin" / "nim_temp".exe + var i = 0 + for fragment in content.split("#!EDIT!#"): + let file = inp.replace(".nim", "_temp.nim") + writeFile(file, fragment) + var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd " + if i == 0: + cmd.add "-f " + cmd.add quoteShell(file) + exec(cmd) + inc i + +proc buildDrNim(args: string) = + if not dirExists("dist/nimz3"): + exec("git clone https://github.com/zevv/nimz3.git dist/nimz3") + when defined(windows): + if not dirExists("dist/dlls"): + exec("git clone -q https://github.com/nim-lang/dlls.git dist/dlls") + copyExe("dist/dlls/libz3.dll", "bin/libz3.dll") + execFold("build drnim", "nim c -o:$1 $2 drnim/drnim" % ["bin/drnim".exe, args]) + else: + if not dirExists("dist/z3"): + exec("git clone -q https://github.com/Z3Prover/z3.git dist/z3") + withDir("dist/z3"): + exec("git fetch") + exec("git checkout " & Z3StableCommit) + createDir("build") + withDir("build"): + exec("""cmake -DZ3_BUILD_LIBZ3_SHARED=FALSE -G "Unix Makefiles" ../""") + exec("make -j4") + execFold("build drnim", "nim cpp --dynlibOverride=libz3 -o:$1 $2 drnim/drnim" % ["bin/drnim".exe, args]) + # always run the tests for now: + exec("testament/testament".exe & " --nim:" & "drnim".exe & " pat drnim/tests") + + +proc hostInfo(): string = + "hostOS: $1, hostCPU: $2, int: $3, float: $4, cpuEndian: $5, cwd: $6" % + [hostOS, hostCPU, $int.sizeof, $float.sizeof, $cpuEndian, getCurrentDir()] + +proc runCI(cmd: string) = + doAssert cmd.len == 0, cmd # avoid silently ignoring + echo "runCI: ", cmd + echo hostInfo() + # boot without -d:nimHasLibFFI to make sure this still works + # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in + # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd). + kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib") + + when false: # debugging: when you need to run only 1 test in CI, use something like this: + execFold("debugging test", "nim r tests/stdlib/tosproc.nim") + doAssert false, "debugging only" + + ## build nimble early on to enable remainder to depend on it if needed + kochExecFold("Build Nimble", "nimble") + + execFold("Install smtp", "nimble install smtp -y") + + let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_") + if getEnv("NIM_TEST_PACKAGES", "0") == "1": + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release") + execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam) + else: + testTools() + + for a in "zip opengl sdl1 jester@#head".split: + let buildDeps = "build"/"deps" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + # if this gives `Additional info: "build/deps" [OSError]`, make sure nimble is >= v0.12.0, + # otherwise `absolutePath` is needed, refs https://github.com/nim-lang/nimble/issues/901 + execFold("", "nimble install -y --nimbleDir:$# $#" % [buildDeps.quoteShell, a]) + + ## run tests + execFold("Test nimscript", "nim e tests/test_nimscript.nims") + when defined(windows): + execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament") + + # main bottleneck here + # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`. + # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343 + execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam) + + block: # nimHasLibFFI: + when defined(posix): # windows can be handled in future PR's + installDeps("libffi") + const nimFFI = "bin/nim.ctffi" + # no need to bootstrap with koch boot (would be slower) + let backend = if doUseCpp(): "cpp" else: "c" + execFold("build with -d:nimHasLibFFI", "nim $1 -d:release --noNimblePath -d:nimHasLibFFI --path:./dist -o:$2 compiler/nim.nim" % [backend, nimFFI]) + execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend]) + + execFold("Run nimdoc tests", "nim r nimdoc/tester") + execFold("Run rst2html tests", "nim r nimdoc/rsttester") + execFold("Run nimpretty tests", "nim r nimpretty/tester.nim") + when defined(posix): + # refs #18385, build with -d:release instead of -d:danger for testing + # We could also skip building nimsuggest in buildTools, or build it with -d:release + # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage + # of rebuilding is this won't affect bin/nimsuggest when running runCI locally + execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest") + execFold("Run nimsuggest tests", "nim r nimsuggest/tester") + + kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib") + + +proc testUnixInstall(cmdLineRest: string) = + csource("-d:danger" & cmdLineRest) + xz(false, cmdLineRest) + let oldCurrentDir = getCurrentDir() try: - copyDir("build/c_code", "../csources/c_code") - copyFile("build/build.sh", "../csources/build.sh") - copyFile("build/build.bat", "../csources/build.bat") - copyFile("build/build64.bat", "../csources/build64.bat") - copyFile("build/makefile", "../csources/makefile") - - setCurrentDir("../csources") - for kind, path in walkDir("c_code"): - if kind == pcDir: - exec("git add " & path / "*.c") - exec("git commit -am \"updated csources to version " & NimVersion & "\"") - exec("git push origin master") - exec("git tag -am \"Version $1\" v$1" % NimVersion) - exec("git push origin v$1" % NimVersion) + let destDir = getTempDir() + copyFile("build/nim-$1.tar.xz" % VersionAsString, + destDir / "nim-$1.tar.xz" % VersionAsString) + setCurrentDir(destDir) + execCleanPath("tar -xJf nim-$1.tar.xz" % VersionAsString) + setCurrentDir("nim-$1" % VersionAsString) + execCleanPath("sh build.sh") + # first test: try if './bin/nim --version' outputs something sane: + let output = execProcess("./bin/nim --version").splitLines + if output.len > 0 and output[0].contains(VersionAsString): + echo "Version check: success" + execCleanPath("./bin/nim c koch.nim") + execCleanPath("./koch boot -d:release", destDir / "bin") + # check the docs build: + execCleanPath("./koch docs", destDir / "bin") + # check nimble builds: + execCleanPath("./koch tools") + # check the tests work: + putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH") + execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin") + else: + echo "Version check: failure" finally: - setCurrentDir(cwd) + setCurrentDir oldCurrentDir proc valgrind(cmd: string) = # somewhat hacky: '=' sign means "pass to valgrind" else "pass to Nim" @@ -530,44 +695,86 @@ proc valgrind(cmd: string) = let supp = getAppDir() / "tools" / "nimgrind.supp" exec("valgrind --suppressions=" & supp & valcmd) -proc showHelp() = +proc showHelp(success: bool) = quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)), - CompileDate, CompileTime], QuitSuccess) + CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure) + +proc branchDone() = + let thisBranch = execProcess("git symbolic-ref --short HEAD").strip() + if thisBranch != "devel" and thisBranch != "": + exec("git checkout devel") + exec("git branch -D " & thisBranch) + exec("git pull --rebase") when isMainModule: var op = initOptParser() - op.next() - case op.kind - of cmdLongOption, cmdShortOption: showHelp() - of cmdArgument: - case normalize(op.key) - of "boot": boot(op.cmdLineRest) - of "clean": clean(op.cmdLineRest) - of "web": web(op.cmdLineRest) - of "doc", "docs": web("--onlyDocs " & op.cmdLineRest) - of "json2": web("--json2 " & op.cmdLineRest) - of "website": website(op.cmdLineRest & gaCode) - of "web0": - # undocumented command for Araq-the-merciful: - web(op.cmdLineRest & gaCode) - of "pdf": pdf() - of "csource", "csources": csource(op.cmdLineRest) - of "zip": zip(op.cmdLineRest) - of "xz": xz(op.cmdLineRest) - of "nsis": nsis(op.cmdLineRest) - of "geninstall": geninstall(op.cmdLineRest) - of "distrohelper": geninstall() - of "install": install(op.cmdLineRest) - of "testinstall": testUnixInstall() - of "test", "tests": tests(op.cmdLineRest) - of "temp": temp(op.cmdLineRest) - of "xtemp": xtemp(op.cmdLineRest) - #of "winrelease": winRelease() - of "wintools": bundleWinTools() - of "nimble": buildNimble(existsDir(".git")) - of "nimsuggest": bundleNimsuggest(buildExe=true) - of "tools": buildTools(existsDir(".git")) - of "pushcsource", "pushcsources": pushCsources() - of "valgrind": valgrind(op.cmdLineRest) - else: showHelp() - of cmdEnd: showHelp() + var + latest = false + localDocsOnly = false + localDocsOut = "" + skipIntegrityCheck = false + while true: + op.next() + case op.kind + of cmdLongOption, cmdShortOption: + case normalize(op.key) + of "help", "h": showHelp(success = true) + of "latest": latest = true + of "stable": latest = false + of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir + of "localdocs": + localDocsOnly = true + if op.val.len > 0: + localDocsOut = op.val.absolutePath + of "skipintegritycheck": + skipIntegrityCheck = true + else: showHelp(success = false) + of cmdArgument: + case normalize(op.key) + of "boot": boot(op.cmdLineRest, skipIntegrityCheck) + of "clean": clean(op.cmdLineRest) + of "doc", "docs": buildDocs(op.cmdLineRest & " --d:nimPreviewSlimSystem " & paCode, localDocsOnly, localDocsOut) + of "doc0", "docs0": + # undocumented command for Araq-the-merciful: + buildDocs(op.cmdLineRest & gaCode) + of "pdf": buildPdfDoc(op.cmdLineRest, "doc/pdf") + of "csource", "csources": csource(op.cmdLineRest) + of "zip": zip(latest, op.cmdLineRest) + of "xz": xz(latest, op.cmdLineRest) + of "nsis": nsis(latest, op.cmdLineRest) + of "geninstall": geninstall(op.cmdLineRest) + of "distrohelper": geninstall() + of "install": install(op.cmdLineRest) + of "testinstall": testUnixInstall(op.cmdLineRest) + of "installdeps": installDeps(op.cmdLineRest) + of "runci": runCI(op.cmdLineRest) + of "test", "tests": tests(op.cmdLineRest) + of "temp": temp(op.cmdLineRest) + of "xtemp": xtemp(op.cmdLineRest) + of "wintools": bundleWinTools(op.cmdLineRest) + of "nimble": bundleNimbleExe(latest, op.cmdLineRest) + of "atlas": bundleAtlasExe(latest, op.cmdLineRest) + of "nimsuggest": bundleNimsuggest(op.cmdLineRest) + # toolsNoNimble is kept for backward compatibility with build scripts + of "toolsnonimble", "toolsnoexternal": + buildTools(op.cmdLineRest) + of "tools": + buildTools(op.cmdLineRest) + bundleNimbleExe(latest, op.cmdLineRest) + bundleAtlasExe(latest, op.cmdLineRest) + of "checksums": + bundleChecksums(latest) + of "pushcsource": + quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim" + of "valgrind": valgrind(op.cmdLineRest) + of "c2nim": bundleC2nim(op.cmdLineRest) + of "drnim": buildDrNim(op.cmdLineRest) + of "fusion": + let suffix = if latest: HeadHash else: FusionStableHash + exec("nimble install -y fusion@$#" % suffix) + of "ic": icTest(op.cmdLineRest) + of "branchdone": branchDone() + else: showHelp(success = false) + break + of cmdEnd: + showHelp(success = false) |