8 files changed, 199 insertions, 108 deletions
diff --git a/ci/action.nim b/ci/action.nim
new file mode 100644
index 000000000..ad0f4df0b
--- /dev/null
+++ b/ci/action.nim
@@ -0,0 +1,26 @@
+import std/[strutils, os, osproc, parseutils, strformat]
+
+
+proc main() =
+  var msg = ""
+  const cmd = "./koch boot --mm:orc -d:release"
+
+  let (output, exitCode) = execCmdEx(cmd)
+
+  doAssert exitCode == 0, output
+
+  let start = rfind(output, "Hint: mm")
+  doAssert parseUntil(output, msg, "; proj", start) > 0, output
+
+  let (commitHash, _) = execCmdEx("""git log --format="%H" -n 1""")
+
+  let welcomeMessage = fmt"""Thanks for your hard work on this PR!
+The lines below are statistics of the Nim compiler built from {commitHash}
+
+{msg}
+"""
+  createDir "ci/nimcache"
+  writeFile "ci/nimcache/results.txt", welcomeMessage
+
+when isMainModule:
+  main()
diff --git a/ci/build.bat b/ci/build.bat
deleted file mode 100644
index cdce8d3d2..000000000
--- a/ci/build.bat
+++ /dev/null
@@ -1,14 +0,0 @@
-REM Some debug info
-echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
-gcc -v
-
-git clone --depth 1 https://github.com/nim-lang/csources.git
-cd csources
-call build64.bat
-cd ..
-set PATH=%CD%\bin;%PATH%
-nim -v
-nim c koch
-koch.exe boot
-copy bin/nim bin/nimd
-koch.exe boot -d:release
diff --git a/ci/build.sh b/ci/build.sh
deleted file mode 100644
index 6321fffba..000000000
--- a/ci/build.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-sh ci/deps.sh
-
-# Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources.git
-cd csources
-sh build.sh
-cd ..
-# Add Nim to the PATH
-export PATH=$(pwd)/bin${PATH:+:$PATH}
-# Bootstrap.
-nim -v
-nim c koch
-./koch boot
-cp bin/nim bin/nimd
-./koch boot -d:release
diff --git a/ci/build_autogen.bat b/ci/build_autogen.bat
new file mode 100644
index 000000000..0b6ca566e
--- /dev/null
+++ b/ci/build_autogen.bat
@@ -0,0 +1,26 @@
+@echo off
+rem DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`
+rem Build development version of the compiler; can be rerun safely
+rem bare bones version of ci/funs.sh adapted for windows.
+
+rem Read in some common shared variables (shared with other tools),
+rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
+for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
+SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe
+echo "building from csources: %nim_csources%"
+
+if not exist %nim_csourcesDir% (
+  git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir%
+)
+
+if not exist %nim_csources% (
+  cd %nim_csourcesDir%
+  git checkout %nim_csourcesHash%
+  echo "%PROCESSOR_ARCHITECTURE%"
+  if "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
+    SET ARCH=64
+  )
+  CALL build.bat
+  cd ..
+  copy /y bin\nim.exe  %nim_csources%
+)
diff --git a/ci/deps.bat b/ci/deps.bat
deleted file mode 100644
index bda1fe14f..000000000
--- a/ci/deps.bat
+++ /dev/null
@@ -1,4 +0,0 @@
-nim e install_nimble.nims
-nim e tests/test_nimscript.nims
-nimble update
-nimble install -y zip opengl sdl1 jester@#head niminst
diff --git a/ci/deps.sh b/ci/deps.sh
deleted file mode 100644
index f0f831a2a..000000000
--- a/ci/deps.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-# Some debug info
-echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS."
-
-# Packages
-apt-get update -qq
-apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs
-
-gcc -v
-
-export PATH=$(pwd)/bin${PATH:+:$PATH}
-
-# Nimble deps
-nim e install_nimble.nims
-nim e tests/test_nimscript.nims
-nimble update
-nimble install zip opengl sdl1 jester@#head niminst
diff --git a/ci/funs.sh b/ci/funs.sh
new file mode 100644
index 000000000..026366198
--- /dev/null
+++ b/ci/funs.sh
@@ -0,0 +1,147 @@
+# Utilities used in CI pipelines and tooling to avoid duplication.
+# Avoid top-level statements.
+# Prefer nim scripts whenever possible.
+# functions starting with `_` are considered internal, less stable.
+
+echo_run () {
+  # echo's a command before running it, which helps understanding logs
+  echo ""
+  echo "cmd: $@" # in azure we could also use this: echo '##[section]"$@"'
+  "$@"
+}
+
+nimGetLastCommit() {
+  git log --no-merges -1 --pretty=format:"%s"
+}
+
+nimIsCiSkip(){
+  # D20210329T004830:here refs https://github.com/microsoft/azure-pipelines-agent/issues/2944
+  # `--no-merges` is needed to avoid merge commits which occur for PR's.
+  # $(Build.SourceVersionMessage) is not helpful
+  # nor is `github.event.head_commit.message` for github actions.
+  # Note: `[skip ci]` is now handled automatically for github actions, see https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
+  commitMsg=$(nimGetLastCommit)
+  echo commitMsg: "$commitMsg"
+  if [[ $commitMsg == *"[skip ci]"* ]]; then
+    echo "skipci: true"
+    return 0
+  else
+    echo "skipci: false"
+    return 1
+  fi
+}
+
+nimInternalInstallDepsWindows(){
+  echo_run mkdir dist
+  echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+  echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+  echo_run 7z x dist/mingw64.7z -odist
+  echo_run 7z x dist/dlls.zip -obin
+}
+
+nimInternalBuildKochAndRunCI(){
+  echo_run nim c koch
+  if ! echo_run ./koch runCI; then
+    echo_run echo "runCI failed"
+    echo_run nim r tools/ci_testresults.nim
+    return 1
+  fi
+}
+
+nimDefineVars(){
+  . config/build_config.txt
+  nim_csources=bin/nim_csources_$nim_csourcesHash
+}
+
+_nimNumCpu(){
+  # linux: $(nproc)
+  # FreeBSD | macOS: $(sysctl -n hw.ncpu)
+  # OpenBSD: $(sysctl -n hw.ncpuonline)
+  # windows: $NUMBER_OF_PROCESSORS ?
+  if env | grep -q '^NIMCORES='; then
+    echo $NIMCORES
+  else
+    echo $(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+  fi
+}
+
+_nimBuildCsourcesIfNeeded(){
+  # if some systems cannot use make or gmake, we could add support for calling `build.sh`
+  # but this is slower (not parallel jobs) and would require making build.sh
+  # understand the arguments passed to the makefile (e.g. `CC=gcc ucpu=amd64 uos=darwin`),
+  # instead of `--cpu amd64 --os darwin`.
+  unamestr=$(uname)
+  # uname values: https://en.wikipedia.org/wiki/Uname
+  if [ "$unamestr" = 'FreeBSD' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'OpenBSD' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'NetBSD' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'CROSSOS' ]; then
+    makeX=gmake
+  elif [ "$unamestr" = 'SunOS' ]; then
+    makeX=gmake
+  else
+    makeX=make
+  fi
+  nCPU=$(_nimNumCpu)
+  echo_run which $makeX
+  # parallel jobs (5X faster on 16 cores: 10s instead of 50s)
+  echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@"
+  # keep $nim_csources in case needed to investigate bootstrap issues
+  # without having to rebuild
+  echo_run cp bin/nim $nim_csources
+}
+
+nimCiSystemInfo(){
+  nimDefineVars
+  echo_run eval echo '$'nim_csources
+  echo_run pwd
+  echo_run date
+  echo_run uname -a
+  echo_run git log --no-merges -1 --pretty=oneline
+  echo_run eval echo '$'PATH
+  echo_run gcc -v
+  echo_run node -v
+  echo_run make -v
+}
+
+nimCsourcesHash(){
+  nimDefineVars
+  echo $nim_csourcesHash
+}
+
+nimBuildCsourcesIfNeeded(){
+  # goal: allow cachine each tagged version independently
+  # to avoid rebuilding, so that tools like `git bisect`
+  # can grab a cached past version without rebuilding.
+  nimDefineVars
+  (
+    set -e
+    # avoid polluting caller scope with internal variable definitions.
+    if test -f "$nim_csources"; then
+      echo "$nim_csources exists."
+    else
+      if test -d "$nim_csourcesDir"; then
+        echo "$nim_csourcesDir exists."
+      else
+        # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g.
+        # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`.
+        echo_run git clone -q --depth 1 -b $nim_csourcesBranch \
+            $nim_csourcesUrl "$nim_csourcesDir"
+        # old `git` versions don't support -C option, using `cd` explicitly:
+        echo_run cd "$nim_csourcesDir"
+        echo_run git checkout $nim_csourcesHash
+        echo_run cd "$OLDPWD"
+        # if needed we could also add: `git reset --hard $nim_csourcesHash`
+      fi
+      _nimBuildCsourcesIfNeeded "$@"
+    fi
+
+    echo_run rm -f bin/nim
+      # fixes bug #17913, but it's unclear why it's needed, maybe specific to MacOS Big Sur 11.3 on M1 arch?
+    echo_run cp $nim_csources bin/nim
+    echo_run $nim_csources -v
+  )
+}
diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat
deleted file mode 100644
index 4806810d7..000000000
--- a/ci/nsis_build.bat
+++ /dev/null
@@ -1,59 +0,0 @@
-REM - Run the full testsuite;  testament\tester all
-
-REM - Uncomment the list of changes in news.txt
-REM - write a news ticker entry
-REM - Update the version
-
-REM - Generate the full docs;  koch web0
-REM - Generate the installers;
-REM - Update the version in system.nim
-REM - Test the installers
-REM - Tag the release
-REM - Merge devel into master
-REM - Update csources
-
-set NIMVER=%1
-
-Rem Build -docs file:
-koch web0
-cd web\upload
-7z a -tzip docs-%NIMVER%.zip *.html
-move /y docs-%NIMVER%.zip download
-cd ..\..
-
-Rem Build csources
-koch csources -d:release || exit /b
-
-rem Grab C sources and nimsuggest
-git clone --depth 1 https://github.com/nim-lang/csources.git
-
-set PATH=%CD%\bin;%PATH%
-
-ReM Build Win32 version:
-
-set PATH=C:\Users\araq\projects\mingw32\bin;%PATH%
-cd csources
-call build.bat
-cd ..
-ReM Rebuilding koch is necessary because it uses its pointer size to determine
-ReM which mingw link to put in the NSIS installer.
-nim c --out:koch_temp koch || exit /b
-koch_temp boot -d:release || exit /b
-koch_temp nsis -d:release || exit /b
-koch_temp zip -d:release || exit /b
-dir build
-move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b
-move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b
-
-
-ReM Build Win64 version:
-set PATH=C:\Users\araq\projects\mingw64\bin;%PATH%
-cd csources
-call build64.bat
-cd ..
-nim c --out:koch_temp koch || exit /b
-koch_temp boot -d:release || exit /b
-koch_temp nsis -d:release || exit /b
-koch_temp zip -d:release || exit /b
-move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b
-move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b