Incremental compilation (IC): Improvements (#11881)

* IC: C codegen is aware of IC
* manual: minor change to make VSCode's RST plugin render it properly
* IC: minor refactoring
* testament: code refactorings
* rodutils: removed dead code
* IC: always build the compiler with the IC feature
* IC: C codegen improvements
* IC: implement the undocumented -d:nimMustCache option for testing purposes
* IC: added first basic tests
* IC: extensive testing of the deserialization feature
* testament: refactoring; better IC tests
* IC: removes 'nimMustCache' flag; readonly does the same
* testament: minor refactoring
* update Nimble version
* testament: removed dead code and imports; IC: added simple test
* IC: progress

7 files changed, 87 insertions, 77 deletions

diff --git a/compiler/cgen.nim b/compiler/cgen.nim
index c8906f380..c0ef4e1e3 100644
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -1108,7 +1108,8 @@ proc genProcNoForward(m: BModule, prc: PSym) =
     # a check for ``m.declaredThings``.
     if not containsOrIncl(m.declaredThings, prc.id):
       #if prc.loc.k == locNone:
-      # mangle the inline proc based on the module where it is defined - not on the first module that uses it
+      # mangle the inline proc based on the module where it is defined -
+      # not on the first module that uses it
       fillProcLoc(findPendingModule(m, prc), prc.ast[namePos])
       #elif {sfExportc, sfImportc} * prc.flags == {}:
       #  # reset name to restore consistency in case of hashing collisions:
@@ -1781,10 +1782,6 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
                 else: AbsoluteFile""
   open(result.ndi, ndiName, g.config)
 
-proc nullify[T](arr: var T) =
-  for i in low(arr)..high(arr):
-    arr[i] = Rope(nil)
-
 proc rawNewModule(g: BModuleList; module: PSym; conf: ConfigRef): BModule =
   result = rawNewModule(g, module, AbsoluteFile toFullPath(conf, module.position.FileIndex))
 
@@ -1875,7 +1872,9 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
   result = n
   if b == nil: return
   var m = BModule(b)
-  if passes.skipCodegen(m.config, n): return
+  if passes.skipCodegen(m.config, n) or
+      not moduleHasChanged(m.g.graph, m.module):
+    return
   m.initProc.options = initProcOptions(m)
   #softRnl = if optLineDir in m.config.options: noRnl else: rnl
   # XXX replicate this logic!
@@ -1886,9 +1885,10 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
     genStmts(m.initProc, transformedN)
 
 proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
-  result = true
   if optForceFullMake notin m.config.globalOptions:
-    if not equalsFile(code, cfile.cname):
+    if not moduleHasChanged(m.g.graph, m.module):
+      result = false
+    elif not equalsFile(code, cfile.cname):
       if false:
         #m.config.symbolFiles == readOnlySf: #isDefined(m.config, "nimdiff"):
         if fileExists(cfile.cname):
@@ -1898,12 +1898,15 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
           echo "new file ", cfile.cname.string
       if not writeRope(code, cfile.cname):
         rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
-      return
-    if fileExists(cfile.obj) and os.fileNewer(cfile.obj.string, cfile.cname.string):
+      result = true
+    elif fileExists(cfile.obj) and os.fileNewer(cfile.obj.string, cfile.cname.string):
       result = false
+    else:
+      result = true
   else:
     if not writeRope(code, cfile.cname):
       rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
+    result = true
 
 # We need 2 different logics here: pending modules (including
 # 'nim__dat') may require file merging for the combination of dead code
@@ -1915,18 +1918,19 @@ proc writeModule(m: BModule, pending: bool) =
   let cfile = getCFile(m)
 
   if true or optForceFullMake in m.config.globalOptions:
-    genInitCode(m)
-    finishTypeDescriptions(m)
-    if sfMainModule in m.module.flags:
-      # generate main file:
-      genMainProc(m)
-      add(m.s[cfsProcHeaders], m.g.mainModProcs)
-      generateThreadVarsSize(m)
+    if moduleHasChanged(m.g.graph, m.module):
+      genInitCode(m)
+      finishTypeDescriptions(m)
+      if sfMainModule in m.module.flags:
+        # generate main file:
+        genMainProc(m)
+        add(m.s[cfsProcHeaders], m.g.mainModProcs)
+        generateThreadVarsSize(m)
 
     var cf = Cfile(nimname: m.module.name.s, cname: cfile,
                    obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
     var code = genModule(m, cf)
-    if code != nil:
+    if code != nil or m.config.symbolFiles != disabledSf:
       when hasTinyCBackend:
         if conf.cmd == cmdRun:
           tccgen.compileCCode($code)
@@ -1983,46 +1987,47 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     for destructorCall in graph.globalDestructors:
       n.add destructorCall
   if passes.skipCodegen(m.config, n): return
-  # if the module is cached, we don't regenerate the main proc
-  # nor the dispatchers? But if the dispatchers changed?
-  # XXX emit the dispatchers into its own .c file?
-  if n != nil:
-    m.initProc.options = initProcOptions(m)
-    genStmts(m.initProc, n)
+  if moduleHasChanged(graph, m.module):
+    # if the module is cached, we don't regenerate the main proc
+    # nor the dispatchers? But if the dispatchers changed?
+    # XXX emit the dispatchers into its own .c file?
+    if n != nil:
+      m.initProc.options = initProcOptions(m)
+      genStmts(m.initProc, n)
 
-  if m.hcrOn:
-    # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
-    discard cgsym(m, "programResult")
-    if m.inHcrInitGuard:
-      endBlock(m.initProc)
-
-  if sfMainModule in m.module.flags:
     if m.hcrOn:
-      # pull ("define" since they are inline when HCR is on) these functions in the main file
-      # so it can load the HCR runtime and later pass the library handle to the HCR runtime which
-      # will in turn pass it to the other modules it initializes so they can initialize the
-      # register/get procs so they don't have to have the definitions of these functions as well
-      discard cgsym(m, "nimLoadLibrary")
-      discard cgsym(m, "nimLoadLibraryError")
-      discard cgsym(m, "nimGetProcAddr")
-      discard cgsym(m, "procAddrError")
-      discard cgsym(m, "rawWrite")
-
-    # raise dependencies on behalf of genMainProc
-    if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
-      discard cgsym(m, "initStackBottomWith")
-    if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
-      discard cgsym(m, "initThreadVarsEmulation")
+      # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init)
+      discard cgsym(m, "programResult")
+      if m.inHcrInitGuard:
+        endBlock(m.initProc)
 
-    if m.g.breakpoints != nil:
-      discard cgsym(m, "dbgRegisterBreakpoint")
-    if optEndb in m.config.options:
-      discard cgsym(m, "dbgRegisterFilename")
-
-    if m.g.forwardedProcs.len == 0:
-      incl m.flags, objHasKidsValid
-    let disp = generateMethodDispatchers(graph)
-    for x in disp: genProcAux(m, x.sym)
+    if sfMainModule in m.module.flags:
+      if m.hcrOn:
+        # pull ("define" since they are inline when HCR is on) these functions in the main file
+        # so it can load the HCR runtime and later pass the library handle to the HCR runtime which
+        # will in turn pass it to the other modules it initializes so they can initialize the
+        # register/get procs so they don't have to have the definitions of these functions as well
+        discard cgsym(m, "nimLoadLibrary")
+        discard cgsym(m, "nimLoadLibraryError")
+        discard cgsym(m, "nimGetProcAddr")
+        discard cgsym(m, "procAddrError")
+        discard cgsym(m, "rawWrite")
+
+      # raise dependencies on behalf of genMainProc
+      if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
+        discard cgsym(m, "initStackBottomWith")
+      if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
+        discard cgsym(m, "initThreadVarsEmulation")
+
+      if m.g.breakpoints != nil:
+        discard cgsym(m, "dbgRegisterBreakpoint")
+      if optEndb in m.config.options:
+        discard cgsym(m, "dbgRegisterFilename")
+
+      if m.g.forwardedProcs.len == 0:
+        incl m.flags, objHasKidsValid
+      let disp = generateMethodDispatchers(graph)
+      for x in disp: genProcAux(m, x.sym)
 
   m.g.modulesClosed.add m
 
diff --git a/compiler/commands.nim b/compiler/commands.nim
index ed0320a7c..87ded61db 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -30,6 +30,8 @@ import
   wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
   pathutils, strtabs
 
+from incremental import nimIncremental
+
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
 bootSwitch(usedNativeStacktrace,
@@ -669,7 +671,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     helpOnError(conf, pass)
   of "symbolfiles": discard "ignore for backwards compat"
   of "incremental":
-    when not defined(nimIncremental):
+    when not nimIncremental:
       localError(conf, info, "the compiler was not built with " &
         "incremental compilation features; bootstrap with " &
         "-d:nimIncremental to enable")
diff --git a/compiler/incremental.nim b/compiler/incremental.nim
index 29528bbd3..6cc085020 100644
--- a/compiler/incremental.nim
+++ b/compiler/incremental.nim
@@ -10,13 +10,14 @@
 ## Basic type definitions the module graph needs in order to support
 ## incremental compilations.
 
-const nimIncremental* = defined(nimIncremental)
+const nimIncremental* = true # defined(nimIncremental)
 
 import options, lineinfos
 
 when nimIncremental:
   import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
   from strutils import parseInt
+  from os import isAbsolute
 
   type
     Writer* = object
@@ -47,7 +48,7 @@ when nimIncremental:
 
   proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
     result = msgs.getHash(conf, fileIdx)
-    if result.len == 0:
+    if result.len == 0 and isAbsolute(string fullpath):
       result = $secureHashFile(string fullpath)
       msgs.setHash(conf, fileIdx, result)
 
diff --git a/compiler/modules.nim b/compiler/modules.nim
index 40d9a904c..13845e6e9 100644
--- a/compiler/modules.nim
+++ b/compiler/modules.nim
@@ -77,8 +77,6 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     if result == nil:
       result = newModule(graph, fileIdx)
       result.flags = result.flags + flags
-      if sfMainModule in result.flags:
-        graph.config.mainPackageId = result.owner.id
       result.id = id
       registerModule(graph, result)
     else:
diff --git a/compiler/passes.nim b/compiler/passes.nim
index e917c1ce4..d6bbd4d14 100644
--- a/compiler/passes.nim
+++ b/compiler/passes.nim
@@ -113,6 +113,8 @@ const
     nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
 
 proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
+  if sfMainModule in module.flags:
+    graph.config.mainPackageId = module.owner.id
   # don't be verbose unless the module belongs to the main package:
   if module.owner.id == graph.config.mainPackageId:
     graph.config.notes = graph.config.mainPackageNotes
@@ -121,7 +123,7 @@ proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
     graph.config.notes = graph.config.foreignPackageNotes
 
 proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
-  result = module.id >= 0
+  result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
 
 proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
@@ -131,7 +133,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
     s: PLLStream
     fileIdx = module.fileIdx
   prepareConfigNotes(graph, module)
-  if not moduleHasChanged(graph, module):
+  if module.id < 0:
     # new module caching mechanism:
     for i in 0 ..< graph.passes.len:
       if not isNil(graph.passes[i].open) and not graph.passes[i].isFrontend:
diff --git a/compiler/rodimpl.nim b/compiler/rodimpl.nim
index f5f689939..69b50c391 100644
--- a/compiler/rodimpl.nim
+++ b/compiler/rodimpl.nim
@@ -17,7 +17,6 @@ import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
 ## - Dependency computation should use *signature* hashes in order to
 ##   avoid recompiling dependent modules.
 ## - Patch the rest of the compiler to do lazy loading of proc bodies.
-## - Patch the C codegen to cache proc bodies and maybe types.
 
 template db(): DbConn = g.incr.db
 
@@ -72,9 +71,12 @@ proc getModuleId(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile): in
       # not changed, so use the cached AST:
       doAssert(result != 0)
       var cycleCheck = initIntSet()
-      if not needsRecompile(g, fileIdx, fullpath, cycleCheck) and not g.incr.configChanged:
-        echo "cached successfully! ", string fullpath
-        return -result
+      if not needsRecompile(g, fileIdx, fullpath, cycleCheck):
+        if not g.incr.configChanged or g.config.symbolFiles == readOnlySf:
+          #echo "cached successfully! ", string fullpath
+          return -result
+      elif g.config.symbolFiles == readOnlySf:
+        internalError(g.config, "file needs to be recompiled: " & (string fullpath))
     db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
     db.exec(sql"delete from deps where module = ?", module[0])
     db.exec(sql"delete from types where module = ?", module[0])
@@ -296,7 +298,7 @@ proc encodeSym(g: ModuleGraph, s: PSym, result: var string) =
     pushSym(w, s.owner)
   if s.flags != {}:
     result.add('$')
-    encodeVInt(cast[int32](s.flags), result)
+    encodeVBiggestInt(cast[int64](s.flags), result)
   if s.magic != mNone:
     result.add('@')
     encodeVInt(ord(s.magic), result)
@@ -723,7 +725,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
     result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info)
   if b.s[b.pos] == '$':
     inc(b.pos)
-    result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos)))
+    result.flags = cast[TSymFlags](decodeVBiggestInt(b.s, b.pos))
   if b.s[b.pos] == '@':
     inc(b.pos)
     result.magic = TMagic(decodeVInt(b.s, b.pos))
@@ -756,6 +758,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
     if b.s[b.pos] == '\24':
       inc b.pos
       result.transformedBody = decodeNode(g, b, result.info)
+      #result.transformedBody = nil
   of skModule, skPackage:
     decodeInstantiations(g, b, result.info, result.usedGenerics)
   of skLet, skVar, skField, skForVar:
@@ -769,7 +772,7 @@ proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
 
   if b.s[b.pos] == '(':
     #if result.kind in routineKinds:
-    #  result.ast = decodeNodeLazyBody(b, result.info, result)
+    #  result.ast = nil
     #else:
     result.ast = decodeNode(g, b, result.info)
   if sfCompilerProc in result.flags:
@@ -886,6 +889,10 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
       internalAssert g.config, imported.id < 0
   of nkStmtList, nkStmtListExpr:
     for x in n: replay(g, module, x)
+  of nkExportStmt:
+    for x in n:
+      doAssert x.kind == nkSym
+      strTableAdd(module.tab, x.sym)
   else: discard "nothing to do for this node"
 
 proc loadNode*(g: ModuleGraph; module: PSym): PNode =
diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim
index 43fb42de9..17c4832bf 100644
--- a/compiler/rodutils.nim
+++ b/compiler/rodutils.nim
@@ -46,14 +46,9 @@ proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   of fcNegInf:
     result = "-INF"
   else:
-    when defined(nimNoArrayToCstringConversion):
-      result = newString(81)
-      let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-      setLen(result, n)
-    else:
-      var buf: array[0..80, char]
-      discard c_snprintf(buf.cstring, buf.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-      result = $buf.cstring
+    result = newString(81)
+    let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
+    setLen(result, n)
 
 proc encodeStr*(s: string, result: var string) =
   for i in 0 ..< len(s):