summary refs log tree commit diff stats
path: root/compiler/main.nim
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/main.nim')
-rwxr-xr-xcompiler/main.nim430
1 files changed, 345 insertions, 85 deletions
diff --git a/compiler/main.nim b/compiler/main.nim
index 0a76b967a..ac37ab5f3 100755
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -12,11 +12,11 @@
 
 import 
   llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs, 
-  os, lists, condsyms, rodread, rodwrite, ropes, trees, 
+  os, lists, condsyms, rodread, rodwrite, ropes, trees, times,
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
-  cgen, ecmasgen,
+  cgen, ecmasgen, cgendata,
   platform, nimconf, importer, passaux, depends, evals, types, idgen,
-  tables, docgen2, service
+  tables, docgen2, service, magicsys, parser, crc, ccgutils
 
 const
   has_LLVM_Backend = false
@@ -28,60 +28,166 @@ proc MainCommand*()
 
 # ------------------ module handling -----------------------------------------
 
+type
+  TNeedRecompile = enum Maybe, No, Yes, Probing, Recompiled
+  TCrcStatus = enum crcNotTaken, crcCached, crcHasChanged, crcNotChanged
+
+  TModuleInMemory = object
+    compiledAt: float
+    crc: TCrc32
+    deps: seq[int32] ## XXX: slurped files are not currently tracked
+    needsRecompile: TNeedRecompile
+    crcStatus: TCrcStatus
+
 var
-  compMods = initTable[string, PSym]() # all compiled modules
+  gCompiledModules: seq[PSym] = @[]
+  gMemCacheData: seq[TModuleInMemory] = @[]
+    ## XXX: we should implement recycling of file IDs
+    ## if the user keeps renaming modules, the file IDs will keep growing
+
+proc getModule(fileIdx: int32): PSym =
+  if fileIdx >= 0 and fileIdx < gCompiledModules.len:
+    result = gCompiledModules[fileIdx]
+  else:
+    result = nil
+
+template compiledAt(x: PSym): expr =
+  gMemCacheData[x.position].compiledAt
+
+template crc(x: PSym): expr =
+  gMemCacheData[x.position].crc
+
+proc crcChanged(fileIdx: int32): bool =
+  InternalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
+  
+  template updateStatus =
+    gMemCacheData[fileIdx].crcStatus = if result: crcHasChanged
+                                       else: crcNotChanged
+    # echo "TESTING CRC: ", fileIdx.toFilename, " ", result
+    
+  case gMemCacheData[fileIdx].crcStatus:
+  of crcHasChanged:
+    result = true
+  of crcNotChanged:
+    result = false
+  of crcCached:
+    let newCrc = crcFromFile(fileIdx.toFilename)
+    result = newCrc != gMemCacheData[fileIdx].crc
+    gMemCacheData[fileIdx].crc = newCrc
+    updateStatus()
+  of crcNotTaken:
+    gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename)
+    result = true
+    updateStatus()
+
+proc doCRC(fileIdx: int32) =
+  if gMemCacheData[fileIdx].crcStatus == crcNotTaken:
+    # echo "FIRST CRC: ", fileIdx.ToFilename
+    gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename)
+
+proc addDep(x: Psym, dep: int32) =
+  growCache gMemCacheData, dep
+  gMemCacheData[x.position].deps.safeAdd(dep)
+
+proc ResetModule(fileIdx: int32) =
+  echo "HARD RESETTING ", fileIdx.toFilename
+  gMemCacheData[fileIdx].needsRecompile = Yes
+  gCompiledModules[fileIdx] = nil
+  cgendata.gModules[fileIdx] = nil
+
+proc ResetAllModules =
+  for i in 0..gCompiledModules.high:
+    if gCompiledModules[i] != nil:
+      ResetModule(i.int32)
+
+  for m in cgenModules():
+    echo "CGEN MODULE FOUND"
+
+proc checkDepMem(fileIdx: int32): TNeedRecompile  =
+  template markDirty =
+    ResetModule(fileIdx)
+    return Yes
 
-# This expects a normalized module path
-proc registerModule(filename: string, module: PSym) =
-  compMods[filename] = module
+  if gMemCacheData[fileIdx].needsRecompile != Maybe:
+    return gMemCacheData[fileIdx].needsRecompile
 
-# This expects a normalized module path
-proc getModule(filename: string): PSym =
-  result = compMods[filename]
+  if optForceFullMake in gGlobalOptions or
+     curCaasCmd != lastCaasCmd or
+     crcChanged(fileIdx): markDirty
+  
+  if gMemCacheData[fileIdx].deps != nil:
+    gMemCacheData[fileIdx].needsRecompile = Probing
+    for dep in gMemCacheData[fileIdx].deps:
+      let d = checkDepMem(dep)
+      if d in { Yes, Recompiled }:
+        echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
+        markDirty
+  
+  gMemCacheData[fileIdx].needsRecompile = No
+  return No
 
-var gModulesCount = 0
-proc newModule(filename: string): PSym = 
+proc newModule(fileIdx: int32): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID. 
   new(result)
   result.id = - 1             # for better error checking
   result.kind = skModule
+  let filename = fileIdx.toFilename
   result.name = getIdent(splitFile(filename).name)
   if not isNimrodIdentifier(result.name.s):
     rawMessage(errInvalidModuleName, result.name.s)
   
   result.owner = result       # a module belongs to itself
-  result.info = newLineInfo(filename, 1, 1)
-  result.position = gModulesCount
-  inc gModulesCount
+  result.info = newLineInfo(fileIdx, 1, 1)
+  result.position = fileIdx
+  
+  growCache gMemCacheData, fileIdx
+  growCache gCompiledModules, fileIdx
+  gCompiledModules[result.position] = result
+  
   incl(result.flags, sfUsed)
   initStrTable(result.tab)
-  RegisterModule(filename, result)
   StrTableAdd(result.tab, result) # a module knows itself
-  
-proc CompileModule(filename: string, flags: TSymFlags): PSym
-proc importModule(filename: string): PSym = 
+
+proc compileModule(fileIdx: int32, flags: TSymFlags): PSym =
+  result = getModule(fileIdx)
+  if result == nil:
+    growCache gMemCacheData, fileIdx
+    gMemCacheData[fileIdx].needsRecompile = Probing
+    result = newModule(fileIdx)
+    var rd = handleSymbolFile(result)
+    result.flags = result.flags + flags
+    if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
+      rd = handleSymbolFile(result)
+      if result.id < 0: 
+        InternalError("handleSymbolFile should have set the module\'s ID")
+        return
+    else:
+      result.id = getID()
+    processModule(result, nil, rd)
+    if optCaasEnabled in gGlobalOptions:
+      gMemCacheData[fileIdx].compiledAt = gLastCmdTime
+      gMemCacheData[fileIdx].needsRecompile = Recompiled
+      doCRC fileIdx
+  else:
+    if checkDepMem(fileIdx) == Yes:
+      result = CompileModule(fileIdx, flags)
+    else:
+      result = gCompiledModules[fileIdx]
+
+proc importModule(s: PSym, fileIdx: int32): PSym =
   # this is called by the semantic checking phase
-  result = getModule(filename)
-  if result == nil: 
-    # compile the module
-    result = compileModule(filename, {})
-  elif sfSystemModule in result.flags: 
+  result = compileModule(fileIdx, {})
+  if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
+  if sfSystemModule in result.flags:
     LocalError(result.info, errAttemptToRedefine, result.Name.s)
-  
-proc CompileModule(filename: string, flags: TSymFlags): PSym =
-  var rd: PRodReader = nil
-  var f = addFileExt(filename, nimExt)
-  result = newModule(f)
-  result.flags = result.flags + flags
-  if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: 
-    rd = handleSymbolFile(result, f)
-    if result.id < 0: 
-      InternalError("handleSymbolFile should have set the module\'s ID")
-      return
-  else:
-    result.id = getID()
-  processModule(result, f, nil, rd)
+
+proc includeModule(s: PSym, fileIdx: int32): PNode =
+  result = syntaxes.parseFile(fileIdx)
+  if optCaasEnabled in gGlobalOptions:
+    growCache gMemCacheData, fileIdx
+    addDep(s, fileIdx)
+    doCrc(fileIdx)
 
 proc `==^`(a, b: string): bool =
   try:
@@ -89,22 +195,31 @@ proc `==^`(a, b: string): bool =
   except EOS:
     result = false
 
-proc CompileProject(projectFile = gProjectFull) =
-  let systemFile = options.libpath / "system"
-  if projectFile.addFileExt(nimExt) ==^ systemFile.addFileExt(nimExt):
+proc compileSystemModule =
+  if magicsys.SystemModule == nil:
+    SystemFileIdx = fileInfoIdx(options.libpath/"system.nim")
+    discard CompileModule(SystemFileIdx, {sfSystemModule})
+
+proc CompileProject(projectFile = gProjectMainIdx) =
+  let systemFileIdx = fileInfoIdx(options.libpath / "system.nim")
+  if projectFile == SystemFileIdx:
     discard CompileModule(projectFile, {sfMainModule, sfSystemModule})
   else:
-    discard CompileModule(systemFile, {sfSystemModule})
+    compileSystemModule()
     discard CompileModule(projectFile, {sfMainModule})
 
+proc rodPass =
+  if optSymbolFiles in gGlobalOptions:
+    registerPass(rodwritePass)
+
 proc semanticPasses =
-  registerPass(verbosePass())
-  registerPass(sem.semPass())
+  registerPass verbosePass
+  registerPass semPass
 
 proc CommandGenDepend =
   semanticPasses()
-  registerPass(genDependPass())
-  registerPass(cleanupPass())
+  registerPass(genDependPass)
+  registerPass(cleanupPass)
   compileProject()
   generateDot(gProjectFull)
   execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") &
@@ -113,31 +228,76 @@ proc CommandGenDepend =
 proc CommandCheck =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()            # use an empty backend for semantic checking only
-  registerPass(rodwrite.rodwritePass())
-  compileProject(mainCommandArg())
+  rodPass()
+  compileProject()
 
 proc CommandDoc2 =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()
-  registerPass(docgen2Pass())
+  registerPass(docgen2Pass)
   #registerPass(cleanupPass())
-  compileProject(mainCommandArg())
-  finishDoc2Pass(gProjectFull)
+  compileProject()
+  finishDoc2Pass(gProjectName)
 
 proc CommandCompileToC =
   semanticPasses()
-  registerPass(cgen.cgenPass())
-  registerPass(rodwrite.rodwritePass())
+  registerPass(cgenPass)
+  rodPass()
   #registerPass(cleanupPass())
+  if optCaasEnabled in gGlobalOptions:
+    # echo "BEFORE CHECK DEP"
+    # discard checkDepMem(gProjectMainIdx)
+    # echo "CHECK DEP COMPLETE"
+
   compileProject()
+
+  if optCaasEnabled in gGlobalOptions:
+    cgenCaasUpdate()
+
   if gCmd != cmdRun:
     extccomp.CallCCompiler(changeFileExt(gProjectFull, ""))
 
+  if optCaasEnabled in gGlobalOptions:
+    # caas will keep track only of the compilation commands
+    lastCaasCmd = curCaasCmd
+    resetCgenModules()
+    for i in 0 .. <gMemCacheData.len:
+      gMemCacheData[i].crcStatus = crcCached
+      gMemCacheData[i].needsRecompile = Maybe
+
+      # XXX: clean these global vars
+      # ccgstmts.gBreakpoints
+      # ccgthreadvars.nimtv
+      # ccgthreadvars.nimtVDeps
+      # ccgthreadvars.nimtvDeclared
+      # cgendata
+      # cgmeth?
+      # condsyms?
+      # depends?
+      # lexer.gLinesCompiled
+      # msgs - error counts
+      # magicsys, when system.nim changes
+      # rodread.rodcompilerProcs
+      # rodread.gTypeTable
+      # rodread.gMods
+      
+      # !! ropes.cache
+      # semthreads.computed?
+      #
+      # suggest.usageSym
+      #
+      # XXX: can we run out of IDs?
+      # XXX: detect config reloading (implement as error/require restart)
+      # XXX: options are appended (they will accumulate over time)
+    resetCompilationLists()
+    ccgutils.resetCaches()
+    GC_fullCollect()
+
 when has_LLVM_Backend:
   proc CommandCompileToLLVM =
     semanticPasses()
     registerPass(llvmgen.llvmgenPass())
-    registerPass(rodwrite.rodwritePass())
+    rodPass()
     #registerPass(cleanupPass())
     compileProject()
 
@@ -148,31 +308,52 @@ proc CommandCompileToEcmaScript =
   DefineSymbol("nimrod") # 'nimrod' is always defined
   DefineSymbol("ecmascript")
   semanticPasses()
-  registerPass(ecmasgenPass())
+  registerPass(ecmasgenPass)
   compileProject()
 
-proc CommandInteractive =
-  msgs.gErrorMax = high(int)  # do not stop after first error
+proc InteractivePasses =
+  incl(gGlobalOptions, optSafeCode)
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
   DefineSymbol("nimrodvm")
-  when hasFFI:
-    DefineSymbol("nimffi")
+  when hasFFI: DefineSymbol("nimffi")
+  registerPass(verbosePass)
+  registerPass(semPass)
+  registerPass(evalPass)
+
+var stdinModule: PSym
+proc makeStdinModule: PSym =
+  if stdinModule == nil:
+    stdinModule = newModule(fileInfoIdx"stdin")
+    stdinModule.id = getID()
+  result = stdinModule
 
-  registerPass(verbosePass())
-  registerPass(sem.semPass())
-  registerPass(evals.evalPass()) # load system module:
-  discard CompileModule(options.libpath /"system", {sfSystemModule})
+proc CommandInteractive =
+  msgs.gErrorMax = high(int)  # do not stop after first error
+  InteractivePasses()
+  compileSystemModule()
   if commandArgs.len > 0:
-    discard CompileModule(mainCommandArg(), {})
+    discard CompileModule(fileInfoIdx(gProjectFull), {})
   else:
-    var m = newModule("stdin")
-    m.id = getID()
+    var m = makeStdinModule()
     incl(m.flags, sfMainModule)
-    processModule(m, "stdin", LLStreamOpenStdIn(), nil)
+    processModule(m, LLStreamOpenStdIn(), nil)
+
+const evalPasses = [verbosePass, semPass, evalPass]
+
+proc evalNim(nodes: PNode, module: PSym) =
+  carryPasses(nodes, module, evalPasses)
+
+proc commandEval(exp: string) =
+  if SystemModule == nil:
+    InteractivePasses()
+    compileSystemModule()
+  var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
+  evalNim(echoExp.parseString, makeStdinModule())
 
 proc CommandPretty =
-  var module = parseFile(addFileExt(mainCommandArg(), NimExt))
+  var projectFile = addFileExt(mainCommandArg(), NimExt)
+  var module = parseFile(projectFile.fileInfoIdx)
   if module != nil: 
     renderModule(module, getOutFile(mainCommandArg(), "pretty." & NimExt))
   
@@ -196,20 +377,82 @@ proc CommandScan =
 proc CommandSuggest =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()
-  registerPass(rodwrite.rodwritePass())
+  rodPass()
   compileProject()
 
 proc wantMainModule =
   if gProjectFull.len == 0:
-    Fatal(gCmdLineInfo, errCommandExpectsFilename)
+    if optMainModule.len == 0:
+      Fatal(gCmdLineInfo, errCommandExpectsFilename)
+    else:
+      gProjectName = optMainModule
+      gProjectFull = gProjectPath / gProjectName
+
+  gProjectMainIdx = addFileExt(gProjectFull, nimExt).fileInfoIdx
+
+proc resetMemory =
+  resetCompilationLists()
+  ccgutils.resetCaches()
+  ResetAllModules()
+  resetRopeCache()
+  resetSysTypes()
+  gOwners = @[]
+  rangeDestructorProc = nil
+  for i in low(buckets)..high(buckets):
+    buckets[i] = nil
+  idAnon = nil
+  
+  # XXX: clean these global vars
+  # ccgstmts.gBreakpoints
+  # ccgthreadvars.nimtv
+  # ccgthreadvars.nimtVDeps
+  # ccgthreadvars.nimtvDeclared
+  # cgendata
+  # cgmeth?
+  # condsyms?
+  # depends?
+  # lexer.gLinesCompiled
+  # msgs - error counts
+  # magicsys, when system.nim changes
+  # rodread.rodcompilerProcs
+  # rodread.gTypeTable
+  # rodread.gMods
   
+  # !! ropes.cache
+  # semthreads.computed?
+  #
+  # suggest.usageSym
+  #
+  # XXX: can we run out of IDs?
+  # XXX: detect config reloading (implement as error/require restart)
+  # XXX: options are appended (they will accumulate over time)
+  # vis = visimpl
+  gcDebugging = true
+  echo "COLLECT 1"
+  GC_fullCollect()
+  echo "COLLECT 2"
+  GC_fullCollect()
+  echo "COLLECT 3"
+  GC_fullCollect()
+  echo GC_getStatistics()
+
+const
+  SimiluateCaasMemReset = false
+  PrintRopeCacheStats = false
+
 proc MainCommand =
+  when SimiluateCaasMemReset:
+    gGlobalOptions.incl(optCaasEnabled)
+      
+  # In "nimrod serve" scenario, each command must reset the registered passes
+  clearPasses()
+  gLastCmdTime = epochTime()
   appendStr(searchPaths, options.libpath)
   if gProjectFull.len != 0:
     # current path is always looked first for modules
     prependStr(searchPaths, gProjectPath)
   setID(100)
-  passes.gIncludeFile = syntaxes.parseFile
+  passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   case command.normalize
   of "c", "cc", "compile", "compiletoc": 
@@ -292,7 +535,7 @@ proc MainCommand =
   of "parse": 
     gCmd = cmdParse
     wantMainModule()
-    discard parseFile(addFileExt(gProjectFull, nimExt))
+    discard parseFile(gProjectMainIdx)
   of "scan": 
     gCmd = cmdScan
     wantMainModule()
@@ -301,20 +544,37 @@ proc MainCommand =
   of "i": 
     gCmd = cmdInteractive
     CommandInteractive()
+  of "e":
+    # XXX: temporary command for easier testing
+    commandEval(mainCommandArg())
+  of "reset":
+    resetMemory()
   of "idetools":
     gCmd = cmdIdeTools
-    wantMainModule()
-    CommandSuggest()
+    if gEvalExpr != "":
+      commandEval(gEvalExpr)
+    else:
+      wantMainModule()
+      CommandSuggest()
   of "serve":
-    gCmd = cmdIdeTools
-    msgs.gErrorMax = high(int)  # do not stop after first error
-    semanticPasses()
-    # no need to write rod files and would slow down things:
-    #registerPass(rodwrite.rodwritePass())
-    discard CompileModule(options.libpath / "system", {sfSystemModule})
-    service.serve(proc () =
-      let projectFile = mainCommandArg()
-      discard CompileModule(projectFile, {sfMainModule})
-    )
-  else: rawMessage(errInvalidCommandX, command)
+    gGlobalOptions.incl(optCaasEnabled)
+    msgs.gErrorMax = high(int)  # do not stop after first error     
+    serve(MainCommand)
+  else:
+    rawMessage(errInvalidCommandX, command)
+  
+  if msgs.gErrorCounter == 0 and gCmd notin {cmdInterpret, cmdRun}:
+    rawMessage(hintSuccessX, [$gLinesCompiled,
+               formatFloat(epochTime() - gLastCmdTime, ffDecimal, 3),
+               formatSize(getTotalMem())])
+
+  when PrintRopeCacheStats:
+    echo "rope cache stats: "
+    echo "  tries : ", gCacheTries
+    echo "  misses: ", gCacheMisses
+    echo "  int tries: ", gCacheIntTries
+    echo "  efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float), ffDecimal, 3)
+
+  when SimiluateCaasMemReset:
+    resetMemory()
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-07-06 18:15:16 +0000