bugfixes for ROD file generation; nimcache dir is now flat

11 files changed, 103 insertions, 75 deletions

diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim
index 37df89cfd..aae006208 100755
--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -46,10 +46,13 @@ proc mangleName(s: PSym): PRope =
     app(result, toRope(mangle(s.name.s)))
     app(result, "_")
     app(result, toRope(s.id))
-    if optGenMapping in gGlobalOptions: 
-      if s.owner != nil: 
-        appf(gMapping, "r\"$1.$2\": $3$n", 
-             [toRope(s.owner.Name.s), toRope(s.name.s), result])
+    when false:
+      # deactivated to make mapping file smaller which is currently only used
+      # for the list of generated C files
+      if optGenMapping in gGlobalOptions: 
+        if s.owner != nil: 
+          appf(gMapping, "r\"$1.$2\": $3$n", 
+               [toRope(s.owner.Name.s), toRope(s.name.s), result])
     s.loc.r = result
 
 proc getTypeName(typ: PType): PRope = 
diff --git a/compiler/cgen.nim b/compiler/cgen.nim
index d4a9dd278..7cfcbe098 100755
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -920,7 +920,7 @@ proc newModule(module: PSym, filename: string): BModule =
 proc registerTypeInfoModule() = 
   const moduleName = "nim__dat"
   var s = NewSym(skModule, getIdent(moduleName), nil)
-  gNimDat = rawNewModule(s, joinPath(options.projectPath, moduleName) & ".nim")
+  gNimDat = rawNewModule(s, (options.projectPath / moduleName) & ".nim")
   gNimDat.PreventStackTrace = true
   addPendingModule(gNimDat)
   appff(mainModProcs, "N_NOINLINE(void, $1)(void);$n", 
@@ -934,13 +934,17 @@ proc myOpenCached(module: PSym, filename: string,
                   rd: PRodReader): PPassContext = 
   if gNimDat == nil: 
     registerTypeInfoModule()
-  var cfile = changeFileExt(completeCFilePath(filename), cExt)
-  var cfilenoext = changeFileExt(cfile, "")
-  addFileToLink(cfilenoext)
-  registerModuleToMain(module)
+    # XXX load old nimcache data here!
+    gNimDat.fromCache = true
+  result = newModule(module, filename)
+  #if gNimDat == nil: registerTypeInfoModule()
+  #var cfile = changeFileExt(completeCFilePath(filename), cExt)
+  #var cfilenoext = changeFileExt(cfile, "")
+  #addFileToLink(cfilenoext)
+  #registerModuleToMain(module)
   # XXX: this cannot be right here, initalization has to be appended during
   # the ``myClose`` call
-  result = nil
+  #result = nil
 
 proc shouldRecompile(code: PRope, cfile, cfilenoext: string): bool = 
   result = true
@@ -989,8 +993,9 @@ proc writeModule(m: BModule) =
       tccgen.compileCCode(ropeToStr(code))
       return
   
-  if shouldRecompile(code, changeFileExt(cfile, cExt), cfilenoext): 
-    addFileToCompile(cfilenoext)
+  if not m.fromCache:
+    if shouldRecompile(code, changeFileExt(cfile, cExt), cfilenoext): 
+      addFileToCompile(cfilenoext)
   addFileToLink(cfilenoext)
 
 proc myClose(b: PPassContext, n: PNode): PNode = 
@@ -1000,7 +1005,9 @@ proc myClose(b: PPassContext, n: PNode): PNode =
   if n != nil: 
     m.initProc.options = gOptions
     genStmts(m.initProc, n)
+  # cached modules need to registered too: 
   registerModuleToMain(m.module)
+  
   if not (optDeadCodeElim in gGlobalOptions) and
       not (sfDeadCodeElim in m.module.flags): 
     finishModule(m)
@@ -1014,13 +1021,14 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     while gForwardedProcsCounter > 0: 
       for i in countup(0, high(gPendingModules)): 
         finishModule(gPendingModules[i])
-    for i in countup(0, high(gPendingModules)): writeModule(gPendingModules[i])
+    for i in countup(0, high(gPendingModules)): 
+      writeModule(gPendingModules[i])
     setlen(gPendingModules, 0)
   if not (optDeadCodeElim in gGlobalOptions) and
       not (sfDeadCodeElim in m.module.flags): 
     writeModule(m)
   if sfMainModule in m.module.flags: writeMapping(gMapping)
-  
+      
 proc cgenPass(): TPass = 
   initPass(result)
   result.open = myOpen
diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim
index cf4d85dc8..b48c65f87 100755
--- a/compiler/extccomp.nim
+++ b/compiler/extccomp.nim
@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2010 Andreas Rumpf
+#        (c) Copyright 2011 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 7fe6f649d..65b7f9049 100755
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -572,7 +572,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     frmt = posErrorFormat
     # we try to filter error messages so that not two error message
     # in the same file and line are produced:
-    ignoreMsg = lastError == info
+    ignoreMsg = lastError == info and eh != doAbort
     lastError = info
   of warnMin..warnMax: 
     ignoreMsg = optWarns notin gOptions or msg notin gNotes
diff --git a/compiler/options.nim b/compiler/options.nim
index 1af06953d..55ed8cb59 100755
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -82,8 +82,6 @@ var
   gVerbosity*: int            # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors
 
-proc FindFile*(f: string): string
-
 const 
   genSubDir* = "nimcache"
   NimExt* = "nim"
@@ -94,12 +92,6 @@ const
   DocConfig* = "nimdoc.cfg"
   DocTexConfig* = "nimdoc.tex.cfg"
 
-proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string
-proc toGeneratedFile*(path, ext: string): string
-  # converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
-proc getPrefixDir*(): string
-  # gets the application directory
-
 # additional configuration variables:
 var 
   gConfigVars*: PStringTable
@@ -110,36 +102,30 @@ var
   gKeepComments*: bool = true # whether the parser needs to keep comments
   gImplicitMods*: TStringSeq = @[] # modules that are to be implicitly imported
 
-proc existsConfigVar*(key: string): bool
-proc getConfigVar*(key: string): string
-proc setConfigVar*(key, val: string)
-proc addImplicitMod*(filename: string)
-proc binaryStrSearch*(x: openarray[string], y: string): int
-# implementation
-
-proc existsConfigVar(key: string): bool = 
+proc existsConfigVar*(key: string): bool = 
   result = hasKey(gConfigVars, key)
 
-proc getConfigVar(key: string): string = 
+proc getConfigVar*(key: string): string = 
   result = gConfigVars[key]
 
-proc setConfigVar(key, val: string) = 
+proc setConfigVar*(key, val: string) = 
   gConfigVars[key] = val
 
 proc getOutFile*(filename, ext: string): string = 
   if options.outFile != "": result = options.outFile
   else: result = changeFileExt(filename, ext)
   
-proc addImplicitMod(filename: string) = 
+proc addImplicitMod*(filename: string) = 
   var length = len(gImplicitMods)
   setlen(gImplicitMods, length + 1)
   gImplicitMods[length] = filename
 
-proc getPrefixDir(): string = 
+proc getPrefixDir*(): string = 
+  ## gets the application directory
   result = SplitPath(getAppDir()).head
 
 proc shortenDir(dir: string): string = 
-  # returns the interesting part of a dir
+  ## returns the interesting part of a dir
   var prefix = getPrefixDir() & dirSep
   if startsWith(dir, prefix): 
     return substr(dir, len(prefix))
@@ -161,15 +147,17 @@ proc removeTrailingDirSep*(path: string): string =
 proc getGeneratedPath: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: projectPath / genSubDir
   
-proc toGeneratedFile(path, ext: string): string = 
+proc toGeneratedFile*(path, ext: string): string = 
+  ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
   var (head, tail) = splitPath(path)
-  if len(head) > 0: head = shortenDir(head & dirSep)
-  result = joinPath([getGeneratedPath(), head, changeFileExt(tail, ext)])
+  #if len(head) > 0: head = shortenDir(head & dirSep)
+  result = joinPath([getGeneratedPath(), changeFileExt(tail, ext)])
+  #echo "toGeneratedFile(", path, ", ", ext, ") = ", result
 
-proc completeGeneratedFilePath(f: string, createSubDir: bool = true): string = 
+proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string = 
   var (head, tail) = splitPath(f)
-  if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
-  var subdir = getGeneratedPath() / head
+  #if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
+  var subdir = getGeneratedPath() # / head
   if createSubDir:
     try: 
       createDir(subdir)
@@ -177,6 +165,7 @@ proc completeGeneratedFilePath(f: string, createSubDir: bool = true): string =
       writeln(stdout, "cannot create directory: " & subdir)
       quit(1)
   result = joinPath(subdir, tail)
+  #echo "completeGeneratedFilePath(", f, ") = ", result
 
 iterator iterSearchPath*(): string = 
   var it = PStrEntry(SearchPaths.head)
@@ -193,11 +182,11 @@ proc rawFindFile(f: string): string =
       if ExistsFile(result): return
     result = ""
 
-proc FindFile(f: string): string = 
+proc FindFile*(f: string): string = 
   result = rawFindFile(f)
   if len(result) == 0: result = rawFindFile(toLower(f))
   
-proc binaryStrSearch(x: openarray[string], y: string): int = 
+proc binaryStrSearch*(x: openarray[string], y: string): int = 
   var a = 0
   var b = len(x) - 1
   while a <= b: 
diff --git a/compiler/passes.nim b/compiler/passes.nim
index d5aeb50d1..1ce5ad2d3 100755
--- a/compiler/passes.nim
+++ b/compiler/passes.nim
@@ -16,7 +16,14 @@ import
   nimsets, syntaxes, times, rodread, semthreads, idgen
 
 type
+  TPassComm* = object {.pure.} ## communication object between passes
+    optimizers*: TSymSeq       ## filled by semantic pass; used in HLO
+  PPassComm* = ref TPassComm
+  
   TPassContext* = object of TObject # the pass's context
+    comm*: PPassComm
+    fromCache*: bool  # true if created by "openCached"
+    
   PPassContext* = ref TPassContext
   TPass* = tuple[
     open: proc (module: PSym, filename: string): PPassContext, 
@@ -78,17 +85,28 @@ proc registerPass(p: TPass) =
   gPasses[gPassesLen] = p
   inc(gPassesLen)
 
+proc newPassComm(): PPassComm =
+  new(result)
+  result.optimizers = @[]
+
 proc openPasses(a: var TPassContextArray, module: PSym, filename: string) = 
+  var comm = newPassComm()
   for i in countup(0, gPassesLen - 1): 
-    if not isNil(gPasses[i].open): a[i] = gPasses[i].open(module, filename)
+    if not isNil(gPasses[i].open): 
+      a[i] = gPasses[i].open(module, filename)
+      if a[i] != nil: a[i].comm = comm
     else: a[i] = nil
   
 proc openPassesCached(a: var TPassContextArray, module: PSym, filename: string, 
                       rd: PRodReader) = 
+  var comm = newPassComm()
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].openCached): 
       a[i] = gPasses[i].openCached(module, filename, rd)
-    else: 
+      if a[i] != nil: 
+        a[i].comm = comm
+        a[i].fromCache = true
+    else:
       a[i] = nil
   
 proc closePasses(a: var TPassContextArray) = 
@@ -110,7 +128,7 @@ proc processTopLevelStmtCached(n: PNode, a: var TPassContextArray) =
     if not isNil(gPasses[i].openCached): m = gPasses[i].process(a[i], m)
   
 proc closePassesCached(a: var TPassContextArray) = 
-  var m = ast.emptyNode
+  var m: PNode = nil
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].openCached) and not isNil(gPasses[i].close): 
       m = gPasses[i].close(a[i], m)
@@ -144,7 +162,7 @@ proc processModule(module: PSym, filename: string, stream: PLLStream,
     IDsynchronizationPoint(1000)
   else: 
     openPassesCached(a, module, filename, rd)
-    var n = loadInitSection(rd)   #MessageOut('init section' + renderTree(n));
+    var n = loadInitSection(rd)
     for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
     closePassesCached(a)
 
diff --git a/compiler/rodread.nim b/compiler/rodread.nim
index 1a59916a4..eb14a4ddb 100755
--- a/compiler/rodread.nim
+++ b/compiler/rodread.nim
@@ -96,7 +96,7 @@ const
     "list of options changed for: $1", "an include file edited: $1", 
     "a module $1 depends on has changed"]
 
-type 
+type
   TIndex*{.final.} = object   # an index with compression
     lastIdxKey*, lastIdxVal*: int
     tab*: TIITable
@@ -123,13 +123,12 @@ type
   PRodReader* = ref TRodReader
 
 const 
-  FileVersion* = "1012"       # modify this if the rod-format changes!
+  FileVersion* = "1018"       # modify this if the rod-format changes!
 
 var rodCompilerprocs*: TStrTable
 
 proc handleSymbolFile*(module: PSym, filename: string): PRodReader
-  # global because this is needed by magicsys
-proc GetCRC*(filename: string): TCrc32
+# global because this is needed by magicsys
 proc loadInitSection*(r: PRodReader): PNode
 proc loadStub*(s: PSym)
 
@@ -144,11 +143,11 @@ proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType
 proc decodeLineInfo(r: PRodReader, info: var TLineInfo) = 
   if r.s[r.pos] == '?': 
     inc(r.pos)
-    if r.s[r.pos] == ',': info.col = int16(- 1)
+    if r.s[r.pos] == ',': info.col = -1'i16
     else: info.col = int16(decodeVInt(r.s, r.pos))
     if r.s[r.pos] == ',': 
       inc(r.pos)
-      if r.s[r.pos] == ',': info.line = int16(- 1)
+      if r.s[r.pos] == ',': info.line = -1'i16
       else: info.line = int16(decodeVInt(r.s, r.pos))
       if r.s[r.pos] == ',': 
         inc(r.pos)
@@ -671,7 +670,7 @@ proc loadConverters(r: PRodReader) =
   if r.convertersIdx == 0 or r.dataIdx == 0: 
     InternalError("importConverters")
   r.pos = r.convertersIdx
-  while (r.s[r.pos] > '\x0A'): 
+  while r.s[r.pos] > '\x0A': 
     var d = decodeVInt(r.s, r.pos)
     discard rrGetSym(r, d, UnknownLineInfo())
     if r.s[r.pos] == ' ': inc(r.pos)
@@ -741,13 +740,12 @@ proc handleSymbolFile(module: PSym, filename: string): PRodReader =
   else: 
     module.id = getID()
   
-proc GetCRC(filename: string): TCrc32 = 
+proc GetCRC*(filename: string): TCrc32 = 
   var idx = getModuleIdx(filename)
   result = gMods[idx].crc
 
-proc loadStub(s: PSym) = 
-  if s.kind != skStub: 
-    InternalError("loadStub") #MessageOut('loading stub: ' + s.name.s);
+proc loadStub(s: PSym) =
+  if s.kind != skStub: InternalError("loadStub")
   var rd = gMods[s.position].rd
   var theId = s.id                # used for later check
   var d = IITableGet(rd.index.tab, s.id)
diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim
index 037bf450d..5b6ffc57f 100755
--- a/compiler/rodutils.nim
+++ b/compiler/rodutils.nim
@@ -58,6 +58,12 @@ proc decodeStr*(s: cstring, pos: var int): string =
 const
   chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
+# since negative numbers require a leading '-' they use up 1 byte. Thus we
+# subtract/add vintDelta here to save space for little negative numbers
+# which are common in ROD files: 
+const
+  vintDelta = 5
+
 template encodeIntImpl(self: expr) =
   var d: char
   var v = x
@@ -74,13 +80,22 @@ template encodeIntImpl(self: expr) =
   if v != 0: self(v, result)
   add(result, d)
 
+proc encodeVBiggestIntAux(x: BiggestInt, result: var string) =
+  ## encode a biggest int as a variable length base 190 int.
+  encodeIntImpl(encodeVBiggestIntAux)
+
 proc encodeVBiggestInt*(x: BiggestInt, result: var string) =
   ## encode a biggest int as a variable length base 190 int.
-  encodeIntImpl(encodeVBiggestInt)
+  encodeVBiggestIntAux(x +% vintDelta, result)
+  #  encodeIntImpl(encodeVBiggestInt)
+
+proc encodeVIntAux(x: int, result: var string) = 
+  ## encode an int as a variable length base 190 int.
+  encodeIntImpl(encodeVIntAux)
   
 proc encodeVInt*(x: int, result: var string) = 
   ## encode an int as a variable length base 190 int.
-  encodeIntImpl(encodeVInt)
+  encodeVIntAux(x +% vintDelta, result)
 
 template decodeIntImpl() =
   var i = pos
@@ -98,7 +113,7 @@ template decodeIntImpl() =
     of '\x80'..'\xFF': result = result * 190 - (ord(s[i]) - 128 + 62)
     else: break
     inc(i)
-  result = result * sign
+  result = result * sign -% vintDelta
   pos = i
 
 proc decodeVInt*(s: cstring, pos: var int): int = 
diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim
index 3c2fa3c0d..ed81708d9 100755
--- a/compiler/rodwrite.nim
+++ b/compiler/rodwrite.nim
@@ -459,11 +459,11 @@ proc writeRod(w: PRodWriter) =
   f.write(w.data)
   f.write(')' & rodNL)
   f.close()
-  #MessageOut('interf ' + ToString(ropeLen(w.interf)))
-  #MessageOut('index ' + ToString(ropeLen(w.indexRope)))
-  #MessageOut('init ' + ToString(ropeLen(w.init)))
-  #MessageOut('data ' + ToString(ropeLen(w.data)))
-  return 22 + 33
+  
+  #echo "interf: ", w.interf.len
+  #echo "index:  ", w.index.r.len
+  #echo "init:   ", w.init.len
+  #echo "data:   ", w.data.len
 
 proc process(c: PPassContext, n: PNode): PNode = 
   result = n
diff --git a/compiler/sem.nim b/compiler/sem.nim
index 1cb0e3b68..c59ac96a9 100755
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -157,9 +157,7 @@ proc myOpen(module: PSym, filename: string): PPassContext =
 
 proc myOpenCached(module: PSym, filename: string, 
                   rd: PRodReader): PPassContext = 
-  var c = PContext(myOpen(module, filename))
-  c.fromCache = true
-  result = c
+  result = myOpen(module, filename)
 
 proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = 
   result = semStmt(c, n)
@@ -199,11 +197,11 @@ proc checkThreads(c: PContext) =
   
 proc myClose(context: PPassContext, n: PNode): PNode = 
   var c = PContext(context)
-  closeScope(c.tab)           # close module's scope
-  rawCloseScope(c.tab)        # imported symbols; don't check for unused ones!
+  closeScope(c.tab)         # close module's scope
+  rawCloseScope(c.tab)      # imported symbols; don't check for unused ones!
   if n == nil: 
     result = newNode(nkStmtList)
-  else: 
+  else:
     InternalError(n.info, "n is not nil") #result := n;
   addCodeForGenerics(c, result)
   checkThreads(c)
diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index e9c0567d1..50100160b 100755
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -50,7 +50,6 @@ type
     converters*: TSymSeq      # sequence of converters
     optionStack*: TLinkedList
     libs*: TLinkedList        # all libs used by this module
-    fromCache*: bool          # is the module read from a cache?
     semConstExpr*: proc (c: PContext, n: PNode): PNode # for the pragmas
     semExpr*: proc (c: PContext, n: PNode): PNode      # for the pragmas
     includedFiles*: TIntSet   # used to detect recursive include files