small steps for making --symbolFiles:on work again

3 files changed, 108 insertions, 106 deletions

diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim
index 6f37fe756..c771155af 100644
--- a/compiler/pragmas.nim
+++ b/compiler/pragmas.nim
@@ -276,7 +276,8 @@ proc processNote(c: PContext, n: PNode) =
   if (n.kind == nkExprColonExpr) and (sonsLen(n) == 2) and
       (n.sons[0].kind == nkBracketExpr) and
       (n.sons[0].sons[1].kind == nkIdent) and
-      (n.sons[0].sons[0].kind == nkIdent) and (n.sons[1].kind == nkIdent):
+      (n.sons[0].sons[0].kind == nkIdent):
+      #and (n.sons[1].kind == nkIdent):
     var nk: TNoteKind
     case whichKeyword(n.sons[0].sons[0].ident)
     of wHint:
diff --git a/compiler/rodread.nim b/compiler/rodread.nim
index 02e909aac..92ce00240 100644
--- a/compiler/rodread.nim
+++ b/compiler/rodread.nim
@@ -90,7 +90,7 @@
 
 import
   os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms,
-  ropes, idents, securehash, idgen, types, rodutils, memfiles
+  ropes, idents, securehash, idgen, types, rodutils, memfiles, tables
 
 type
   TReasonForRecompile* = enum ## all the reasons that can trigger recompilation
@@ -136,7 +136,7 @@ type
     readerIndex: int
     line: int            # only used for debugging, but is always in the code
     moduleID: int
-    syms: TIdTable       # already processed symbols
+    syms: Table[int, PSym]       # already processed symbols
     memfile: MemFile     # unfortunately there is no point in time where we
                          # can close this! XXX
     methods*: TSymSeq
@@ -372,11 +372,11 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
   else:
     internalError(info, "decodeSym: no ident")
   #echo "decoding: {", ident.s
-  result = PSym(idTableGet(r.syms, id))
+  result = r.syms[id]
   if result == nil:
     new(result)
     result.id = id
-    idTablePut(r.syms, result, result)
+    r.syms[result.id] = result
     if debugIds: registerID(result)
   elif result.id != id:
     internalError(info, "decodeSym: wrong id")
@@ -481,7 +481,7 @@ proc processInterf(r: PRodReader, module: PSym) =
     var s = newStub(r, w, key)
     s.owner = module
     strTableAdd(module.tab, s)
-    idTablePut(r.syms, s, s)
+    r.syms[s.id] = s
 
 proc processCompilerProcs(r: PRodReader, module: PSym) =
   if r.compilerProcsIdx == 0: internalError("processCompilerProcs")
@@ -491,11 +491,11 @@ proc processCompilerProcs(r: PRodReader, module: PSym) =
     inc(r.pos)
     var key = decodeVInt(r.s, r.pos)
     inc(r.pos)                # #10
-    var s = PSym(idTableGet(r.syms, key))
+    var s = r.syms[key]
     if s == nil:
       s = newStub(r, w, key)
       s.owner = module
-      idTablePut(r.syms, s, s)
+      r.syms[s.id] = s
     strTableAdd(rodCompilerprocs, s)
 
 proc processIndex(r: PRodReader; idx: var TIndex; outf: File = nil) =
@@ -667,7 +667,7 @@ proc newRodReader(modfilename: string, hash: SecureHash,
   r.line = 1
   r.readerIndex = readerIndex
   r.filename = modfilename
-  initIdTable(r.syms)
+  r.syms = initTable[int, PSym]()
   # we terminate the file explicitly with ``\0``, so the cast to `cstring`
   # is safe:
   r.s = cast[cstring](r.memfile.mem)
@@ -737,7 +737,7 @@ proc getReader(moduleId: int): PRodReader =
   return nil
 
 proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
-  result = PSym(idTableGet(r.syms, id))
+  result = r.syms[id]
   if result == nil:
     # load the symbol:
     var d = iiTableGet(r.index.tab, id)
@@ -802,7 +802,7 @@ proc getHash*(fileIdx: int32): SecureHash =
   if gMods[fileIdx].hashDone:
     return gMods[fileIdx].hash
 
-  result = secureHashFile(fileIdx.toFilename)
+  result = secureHashFile(fileIdx.toFullPath)
   gMods[fileIdx].hash = result
 
 template growCache*(cache, pos) =
@@ -859,7 +859,7 @@ proc handleSymbolFile(module: PSym): PRodReader =
   result = gMods[fileIdx].rd
   if result != nil:
     module.id = result.moduleID
-    idTablePut(result.syms, module, module)
+    result.syms[module.id] = module
     processInterf(result, module)
     processCompilerProcs(result, module)
     loadConverters(result)
diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim
index 6f7000efc..d48a9ba40 100644
--- a/compiler/rodwrite.nim
+++ b/compiler/rodwrite.nim
@@ -11,14 +11,14 @@
 # rod files is a pass, reading of rod files is not! This is why reading and
 # writing of rod files is split into two different modules.
 
-import 
+import
   intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
   condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
   rodutils
 
 # implementation
 
-type 
+type
   TRodWriter = object of TPassContext
     module: PSym
     hash: SecureHash
@@ -46,15 +46,15 @@ proc addInterfaceSym(w: PRodWriter, s: PSym)
 proc addStmt(w: PRodWriter, n: PNode)
 proc writeRod(w: PRodWriter)
 
-proc getDefines(): string = 
+proc getDefines(): string =
   result = ""
   for d in definedSymbolNames():
     if result.len != 0: add(result, " ")
     add(result, d)
 
-proc fileIdx(w: PRodWriter, filename: string): int = 
-  for i in countup(0, high(w.files)): 
-    if w.files[i] == filename: 
+proc fileIdx(w: PRodWriter, filename: string): int =
+  for i in countup(0, high(w.files)):
+    if w.files[i] == filename:
       return i
   result = len(w.files)
   setLen(w.files, result + 1)
@@ -63,7 +63,7 @@ proc fileIdx(w: PRodWriter, filename: string): int =
 template filename*(w: PRodWriter): string =
   w.module.filename
 
-proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter = 
+proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter =
   new(result)
   result.sstack = @[]
   result.tstack = @[]
@@ -85,12 +85,12 @@ proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter =
   result.init = ""
   result.origFile = module.info.toFilename
   result.data = newStringOfCap(12_000)
-  
+
 proc addModDep(w: PRodWriter, dep: string) =
   if w.modDeps.len != 0: add(w.modDeps, ' ')
   encodeVInt(fileIdx(w, dep), w.modDeps)
 
-const 
+const
   rodNL = "\x0A"
 
 proc addInclDep(w: PRodWriter, dep: string) =
@@ -110,18 +110,18 @@ proc pushSym(w: PRodWriter, s: PSym) =
   if iiTableGet(w.index.tab, s.id) == InvalidKey:
     w.sstack.add(s)
 
-proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, 
-                result: var string) = 
-  if n == nil: 
+proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
+                result: var string) =
+  if n == nil:
     # nil nodes have to be stored too:
     result.add("()")
     return
   result.add('(')
-  encodeVInt(ord(n.kind), result) 
+  encodeVInt(ord(n.kind), result)
   # we do not write comments for now
   # Line information takes easily 20% or more of the filesize! Therefore we
   # omit line information if it is the same as the father's line information:
-  if fInfo.fileIndex != n.info.fileIndex: 
+  if fInfo.fileIndex != n.info.fileIndex:
     result.add('?')
     encodeVInt(n.info.col, result)
     result.add(',')
@@ -139,7 +139,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
   # No need to output the file index, as this is the serialization of one
   # file.
   var f = n.flags * PersistentNodeFlags
-  if f != {}: 
+  if f != {}:
     result.add('$')
     encodeVInt(cast[int32](f), result)
   if n.typ != nil:
@@ -147,16 +147,16 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.typ.id, result)
     pushType(w, n.typ)
   case n.kind
-  of nkCharLit..nkInt64Lit: 
+  of nkCharLit..nkInt64Lit:
     if n.intVal != 0:
       result.add('!')
       encodeVBiggestInt(n.intVal, result)
-  of nkFloatLit..nkFloat64Lit: 
-    if n.floatVal != 0.0: 
+  of nkFloatLit..nkFloat64Lit:
+    if n.floatVal != 0.0:
       result.add('!')
       encodeStr($n.floatVal, result)
   of nkStrLit..nkTripleStrLit:
-    if n.strVal != "": 
+    if n.strVal != "":
       result.add('!')
       encodeStr(n.strVal, result)
   of nkIdent:
@@ -167,25 +167,25 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.sym.id, result)
     pushSym(w, n.sym)
   else:
-    for i in countup(0, sonsLen(n) - 1): 
+    for i in countup(0, sonsLen(n) - 1):
       encodeNode(w, n.info, n.sons[i], result)
   add(result, ')')
 
-proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) = 
+proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
   var oldLen = result.len
   result.add('<')
   if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
-  if loc.s != low(loc.s): 
+  if loc.s != low(loc.s):
     add(result, '*')
     encodeVInt(ord(loc.s), result)
-  if loc.flags != {}: 
+  if loc.flags != {}:
     add(result, '$')
     encodeVInt(cast[int32](loc.flags), result)
   if loc.t != nil:
     add(result, '^')
     encodeVInt(cast[int32](loc.t.id), result)
     pushType(w, loc.t)
-  if loc.r != nil: 
+  if loc.r != nil:
     add(result, '!')
     encodeStr($loc.r, result)
   if oldLen + 1 == result.len:
@@ -193,9 +193,9 @@ proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
     setLen(result, oldLen)
   else:
     add(result, '>')
-  
-proc encodeType(w: PRodWriter, t: PType, result: var string) = 
-  if t == nil: 
+
+proc encodeType(w: PRodWriter, t: PType, result: var string) =
+  if t == nil:
     # nil nodes have to be stored too:
     result.add("[]")
     return
@@ -207,38 +207,38 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   encodeVInt(ord(t.kind), result)
   add(result, '+')
   encodeVInt(t.id, result)
-  if t.n != nil: 
+  if t.n != nil:
     encodeNode(w, unknownLineInfo(), t.n, result)
-  if t.flags != {}: 
+  if t.flags != {}:
     add(result, '$')
     encodeVInt(cast[int32](t.flags), result)
-  if t.callConv != low(t.callConv): 
+  if t.callConv != low(t.callConv):
     add(result, '?')
     encodeVInt(ord(t.callConv), result)
-  if t.owner != nil: 
+  if t.owner != nil:
     add(result, '*')
     encodeVInt(t.owner.id, result)
     pushSym(w, t.owner)
-  if t.sym != nil: 
+  if t.sym != nil:
     add(result, '&')
     encodeVInt(t.sym.id, result)
     pushSym(w, t.sym)
-  if t.size != - 1: 
+  if t.size != - 1:
     add(result, '/')
     encodeVBiggestInt(t.size, result)
-  if t.align != 2: 
+  if t.align != 2:
     add(result, '=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)
-  for i in countup(0, sonsLen(t) - 1): 
-    if t.sons[i] == nil: 
+  for i in countup(0, sonsLen(t) - 1):
+    if t.sons[i] == nil:
       add(result, "^()")
-    else: 
-      add(result, '^') 
+    else:
+      add(result, '^')
       encodeVInt(t.sons[i].id, result)
       pushType(w, t.sons[i])
 
-proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = 
+proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
   add(result, '|')
   encodeVInt(ord(lib.kind), result)
   add(result, '|')
@@ -277,10 +277,10 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   if s.magic != mNone:
     result.add('@')
     encodeVInt(ord(s.magic), result)
-  if s.options != w.options: 
+  if s.options != w.options:
     result.add('!')
     encodeVInt(cast[int32](s.options), result)
-  if s.position != 0: 
+  if s.position != 0:
     result.add('%')
     encodeVInt(s.position, result)
   if s.offset != - 1:
@@ -308,7 +308,7 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
       if codeAst != nil:
         # resore the AST:
         s.ast.sons[codePos] = codeAst
-  
+
 proc addToIndex(w: var TIndex, key, val: int) =
   if key - w.lastIdxKey == 1:
     # we do not store a key-diff of 1 to safe space
@@ -329,24 +329,25 @@ when debugWrittenIds:
 
 proc symStack(w: PRodWriter): int =
   var i = 0
-  while i < len(w.sstack): 
+  while i < len(w.sstack):
     var s = w.sstack[i]
     if sfForward in s.flags:
       w.sstack[result] = s
       inc result
     elif iiTableGet(w.index.tab, s.id) == InvalidKey:
       var m = getModule(s)
-      if m == nil: internalError("symStack: module nil: " & s.name.s)
-      if (m.id == w.module.id) or (sfFromGeneric in s.flags): 
+      if m == nil and s.kind != skPackage:
+        internalError("symStack: module nil: " & s.name.s)
+      if s.kind == skPackage or m.id == w.module.id or sfFromGeneric in s.flags:
         # put definition in here
         var L = w.data.len
-        addToIndex(w.index, s.id, L) 
+        addToIndex(w.index, s.id, L)
         when debugWrittenIds: incl(debugWritten, s.id)
         encodeSym(w, s, w.data)
         add(w.data, rodNL)
         # put into interface section if appropriate:
-        if {sfExported, sfFromGeneric} * s.flags == {sfExported} and 
-            s.kind in ExportableSymKinds: 
+        if {sfExported, sfFromGeneric} * s.flags == {sfExported} and
+            s.kind in ExportableSymKinds:
           encodeStr(s.name.s, w.interf)
           add(w.interf, ' ')
           encodeVInt(s.id, w.interf)
@@ -362,26 +363,26 @@ proc symStack(w: PRodWriter): int =
         if s.kind == skMethod and sfDispatcher notin s.flags:
           if w.methods.len != 0: add(w.methods, ' ')
           encodeVInt(s.id, w.methods)
-      elif iiTableGet(w.imports.tab, s.id) == InvalidKey: 
+      elif iiTableGet(w.imports.tab, s.id) == InvalidKey:
         addToIndex(w.imports, s.id, m.id)
         when debugWrittenIds:
-          if not Contains(debugWritten, s.id):
+          if not contains(debugWritten, s.id):
             echo(w.filename)
             debug(s)
             debug(s.owner)
             debug(m)
-            InternalError("Symbol referred to but never written")
+            internalError("Symbol referred to but never written")
     inc(i)
   setLen(w.sstack, result)
 
-proc typeStack(w: PRodWriter): int = 
+proc typeStack(w: PRodWriter): int =
   var i = 0
-  while i < len(w.tstack): 
+  while i < len(w.tstack):
     var t = w.tstack[i]
     if t.kind == tyForward:
       w.tstack[result] = t
       inc result
-    elif iiTableGet(w.index.tab, t.id) == InvalidKey: 
+    elif iiTableGet(w.index.tab, t.id) == InvalidKey:
       var L = w.data.len
       addToIndex(w.index, t.id, L)
       encodeType(w, t, w.data)
@@ -401,24 +402,24 @@ proc processStacks(w: PRodWriter, finalPass: bool) =
   if finalPass and (oldS != 0 or oldT != 0):
     internalError("could not serialize some forwarded symbols/types")
 
-proc rawAddInterfaceSym(w: PRodWriter, s: PSym) = 
+proc rawAddInterfaceSym(w: PRodWriter, s: PSym) =
   pushSym(w, s)
   processStacks(w, false)
 
-proc addInterfaceSym(w: PRodWriter, s: PSym) = 
-  if w == nil: return 
-  if s.kind in ExportableSymKinds and 
-      {sfExported, sfCompilerProc} * s.flags != {}: 
+proc addInterfaceSym(w: PRodWriter, s: PSym) =
+  if w == nil: return
+  if s.kind in ExportableSymKinds and
+      {sfExported, sfCompilerProc} * s.flags != {}:
     rawAddInterfaceSym(w, s)
 
-proc addStmt(w: PRodWriter, n: PNode) = 
+proc addStmt(w: PRodWriter, n: PNode) =
   encodeVInt(w.data.len, w.init)
   add(w.init, rodNL)
   encodeNode(w, unknownLineInfo(), n, w.data)
   add(w.data, rodNL)
   processStacks(w, false)
 
-proc writeRod(w: PRodWriter) = 
+proc writeRod(w: PRodWriter) =
   processStacks(w, true)
   var f: File
   if not open(f, completeGeneratedFilePath(changeFileExt(
@@ -439,12 +440,12 @@ proc writeRod(w: PRodWriter) =
   encodeStr(w.origFile, orig)
   f.write(orig)
   f.write(rodNL)
-  
+
   var hash = "HASH:"
   encodeStr($w.hash, hash)
   f.write(hash)
   f.write(rodNL)
-  
+
   var options = "OPTIONS:"
   encodeVInt(cast[int32](w.options), options)
   f.write(options)
@@ -458,31 +459,31 @@ proc writeRod(w: PRodWriter) =
   var cmd = "CMD:"
   encodeVInt(cast[int32](gCmd), cmd)
   f.write(cmd)
-  f.write(rodNL)  
-  
+  f.write(rodNL)
+
   f.write("DEFINES:")
   f.write(w.defines)
   f.write(rodNL)
-  
+
   var files = "FILES(" & rodNL
-  for i in countup(0, high(w.files)): 
+  for i in countup(0, high(w.files)):
     encodeStr(w.files[i], files)
     files.add(rodNL)
   f.write(files)
   f.write(')' & rodNL)
-  
+
   f.write("INCLUDES(" & rodNL)
   f.write(w.inclDeps)
   f.write(')' & rodNL)
-  
+
   f.write("DEPS:")
   f.write(w.modDeps)
   f.write(rodNL)
-  
+
   f.write("INTERF(" & rodNL)
   f.write(w.interf)
   f.write(')' & rodNL)
-  
+
   f.write("COMPILERPROCS(" & rodNL)
   f.write(w.compilerProcs)
   f.write(')' & rodNL)
@@ -490,11 +491,11 @@ proc writeRod(w: PRodWriter) =
   f.write("INDEX(" & rodNL)
   f.write(w.index.r)
   f.write(')' & rodNL)
-  
+
   f.write("IMPORTS(" & rodNL)
   f.write(w.imports.r)
   f.write(')' & rodNL)
-  
+
   f.write("CONVERTERS:")
   f.write(w.converters)
   f.write(rodNL)
@@ -502,11 +503,11 @@ proc writeRod(w: PRodWriter) =
   f.write("METHODS:")
   f.write(w.methods)
   f.write(rodNL)
-  
+
   f.write("INIT(" & rodNL)
   f.write(w.init)
   f.write(')' & rodNL)
-  
+
   f.write("DATA(" & rodNL)
   f.write(w.data)
   f.write(')' & rodNL)
@@ -514,23 +515,23 @@ proc writeRod(w: PRodWriter) =
   # for reading:
   f.write("\0")
   f.close()
-  
+
   #echo "interf: ", w.interf.len
   #echo "index:  ", w.index.r.len
   #echo "init:   ", w.init.len
   #echo "data:   ", w.data.len
 
-proc process(c: PPassContext, n: PNode): PNode = 
+proc process(c: PPassContext, n: PNode): PNode =
   result = n
-  if c == nil: return 
+  if c == nil: return
   var w = PRodWriter(c)
   case n.kind
   of nkStmtList:
     for i in countup(0, sonsLen(n) - 1): discard process(c, n.sons[i])
     #var s = n.sons[namePos].sym
     #addInterfaceSym(w, s)
-  of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef, 
-      nkTemplateDef, nkMacroDef: 
+  of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef,
+      nkTemplateDef, nkMacroDef:
     var s = n.sons[namePos].sym
     if s == nil: internalError(n.info, "rodwrite.process")
     if n.sons[bodyPos] == nil:
@@ -539,17 +540,17 @@ proc process(c: PPassContext, n: PNode): PNode =
         sfForward notin s.flags:
       addInterfaceSym(w, s)
   of nkVarSection, nkLetSection, nkConstSection:
-    for i in countup(0, sonsLen(n) - 1): 
+    for i in countup(0, sonsLen(n) - 1):
       var a = n.sons[i]
       if a.kind == nkCommentStmt: continue
       addInterfaceSym(w, a.sons[0].sym)
-  of nkTypeSection: 
-    for i in countup(0, sonsLen(n) - 1): 
+  of nkTypeSection:
+    for i in countup(0, sonsLen(n) - 1):
       var a = n.sons[i]
-      if a.kind == nkCommentStmt: continue 
+      if a.kind == nkCommentStmt: continue
       if a.sons[0].kind != nkSym: internalError(a.info, "rodwrite.process")
       var s = a.sons[0].sym
-      addInterfaceSym(w, s) 
+      addInterfaceSym(w, s)
       # this takes care of enum fields too
       # Note: The check for ``s.typ.kind = tyEnum`` is wrong for enum
       # type aliasing! Otherwise the same enum symbol would be included
@@ -557,20 +558,20 @@ proc process(c: PPassContext, n: PNode): PNode =
       #
       #        if (a.sons[2] <> nil) and (a.sons[2].kind = nkEnumTy) then begin
       #          a := s.typ.n;
-      #          for j := 0 to sonsLen(a)-1 do 
-      #            addInterfaceSym(w, a.sons[j].sym);        
-      #        end 
-  of nkImportStmt: 
+      #          for j := 0 to sonsLen(a)-1 do
+      #            addInterfaceSym(w, a.sons[j].sym);
+      #        end
+  of nkImportStmt:
     for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleName(n.sons[i]))
     addStmt(w, n)
-  of nkFromStmt: 
+  of nkFromStmt:
     addModDep(w, getModuleName(n.sons[0]))
     addStmt(w, n)
-  of nkIncludeStmt: 
+  of nkIncludeStmt:
     for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleName(n.sons[i]))
-  of nkPragma: 
+  of nkPragma:
     addStmt(w, n)
-  else: 
+  else:
     discard
 
 proc myOpen(module: PSym): PPassContext =
@@ -579,7 +580,7 @@ proc myOpen(module: PSym): PPassContext =
   rawAddInterfaceSym(w, module)
   result = w
 
-proc myClose(c: PPassContext, n: PNode): PNode = 
+proc myClose(c: PPassContext, n: PNode): PNode =
   result = process(c, n)
   var w = PRodWriter(c)
   writeRod(w)