adds an option to interleave the generated code with snippets from the original source

Lines from the original source are outputted as comments next to line directives.
Hopefully, this will make debugging codegen problems easier.

Other changes:
The frame setup code now uses a single-line C macro. My motivation was to reduce
the noise in the generated output and make it easier to step over the boiler-plate
code, but counter-intuitively this also improved the overall compilation speed a
little bit so I applied the same treatment to line tracking too (this reduces the size of
the generated files and the explanation is that probably the I/O overhead dominates
the macro expansion costs).

7 files changed, 136 insertions, 85 deletions

diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim
index 90a1c5d81..90696825b 100755
--- a/compiler/ccgutils.nim
+++ b/compiler/ccgutils.nim
@@ -154,36 +154,11 @@ proc TableGetType*(tab: TIdTable, key: PType): PObject =
         if sameType(t, key): 
           return tab.data[h].val
 
-proc toCChar*(c: Char): string = 
-  case c
-  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
-  of '\'', '\"', '\\': result = '\\' & c
-  else: result = $(c)
-
 proc makeSingleLineCString*(s: string): string =
   result = "\""
   for c in items(s):
     result.add(c.toCChar)
   result.add('\"')
-  
-proc makeCString*(s: string): PRope = 
-  # BUGFIX: We have to split long strings into many ropes. Otherwise
-  # this could trigger an InternalError(). See the ropes module for
-  # further information.
-  const 
-    MaxLineLength = 64
-  result = nil
-  var res = "\""
-  for i in countup(0, len(s) - 1):
-    if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, tnl)
-      app(result, toRope(res)) # reset:
-      setlen(res, 1)
-      res[0] = '\"'
-    add(res, toCChar(s[i]))
-  add(res, '\"')
-  app(result, toRope(res))
 
 proc makeLLVMString*(s: string): PRope = 
   const MaxLineLength = 64
diff --git a/compiler/cgen.nim b/compiler/cgen.nim
index c7b3662a0..fafa08ef4 100755
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -271,15 +271,18 @@ proc genCLineDir(r: var PRope, info: TLineInfo) =
 
 proc genLineDir(p: BProc, t: PNode) = 
   var line = t.info.safeLineNm
+  if optEmbedOrigSrc in gGlobalOptions:
+    app(p.s(cpsStmts), con(~"//", t.info.sourceLine, rnl))
   genCLineDir(p.s(cpsStmts), t.info.toFullPath, line)
   if ({optStackTrace, optEndb} * p.Options == {optStackTrace, optEndb}) and
       (p.prc == nil or sfPure notin p.prc.flags): 
     linefmt(p, cpsStmts, "#endb($1);$n", toRope(line))
   elif ({optLineTrace, optStackTrace} * p.Options ==
       {optLineTrace, optStackTrace}) and
-      (p.prc == nil or sfPure notin p.prc.flags): 
-    lineF(p, cpsStmts, "F.line = $1;F.filename = $2;$n", 
-        [toRope(line), makeCString(toFilename(t.info).extractFilename)])
+      (p.prc == nil or sfPure notin p.prc.flags):
+   
+    linefmt(p, cpsStmts, "nimln($1, $2);$n",
+            line.toRope, t.info.quotedFilename)
 
 include "ccgtypes.nim"
 
@@ -696,33 +699,17 @@ proc generateHeaders(m: BModule) =
       appf(m.s[cfsHeaders], "$N#include $1$N", [toRope(it.data)])
     it = PStrEntry(it.Next)
 
-proc getFrameDecl(p: BProc) = 
-  var slots: PRope
-  if p.frameLen > 0: 
-    discard cgsym(p.module, "TVarSlot")
-    slots = ropeff("  TVarSlot s[$1];$n", ", [$1 x %TVarSlot]", 
-                   [toRope(p.frameLen)])
-  else: 
-    slots = nil
-  lineFF(p, cpsLocals, "volatile struct {TFrame* prev;" &
-      "NCSTRING procname;NI line;NCSTRING filename;" &
-      "NI len;$1} F;$n",
-      "%TF = type {%TFrame*, i8*, %NI, %NI$1}$n" & 
-      "%F = alloca %TF$n", [slots])
-  inc(p.labels)
-  prepend(p.s(cpsInit), indentLine(p, ropeff("F.len = $1;$n", 
-      "%LOC$2 = getelementptr %TF %F, %NI 4$n" &
-      "store %NI $1, %NI* %LOC$2$n", [toRope(p.frameLen), toRope(p.labels)])))
-
 proc retIsNotVoid(s: PSym): bool = 
   result = (s.typ.sons[0] != nil) and not isInvalidReturnType(s.typ.sons[0])
 
-proc initFrame(p: BProc, procname, filename: PRope): PRope = 
-  result = ropecg(p.module, 
-    "\tF.procname = $1;$n" &
-    "\tF.filename = $2;$n" & 
-    "\tF.line = 0;$n" & 
-    "\t#pushFrame((TFrame*)&F);$n", [procname, filename])
+proc initFrame(p: BProc, procname, filename: PRope): PRope =
+  discard cgsym(p.module, "pushFrame")
+  if p.frameLen > 0:
+    discard cgsym(p.module, "TVarSlot")
+    result = rfmt(nil, "\tnimfrs($1, $2, $3)$N",
+                  procname, filename, p.frameLen.toRope)
+  else:
+    result = rfmt(nil, "\tnimfr($1, $2)$N", procname, filename)
 
 proc deinitFrame(p: BProc): PRope =
   result = rfmt(p.module, "\t#popFrame();$n")
@@ -774,11 +761,9 @@ proc genProcAux(m: BModule, prc: PSym) =
     generatedProc = rfmt(nil, "$N$1 {$N", header)
     app(generatedProc, initGCFrame(p))
     if optStackTrace in prc.options: 
-      getFrameDecl(p)
       app(generatedProc, p.s(cpsLocals))
       var procname = CStringLit(p, generatedProc, prc.name.s)
-      var filename = CStringLit(p, generatedProc, toFilename(prc.info))
-      app(generatedProc, initFrame(p, procname, filename))
+      app(generatedProc, initFrame(p, procname, prc.info.quotedFilename))
     else: 
       app(generatedProc, p.s(cpsLocals))
     if (optProfiler in prc.options) and (gCmd != cmdCompileToLLVM):
@@ -1014,11 +999,6 @@ proc genInitCode(m: BModule) =
   if m.nimTypes > 0: 
     appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", 
           [m.nimTypesName, toRope(m.nimTypes)])
-  if optStackTrace in m.initProc.options and not m.FrameDeclared:
-    # BUT: the generated init code might depend on a current frame, so
-    # declare it nevertheless:
-    m.FrameDeclared = true
-    getFrameDecl(m.initProc)
   
   app(prc, initGCFrame(m.initProc))
  
@@ -1027,11 +1007,16 @@ proc genInitCode(m: BModule) =
   app(prc, m.preInitProc.s(cpsLocals))
   app(prc, genSectionEnd(cpsLocals))
 
-  if optStackTrace in m.initProc.options and not m.PreventStackTrace: 
-    var procname = CStringLit(m.initProc, prc, m.module.name.s)
-    var filename = CStringLit(m.initProc, prc, toFilename(m.module.info))
-    app(prc, initFrame(m.initProc, procname, filename))
- 
+  if optStackTrace in m.initProc.options and not m.FrameDeclared:
+    # BUT: the generated init code might depend on a current frame, so
+    # declare it nevertheless:
+    m.FrameDeclared = true
+    if not m.PreventStackTrace:
+      var procname = CStringLit(m.initProc, prc, m.module.name.s)
+      app(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
+    else:
+      app(prc, ~"\tvolatile TFrame F; F.len = 0;$N")
+    
   app(prc, genSectionStart(cpsInit))
   app(prc, m.preInitProc.s(cpsInit))
   app(prc, m.initProc.s(cpsInit))
diff --git a/compiler/commands.nim b/compiler/commands.nim
index 5043a60ec..a1a9f0791 100755
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -257,6 +257,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "debuginfo": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optCDebug)
+  of "embedsrc":
+    expectNoArg(switch, arg, pass, info)
+    incl(gGlobalOptions, optEmbedOrigSrc)
   of "compileonly", "c": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optCompileOnly)
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index c702cece8..89a9497df 100755
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -508,16 +508,31 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
     if (xi <= 255): add(tok.literal, Chr(xi))
     else: lexMessage(L, errInvalidCharacterConstant)
   else: lexMessage(L, errInvalidCharacterConstant)
+
+proc newString(s: cstring, l: int): string =
+  ## XXX, how come there is no support for this?
+  result = newString(l)
+  for i in 0 .. <l:
+    result[i] = s[i]
+
+proc HandleCRLF(L: var TLexer, pos: int): int =
+  template registerLine =
+    let col = L.getColNumber(pos)
+    
+    if col > MaxLineLength:
+      lexMessagePos(L, hintLineTooLong, pos)
+
+    if optEmbedOrigSrc in gGlobalOptions:
+      let lineStart = cast[TAddress](L.buf) + L.lineStart
+      let line = newString(cast[cstring](lineStart), col)
+      addSourceLine(L.fileIdx, line)
   
-proc HandleCRLF(L: var TLexer, pos: int): int = 
   case L.buf[pos]
-  of CR: 
-    if getColNumber(L, pos) > MaxLineLength: 
-      lexMessagePos(L, hintLineTooLong, pos)
+  of CR:
+    registerLine()
     result = lexbase.HandleCR(L, pos)
-  of LF: 
-    if getColNumber(L, pos) > MaxLineLength: 
-      lexMessagePos(L, hintLineTooLong, pos)
+  of LF:
+    registerLine()
     result = lexbase.HandleLF(L, pos)
   else: result = pos
   
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 030a14e4c..e45a1b0ef 100755
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, sockets
+  options, strutils, os, tables, sockets, ropes, platform
 
 type 
   TMsgKind* = enum 
@@ -403,6 +403,14 @@ type
   TFileInfo*{.final.} = object 
     fullPath*: string          # This is a canonical full filesystem path
     projPath*: string          # This is relative to the project's root
+    
+    quotedName*: PRope         # cached quoted short name for codegen
+                               # purpoes
+    
+    lines*: seq[PRope]         # the source code of the module
+                               #   used for better error messages and
+                               #   embedding the original source in the
+                               #   generated code
 
   TLineInfo*{.final.} = object # This is designed to be as small as possible,
                                # because it is used
@@ -424,11 +432,40 @@ var
   fileInfos*: seq[TFileInfo] = @[]
   SystemFileIdx*: int32
 
+proc toCChar*(c: Char): string = 
+  case c
+  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
+  of '\'', '\"', '\\': result = '\\' & c
+  else: result = $(c)
+
+proc makeCString*(s: string): PRope =
+  # BUGFIX: We have to split long strings into many ropes. Otherwise
+  # this could trigger an InternalError(). See the ropes module for
+  # further information.
+  const 
+    MaxLineLength = 64
+  result = nil
+  var res = "\""
+  for i in countup(0, len(s) - 1):
+    if (i + 1) mod MaxLineLength == 0:
+      add(res, '\"')
+      add(res, tnl)
+      app(result, toRope(res)) # reset:
+      setlen(res, 1)
+      res[0] = '\"'
+    add(res, toCChar(s[i]))
+  add(res, '\"')
+  app(result, toRope(res))
+
+
 proc newFileInfo(fullPath, projPath: string): TFileInfo =
   result.fullPath = fullPath
   #shallow(result.fullPath)
   result.projPath = projPath
   #shallow(result.projPath)
+  result.quotedName = projPath.extractFilename.makeCString
+  if optEmbedOrigSrc in gGlobalOptions or true:
+    result.lines = @[]
 
 proc fileInfoIdx*(filename: string): int32 =
   var
@@ -524,11 +561,11 @@ proc getInfoContext*(index: int): TLineInfo =
   if i >=% L: result = UnknownLineInfo()
   else: result = msgContext[i]
 
-proc ToFilename*(info: TLineInfo): string =
+proc toFilename*(info: TLineInfo): string =
   if info.fileIndex < 0: result = "???"
   else: result = fileInfos[info.fileIndex].projPath
 
-proc ToFilename*(fileIdx: int32): string =
+proc toFilename*(fileIdx: int32): string =
   if fileIdx < 0: result = "???"
   else: result = fileInfos[fileIdx].projPath
 
@@ -536,7 +573,7 @@ proc toFullPath*(info: TLineInfo): string =
   if info.fileIndex < 0: result = "???"
   else: result = fileInfos[info.fileIndex].fullPath
   
-proc ToLinenumber*(info: TLineInfo): int {.inline.} = 
+proc toLinenumber*(info: TLineInfo): int {.inline.} = 
   result = info.line
 
 proc toColumn*(info: TLineInfo): int {.inline.} = 
@@ -714,3 +751,28 @@ template AssertNotNil*(e: expr): expr =
 
 template InternalAssert*(e: bool): stmt =
   if not e: InternalError($InstantiationInfo())
+
+proc addSourceLine*(fileIdx: int32, line: string) =
+  fileInfos[fileIdx].lines.add line.toRope
+
+proc sourceLine*(i: TLineInfo): PRope =
+  if i.fileIndex < 0: return nil
+  InternalAssert i.fileIndex < fileInfos.len and
+                 i.line <= fileInfos[i.fileIndex].lines.len
+
+  result = fileInfos[i.fileIndex].lines[i.line-1]
+
+proc quotedFilename*(i: TLineInfo): PRope =
+  InternalAssert i.fileIndex >= 0
+  result = fileInfos[i.fileIndex].quotedName
+
+ropes.ErrorHandler = proc (err: TRopesError, msg: string, useWarning: bool) =
+  case err
+  of rInvalidFormatStr:
+    internalError("ropes: invalid format string: " & msg)
+  of rTokenTooLong:
+    internalError("ropes: token too long: " & msg)
+  of rCannotOpenFile:
+    rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile,
+               msg)
+ 
diff --git a/compiler/options.nim b/compiler/options.nim
index 588d716c3..2d45201dc 100755
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -30,6 +30,7 @@ type                          # please make sure we have under 32 options
     optImplicitStatic,        # optimization: implicit at compile time
                               # evaluation
     optPatterns               # en/disable pattern matching
+
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
     gloptNone, optForceFullMake, optBoehmGC, optRefcGC, optDeadCodeElim, 
@@ -59,8 +60,9 @@ type                          # please make sure we have under 32 options
     optTaintMode,             # taint mode turned on
     optTlsEmulation,          # thread var emulation turned on
     optGenIndex               # generate index file for documentation;
+    optEmbedOrigSrc           # embed the original source in the generated code
                               # also: generate header file
-
+   
   TGlobalOptions* = set[TGlobalOption]
   TCommands* = enum           # Nimrod's commands
                               # **keep binary compatible**
@@ -236,4 +238,5 @@ proc binaryStrSearch*(x: openarray[string], y: string): int =
 template nimdbg*: expr = c.module.fileIdx == gProjectMainIdx
 template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx
 template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx
+template lnimdbg*: expr = L.fileIdx == gProjectMainIdx
 
diff --git a/compiler/ropes.nim b/compiler/ropes.nim
index 18c10fabd..707c29123 100755
--- a/compiler/ropes.nim
+++ b/compiler/ropes.nim
@@ -56,7 +56,7 @@
 #  To cache them they are inserted in a `cache` array.
 
 import 
-  msgs, strutils, platform, hashes, crc, options
+  strutils, platform, hashes, crc, options
 
 type
   TFormatStr* = string # later we may change it to CString for better
@@ -72,6 +72,11 @@ type
   
   TRopeSeq* = seq[PRope]
 
+  TRopesError* = enum
+    rCannotOpenFile
+    rInvalidFormatStr
+    rTokenTooLong
+
 proc con*(a, b: PRope): PRope
 proc con*(a: PRope, b: string): PRope
 proc con*(a: string, b: PRope): PRope
@@ -92,6 +97,9 @@ proc RopeInvariant*(r: PRope): bool
   # exported for debugging
 # implementation
 
+var ErrorHandler*: proc(err: TRopesError, msg: string, useWarning = false)
+  # avoid dependency on msgs.nim
+  
 proc ropeLen(a: PRope): int = 
   if a == nil: result = 0
   else: result = a.length
@@ -225,8 +233,7 @@ proc WriteRope*(head: PRope, filename: string, useWarning = false) =
     if head != nil: WriteRope(f, head)
     close(f)
   else:
-    rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile,
-               filename)
+    ErrorHandler(rCannotOpenFile, filename, useWarning)
 
 var
   rnl* = tnl.newRope
@@ -255,8 +262,8 @@ proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope =
           inc(i)
           if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break 
         num = j
-        if j > high(args) + 1: 
-          internalError("ropes: invalid format string $" & $(j))
+        if j > high(args) + 1:
+          ErrorHandler(rInvalidFormatStr, $(j))
         else:
           app(result, args[j - 1])
       of 'n':
@@ -265,7 +272,8 @@ proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope =
       of 'N':
         app(result, rnl)
         inc(i)
-      else: InternalError("ropes: invalid format string $" & frmt[i])
+      else:
+        ErrorHandler(rInvalidFormatStr, $(frmt[i]))
     var start = i
     while i < length:
       if frmt[i] != '$': inc(i)
@@ -290,8 +298,8 @@ const
 
 proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = 
   if r.data != nil:
-    if r.length > bufSize: 
-      internalError("ropes: token too long")
+    if r.length > bufSize:
+      ErrorHandler(rTokenTooLong, r.data)
       return
     var readBytes = readBuffer(bin, buf, r.length)
     result = readBytes == r.length and