Multiple C lines corresponding to a single nimrod line are joined together

This patch greatly improves the "step over" operation available in debuggers.
In practice, there are often 4-8 lines of C code generated for each nimrod line
Each such line will be responsible to a single step in the debugger that is
a) not expected by the user
b) taking the user to an incorrect line in the nimrod code

To keep this working, all code generation should use the rope formatting
facilities when producing new lines (i.e. $n and $N).
New semantics for the format string are introduced:
$n means "soft new line" that could be joined/broken when lineDir is enabled.
$N means "hard new line" that will always appear as a new line.

As an alternative to this approach, I also tested producing code like this:
#line "code.nim" 154
foo = bar; \
foo(bar) \

This is better for readability of the final output, but unfortunately it didn't
produce the desired result across all compilers/debuggers.

6 files changed, 81 insertions, 61 deletions

diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 743357ba7..03c5444fd 100755
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -105,7 +105,7 @@ proc bitSetToWord(s: TBitSet, size: int): BiggestInt =
 proc genRawSetData(cs: TBitSet, size: int): PRope =
   var frmt: TFormatStr
   if size > 8:
-    result = toRope('{' & tnl)
+    result = ropef("{$n")
     for i in countup(0, size - 1):
       if i < size - 1:
         # not last iteration?
@@ -797,14 +797,14 @@ proc fixupCall(p: BProc, t: PNode, d: var TLoc, pl: PRope) =
         app(pl, addrLoc(d))
         app(pl, ")")
         app(p.s[cpsStmts], pl)
-        app(p.s[cpsStmts], ';' & tnl)
+        appf(p.s[cpsStmts], ";$n")
       else:
         var tmp: TLoc
         getTemp(p, typ.sons[0], tmp)
         app(pl, addrLoc(tmp))
         app(pl, ")")
         app(p.s[cpsStmts], pl)
-        app(p.s[cpsStmts], ';' & tnl)
+        appf(p.s[cpsStmts], ";$n")
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
       app(pl, ")")
@@ -817,7 +817,7 @@ proc fixupCall(p: BProc, t: PNode, d: var TLoc, pl: PRope) =
   else:
     app(pl, ")")
     app(p.s[cpsStmts], pl)
-    app(p.s[cpsStmts], ';' & tnl)
+    appf(p.s[cpsStmts], ";$n")
 
 proc openArrayLoc(a: TLoc): PRope =
   case skipTypes(a.t, abstractVar).kind
@@ -926,14 +926,14 @@ proc genNamedParamCall(p: BProc, t: PNode, d: var TLoc) =
         app(pl, addrLoc(d))
         app(pl, "]")
         app(p.s[cpsStmts], pl)
-        app(p.s[cpsStmts], ';' & tnl)
+        appf(p.s[cpsStmts], ";$n")
       else:
         var tmp: TLoc
         getTemp(p, typ.sons[0], tmp)
         app(pl, addrLoc(tmp))
         app(pl, "]")
         app(p.s[cpsStmts], pl)
-        app(p.s[cpsStmts], ';' & tnl)
+        appf(p.s[cpsStmts], ";$n")
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
       app(pl, "]")
@@ -946,7 +946,7 @@ proc genNamedParamCall(p: BProc, t: PNode, d: var TLoc) =
   else:
     app(pl, "]")
     app(p.s[cpsStmts], pl)
-    app(p.s[cpsStmts], ';' & tnl)
+    appf(p.s[cpsStmts], ";$n")
 
 proc genStrConcat(p: BProc, e: PNode, d: var TLoc) =
   #   <Nimrod code>
@@ -1839,7 +1839,7 @@ proc genConstSimpleList(p: BProc, n: PNode): PRope =
   for i in countup(0, length - 2):
     appf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])])
   if length > 0: app(result, genNamedConstExpr(p, n.sons[length - 1]))
-  app(result, '}' & tnl)
+  appf(result, "}$n")
 
 proc genConstExpr(p: BProc, n: PNode): PRope =
   case n.Kind
diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim
index 01813d158..f982f2c22 100755
--- a/compiler/ccgstmts.nim
+++ b/compiler/ccgstmts.nim
@@ -13,22 +13,6 @@ const
   stringCaseThreshold = 8
     # above X strings a hash-switch for strings is generated
 
-proc genLineDir(p: BProc, t: PNode) = 
-  var line = toLinenumber(t.info) # BUGFIX
-  if line < 0: 
-    line = 0                  # negative numbers are not allowed in #line
-  if optLineDir in p.Options and line > 0: 
-    appff(p.s[cpsStmts], "#line $2 $1$n", "; line $2 \"$1\"$n", 
-          [toRope(makeSingleLineCString(toFullPath(t.info))), toRope(line)])
-  if ({optStackTrace, optEndb} * p.Options == {optStackTrace, optEndb}) and
-      (p.prc == nil or sfPure notin p.prc.flags): 
-    appcg(p, cpsStmts, "#endb($1);$n", [toRope(line)])
-  elif ({optLineTrace, optStackTrace} * p.Options ==
-      {optLineTrace, optStackTrace}) and
-      (p.prc == nil or sfPure notin p.prc.flags): 
-    appf(p.s[cpsStmts], "F.line = $1;F.filename = $2;$n", 
-        [toRope(line), makeCString(toFilename(t.info).extractFilename)])
-
 proc genVarTuple(p: BProc, n: PNode) = 
   var tup, field: TLoc
   if n.kind != nkVarTuple: InternalError(n.info, "genVarTuple")
@@ -171,14 +155,14 @@ proc genWhileStmt(p: BProc, t: PNode) =
   setlen(p.blocks, length + 1)
   p.blocks[length].id = - p.labels # negative because it isn't used yet
   p.blocks[length].nestedTryStmts = p.nestedTryStmts.len
-  app(p.s[cpsStmts], "while (1) {" & tnl)
+  appf(p.s[cpsStmts], "while (1) {$n")
   initLocExpr(p, t.sons[0], a)
   if (t.sons[0].kind != nkIntLit) or (t.sons[0].intVal == 0): 
     p.blocks[length].id = abs(p.blocks[length].id)
     appf(p.s[cpsStmts], "if (!$1) goto $2;$n", [rdLoc(a), Labl])
   genStmts(p, t.sons[1])
   if p.blocks[length].id > 0: appf(p.s[cpsStmts], "} $1: ;$n", [Labl])
-  else: app(p.s[cpsStmts], '}' & tnl)
+  else: appf(p.s[cpsStmts], "}$n")
   setlen(p.blocks, len(p.blocks) - 1)
   dec(p.withinLoop)
 
@@ -231,9 +215,9 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     genLineDir(p, t)
     # reraise the last exception:
     #if gCmd == cmdCompileToCpp: 
-    #  appcg(p, cpsStmts, "throw;" & tnl)
+    #  appcg(p, cpsStmts, "throw;$n")
     #else: 
-    appcg(p, cpsStmts, "#reraiseException();" & tnl)
+    appcg(p, cpsStmts, "#reraiseException();$n")
 
 proc genCaseGenericBranch(p: BProc, b: PNode, e: TLoc, 
                           rangeFormat, eqFormat: TFormatStr, labl: TLabel) = 
@@ -326,7 +310,7 @@ proc genStringCase(p: BProc, t: PNode) =
       if branches[j] != nil: 
         appf(p.s[cpsStmts], "case $1: $n$2break;$n", 
              [intLiteral(j), branches[j]])
-    app(p.s[cpsStmts], '}' & tnl) # else statement:
+    appf(p.s[cpsStmts], "}$n") # else statement:
     if t.sons[sonsLen(t) - 1].kind != nkOfBranch: 
       appf(p.s[cpsStmts], "goto LA$1;$n", [toRope(p.labels)]) 
     # third pass: generate statements
@@ -388,13 +372,13 @@ proc genOrdinalCase(p: BProc, n: PNode) =
         genStmts(p, branch[length-1])
       else: 
         # else part of case statement:
-        app(p.s[cpsStmts], "default:" & tnl)
+        appf(p.s[cpsStmts], "default:$n")
         genStmts(p, branch[0])
         hasDefault = true
-      app(p.s[cpsStmts], "break;" & tnl)
+      appf(p.s[cpsStmts], "break;$n")
     if (hasAssume in CC[ccompiler].props) and not hasDefault: 
-      app(p.s[cpsStmts], "default: __assume(0);" & tnl)
-    app(p.s[cpsStmts], '}' & tnl)
+      appf(p.s[cpsStmts], "default: __assume(0);$n")
+    appf(p.s[cpsStmts], "}$n")
   if Lend != nil: fixLabel(p, Lend)
   
 proc genCaseStmt(p: BProc, t: PNode) = 
@@ -453,7 +437,7 @@ proc genTryStmtCpp(p: BProc, t: PNode) =
     appf(p.s[cpsLocals], "volatile NIM_BOOL $1 = NIM_FALSE;$n", [rethrowFlag])
   if optStackTrace in p.Options: 
     appcg(p, cpsStmts, "#setFrame((TFrame*)&F);$n")
-  app(p.s[cpsStmts], "try {" & tnl)
+  appf(p.s[cpsStmts], "try {$n")
   add(p.nestedTryStmts, t)
   genStmts(p, t.sons[0])
   length = sonsLen(t)
@@ -467,7 +451,7 @@ proc genTryStmtCpp(p: BProc, t: PNode) =
     blen = sonsLen(t.sons[i])
     if blen == 1: 
       # general except section:
-      app(p.s[cpsStmts], "default: " & tnl)
+      appf(p.s[cpsStmts], "default:$n")
       genStmts(p, t.sons[i].sons[0])
     else: 
       for j in countup(0, blen - 2): 
@@ -476,10 +460,10 @@ proc genTryStmtCpp(p: BProc, t: PNode) =
       genStmts(p, t.sons[i].sons[blen - 1])
     if rethrowFlag != nil: 
       appf(p.s[cpsStmts], "$1 = NIM_FALSE;  ", [rethrowFlag])
-    app(p.s[cpsStmts], "break;" & tnl)
+    appf(p.s[cpsStmts], "break;$n")
     inc(i)
   if t.sons[1].kind == nkExceptBranch: 
-    app(p.s[cpsStmts], "}}" & tnl) # end of catch-switch statement
+    appf(p.s[cpsStmts], "}}$n") # end of catch-switch statement
   appcg(p, cpsStmts, "#popSafePoint();")
   discard pop(p.nestedTryStmts)
   if (i < length) and (t.sons[i].kind == nkFinally): 
@@ -528,10 +512,10 @@ proc genTryStmt(p: BProc, t: PNode) =
     var blen = sonsLen(t.sons[i])
     if blen == 1: 
       # general except section:
-      if i > 1: app(p.s[cpsStmts], "else {" & tnl)
+      if i > 1: appf(p.s[cpsStmts], "else {$n")
       genStmts(p, t.sons[i].sons[0])
       appcg(p, cpsStmts, "$1.status = 0;#popCurrentException();$n", [safePoint])
-      if i > 1: app(p.s[cpsStmts], '}' & tnl)
+      if i > 1: appf(p.s[cpsStmts], "}$n")
     else: 
       var orExpr: PRope = nil
       for j in countup(0, blen - 2): 
@@ -547,7 +531,7 @@ proc genTryStmt(p: BProc, t: PNode) =
       appcg(p, cpsStmts, "$1.status = 0;#popCurrentException();}$n",
            [safePoint])
     inc(i)
-  app(p.s[cpsStmts], '}' & tnl) # end of if statement
+  appf(p.s[cpsStmts], "}$n") # end of if statement
   discard pop(p.nestedTryStmts)
   if i < length and t.sons[i].kind == nkFinally: 
     genStmts(p, t.sons[i].sons[0])
diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim
index 7237fec8b..99f9ba3b0 100755
--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -520,7 +520,7 @@ proc finishTypeDescriptions(m: BModule) =
   while i < len(m.typeStack): 
     discard getTypeDesc(m, m.typeStack[i])
     inc(i)
-
+  
 proc genProcHeader(m: BModule, prc: PSym): PRope = 
   var 
     rettype, params: PRope
@@ -529,6 +529,7 @@ proc genProcHeader(m: BModule, prc: PSym): PRope =
   var check = initIntSet()
   fillLoc(prc.loc, locProc, prc.typ, mangleName(prc), OnUnknown)
   genProcParams(m, prc.typ, rettype, params, check)
+  genCLineDir(result, prc.ast.sons[codePos])
   appf(result, "$1($2, $3)$4", 
        [toRope(CallingConvToStr[prc.typ.callConv]), rettype, prc.loc.r, params])
 
diff --git a/compiler/cgen.nim b/compiler/cgen.nim
index 2a29381b6..4188e7df0 100755
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -204,7 +204,10 @@ proc ropecg(m: BModule, frmt: TFormatStr, args: openarray[PRope]): PRope =
         if j > high(args) + 1: 
           internalError("ropes: invalid format string $" & $(j))
         app(result, args[j - 1])
-      of 'N', 'n': 
+      of 'n':
+        if not (optLineDir in gOptions): app(result, tnl)
+        inc(i)
+      of 'N': 
         app(result, tnl)
         inc(i)
       else: InternalError("ropes: invalid format string $" & frmt[i])
@@ -241,6 +244,30 @@ proc appcg(p: BProc, s: TCProcSection, frmt: TFormatStr,
            args: openarray[PRope]) = 
   app(p.s[s], ropecg(p.module, frmt, args))
 
+proc safeLineNm(t: PNode) : int =
+  result = toLinenumber(t.info)   # BUGFIX
+  if result < 0: result = 0       # negative numbers are not allowed in #line
+
+proc genCLineDir(r: var PRope, filename: string, line: int) =
+  assert line >= 0
+  if optLineDir in gOptions:
+    appff(r, "$N#line $2 $1$N", "; line $2 \"$1\"$n",
+          [toRope(makeSingleLineCString(filename)), toRope(line)])
+
+proc genCLineDir(r: var PRope, t: PNode) = 
+  genCLineDir(r, t.info.toFullPath, t.safeLineNm)
+
+proc genLineDir(p: BProc, t: PNode) = 
+  var line = t.safeLineNm
+  genCLineDir(p.s[cpsStmts], t.info.toFullPath, line)
+  if ({optStackTrace, optEndb} * p.Options == {optStackTrace, optEndb}) and
+      (p.prc == nil or sfPure notin p.prc.flags): 
+    appcg(p, cpsStmts, "#endb($1);$n", [toRope(line)])
+  elif ({optLineTrace, optStackTrace} * p.Options ==
+      {optLineTrace, optStackTrace}) and
+      (p.prc == nil or sfPure notin p.prc.flags): 
+    appf(p.s[cpsStmts], "F.line = $1;F.filename = $2;$n", 
+        [toRope(line), makeCString(toFilename(t.info).extractFilename)])
 
 include "ccgtypes.nim"
 
@@ -546,13 +573,13 @@ proc cgsym(m: BModule, name: string): PRope =
   result = sym.loc.r
   
 proc generateHeaders(m: BModule) = 
-  app(m.s[cfsHeaders], "#include \"nimbase.h\"" & tnl & tnl)
+  app(m.s[cfsHeaders], tnl & "#include \"nimbase.h\"" & tnl)
   var it = PStrEntry(m.headerFiles.head)
   while it != nil: 
     if it.data[0] notin {'\"', '<'}: 
-      appf(m.s[cfsHeaders], "#include \"$1\"$n", [toRope(it.data)])
+      appf(m.s[cfsHeaders], "$N#include \"$1\"$N", [toRope(it.data)])
     else: 
-      appf(m.s[cfsHeaders], "#include $1$n", [toRope(it.data)])
+      appf(m.s[cfsHeaders], "$N#include $1$N", [toRope(it.data)])
     it = PStrEntry(it.Next)
 
 proc getFrameDecl(p: BProc) = 
@@ -629,24 +656,24 @@ proc genProcAux(m: BModule, prc: PSym) =
       if gProcProfile >= 64 * 1024: 
         InternalError(prc.info, "too many procedures for profiling")
       discard cgsym(m, "profileData")
-      app(p.s[cpsLocals], "ticks NIM_profilingStart;" & tnl)
+      appf(p.s[cpsLocals], "ticks NIM_profilingStart;$n")
       if prc.loc.a < 0: 
         appf(m.s[cfsDebugInit], "profileData[$1].procname = $2;$n", [
             toRope(gProcProfile), 
             makeCString(prc.name.s)])
         prc.loc.a = gProcProfile
         inc(gProcProfile)
-      prepend(p.s[cpsInit], toRope("NIM_profilingStart = getticks();" & tnl))
+      prepend(p.s[cpsInit], ropef("NIM_profilingStart = getticks();$n"))
     app(generatedProc, p.s[cpsInit])
     app(generatedProc, p.s[cpsStmts])
-    if p.beforeRetNeeded: app(generatedProc, "BeforeRet: ;" & tnl)
+    if p.beforeRetNeeded: appf(generatedProc, "BeforeRet: $n;")
     if optStackTrace in prc.options: app(generatedProc, deinitFrame(p))
     if (optProfiler in prc.options) and (gCmd != cmdCompileToLLVM): 
       appf(generatedProc, 
         "profileData[$1].total += elapsed(getticks(), NIM_profilingStart);$n", 
         [toRope(prc.loc.a)])
     app(generatedProc, returnStmt)
-    app(generatedProc, '}' & tnl)
+    appf(generatedProc, "}$n")
   app(m.s[cfsProcs], generatedProc)
   
 proc genProcPrototype(m: BModule, sym: PSym) = 
@@ -667,8 +694,8 @@ proc genProcNoForward(m: BModule, prc: PSym) =
   if lfImportCompilerProc in prc.loc.flags:
     # dependency to a compilerproc:
     discard cgsym(m, prc.name.s)
-    return
-  genProcPrototype(m, prc)
+    return  
+  genProcPrototype(m, prc)  
   if lfNoDecl in prc.loc.Flags: nil
   elif prc.typ.callConv == ccInline:
     # We add inline procs to the calling module to enable C based inlining.
@@ -854,7 +881,7 @@ proc genInitCode(m: BModule) =
   app(prc, m.initProc.s[cpsStmts])
   if optStackTrace in m.initProc.options and not m.PreventStackTrace: 
     app(prc, deinitFrame(m.initProc))
-  app(prc, '}' & tnl & tnl)
+  appf(prc, "}$n$n")
   app(m.s[cfsProcs], prc)
 
 proc genModule(m: BModule, cfilenoext: string): PRope = 
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 5d0bde12e..7c0f11181 100755
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -414,7 +414,7 @@ proc UnknownLineInfo*(): TLineInfo =
   result.fileIndex = -1
 
 var 
-  filenames: seq[string] = @[]
+  filenames: seq[tuple[filename: string, fullpath: string]] = @[]
   msgContext: seq[TLineInfo] = @[]
 
 proc pushInfoContext*(info: TLineInfo) = 
@@ -425,10 +425,16 @@ proc popInfoContext*() =
 
 proc includeFilename*(f: string): int = 
   for i in countdown(high(filenames), low(filenames)): 
-    if filenames[i] == f: 
+    if filenames[i].filename == f:
       return i
+  
   result = len(filenames)
-  filenames.add(f)
+  
+  var fullpath: string
+  try: fullpath = expandFilename(f)
+  except: fullpath = ""
+
+  filenames.add((filename: f, fullpath: fullpath))
 
 proc newLineInfo*(filename: string, line, col: int): TLineInfo = 
   result.fileIndex = includeFilename(filename)
@@ -437,12 +443,11 @@ proc newLineInfo*(filename: string, line, col: int): TLineInfo =
 
 proc ToFilename*(info: TLineInfo): string = 
   if info.fileIndex < 0: result = "???"
-  else: result = filenames[info.fileIndex]
+  else: result = filenames[info.fileIndex].filename
 
 proc toFullPath*(info: TLineInfo): string =
-  result = info.toFilename
-  if not isAbsolute(result):
-    result = JoinPath(options.projectPath, result)
+  if info.fileIndex < 0: result = "???"
+  else: result = filenames[info.fileIndex].fullpath
   
 proc ToLinenumber*(info: TLineInfo): int {.inline.} = 
   result = info.line
diff --git a/compiler/ropes.nim b/compiler/ropes.nim
index 9b8a8466c..7e0d6c8e2 100755
--- a/compiler/ropes.nim
+++ b/compiler/ropes.nim
@@ -59,7 +59,7 @@
 #
 
 import 
-  msgs, strutils, platform, hashes, crc
+  msgs, strutils, platform, hashes, crc, options
 
 const 
   CacheLeafs* = true
@@ -319,7 +319,10 @@ proc ropef(frmt: TFormatStr, args: openarray[PRope]): PRope =
         if j > high(args) + 1: 
           internalError("ropes: invalid format string $" & $(j))
         app(result, args[j - 1])
-      of 'N', 'n': 
+      of 'n':
+        if not (optLineDir in gOptions): app(result, tnl)
+        inc i
+      of 'N': 
         app(result, tnl)
         inc(i)
       else: InternalError("ropes: invalid format string $" & frmt[i])