Moved the parseAST magics to evals.nim

Added string interpolation helper routines in parserutils

Added a proof-of-concept string interpolation user-land macros (currently, only as a test case):
  the interpolated expression could either be transformed to
  concat("literal string ", $(interpolated), " end") or
  "literal string $1 end" % [$(interpolated)]

Added a very initial definition of Optional[T] generic type

A new overload of ParseIdent was added in hope to get around the fact that the 
old one doesn't work correctly in macros, but the problem persists.

6 files changed, 215 insertions, 39 deletions

diff --git a/compiler/evals.nim b/compiler/evals.nim
index a61bd24cf..65c64f4d3 100755
--- a/compiler/evals.nim
+++ b/compiler/evals.nim
@@ -16,7 +16,7 @@
 import 
   strutils, magicsys, lists, options, ast, astalgo, trees, treetab, nimsets, 
   msgs, os, condsyms, idents, renderer, types, passes, semfold, transf, 
-  ropes
+  parser, ropes
 
 type 
   PStackFrame* = ref TStackFrame
@@ -752,6 +752,31 @@ proc evalRepr(c: PEvalContext, n: PNode): PNode =
 proc isEmpty(n: PNode): bool = 
   result = (n != nil) and (n.kind == nkEmpty)
 
+# The lexer marks multi-line strings as residing at the line where they are closed
+# This function returns the line where the string begins
+# Maybe the lexer should mark both the beginning and the end of expressions, then
+# this function could be removed
+proc stringStartingLine(s: PNode): int =
+  var totalLines = 0
+  for ln in splitLines(s.strVal): inc totalLines
+
+  result = s.info.line - totalLines
+
+proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
+  var code = evalAux(c, n.sons[1], {})
+  var ast = parseString(code.getStrValue, code.info.toFilename, code.stringStartingLine)
+
+  if sonsLen(ast) != 1:
+    GlobalError(code.info, errExprExpected, "multiple statements")
+
+  result = ast.sons[0]
+  result.typ = newType(tyExpr, c.module)
+
+proc evalParseStmt(c: PEvalContext, n: Pnode): Pnode =
+  var code = evalAux(c, n.sons[1], {})
+  result = parseString(code.getStrValue, code.info.toFilename, code.stringStartingLine)
+  result.typ = newType(tyStmt, c.module)
+
 proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode = 
   var m = getMagic(n)
   case m
@@ -776,6 +801,8 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
   of mAppendStrCh: result = evalAppendStrCh(c, n)
   of mAppendStrStr: result = evalAppendStrStr(c, n)
   of mAppendSeqElem: result = evalAppendSeqElem(c, n)
+  of mParseExprToAst: result = evalParseExpr(c, n)
+  of mParseStmtToAst: result = evalParseStmt(c, n)
   of mNLen: 
     result = evalAux(c, n.sons[1], {efLValue})
     if isSpecial(result): return 
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 5f4c7749a..af4d4b6bc 100755
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -889,44 +889,16 @@ proc setMs(n: PNode, s: PSym): PNode =
   n.sons[0].info = n.info
 
 proc expectStringArg(c: PContext, n: PNode, i: int): PNode =
-  result = c.semConstExpr(c, n.sons[i+1])
+  result = c.semAndEvalConstExpr(n.sons[i+1])
 
   if result.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit}:
-     GlobalError(result.info, errStringLiteralExpected)
+    GlobalError(result.info, errStringLiteralExpected)
 
-# The lexer marks multi-line strings as residing at the line where they are closed
-# This function returns the line where the string begins
-# Maybe the lexer should mark both the beginning and the end of expressions, then
-# this function could be removed
-proc stringStartingLine(s: PNode): int =
-  var totalLines = 0
-  for ln in splitLines(s.strVal): inc totalLines
-
-  result = s.info.line - totalLines
-
-proc semParseExprToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
-  if sonsLen(n) == 2:
-    var code = expectStringArg(c, n, 0)
-    var ast = parseString(code.strVal, code.info.toFilename, code.stringStartingLine)
-
-    if sonsLen(ast) != 1:
-      GlobalError(code.info, errExprExpected, "multiple statements")
-
-    result = newMetaNodeIT(ast.sons[0], code.info, newTypeS(tyExpr, c))
-  else:
-    result = semDirectOp(c, n, flags)
-
-proc semParseStmtToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
+proc semExpandMacroToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
   if sonsLen(n) == 2:
-    var code = expectStringArg(c, n, 0)
-    var ast = parseString(code.strVal, code.info.toFilename, code.stringStartingLine)
+    if not isCallExpr(n.sons[1]):
+      GlobalError(n.info, errXisNoMacroOrTemplate, n.renderTree)
 
-    result = newMetaNodeIT(ast, code.info, newTypeS(tyStmt, c))
-  else:
-    result = semDirectOp(c, n, flags)
-
-proc semExpandMacroToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
-  if sonsLen(n) == 2 and isCallExpr(n.sons[1]):
     var macroCall = n.sons[1]
 
     var s = qualifiedLookup(c, macroCall.sons[0], {checkUndeclared})
@@ -943,7 +915,7 @@ proc semExpandMacroToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
     var macroRetType = newTypeS(s.typ.sons[0].kind, c)
     result = newMetaNodeIT(expanded, n.info, macroRetType)
   else:
-    GlobalError(n.info, errXisNoMacroOrTemplate, n.renderTree)
+    result = semDirectOp(c, n, flags)
 
 proc semSlurp(c: PContext, n: PNode, flags: TExprFlags): PNode = 
   if sonsLen(n) == 2:
@@ -981,8 +953,6 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     else:
       result = semDirectOp(c, n, flags)
   of mSlurp: result = semSlurp(c, n, flags)
-  of mParseExprToAst: result = semParseExprToAst(c, n, flags)
-  of mParseStmtToAst: result = semParseStmtToAst(c, n, flags)
   of mExpandMacroToAst: result = semExpandMacroToAst(c, n, flags)
   else: result = semDirectOp(c, n, flags)
 
diff --git a/compiler/semfold.nim b/compiler/semfold.nim
index 61e63a69f..570656a39 100755
--- a/compiler/semfold.nim
+++ b/compiler/semfold.nim
@@ -208,6 +208,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mNewString, mNewStringOfCap, 
      mExit, mInc, ast.mDec, mEcho, mAssert, mSwap, mAppendStrCh, 
      mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq, 
+     mParseExprToAst, mParseStmtToAst,
      mNLen..mNError, mEqRef: 
     nil
   else: InternalError(a.info, "evalOp(" & $m & ')')
diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim
index d3346ecde..a046aff36 100755
--- a/lib/pure/parseutils.nim
+++ b/lib/pure/parseutils.nim
@@ -76,6 +76,19 @@ proc parseIdent*(s: string, ident: var string, start = 0): int =
     ident = substr(s, start, i-1)
     result = i-start
 
+proc parseIdent*(s: string, start = 0): TOptional[string] =
+  ## parses an identifier and stores it in ``ident``. Returns
+  ## the number of the parsed characters or 0 in case of an error.
+  result.hasValue = false
+  var i = start
+
+  if s[i] in IdentStartChars:
+    inc(i)
+    while s[i] in IdentChars: inc(i)
+    
+    result.hasValue = true
+    result.value = substr(s, start, i-1)
+
 proc parseToken*(s: string, token: var string, validChars: set[char],
                  start = 0): int {.inline, deprecated.} =
   ## parses a token and stores it in ``token``. Returns
@@ -254,4 +267,82 @@ proc parseFloat*(s: string, number: var float, start = 0): int {.
   result = parseBiggestFloat(s, bf, start)
   number = bf
   
+proc isEscaped*(s: string, pos: int) : bool =
+  assert pos >= 0 and pos < s.len
+
+  var
+    backslashes = 0
+    j = pos - 1
+
+  while j >= 0:
+    if s[j] == '\\':
+      inc backslashes
+      dec j
+    else:
+      break
+
+  return backslashes mod 2 != 0
+
+type
+  TInterpStrFragment* = tuple[interpStart, interpEnd, exprStart, exprEnd: int]
+
+iterator interpolatedFragments*(s: string): TInterpStrFragment =
+  var i = 0
+  while i < s.len:
+    # The $ sign marks the start of an interpolation.
+    #
+    # It's followed either by a varialbe name or an opening bracket 
+    # (so it should be before the end of the string)
+    # if the dollar sign is escaped, don't trigger interpolation
+    if s[i] == '$' and i < (s.len - 1) and not isEscaped(s, i):
+      var next = s[i+1]
+      
+      if next == '{':
+        inc i
+
+        var
+          brackets = {'{', '}'}
+          nestingCount = 1
+          start = i + 1
+
+        # find closing braket, while respecting any nested brackets
+        while i < s.len:
+          inc i, skipUntil(s, brackets, i+1) + 1
+          
+          if not isEscaped(s, i):
+            if s[i] == '}':
+              dec nestingCount
+              if nestingCount == 0: break
+            else:
+              inc nestingCount
+
+        var t : TInterpStrFragment
+        t.interpStart = start - 2
+        t.interpEnd = i
+        t.exprStart = start
+        t.exprEnd = i - 1
+
+        yield t
+        
+      else:
+        var 
+          start = i + 1
+          identifier = parseIdent(s, i+1)
+        
+        if identifier.hasValue:
+          inc i, identifier.value.len
+
+          var t : TInterpStrFragment
+          t.interpStart = start - 1
+          t.interpEnd = i
+          t.exprStart = start
+          t.exprEnd = i
+
+          yield t
+
+        else:
+          raise newException(EInvalidValue, "Unable to parse a varible name at " & s[i..s.len])
+       
+    inc i
+
 {.pop.}
diff --git a/lib/system.nim b/lib/system.nim
index 2e754ece7..3fc4733b2 100755
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -710,7 +710,7 @@ proc `&` * (x, y: string): string {.
 proc `&` * (x: char, y: string): string {.
   magic: "ConStrStr", noSideEffect, merge.}
   ## is the `concatenation operator`. It concatenates `x` and `y`.
-  
+
 # implementation note: These must all have the same magic value "ConStrStr" so
 # that the merge optimization works properly. 
 
@@ -895,7 +895,12 @@ type # these work for most platforms:
   PFloat64* = ptr Float64 ## an alias for ``ptr float64``
   PInt64* = ptr Int64 ## an alias for ``ptr int64``
   PInt32* = ptr Int32 ## an alias for ``ptr int32``
-  
+
+type TOptional*[T] = object
+  case hasValue* : bool
+  of true: value*: T
+  of false: nil
+ 
 proc toFloat*(i: int): float {.
   magic: "ToFloat", noSideEffect, importc: "toFloat".}
   ## converts an integer `i` into a ``float``. If the conversion
diff --git a/tests/accept/run/tstringinterp.nim b/tests/accept/run/tstringinterp.nim
new file mode 100644
index 000000000..83bd37709
--- /dev/null
+++ b/tests/accept/run/tstringinterp.nim
@@ -0,0 +1,82 @@
+discard """
+  file: "tstringinterp.nim"
+  output: "Hello Alice \$ 64 | Hello Bob, 10"
+"""
+
+import macros, parseutils, strutils
+
+proc concat(strings: openarray[string]) : string =
+  result = newString(0)
+  for s in items(strings): result.add(s)
+
+# This will run though the intee
+template ProcessInterpolations(e: expr) =
+  var 
+    s = e[1].strVal
+    stringStart = 0
+ 
+  for i in interpolatedFragments(s):
+    var leadingString = s[stringStart..i.interpStart-1]
+    var interpolatedExpr = s[i.exprStart..i.exprEnd]
+
+    addString(leadingString)
+
+    var interpTargetAst = parseExpr("$(x)")
+    interpTargetAst[1][0] = parseExpr(interpolatedExpr)
+    addExpr(interpTargetAst)
+    
+    stringStart = i.interpEnd + 1
+
+  if stringStart != s.len:
+    var endingString = s[stringStart..s.len]
+    addString(endingString)
+
+macro formatStyleInterpolation(e: expr): expr =
+  var 
+    formatString = ""
+    arrayNode = newNimNode(nnkBracket)
+    idx = 1
+
+  proc addString(s: string) =
+    formatString.add(s)
+
+  proc addExpr(e: expr) =
+    arrayNode.add(e)
+    formatString.add("$" & $(idx))
+    inc idx
+    
+  ProcessInterpolations(e)
+
+  result = parseExpr("\"x\" % [y]")
+  result[1].strVal = formatString
+  result[2] = arrayNode
+
+macro concatStyleInterpolation(e: expr): expr =
+  var args : seq[PNimrodNode]
+  newSeq(args, 0)
+
+  proc addString(s: string)  = args.add(newStrLitNode(s))
+  proc addExpr(e: expr)      = args.add(e)
+
+  ProcessInterpolations(e)
+
+  result = newCall("concat", args)
+
+###
+
+proc sum(a, b, c: int): int =
+  return (a + b + c)
+
+var 
+  alice = "Alice"
+  bob = "Bob"
+  a = 10
+  b = 20
+  c = 34
+
+var
+  s1 = concatStyleInterpolation"Hello ${alice} \$ ${sum (a, b, c)}"
+  s2 = formatStyleInterpolation"Hello ${bob}, ${sum (alice.len, bob.len, 2)}"
+
+write(stdout, s1 & " | " & s2)
+