Merge pull request #58 from zah/getast-unittests

getAst operational. Unit testing library based on it.

9 files changed, 184 insertions, 85 deletions

diff --git a/compiler/ast.nim b/compiler/ast.nim
index b33d99554..0d920b835 100755
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -327,7 +327,7 @@ type
   TMagic* = enum # symbols that require compiler magic:
     mNone, mDefined, mDefinedInScope, mLow, mHigh, mSizeOf, mIs, mOf,
     mEcho, mShallowCopy, mSlurp,
-    mAstToYaml, mParseExprToAst, mParseStmtToAst, mExpandMacroToAst,
+    mAstToYaml, mParseExprToAst, mParseStmtToAst, mExpandToAst,
     mUnaryLt, mSucc, 
     mPred, mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mLengthOpenArray, 
     mLengthStr, mLengthArray, mLengthSeq, mIncl, mExcl, mCard, mChr, mGCref, 
@@ -369,7 +369,7 @@ type
     mCompileOption, mCompileOptionArg,
     mNLen, mNChild, mNSetChild, mNAdd, mNAddMultiple, mNDel, mNKind, 
     mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, 
-    mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, 
+    mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo,
     mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mIdentToStr, 
     mEqIdent, mEqNimrodNode, mNHint, mNWarning, mNError, mGetTypeInfo
 
diff --git a/compiler/evals.nim b/compiler/evals.nim
index 1d443a404..d2559176e 100755
--- a/compiler/evals.nim
+++ b/compiler/evals.nim
@@ -65,6 +65,7 @@ proc popStackFrame*(c: PEvalContext) {.inline.} =
   if (c.tos == nil): InternalError("popStackFrame")
   c.tos = c.tos.next
 
+proc eval*(c: PEvalContext, n: PNode): PNode
 proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode
 
 proc stackTraceAux(x: PStackFrame) =
@@ -764,7 +765,7 @@ proc isEmpty(n: PNode): bool =
 proc stringStartingLine(s: PNode): int =
   result = s.info.line - countLines(s.strVal)
 
-proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
+proc evalParseExpr(c: PEvalContext, n: PNode): PNode =
   var code = evalAux(c, n.sons[1], {})
   var ast = parseString(code.getStrValue, code.info.toFilename,
                         code.stringStartingLine)
@@ -773,12 +774,108 @@ proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
   result = ast.sons[0]
   result.typ = newType(tyExpr, c.module)
 
-proc evalParseStmt(c: PEvalContext, n: Pnode): Pnode =
+proc evalParseStmt(c: PEvalContext, n: PNode): PNode =
   var code = evalAux(c, n.sons[1], {})
   result = parseString(code.getStrValue, code.info.toFilename,
                        code.stringStartingLine)
   result.typ = newType(tyStmt, c.module)
 
+proc evalTemplateAux*(templ, actual: PNode, sym: PSym): PNode = 
+  case templ.kind
+  of nkSym: 
+    var p = templ.sym
+    if (p.kind == skParam) and (p.owner.id == sym.id): 
+      result = copyTree(actual.sons[p.position])
+    else: 
+      result = copyNode(templ)
+  of nkNone..nkIdent, nkType..nkNilLit: # atom
+    result = copyNode(templ)
+  else: 
+    result = copyNode(templ)
+    newSons(result, sonsLen(templ))
+    for i in countup(0, sonsLen(templ) - 1): 
+      result.sons[i] = evalTemplateAux(templ.sons[i], actual, sym)
+
+proc evalTemplateArgs(n: PNode, s: PSym): PNode =
+  var 
+    f, a: int
+    arg: PNode
+    
+  f = sonsLen(s.typ)
+
+  # if the template has zero arguments, it can be called without ``()``
+  # `n` is then a nkSym or something similar
+  case n.kind
+  of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit:
+    a = sonsLen(n)
+  else: a = 0
+  
+  if a > f: GlobalError(n.info, errWrongNumberOfArguments)
+
+  result = copyNode(n)
+  for i in countup(1, f - 1):
+    if i < a:
+      arg = n.sons[i]
+    else:
+      arg = copyTree(s.typ.n.sons[i].sym.ast)
+
+    addSon(result, arg)
+
+var evalTemplateCounter = 0
+  # to prevend endless recursion in templates instantation
+
+proc evalTemplate(n: PNode, sym: PSym): PNode = 
+  inc(evalTemplateCounter)
+  if evalTemplateCounter > 100:
+    GlobalError(n.info, errTemplateInstantiationTooNested)
+
+  # replace each param by the corresponding node:
+  var args = evalTemplateArgs(n, sym)
+  result = evalTemplateAux(sym.ast.sons[codePos], args, sym)
+
+  dec(evalTemplateCounter)
+
+proc evalMacroCall*(c: PEvalContext, n: PNode, sym: PSym): PNode =
+  inc(evalTemplateCounter)
+  if evalTemplateCounter > 100: 
+    GlobalError(n.info, errTemplateInstantiationTooNested)
+
+  var s = newStackFrame()
+  s.call = n
+  setlen(s.params, 2)
+  s.params[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
+  s.params[1] = n
+  pushStackFrame(c, s)
+  discard eval(c, sym.ast.sons[codePos])
+  result = s.params[0]
+  popStackFrame(c)
+  if cyclicTree(result): GlobalError(n.info, errCyclicTree)
+
+  dec(evalTemplateCounter)
+  
+proc evalExpandToAst(c: PEvalContext, original: PNode): PNode =
+  var
+    n = original.copyTree
+    macroCall = n.sons[1]
+    expandedSym = macroCall.sons[0].sym
+
+  for i in countup(1, macroCall.sonsLen - 1):
+    macroCall.sons[i] = evalAux(c, macroCall.sons[i], {})
+
+  case expandedSym.kind
+  of skTemplate:
+    result = evalTemplate(macroCall, expandedSym)
+  of skMacro:
+    # At this point macroCall.sons[0] is nkSym node.
+    # To be completely compatible with normal macro invocation,
+    # we want to replace it with nkIdent node featuring
+    # the original unmangled macro name.
+    macroCall.sons[0] = newIdentNode(expandedSym.name, expandedSym.info)
+    result = evalMacroCall(c, macroCall, expandedSym)
+  else:
+    InternalError(macroCall.info,
+      "ExpandToAst: expanded symbol is no macro or template")
+
 proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode = 
   var m = getMagic(n)
   case m
@@ -805,7 +902,8 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
   of mAppendSeqElem: result = evalAppendSeqElem(c, n)
   of mParseExprToAst: result = evalParseExpr(c, n)
   of mParseStmtToAst: result = evalParseStmt(c, n)
-  of mNLen: 
+  of mExpandToAst: result = evalExpandToAst(c, n)
+  of mNLen:
     result = evalAux(c, n.sons[1], {efLValue})
     if isSpecial(result): return 
     var a = result
@@ -1011,6 +1109,10 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
     if (a == b) or
         (b.kind in {nkNilLit, nkEmpty}) and (a.kind in {nkNilLit, nkEmpty}): 
       result.intVal = 1
+  of mNLineInfo:
+    result = evalAux(c, n.sons[1], {})
+    if isSpecial(result): return
+    result = newStrNodeT(result.info.toFileLineCol, n)
   of mAstToYaml:
     var ast = evalAux(c, n.sons[1], {efLValue})
     result = newStrNode(nkStrLit, ast.treeToYaml.ropeToStr)
diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index d34c6b410..4f8a21f54 100755
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -456,6 +456,12 @@ proc ToLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} = 
   result = info.col
 
+proc toFileLine*(info: TLineInfo): string {.inline.} =
+  result = info.toFilename & ":" & $info.line
+
+proc toFileLineCol*(info: TLineInfo): string {.inline.} =
+  result = info.toFilename & "(" & $info.line & "," & $info.col & ")"
+
 var checkPoints: seq[TLineInfo] = @[]
 
 proc addCheckpoint*(info: TLineInfo) = 
diff --git a/compiler/sem.nim b/compiler/sem.nim
index dcbdac157..7b9f7c4e1 100755
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -10,29 +10,15 @@
 # This module implements the semantic checking pass.
 
 import
-  strutils, hashes, lists, options, lexer, ast, astalgo, trees, treetab, 
-  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math, 
-  magicsys, parser, nversion, nimsets, semdata, evals, semfold, importer, 
+  strutils, hashes, lists, options, lexer, ast, astalgo, trees, treetab,
+  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
+  magicsys, parser, nversion, semdata, nimsets, semfold, importer,
   procfind, lookups, rodread, pragmas, passes, semtypinst, sigmatch, suggest,
-  semthreads, intsets, transf
+  semthreads, intsets, transf, evals
 
 proc semPass*(): TPass
 # implementation
 
-proc isTopLevel(c: PContext): bool {.inline.} = 
-  result = c.tab.tos <= 2
-
-proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = 
-  result = newSym(kind, considerAcc(n), getCurrOwner())
-  result.info = n.info
-  
-proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
-                 allowed: TSymFlags): PSym
-  # identifier with visability
-proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, 
-                        allowed: TSymFlags): PSym
-proc semStmtScope(c: PContext, n: PNode): PNode
-
 type 
   TExprFlag = enum 
     efAllowType, efLValue, efWantIterator, efInTypeof
@@ -50,10 +36,36 @@ proc addResult(c: PContext, t: PType, info: TLineInfo)
 proc addResultNode(c: PContext, n: PNode)
 proc instGenericContainer(c: PContext, n: PNode, header: PType): PType
 
+proc typeMismatch(n: PNode, formal, actual: PType) = 
+  GlobalError(n.Info, errGenerated, msgKindToString(errTypeMismatch) &
+      typeToString(actual) & ") " &
+      `%`(msgKindToString(errButExpectedX), [typeToString(formal)]))
+
+proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = 
+  result = IndexTypesMatch(c, formal, arg.typ, arg)
+  if result == nil:
+    typeMismatch(arg, formal, arg.typ)
+
+proc isTopLevel(c: PContext): bool {.inline.} = 
+  result = c.tab.tos <= 2
+
+proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = 
+  result = newSym(kind, considerAcc(n), getCurrOwner())
+  result.info = n.info
+  
+proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
+                 allowed: TSymFlags): PSym
+  # identifier with visability
+proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, 
+                        allowed: TSymFlags): PSym
+proc semStmtScope(c: PContext, n: PNode): PNode
+
 proc ParamsTypeCheck(c: PContext, typ: PType) {.inline.} =
   if not typeAllowed(typ, skConst):
     GlobalError(typ.n.info, errXisNoType, typeToString(typ))
 
+include semtempl
+
 proc semConstExpr(c: PContext, n: PNode): PNode = 
   var e = semExprWithType(c, n)
   if e == nil: 
@@ -76,17 +88,7 @@ proc semAndEvalConstExpr(c: PContext, n: PNode): PNode =
   result = semConstExpr(c, n)
   
 include seminst, semcall
-  
-proc typeMismatch(n: PNode, formal, actual: PType) = 
-  GlobalError(n.Info, errGenerated, msgKindToString(errTypeMismatch) &
-      typeToString(actual) & ") " &
-      `%`(msgKindToString(errButExpectedX), [typeToString(formal)]))
-
-proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = 
-  result = IndexTypesMatch(c, formal, arg.typ, arg)
-  if result == nil:
-    typeMismatch(arg, formal, arg.typ)
-  
+    
 proc semAfterMacroCall(c: PContext, n: PNode, s: PSym): PNode = 
   result = n
   case s.typ.sons[0].kind
@@ -101,28 +103,13 @@ proc semAfterMacroCall(c: PContext, n: PNode, s: PSym): PNode =
     result = semExpr(c, result)
     result = fitNode(c, s.typ.sons[0], result)
     #GlobalError(s.info, errInvalidParamKindX, typeToString(s.typ.sons[0]))
-  
-include "semtempl.nim"
 
 proc semMacroExpr(c: PContext, n: PNode, sym: PSym, 
                   semCheck: bool = true): PNode = 
-  inc(evalTemplateCounter)
-  if evalTemplateCounter > 100: 
-    GlobalError(n.info, errTemplateInstantiationTooNested)
   markUsed(n, sym)
   var p = newEvalContext(c.module, "", false)
-  var s = newStackFrame()
-  s.call = n
-  setlen(s.params, 2)
-  s.params[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
-  s.params[1] = n
-  pushStackFrame(p, s)
-  discard eval(p, sym.ast.sons[codePos])
-  result = s.params[0]
-  popStackFrame(p)
-  if cyclicTree(result): GlobalError(n.info, errCyclicTree)
+  result = evalMacroCall(p, n, sym)
   if semCheck: result = semAfterMacroCall(c, result, sym)
-  dec(evalTemplateCounter)
 
 proc forceBool(c: PContext, n: PNode): PNode = 
   result = fitNode(c, getSysType(tyBool), n)
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 356f1c196..54f0af9df 100755
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -904,26 +904,37 @@ proc expectStringArg(c: PContext, n: PNode, i: int): PNode =
   if result.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit}:
     GlobalError(result.info, errStringLiteralExpected)
 
-proc semExpandMacroToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
+proc isAstValue(n: PNode): bool =
+  result = n.typ.sym.name.s in [ "expr", "stmt", "PNimrodNode" ]
+
+proc semExpandToAst(c: PContext, n: PNode, magicSym: PSym, flags: TExprFlags): PNode =
   if sonsLen(n) == 2:
     if not isCallExpr(n.sons[1]):
       GlobalError(n.info, errXisNoMacroOrTemplate, n.renderTree)
 
     var macroCall = n.sons[1]
 
-    var s = qualifiedLookup(c, macroCall.sons[0], {checkUndeclared})
-    if s == nil:
+    var expandedSym = qualifiedLookup(c, macroCall.sons[0], {checkUndeclared})
+    if expandedSym == nil:
       GlobalError(n.info, errUndeclaredIdentifier, macroCall.sons[0].renderTree)
 
-    var expanded : Pnode
+    if not (expandedSym.kind in { skMacro, skTemplate }):
+      GlobalError(n.info, errXisNoMacroOrTemplate, expandedSym.name.s)
 
-    case s.kind
-    of skMacro: expanded = semMacroExpr(c, macroCall, s, false)
-    of skTemplate: expanded = semTemplateExpr(c, macroCall, s, false)
-    else: GlobalError(n.info, errXisNoMacroOrTemplate, s.name.s)
+    macroCall.sons[0] = newNodeI(nkSym, macroCall.info)
+    macroCall.sons[0].sym = expandedSym
+    markUsed(n, expandedSym)
 
-    var macroRetType = newTypeS(s.typ.sons[0].kind, c)
-    result = newMetaNodeIT(expanded, n.info, macroRetType)
+    for i in countup(1, macroCall.sonsLen - 1):
+      macroCall.sons[i] = semExprWithType(c, macroCall.sons[i], {efAllowType})
+
+    # Preserve the magic symbol in order to handled in evals.nim
+    n.sons[0] = newNodeI(nkSym, n.info)
+    n.sons[0].sym = magicSym
+   
+    n.typ = expandedSym.getReturnType
+
+    result = n
   else:
     result = semDirectOp(c, n, flags)
 
@@ -963,7 +974,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     else:
       result = semDirectOp(c, n, flags)
   of mSlurp: result = semSlurp(c, n, flags)
-  of mExpandMacroToAst: result = semExpandMacroToAst(c, n, flags)
+  of mExpandToAst: result = semExpandToAst(c, n, s, flags)
   else: result = semDirectOp(c, n, flags)
 
 proc semIfExpr(c: PContext, n: PNode): PNode = 
diff --git a/compiler/semfold.nim b/compiler/semfold.nim
index 77d84b6f8..d51f69bc1 100755
--- a/compiler/semfold.nim
+++ b/compiler/semfold.nim
@@ -206,7 +206,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mNewString, mNewStringOfCap, 
      mExit, mInc, ast.mDec, mEcho, mAssert, mSwap, mAppendStrCh, 
      mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq, 
-     mParseExprToAst, mParseStmtToAst,
+     mParseExprToAst, mParseStmtToAst, mExpandToAst,
      mNLen..mNError, mEqRef: 
     nil
   else: InternalError(a.info, "evalOp(" & $m & ')')
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index c00b68bb5..243c2ce00 100755
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -428,7 +428,7 @@ proc semRaise(c: PContext, n: PNode): PNode =
     var typ = n.sons[0].typ
     if typ.kind != tyRef or typ.sons[0].kind != tyObject: 
       localError(n.info, errExprCannotBeRaised)
-  
+
 proc semTry(c: PContext, n: PNode): PNode = 
   result = n
   checkMinSonsLen(n, 2)
@@ -438,15 +438,19 @@ proc semTry(c: PContext, n: PNode): PNode =
     var a = n.sons[i]
     checkMinSonsLen(a, 1)
     var length = sonsLen(a)
-    if a.kind == nkExceptBranch: 
-      for j in countup(0, length - 2): 
+    if a.kind == nkExceptBranch:
+      if length == 2 and a.sons[0].kind == nkBracket:
+        a.sons[0..0] = a.sons[0].sons
+        length = a.sonsLen
+
+      for j in countup(0, length - 2):
         var typ = semTypeNode(c, a.sons[j], nil)
         if typ.kind == tyRef: typ = typ.sons[0]
-        if typ.kind != tyObject: 
+        if typ.kind != tyObject:
           GlobalError(a.sons[j].info, errExprCannotBeRaised)
         a.sons[j] = newNodeI(nkType, a.sons[j].info)
         a.sons[j].typ = typ
-        if ContainsOrIncl(check, typ.id): 
+        if ContainsOrIncl(check, typ.id):
           localError(a.sons[j].info, errExceptionAlreadyHandled)
     elif a.kind != nkFinally: 
       illFormedAst(n) 
diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim
index 295aaac03..ff2dd3bb1 100755
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -33,22 +33,6 @@ proc isTypeDesc(n: PNode): bool =
     result = true
   else: result = false
   
-proc evalTemplateAux(c: PContext, templ, actual: PNode, sym: PSym): PNode = 
-  case templ.kind
-  of nkSym: 
-    var p = templ.sym
-    if (p.kind == skParam) and (p.owner.id == sym.id): 
-      result = copyTree(actual.sons[p.position])
-    else: 
-      result = copyNode(templ)
-  of nkNone..nkIdent, nkType..nkNilLit: # atom
-    result = copyNode(templ)
-  else: 
-    result = copyNode(templ)
-    newSons(result, sonsLen(templ))
-    for i in countup(0, sonsLen(templ) - 1): 
-      result.sons[i] = evalTemplateAux(c, templ.sons[i], actual, sym)
-  
 var evalTemplateCounter: int = 0
   # to prevend endless recursion in templates instantation
 
@@ -77,13 +61,13 @@ proc evalTemplateArgs(c: PContext, n: PNode, s: PSym): PNode =
       arg = fitNode(c, s.typ.sons[i], semExprWithType(c, arg))
     addSon(result, arg)
 
-proc evalTemplate(c: PContext, n: PNode, sym: PSym): PNode = 
+proc evalTemplate*(c: PContext, n: PNode, sym: PSym): PNode = 
   var args: PNode
   inc(evalTemplateCounter)
   if evalTemplateCounter <= 100: 
     # replace each param by the corresponding node:
     args = evalTemplateArgs(c, n, sym)
-    result = evalTemplateAux(c, sym.ast.sons[codePos], args, sym)
+    result = evalTemplateAux(sym.ast.sons[codePos], args, sym)
     dec(evalTemplateCounter)
   else:
     GlobalError(n.info, errTemplateInstantiationTooNested)
diff --git a/compiler/types.nim b/compiler/types.nim
index cc1281b6e..f02f5064a 100755
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -925,6 +925,11 @@ proc computeSize(typ: PType): biggestInt =
   var a: biggestInt = 1
   result = computeSizeAux(typ, a)
 
+proc getReturnType*(s: PSym): PType =
+  # Obtains the return type of a iterator/proc/macro/template
+  assert s.kind in { skProc, skTemplate, skMacro, skIterator }
+  result = s.typ.n.sons[0].typ
+
 proc getSize(typ: PType): biggestInt = 
   result = computeSize(typ)
   if result < 0: InternalError("getSize(" & $typ.kind & ')')