fully consisent parsing between the new and the old 'do blocks'

2 files changed, 545 insertions, 78 deletions

diff --git a/compiler/parser.nim b/compiler/parser.nim
index 561200bcc..110f4a43d 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -194,7 +194,6 @@ proc newIdentNodeP(ident: PIdent, p: TParser): PNode =
 proc parseExpr(p: var TParser): PNode
 proc parseStmt(p: var TParser): PNode
 proc parseTypeDesc(p: var TParser): PNode
-proc parseDoBlocks(p: var TParser, call: PNode)
 proc parseParamList(p: var TParser, retColon = true): PNode
 
 proc isSigilLike(tok: TToken): bool {.inline.} =
@@ -669,7 +668,7 @@ proc namedParams(p: var TParser, callee: PNode,
   # progress guaranteed
   exprColonEqExprListAux(p, endTok, result)
 
-proc parseMacroColon(p: var TParser, x: PNode): PNode
+proc postExprBlocks(p: var TParser, x: PNode): PNode
 proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
   #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
   #|       | doBlocks
@@ -696,14 +695,6 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
       result = namedParams(p, result, nkCall, tkParRi)
       if result.len > 1 and result.sons[1].kind == nkExprColonExpr:
         result.kind = nkObjConstr
-      elif p.tok.tokType == tkDo:
-        parseDoBlocks(p, result)
-    of tkDo:
-      # progress guaranteed
-      var a = result
-      result = newNodeP(nkCall, p)
-      addSon(result, a)
-      parseDoBlocks(p, result)
     of tkDot:
       # progress guaranteed
       result = dotExpr(p, result)
@@ -735,10 +726,7 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
             if p.tok.tokType != tkComma: break
             getTok(p)
             optInd(p, x)
-          if p.tok.tokType == tkDo:
-            parseDoBlocks(p, result)
-          else:
-            result = parseMacroColon(p, result)
+          result = postExprBlocks(p, result)
       break
     else:
       break
@@ -977,16 +965,9 @@ proc parseDoBlock(p: var TParser; info: TLineInfo): PNode =
   let params = parseParamList(p, retColon=false)
   let pragmas = optPragmas(p)
   colcom(p, result)
-  result = newProcNode(nkDo, info, parseStmt(p),
-                       params = params,
-                       pragmas = pragmas)
-
-proc parseDoBlocks(p: var TParser, call: PNode) =
-  #| doBlocks = doBlock ^* IND{=}
-  while sameOrNoInd(p) and p.tok.tokType == tkDo:
-    let info = parLineInfo(p)
-    getTok(p)
-    addSon(call, parseDoBlock(p, info))
+  result = parseStmt(p)
+  if params.kind != nkEmpty:
+    result = newProcNode(nkDo, info, result, params = params, pragmas = pragmas)
 
 proc parseProcExpr(p: var TParser, isExpr: bool): PNode =
   #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
@@ -1162,49 +1143,73 @@ proc makeCall(n: PNode): PNode =
     result = newNodeI(nkCall, n.info)
     result.add n
 
-proc parseMacroColon(p: var TParser, x: PNode): PNode =
-  #| macroColon = ':' stmt? ( IND{=} 'of' exprList ':' stmt
-  #|                        | IND{=} 'elif' expr ':' stmt
-  #|                        | IND{=} 'except' exprList ':' stmt
-  #|                        | IND{=} 'else' ':' stmt )*
+proc postExprBlocks(p: var TParser, x: PNode): PNode =
+  #| postExprBlocks = ':' stmt? ( IND{=} doBlock
+  #|                            | IND{=} 'of' exprList ':' stmt
+  #|                            | IND{=} 'elif' expr ':' stmt
+  #|                            | IND{=} 'except' exprList ':' stmt
+  #|                            | IND{=} 'else' ':' stmt )*
   result = x
-  if p.tok.tokType == tkColon and p.tok.indent < 0:
+  if p.tok.indent >= 0: return
+
+  var
+    openingParams = emptyNode
+    openingPragmas = emptyNode
+
+  if p.tok.tokType == tkDo:
+    getTok(p)
+    openingParams = parseParamList(p, retColon=false)
+    openingPragmas = optPragmas(p)
+
+  if p.tok.tokType == tkColon:
     result = makeCall(result)
     getTok(p)
     skipComment(p, result)
     var stmtList = newNodeP(nkStmtList, p)
-    if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}:
-      let body = parseStmt(p)
-      stmtList.add body
-      #addSon(result, makeStmtList(body))
-    # progress guaranteed
-    while sameInd(p):
-      var b: PNode
-      case p.tok.tokType
-      of tkOf:
-        b = newNodeP(nkOfBranch, p)
-        exprList(p, tkColon, b)
-      of tkElif:
-        b = newNodeP(nkElifBranch, p)
-        getTok(p)
-        optInd(p, b)
-        addSon(b, parseExpr(p))
-      of tkExcept:
-        b = newNodeP(nkExceptBranch, p)
-        exprList(p, tkColon, b)
-      of tkElse:
-        b = newNodeP(nkElse, p)
-        getTok(p)
-      else: break
-      eat(p, tkColon)
-      addSon(b, parseStmt(p))
-      addSon(stmtList, b)
-      if b.kind == nkElse: break
+    let body = parseStmt(p)
+    stmtList.add body
+
     if stmtList.len == 1 and stmtList[0].kind == nkStmtList:
       # to keep backwards compatibility (see tests/vm/tstringnil)
       stmtList = stmtList[0]
-    result.add newProcNode(nkDo, stmtList.info, stmtList,
-                           params = emptyNode, pragmas = emptyNode)
+
+    if openingParams.kind != nkEmpty:
+      result.add newProcNode(nkDo, stmtList.info, stmtList,
+                             params = openingParams, pragmas = openingPragmas)
+    else:
+      result.add stmtList
+
+    while sameInd(p):
+      var nextBlock: PNode
+      let nextToken = p.tok.tokType
+      if nextToken == tkDo:
+        let info = parLineInfo(p)
+        getTok(p)
+        nextBlock = parseDoBlock(p, info)
+      else:
+        case nextToken:
+        of tkOf:
+          nextBlock = newNodeP(nkOfBranch, p)
+          exprList(p, tkColon, nextBlock)
+        of tkElif:
+          nextBlock = newNodeP(nkElifBranch, p)
+          getTok(p)
+          optInd(p, nextBlock)
+          nextBlock.addSon parseExpr(p)
+        of tkExcept:
+          nextBlock = newNodeP(nkExceptBranch, p)
+          exprList(p, tkColon, nextBlock)
+        of tkElse:
+          nextBlock = newNodeP(nkElse, p)
+          getTok(p)
+        else: break
+        eat(p, tkColon)
+        nextBlock.addSon parseStmt(p)
+      result.add nextBlock
+      if nextBlock.kind == nkElse: break
+  else:
+    if openingParams.kind != nkEmpty:
+      parMessage(p, errTokenExpected, ":")
 
 proc parseExprStmt(p: var TParser): PNode =
   #| exprStmt = simpleExpr
@@ -1219,12 +1224,7 @@ proc parseExprStmt(p: var TParser): PNode =
     getTok(p)
     optInd(p, result)
     var b = parseExpr(p)
-    if p.tok.tokType == tkColon and p.tok.indent < 0:
-      if b.kind != nkEmpty:
-        let call = makeCall(b)
-        call.add parseDoBlock(p, parLineInfo(p))
-        parseDoBlocks(p, call)
-        b = call
+    b = postExprBlocks(p, b)
     addSon(result, a)
     addSon(result, b)
   else:
@@ -1250,11 +1250,7 @@ proc parseExprStmt(p: var TParser): PNode =
         optInd(p, result)
     else:
       result = a
-    if p.tok.tokType == tkDo and p.tok.indent < 0:
-      result = makeCall(result)
-      parseDoBlocks(p, result)
-      return result
-    result = parseMacroColon(p, result)
+    result = postExprBlocks(p, result)
 
 proc parseModuleName(p: var TParser, kind: TNodeKind): PNode =
   result = parseExpr(p)
@@ -1896,14 +1892,7 @@ proc parseVariable(p: var TParser): PNode =
   #| variable = (varTuple / identColonEquals) colonBody? indAndComment
   if p.tok.tokType == tkParLe: result = parseVarTuple(p)
   else: result = parseIdentColonEquals(p, {withPragma})
-  if p.tok.tokType == tkColon and p.tok.indent < 0:
-    let last = result.len-1
-    let ex = result.sons[last]
-    if ex.kind != nkEmpty:
-      let call = makeCall(ex)
-      call.add parseDoBlock(p, parLineInfo(p))
-      parseDoBlocks(p, call)
-      result.sons[last] = call
+  result{-1} = postExprBlocks(p, result{-1})
   indAndComment(p, result)
 
 proc parseBind(p: var TParser, k: TNodeKind): PNode =
diff --git a/tests/parser/tpostexprblocks.nim b/tests/parser/tpostexprblocks.nim
new file mode 100644
index 000000000..785ecdd89
--- /dev/null
+++ b/tests/parser/tpostexprblocks.nim
@@ -0,0 +1,478 @@
+discard """
+nimout: '''
+StmtList
+  Ident !"foo"
+  Call
+    Ident !"foo"
+  Call
+    Ident !"foo"
+    Ident !"x"
+  Command
+    Ident !"foo"
+    Ident !"x"
+  Call
+    Ident !"foo"
+    StmtList
+      DiscardStmt
+        Empty
+  Call
+    Ident !"foo"
+    StmtList
+      DiscardStmt
+        Empty
+  Call
+    Ident !"foo"
+    StrLit test
+    StmtList
+      DiscardStmt
+        Empty
+  Call
+    Ident !"foo"
+    StrLit test
+    StmtList
+      DiscardStmt
+        Empty
+  Command
+    Ident !"foo"
+    StrLit test
+    StmtList
+      DiscardStmt
+        Empty
+  Command
+    Ident !"foo"
+    StrLit test
+    StmtList
+      DiscardStmt
+        Empty
+  Command
+    Ident !"foo"
+    IntLit 1
+    Par
+      Infix
+        Ident !"+"
+        IntLit 2
+        IntLit 3
+    StmtList
+      DiscardStmt
+        Empty
+  Command
+    Ident !"foo"
+    IntLit 1
+    Par
+      Infix
+        Ident !"+"
+        IntLit 2
+        IntLit 3
+    StmtList
+      DiscardStmt
+        Empty
+  Call
+    Ident !"foo"
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Empty
+        IdentDefs
+          Ident !"x"
+          Empty
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+  Call
+    Ident !"foo"
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Empty
+        IdentDefs
+          Ident !"x"
+          Ident !"int"
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+  Call
+    Ident !"foo"
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Ident !"int"
+        IdentDefs
+          Ident !"x"
+          Ident !"int"
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+  Command
+    Ident !"foo"
+    Ident !"x"
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Empty
+        IdentDefs
+          Ident !"y"
+          Empty
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+  Call
+    Ident !"foo"
+    StmtList
+      DiscardStmt
+        Empty
+    Else
+      StmtList
+        DiscardStmt
+          Empty
+  Call
+    Ident !"foo"
+    StmtList
+      DiscardStmt
+        Empty
+    StmtList
+      DiscardStmt
+        Empty
+    Else
+      StmtList
+        DiscardStmt
+          Empty
+  Command
+    Ident !"foo"
+    Ident !"x"
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Empty
+        IdentDefs
+          Ident !"y"
+          Empty
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Ident !"int"
+        IdentDefs
+          Ident !"z"
+          Empty
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+    Do
+      Empty
+      Empty
+      Empty
+      FormalParams
+        Ident !"int"
+        IdentDefs
+          Ident !"w"
+          Ident !"int"
+          Empty
+      Empty
+      Empty
+      StmtList
+        DiscardStmt
+          Empty
+    StmtList
+      DiscardStmt
+        Empty
+    Else
+      StmtList
+        DiscardStmt
+          Empty
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Ident !"foo"
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Call
+        Ident !"foo"
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Call
+        Ident !"foo"
+        Ident !"x"
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Command
+        Ident !"foo"
+        Ident !"x"
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Call
+        Ident !"foo"
+        StmtList
+          DiscardStmt
+            Empty
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Call
+        Ident !"foo"
+        StmtList
+          DiscardStmt
+            Empty
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Call
+        Ident !"foo"
+        StmtList
+          DiscardStmt
+            Empty
+        Else
+          StmtList
+            DiscardStmt
+              Empty
+  VarSection
+    IdentDefs
+      Ident !"a"
+      Empty
+      Command
+        Ident !"foo"
+        Ident !"x"
+        Do
+          Empty
+          Empty
+          Empty
+          FormalParams
+            Empty
+            IdentDefs
+              Ident !"y"
+              Empty
+              Empty
+          Empty
+          Empty
+          StmtList
+            DiscardStmt
+              Empty
+        Else
+          StmtList
+            DiscardStmt
+              Empty
+  Asgn
+    Ident !"a"
+    Ident !"foo"
+  Asgn
+    Ident !"a"
+    Call
+      Ident !"foo"
+  Asgn
+    Ident !"a"
+    Call
+      Ident !"foo"
+      Ident !"x"
+  Asgn
+    Ident !"a"
+    Command
+      Ident !"foo"
+      Ident !"x"
+  Asgn
+    Ident !"a"
+    Call
+      Ident !"foo"
+      StmtList
+        DiscardStmt
+          Empty
+  Asgn
+    Ident !"a"
+    Call
+      Ident !"foo"
+      StmtList
+        DiscardStmt
+          Empty
+  Asgn
+    Ident !"a"
+    Call
+      Ident !"foo"
+      StmtList
+        DiscardStmt
+          Empty
+      Else
+        StmtList
+          DiscardStmt
+            Empty
+  Asgn
+    Ident !"a"
+    Command
+      Ident !"foo"
+      Ident !"x"
+      Do
+        Empty
+        Empty
+        Empty
+        FormalParams
+          Empty
+          IdentDefs
+            Ident !"y"
+            Empty
+            Empty
+        Empty
+        Empty
+        StmtList
+          DiscardStmt
+            Empty
+      Else
+        StmtList
+          DiscardStmt
+            Empty
+'''
+"""
+
+import macros
+
+dumpTree:
+  # simple calls
+  foo
+  foo()
+  foo(x)
+  foo x
+
+  foo:
+    discard
+
+  foo do:
+    discard
+
+  foo("test"):
+    discard
+
+  foo("test") do:
+    discard
+
+  foo "test":
+    discard
+
+  foo "test" do:
+    discard
+
+  # more complicated calls
+  foo 1, (2+3):
+    discard
+
+  foo 1, (2+3) do:
+    discard
+
+  foo do (x):
+    discard
+
+  foo do (x: int):
+    discard
+
+  foo do (x: int) -> int:
+    discard
+
+  foo x do (y):
+    discard
+
+  # extra blocks
+  foo:
+    discard
+  else:
+    discard
+
+  foo do:
+    discard
+  do:
+    discard
+  else:
+    discard
+
+  foo x do (y):
+    discard
+  do (z) -> int:
+    discard
+  do (w: int) -> int:
+    discard
+  do:
+    discard
+  else:
+    discard
+
+  # introduce a variable
+  var a = foo
+  var a = foo()
+  var a = foo(x)
+  var a = foo x
+
+  var a = foo:
+    discard
+
+  var a = foo do:
+    discard
+
+  var a = foo do:
+    discard
+  else:
+    discard
+
+  var a = foo x do (y):
+    discard
+  else:
+    discard
+
+  # assignments
+  a = foo
+  a = foo()
+  a = foo(x)
+  a = foo x
+
+  a = foo:
+    discard
+
+  a = foo do:
+    discard
+
+  a = foo do:
+    discard
+  else:
+    discard
+
+  a = foo x do (y):
+    discard
+  else:
+    discard
+