first steps to the new parser/grammar

5 files changed, 330 insertions, 250 deletions

diff --git a/compiler/commands.nim b/compiler/commands.nim
index 8f8afe206..8b03dfec4 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -196,8 +196,10 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "patterns": result = contains(gOptions, optPatterns)
   else: InvalidCmdLineOption(passCmd1, switch, info)
   
-proc processPath(path: string): string = 
-  result = UnixToNativePath(path % ["nimrod", getPrefixDir(), "lib", libpath,
+proc processPath(path: string, notRelativeToProj = false): string =
+  let p = if notRelativeToProj or os.isAbsolute(path) or '$' in path: path 
+          else: options.gProjectPath / path
+  result = UnixToNativePath(p % ["nimrod", getPrefixDir(), "lib", libpath,
     "home", removeTrailingDirSep(os.getHomeDir()),
     "projectname", options.gProjectName,
     "projectpath", options.gProjectPath])
@@ -229,7 +231,7 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "babelpath":
     if pass in {passCmd2, passPP}:
       expectArg(switch, arg, pass, info)
-      let path = processPath(arg)
+      let path = processPath(arg, notRelativeToProj=true)
       babelpath(path, info)
   of "excludepath":
     expectArg(switch, arg, pass, info)
@@ -451,9 +453,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "genscript": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optGenScript)
-  of "lib": 
+  of "lib":
     expectArg(switch, arg, pass, info)
-    libpath = processPath(arg)
+    libpath = processPath(arg, notRelativeToProj=true)
   of "putenv": 
     expectArg(switch, arg, pass, info)
     splitSwitch(arg, key, val, pass, info)
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 2b7c567c6..9c9d68f5d 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -237,7 +237,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     of tkSymbol: 
       dispA(result, "<span class=\"Identifier\">$1</span>", 
             "\\spanIdentifier{$1}", [toRope(esc(d.target, literal))])
-    of tkInd, tkSad, tkDed, tkSpaces, tkInvalid: 
+    of tkInd, tkSpaces, tkInvalid: 
       app(result, literal)
     of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, 
        tkBracketDotLe, tkBracketDotRi, tkCurlyDotLe, tkCurlyDotRi, tkParDotLe, 
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index 9cf5ccb2b..f28702f5c 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -58,8 +58,7 @@ type
     tkParDotLe, tkParDotRi,   # (. and .)
     tkComma, tkSemiColon,
     tkColon, tkColonColon, tkEquals, tkDot, tkDotDot,
-    tkOpr, tkComment, tkAccent, tkInd, tkSad, 
-    tkDed, # pseudo token types used by the source renderers:
+    tkOpr, tkComment, tkAccent, tkInd,
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr,
     
   TTokTypes* = set[TTokType]
@@ -91,8 +90,8 @@ const
     ")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)",
     ",", ";",
     ":", "::", "=", ".", "..",
-    "tkOpr", "tkComment", "`", "[new indentation]", 
-    "[same indentation]", "[dedentation]", "tkSpaces", "tkInfixOpr", 
+    "tkOpr", "tkComment", "`", "[new indentation]",
+    "tkSpaces", "tkInfixOpr",
     "tkPrefixOpr", "tkPostfixOpr"]
 
 type 
@@ -102,7 +101,8 @@ type
     base2, base8, base16
   TToken* = object            # a Nimrod token
     tokType*: TTokType        # the type of the token
-    indent*: int              # the indentation; only valid if tokType = tkIndent
+    indent*: int              # the indentation; != -1 if the token has been
+                              # preceeded with indentation
     ident*: PIdent            # the parsed identifier
     iNumber*: BiggestInt      # the parsed integer literal
     fNumber*: BiggestFloat    # the parsed floating point literal
@@ -113,8 +113,6 @@ type
   
   TLexer* = object of TBaseLexer
     fileIdx*: int32
-    indentStack*: seq[int]    # the indentation stack
-    dedent*: int              # counter for DED token generation
     indentAhead*: int         # if > 0 an indendation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
@@ -122,9 +120,6 @@ type
 
 var gLinesCompiled*: int  # all lines that have been compiled
 
-proc pushInd*(L: var TLexer, indent: int)
-
-proc popInd*(L: var TLexer)
 proc isKeyword*(kind: TTokType): bool
 proc openLexer*(lex: var TLexer, fileidx: int32, inputstream: PLLStream)
 proc rawGetTok*(L: var TLexer, tok: var TToken)
@@ -154,31 +149,14 @@ proc isNimrodIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc pushInd(L: var TLexer, indent: int) = 
-  var length = len(L.indentStack)
-  setlen(L.indentStack, length + 1)
-  if (indent > L.indentStack[length - 1]): 
-    L.indentstack[length] = indent
-  else: 
-    InternalError("pushInd")
-  
-proc popInd(L: var TLexer) = 
-  var length = len(L.indentStack)
-  setlen(L.indentStack, length - 1)
-
-proc findIdent(L: TLexer, indent: int): bool = 
-  for i in countdown(len(L.indentStack) - 1, 0): 
-    if L.indentStack[i] == indent: 
-      return true
-
 proc tokToStr*(tok: TToken): string = 
   case tok.tokType
   of tkIntLit..tkInt64Lit: result = $tok.iNumber
   of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
   of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
-  of tkParLe..tkColon, tkEof, tkInd, tkSad, tkDed, tkAccent: 
+  of tkParLe..tkColon, tkEof, tkInd, tkAccent: 
     result = tokTypeToStr[tok.tokType]
-  else: 
+  else:
     if tok.ident != nil:
       result = tok.ident.s
     else: 
@@ -216,7 +194,6 @@ proc fillToken(L: var TToken) =
   
 proc openLexer(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) = 
   openBaseLexer(lex, inputstream)
-  lex.indentStack = @[0]
   lex.fileIdx = fileIdx
   lex.indentAhead = - 1
   inc(lex.Linenumber, inputstream.lineOffset) 
@@ -651,23 +628,9 @@ proc getOperator(L: var TLexer, tok: var TToken) =
     Inc(pos)
   endOperator(L, tok, pos, h)
 
-proc handleIndentation(L: var TLexer, tok: var TToken, indent: int) = 
+proc handleIndentation(tok: var TToken, indent: int) {.inline.} = 
   tok.indent = indent
-  var i = high(L.indentStack)
-  if indent > L.indentStack[i]: 
-    tok.tokType = tkInd
-  elif indent == L.indentStack[i]: 
-    tok.tokType = tkSad
-  else: 
-    # check we have the indentation somewhere in the stack:
-    while (i >= 0) and (indent != L.indentStack[i]): 
-      dec(i)
-      inc(L.dedent)
-    dec(L.dedent)
-    tok.tokType = tkDed
-    if i < 0: 
-      tok.tokType = tkSad     # for the parser it is better as SAD
-      lexMessage(L, errInvalidIndentation)
+  tok.tokType = tkInd
 
 proc scanComment(L: var TLexer, tok: var TToken) = 
   var pos = L.bufpos
@@ -705,7 +668,6 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     else:
       if buf[pos] > ' ': 
         L.indentAhead = indent
-        inc(L.dedent)
       break 
   L.bufpos = pos
 
@@ -727,21 +689,17 @@ proc skip(L: var TLexer, tok: var TToken) =
         Inc(pos)
         Inc(indent)
       if (buf[pos] > ' '): 
-        handleIndentation(L, tok, indent)
-        break 
-    else: 
+        handleIndentation(tok, indent)
+        break
+    else:
       break                   # EndOfFile also leaves the loop
   L.bufpos = pos
 
-proc rawGetTok(L: var TLexer, tok: var TToken) = 
+proc rawGetTok(L: var TLexer, tok: var TToken) =
   fillToken(tok)
-  if L.dedent > 0:
-    dec(L.dedent)
-    if L.indentAhead >= 0: 
-      handleIndentation(L, tok, L.indentAhead)
-      L.indentAhead = - 1
-    else:
-      tok.tokType = tkDed
+  if L.indentAhead >= 0:
+    handleIndentation(tok, L.indentAhead)
+    L.indentAhead = - 1
     return
   skip(L, tok)
   # got an documentation comment or tkIndent, return that:
diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim
index 0f0b76827..47d489556 100644
--- a/compiler/nimconf.nim
+++ b/compiler/nimconf.nim
@@ -19,7 +19,7 @@ import
 proc ppGetTok(L: var TLexer, tok: var TToken) = 
   # simple filter
   rawGetTok(L, tok)
-  while tok.tokType in {tkInd, tkSad, tkDed, tkComment}: rawGetTok(L, tok)
+  while tok.tokType in {tkInd, tkComment}: rawGetTok(L, tok)
   
 proc parseExpr(L: var TLexer, tok: var TToken): bool
 proc parseAtom(L: var TLexer, tok: var TToken): bool = 
diff --git a/compiler/parser.nim b/compiler/parser.nim
index 769aa7a3e..c8f45da20 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -12,7 +12,17 @@
 # it uses several helper routines to keep the parser small. A special
 # efficient algorithm is used for the precedence levels. The parser here can
 # be seen as a refinement of the grammar, as it specifies how the AST is built
-# from the grammar and how comments belong to the AST.
+# from the grammar and how comments belong to the AST. 
+
+
+# In fact the grammar is generated from this file:
+when isMainModule:
+  import pegs
+  var outp = open("compiler/grammar.txt", fmWrite)
+  for line in lines("compiler/parser.nim"):
+    if line =~ peg" \s* '#| ' {.*}":
+      outp.writeln matches[0]
+  outp.close
 
 import
   llstream, lexer, idents, strutils, ast, msgs
@@ -22,7 +32,7 @@ type
                               # is being parsed
     lex*: TLexer              # the lexer that is used for parsing
     tok*: TToken              # the current token
-  
+    currInd: int              # current indentation (for skipInd)
 
 proc ParseAll*(p: var TParser): PNode
 proc openParser*(p: var TParser, filename: string, inputstream: PLLStream)
@@ -81,6 +91,15 @@ proc parMessage(p: TParser, msg: TMsgKind, arg: string = "") =
 proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) = 
   lexMessage(p.lex, msg, prettyTok(tok))
 
+template withInd(p: expr, body: stmt) {.immediate.} =
+  let oldInd = p.currInd
+  p.currInd = p.tok.indent
+  body
+  p.currInd = oldInd
+
+template realInd(p): bool = p.tok.tokType == tkInd and p.tok.ident > p.currInd
+template sameInd(p): bool = p.tok.tokType == tkInd and p.tok.ident == p.currInd
+
 proc skipComment(p: var TParser, node: PNode) = 
   if p.tok.tokType == tkComment: 
     if node != nil: 
@@ -90,42 +109,42 @@ proc skipComment(p: var TParser, node: PNode) =
       parMessage(p, errInternal, "skipComment")
     getTok(p)
 
-proc skipInd(p: var TParser) = 
-  if p.tok.tokType == tkInd: getTok(p)
+proc skipInd(p: var TParser) =
+  if realInd(p): getTok(p)
   
-proc optPar(p: var TParser) = 
-  if p.tok.tokType == tkSad or p.tok.tokType == tkInd: getTok(p)
+proc optPar(p: var TParser) =
+  if p.tok.tokType == tkInd and p.tok.indent >= p.currInd: getTok(p)
   
-proc optInd(p: var TParser, n: PNode) = 
+proc optInd(p: var TParser, n: PNode) =
   skipComment(p, n)
   skipInd(p)
 
-proc ExpectNl(p: TParser) = 
-  if p.tok.tokType notin {tkEof, tkSad, tkInd, tkDed, tkComment}: 
+proc ExpectNl(p: TParser) =
+  if p.tok.tokType notin {tkEof, tkInd, tkComment}:
     lexMessage(p.lex, errNewlineExpected, prettyTok(p.tok))
 
-proc expectIdentOrKeyw(p: TParser) = 
-  if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType): 
+proc expectIdentOrKeyw(p: TParser) =
+  if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType):
     lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok))
   
-proc ExpectIdent(p: TParser) = 
-  if p.tok.tokType != tkSymbol: 
+proc ExpectIdent(p: TParser) =
+  if p.tok.tokType != tkSymbol:
     lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok))
   
-proc Eat(p: var TParser, TokType: TTokType) = 
+proc Eat(p: var TParser, TokType: TTokType) =
   if p.tok.TokType == TokType: getTok(p)
   else: lexMessage(p.lex, errTokenExpected, TokTypeToStr[tokType])
   
-proc parLineInfo(p: TParser): TLineInfo = 
+proc parLineInfo(p: TParser): TLineInfo =
   result = getLineInfo(p.lex)
 
-proc indAndComment(p: var TParser, n: PNode) = 
-  if p.tok.tokType == tkInd: 
+proc indAndComment(p: var TParser, n: PNode) =
+  if p.tok.tokType == tkInd and p.tok.indent > p.currInd:
     var info = parLineInfo(p)
     getTok(p)
     if p.tok.tokType == tkComment: skipComment(p, n)
     else: LocalError(info, errInvalidIndentation)
-  else: 
+  else:
     skipComment(p, n)
   
 proc newNodeP(kind: TNodeKind, p: TParser): PNode = 
@@ -195,7 +214,37 @@ proc getPrecedence(tok: TToken): int =
 proc isOperator(tok: TToken): bool = 
   result = getPrecedence(tok) >= 0
 
-proc parseSymbol(p: var TParser): PNode = 
+#| module = stmt? (IND{=} stmt)*
+#|
+#| comma = ',' COMMENT? IND?
+#| semicolon = ';' COMMENT IND?
+#| colon = ':' COMMENT? IND?
+#| colcom = ':' COMMENT?
+#| 
+#| operator =  OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9
+#|          | 'or' | 'xor' | 'and'
+#|          | 'is' | 'isnot' | 'in' | 'notin' | 'of'
+#|          | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'addr' | 'static' | '..'
+#| 
+#| prefixOperator = operator
+#| 
+#| optInd = COMMENT? IND?
+#| optPar = IND{>} | IND{=}
+#| 
+#| lowestExpr = assignExpr (OP0 optInd assignExpr)*
+#| assignExpr = orExpr (OP1 optInd orExpr)*
+#| orExpr = andExpr (OP2 optInd andExpr)*
+#| andExpr = cmpExpr (OP3 optInd cmpExpr)*
+#| cmpExpr = sliceExpr (OP4 optInd sliceExpr)*
+#| sliceExpr = ampExpr (OP5 optInd ampExpr)*
+#| ampExpr = plusExpr (OP6 optInd plusExpr)*
+#| plusExpr = mulExpr (OP7 optInd mulExpr)*
+#| mulExpr = dollarExpr (OP8 optInd dollarExpr)*
+#| dollarExpr = primary (OP9 optInd primary)*
+
+proc parseSymbol(p: var TParser): PNode =
+  #| symbol = '`' (KEYW|IDENT|operator|'(' ')'|'[' ']'|'{' '}'|'='|literal)+ '`'
+  #|        | IDENT
   case p.tok.tokType
   of tkSymbol: 
     result = newIdentNodeP(p.tok.ident, p)
@@ -231,21 +280,23 @@ proc parseSymbol(p: var TParser): PNode =
           parMessage(p, errIdentifierExpected, p.tok)
         break
     eat(p, tkAccent)
-  else: 
+  else:
     parMessage(p, errIdentifierExpected, p.tok)
     getTok(p) # BUGFIX: We must consume a token here to prevent endless loops!
     result = ast.emptyNode
 
 proc indexExpr(p: var TParser): PNode = 
+  #| indexExpr = expr
   result = parseExpr(p)
 
 proc indexExprList(p: var TParser, first: PNode, k: TNodeKind, 
                    endToken: TTokType): PNode = 
+  #| indexExprList = indexExpr (comma indexExpr)* comma?
   result = newNodeP(k, p)
   addSon(result, first)
   getTok(p)
   optInd(p, result)
-  while p.tok.tokType notin {endToken, tkEof, tkSad}:
+  while p.tok.tokType notin {endToken, tkEof}:
     var a = indexExpr(p)
     addSon(result, a)
     if p.tok.tokType != tkComma: break 
@@ -255,6 +306,7 @@ proc indexExprList(p: var TParser, first: PNode, k: TNodeKind,
   eat(p, endToken)
 
 proc exprColonEqExpr(p: var TParser): PNode =
+  #| exprColonEqExpr = expr (':'|'=' expr)?
   var a = parseExpr(p)
   if p.tok.tokType == tkColon:
     result = newNodeP(nkExprColonExpr, p)
@@ -272,6 +324,7 @@ proc exprColonEqExpr(p: var TParser): PNode =
     result = a
 
 proc exprList(p: var TParser, endTok: TTokType, result: PNode) = 
+  #| exprList = expr (comma expr)* comma?
   getTok(p)
   optInd(p, result)
   while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): 
@@ -283,6 +336,7 @@ proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
   eat(p, endTok)
 
 proc dotExpr(p: var TParser, a: PNode): PNode =
+  #| dotExpr = expr '.' optInd ('type' | 'addr' | symbol)
   var info = p.lex.getlineInfo
   getTok(p)
   optInd(p, a)
@@ -301,26 +355,16 @@ proc dotExpr(p: var TParser, a: PNode): PNode =
     addSon(result, parseSymbol(p))
 
 proc qualifiedIdent(p: var TParser): PNode = 
-  result = parseSymbol(p)     #optInd(p, result);
+  #| qualifiedIdent = symbol ('.' optInd ('type' | 'addr' | symbol))?
+  result = parseSymbol(p)
   if p.tok.tokType == tkDot: result = dotExpr(p, result)
 
-proc qualifiedIdentListAux(p: var TParser, endTok: TTokType, result: PNode) = 
-  getTok(p)
-  optInd(p, result)
-  while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): 
-    var a = qualifiedIdent(p)
-    addSon(result, a)         #optInd(p, a);
-    if p.tok.tokType != tkComma: break 
-    getTok(p)
-    optInd(p, a)
-  eat(p, endTok)
-
-proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) = 
+proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
   assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi})
   getTok(p)
   optInd(p, result)
-  while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof) and
-      (p.tok.tokType != tkSad) and (p.tok.tokType != tkInd): 
+  while p.tok.tokType != endTok and p.tok.tokType != tkEof and
+       not (p.tok.tokType == tkInd and p.tok.indent >= p.currInd): 
     var a = exprColonEqExpr(p)
     addSon(result, a)
     if p.tok.tokType != tkComma: break 
@@ -329,12 +373,14 @@ proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
   optPar(p)
   eat(p, endTok)
 
-proc exprColonEqExprList(p: var TParser, kind: TNodeKind, 
-                         endTok: TTokType): PNode = 
+proc exprColonEqExprList(p: var TParser, kind: TNodeKind,
+                         endTok: TTokType): PNode =
+  #| exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)?
   result = newNodeP(kind, p)
   exprColonEqExprListAux(p, endTok, result)
 
 proc setOrTableConstr(p: var TParser): PNode =
+  #| setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}'
   result = newNodeP(nkCurly, p)
   getTok(p) # skip '{'
   optInd(p, result)
@@ -342,7 +388,7 @@ proc setOrTableConstr(p: var TParser): PNode =
     getTok(p) # skip ':'
     result.kind = nkTableConstr
   else:
-    while p.tok.tokType notin {tkCurlyRi, tkEof, tkSad, tkInd}: 
+    while p.tok.tokType notin {tkCurlyRi, tkEof, tkInd}: 
       var a = exprColonEqExpr(p)
       if a.kind == nkExprColonExpr: result.kind = nkTableConstr
       addSon(result, a)
@@ -353,6 +399,7 @@ proc setOrTableConstr(p: var TParser): PNode =
   eat(p, tkCurlyRi) # skip '}'
 
 proc parseCast(p: var TParser): PNode = 
+  #| castExpr = 'cast' '[' optInd typeDesc optPar ']' '(' optInd expr optPar ')'
   result = newNodeP(nkCast, p)
   getTok(p)
   eat(p, tkBracketLe)
@@ -366,15 +413,6 @@ proc parseCast(p: var TParser): PNode =
   optPar(p)
   eat(p, tkParRi)
 
-proc parseAddr(p: var TParser): PNode = 
-  result = newNodeP(nkAddr, p)
-  getTok(p)
-  eat(p, tkParLe)
-  optInd(p, result)
-  addSon(result, parseExpr(p))
-  optPar(p)
-  eat(p, tkParRi)
-
 proc setBaseFlags(n: PNode, base: TNumericalBase) = 
   case base
   of base10: nil
@@ -398,6 +436,18 @@ proc parseGStrLit(p: var TParser, a: PNode): PNode =
     result = a
   
 proc identOrLiteral(p: var TParser): PNode = 
+  #| generalizedLit ::= GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT
+  #| identOrLiteral = generalizedLit | symbol 
+  #|                | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT
+  #|                | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT
+  #|                | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT
+  #|                | STR_LIT | RSTR_LIT | TRIPLESTR_LIT
+  #|                | CHAR_LIT
+  #|                | NIL
+  #|                | tupleConstr | arrayConstr | setOrTableConstr
+  #|                | castExpr
+  #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')'
+  #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']'
   case p.tok.tokType
   of tkSymbol: 
     result = newIdentNodeP(p.tok.ident, p)
@@ -405,7 +455,7 @@ proc identOrLiteral(p: var TParser): PNode =
     result = parseGStrLit(p, result)
   of tkAccent: 
     result = parseSymbol(p)       # literals
-  of tkIntLit: 
+  of tkIntLit:
     result = newIntNodeP(nkIntLit, p.tok.iNumber, p)
     setBaseFlags(result, p.tok.base)
     getTok(p)
@@ -493,6 +543,11 @@ proc identOrLiteral(p: var TParser): PNode =
     result = ast.emptyNode
 
 proc primarySuffix(p: var TParser, r: PNode): PNode =
+  #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
+  #|               | doBlocks
+  #|               | '.' optInd ('type' | 'addr' | symbol) generalizedLit?
+  #|               | '[' optInd indexExprList optPar ']'
+  #|               | '{' optInd indexExprList optPar '}'
   result = r
   while true:
     case p.tok.tokType
@@ -543,12 +598,17 @@ proc lowestExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
     result = a
     opPrec = getPrecedence(p.tok)
   
-proc lowestExpr(p: var TParser, mode = pmNormal): PNode = 
+proc lowestExpr(p: var TParser, mode = pmNormal): PNode =
   result = lowestExprAux(p, -1, mode)
 
-proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode = 
+proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
+  #| condExpr = expr ':' optInd expr optInd
+  #|         ('elif' expr ':' optInd expr optInd)*
+  #|          'else' ':' optInd expr
+  #| ifExpr = 'if' condExpr
+  #| whenExpr = 'when' condExpr
   result = newNodeP(kind, p)
-  while true: 
+  while true:
     getTok(p)                 # skip `if`, `elif`
     var branch = newNodeP(nkElifExpr, p)
     addSon(branch, parseExpr(p))
@@ -565,15 +625,15 @@ proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
   addSon(branch, parseExpr(p))
   addSon(result, branch)
 
-proc parsePragma(p: var TParser): PNode = 
+proc parsePragma(p: var TParser): PNode =
+  #| pragma = '{.' optInd (exprColonExpr comma?)* optPar ('.}' | '}')
   result = newNodeP(nkPragma, p)
   getTok(p)
   optInd(p, result)
-  while (p.tok.tokType != tkCurlyDotRi) and (p.tok.tokType != tkCurlyRi) and
-      (p.tok.tokType != tkEof) and (p.tok.tokType != tkSad): 
+  while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}:
     var a = exprColonEqExpr(p)
     addSon(result, a)
-    if p.tok.tokType == tkComma: 
+    if p.tok.tokType == tkComma:
       getTok(p)
       optInd(p, a)
   optPar(p)
@@ -581,7 +641,7 @@ proc parsePragma(p: var TParser): PNode =
   else: parMessage(p, errTokenExpected, ".}")
   
 proc identVis(p: var TParser): PNode = 
-  # identifier with visability
+  #| identVis = symbol opr?  # postfix position
   var a = parseSymbol(p)
   if p.tok.tokType == tkOpr: 
     result = newNodeP(nkPostfix, p)
@@ -592,6 +652,7 @@ proc identVis(p: var TParser): PNode =
     result = a
   
 proc identWithPragma(p: var TParser): PNode = 
+  #| identWithPragma = identVis pragma?
   var a = identVis(p)
   if p.tok.tokType == tkCurlyDotLe: 
     result = newNodeP(nkPragmaExpr, p)
@@ -599,14 +660,18 @@ proc identWithPragma(p: var TParser): PNode =
     addSon(result, parsePragma(p))
   else: 
     result = a
-  
-type 
+
+type
   TDeclaredIdentFlag = enum 
     withPragma,               # identifier may have pragma
     withBothOptional          # both ':' and '=' parts are optional
   TDeclaredIdentFlags = set[TDeclaredIdentFlag]
 
 proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = 
+  #| declColonEquals = identWithPragma (comma identWithPragma)* comma?
+  #|                   (':' optInd typeDesc)? ('=' optInd expr)?
+  #| identColonEquals = ident (comma ident)* comma?
+  #|      (':' optInd typeDesc)? ('=' optInd expr)?)
   var a: PNode
   result = newNodeP(nkIdentDefs, p)
   while true: 
@@ -635,53 +700,55 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
   else: 
     addSon(result, ast.emptyNode)
   
-proc parseTuple(p: var TParser, indentAllowed = false): PNode = 
+proc parseTuple(p: var TParser, indentAllowed = false): PNode =
+  #| inlTupleDecl = 'tuple'
+  #|     [' optInd  (identColonEquals (comma/semicolon)?)*  optPar ']'
+  #| extTupleDecl = 'tuple'
+  #|     COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?
   result = newNodeP(nkTupleTy, p)
   getTok(p)
   if p.tok.tokType == tkBracketLe:
     getTok(p)
     optInd(p, result)
-    while (p.tok.tokType == tkSymbol) or (p.tok.tokType == tkAccent): 
+    while p.tok.tokType in {tkSymbol, tkAccent}:
       var a = parseIdentColonEquals(p, {})
       addSon(result, a)
-      if p.tok.tokType notin {tkComma, tkSemicolon}: break 
+      if p.tok.tokType notin {tkComma, tkSemicolon}: break
       getTok(p)
       optInd(p, a)
     optPar(p)
     eat(p, tkBracketRi)
   elif indentAllowed:
     skipComment(p, result)
-    if p.tok.tokType == tkInd:
-      pushInd(p.lex, p.tok.indent)
-      getTok(p)
-      skipComment(p, result)
-      while true:
-        case p.tok.tokType
-        of tkSad:
-          getTok(p)
-        of tkSymbol, tkAccent:
-          var a = parseIdentColonEquals(p, {})
-          skipComment(p, a)
-          addSon(result, a)
-        of tkDed:
+    if realInd(p):
+      withInd(p):
+        getTok(p)
+        skipComment(p, result)
+        while true:
+          case p.tok.tokType
+          of tkSymbol, tkAccent:
+            var a = parseIdentColonEquals(p, {})
+            skipComment(p, a)
+            addSon(result, a)
+          of tkEof: break
+          else:
+            parMessage(p, errIdentifierExpected, p.tok)
+            break
+          if not sameInd(p): break
           getTok(p)
-          break
-        of tkEof:
-          break
-        else:
-          parMessage(p, errIdentifierExpected, p.tok)
-          break
-      popInd(p.lex)
 
-proc parseParamList(p: var TParser, retColon = true): PNode = 
+proc parseParamList(p: var TParser, retColon = true): PNode =
+  #| paramList = '(' (identColonEquals (comma/semicolon identColonEquals)*)? ')'
+  #| paramListArrow = paramList? ('->' optInd typeDesc)?
+  #| paramListColon = paramList? (':' optInd typeDesc)?
   var a: PNode
   result = newNodeP(nkFormalParams, p)
   addSon(result, ast.emptyNode) # return type
   if p.tok.tokType == tkParLe:
     getTok(p)
     optInd(p, result)
-    while true: 
-      case p.tok.tokType      #optInd(p, a);
+    while true:
+      case p.tok.tokType
       of tkSymbol, tkAccent: 
         a = parseIdentColonEquals(p, {withBothOptional})
       of tkParRi: 
@@ -707,6 +774,7 @@ proc optPragmas(p: var TParser): PNode =
   else: result = ast.emptyNode
 
 proc parseDoBlock(p: var TParser): PNode =
+  #| doBlock = 'do' paramListArrow pragmas? colcom stmt
   let info = parLineInfo(p)
   getTok(p)
   let params = parseParamList(p, retColon=false)
@@ -718,12 +786,14 @@ proc parseDoBlock(p: var TParser): PNode =
                        pragmas = pragmas)
 
 proc parseDoBlocks(p: var TParser, call: PNode) =
+  #| doBlocks = doBlock*
   while p.tok.tokType == tkDo:
     addSon(call, parseDoBlock(p))
     
 proc parseProcExpr(p: var TParser, isExpr: bool): PNode = 
+  #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
   # either a proc type or a anonymous proc
-  var 
+  var
     pragmas, params: PNode
     info: TLineInfo
   info = parLineInfo(p)
@@ -752,7 +822,8 @@ proc isExprStart(p: TParser): bool =
     result = true
   else: result = false
   
-proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, mode: TPrimaryMode): PNode = 
+proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, 
+                       mode: TPrimaryMode): PNode = 
   result = newNodeP(kind, p)
   getTok(p)
   optInd(p, result)
@@ -760,11 +831,10 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, mode: TPrimaryMode): PNo
     addSon(result, primary(p, mode))
 
 proc parseExpr(p: var TParser): PNode = 
-  #
-  #expr ::= lowestExpr
-  #     | 'if' expr ':' expr ('elif' expr ':' expr)* 'else' ':' expr
-  #     | 'when' expr ':' expr ('elif' expr ':' expr)* 'else' ':' expr
-  #
+  #| expr = lowestExpr
+  #|       | ifExpr
+  #|       | whenExpr
+  #|       | caseExpr
   case p.tok.tokType:
   of tkIf: result = parseIfExpr(p, nkIfExpr)
   of tkWhen: result = parseIfExpr(p, nkWhenExpr)
@@ -778,7 +848,13 @@ proc parseDistinct(p: var TParser): PNode
 proc parseEnum(p: var TParser): PNode
 
 proc primary(p: var TParser, mode: TPrimaryMode): PNode = 
-  # prefix operator?
+  #| typeKeyw = 'var' | 'ref' | 'ptr' | 'shared' | 'type' | 'tuple'
+  #|          | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
+  #| primary = typeKeyw typeDescK
+  #|         /  prefixOperator* identOrLiteral primarySuffix*
+  #|         / 'addr' primary
+  #|         / 'static' primary
+  #|         / 'bind' primary
   if isOperator(p.tok):
     let isSigil = IsSigilLike(p.tok)
     result = newNodeP(nkPrefix, p)
@@ -854,6 +930,14 @@ proc parseTypeDefAux(p: var TParser): PNode =
   result = lowestExpr(p, pmTypeDef)
 
 proc parseExprStmt(p: var TParser): PNode = 
+  #| exprStmt = lowestExpr (
+  #|           '=' optInd expr
+  #|          / doBlocks
+  #|          / ':' stmt? ('of' exprList ':' stmt 
+  #|                      | 'elif' expr ':' stmt
+  #|                      | 'except' exprList ':' stmt
+  #|                      | 'else' ':' stmt )?
+  #|          )
   var a = lowestExpr(p)
   if p.tok.tokType == tkEquals: 
     getTok(p)
@@ -865,6 +949,7 @@ proc parseExprStmt(p: var TParser): PNode =
   else:
     var call = if a.kind == nkCall: a
                else: newNode(nkCommand, a.info, @[a])
+    # XXX this is clearly a bug: p(a, b) c should not parse as p(a, b, c)!
     while true:
       if not isExprStart(p): break 
       var e = parseExpr(p)
@@ -874,19 +959,19 @@ proc parseExprStmt(p: var TParser): PNode =
       optInd(p, a)
     if p.tok.tokType == tkDo:
       parseDoBlocks(p, call)
-      return    
+      return
     result = if call.sonsLen <= 1: a
              else: call
     if p.tok.tokType == tkColon:
       result = call
       getTok(p)
       skipComment(p, result)
-      if p.tok.tokType == tkSad: getTok(p)
+      if sameInd(p): getTok(p)
       if p.tok.TokType notin {tkOf, tkElif, tkElse, tkExcept}:
         let body = parseStmt(p)
         addSon(result, newProcNode(nkDo, body.info, body))
       while true:
-        if p.tok.tokType == tkSad: getTok(p)
+        if sameInd(p): getTok(p)
         var b: PNode
         case p.tok.tokType
         of tkOf: 
@@ -900,7 +985,7 @@ proc parseExprStmt(p: var TParser): PNode =
           eat(p, tkColon)
         of tkExcept: 
           b = newNodeP(nkExceptBranch, p)
-          qualifiedIdentListAux(p, tkColon, b)
+          exprList(p, tkColon, b)
           skipComment(p, b)
         of tkElse: 
           b = newNodeP(nkElse, p)
@@ -912,6 +997,9 @@ proc parseExprStmt(p: var TParser): PNode =
         if b.kind == nkElse: break
 
 proc parseImport(p: var TParser, kind: TNodeKind): PNode =
+  #| importStmt = 'import' optInd expr
+  #|               ((comma expr)*
+  #|               / 'except' optInd expr (comma expr)*)
   result = newNodeP(kind, p)
   getTok(p)                   # skip `import` or `export`
   optInd(p, result)
@@ -922,7 +1010,8 @@ proc parseImport(p: var TParser, kind: TNodeKind): PNode =
       result.kind = succ(kind)
     getTok(p)
     optInd(p, result)
-    while p.tok.tokType notin {tkEof, tkSad, tkDed}:
+    while true:
+      # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}:
       a = parseExpr(p)
       if a.kind == nkEmpty: break 
       addSon(result, a)
@@ -932,10 +1021,12 @@ proc parseImport(p: var TParser, kind: TNodeKind): PNode =
   expectNl(p)
 
 proc parseIncludeStmt(p: var TParser): PNode =
+  #| includeStmt = 'include' optInd expr (comma expr)*
   result = newNodeP(nkIncludeStmt, p)
   getTok(p)                   # skip `import` or `include`
   optInd(p, result)
-  while p.tok.tokType notin {tkEof, tkSad, tkDed}:
+  while true:
+    # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}:
     var a = parseExpr(p)
     if a.kind == nkEmpty: break
     addSon(result, a)
@@ -945,6 +1036,7 @@ proc parseIncludeStmt(p: var TParser): PNode =
   expectNl(p)
 
 proc parseFromStmt(p: var TParser): PNode = 
+  #| fromStmt = 'from' expr 'import' optInd expr (comma expr)*
   result = newNodeP(nkFromStmt, p)
   getTok(p)                   # skip `from`
   optInd(p, result)
@@ -952,7 +1044,8 @@ proc parseFromStmt(p: var TParser): PNode =
   addSon(result, a)           #optInd(p, a);
   eat(p, tkImport)
   optInd(p, result)
-  while p.tok.tokType notin {tkEof, tkSad, tkDed}:
+  while true:
+    # p.tok.tokType notin {tkEof, tkSad, tkDed}:
     a = parseExpr(p)
     if a.kind == nkEmpty: break
     addSon(result, a)
@@ -962,28 +1055,25 @@ proc parseFromStmt(p: var TParser): PNode =
   expectNl(p)
 
 proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode = 
+  #| returnStmt = 'return' optInd expr?
+  #| raiseStmt = 'raise' optInd expr?
+  #| yieldStmt = 'yield' optInd expr?
+  #| discardStmt = 'discard' optInd expr?
+  #| breakStmt = 'break' optInd expr?
+  #| continueStmt = 'break' optInd expr?
   result = newNodeP(kind, p)
   getTok(p)
   optInd(p, result)
   case p.tok.tokType
-  of tkEof, tkSad, tkDed: addSon(result, ast.emptyNode)
+  of tkEof, tkInd: addSon(result, ast.emptyNode)
   else: addSon(result, parseExpr(p))
-  
-proc parseYieldOrDiscard(p: var TParser, kind: TNodeKind): PNode = 
-  result = newNodeP(kind, p)
-  getTok(p)
-  optInd(p, result)
-  addSon(result, parseExpr(p))
-
-proc parseBreakOrContinue(p: var TParser, kind: TNodeKind): PNode =
-  result = newNodeP(kind, p)
-  getTok(p)
-  optInd(p, result)
-  case p.tok.tokType
-  of tkEof, tkSad, tkDed: addSon(result, ast.emptyNode)
-  else: addSon(result, parseSymbol(p))
 
 proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
+  #| condStmt = expr colcom stmt COMMENT?
+  #|            ('elif' expr colcom stmt)*
+  #|            ('else' colcom stmt)?
+  #| ifStmt = 'if' condStmt
+  #| whenStmt = 'when' condStmt
   result = newNodeP(kind, p)
   while true:
     getTok(p)                 # skip `if`, `when`, `elif`
@@ -1004,7 +1094,8 @@ proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
     addSon(branch, parseStmt(p))
     addSon(result, branch)
 
-proc parseWhile(p: var TParser): PNode = 
+proc parseWhile(p: var TParser): PNode =
+  #| whileStmt = 'while' expr colcom stmt
   result = newNodeP(nkWhileStmt, p)
   getTok(p)
   optInd(p, result)
@@ -1013,8 +1104,15 @@ proc parseWhile(p: var TParser): PNode =
   skipComment(p, result)
   addSon(result, parseStmt(p))
 
-proc parseCase(p: var TParser): PNode = 
-  var 
+proc parseCase(p: var TParser): PNode =
+  #| ofBranch = 'of' exprList colcom stmt
+  #| ofBranches = ofBranch (IND{=} ofBranch)*
+  #|                       (IND{=} 'elif' expr colcom stmt)*
+  #|                       (IND{=} 'else' colcom stmt)?
+  #| caseStmt = 'case' expr ':'? COMMENT?
+  #|             (IND{>} ofBranches
+  #|             | IND{=} ofBranches)
+  var
     b: PNode
     inElif= false
     wasIndented = false
@@ -1024,40 +1122,43 @@ proc parseCase(p: var TParser): PNode =
   if p.tok.tokType == tkColon: getTok(p)
   skipComment(p, result)
   
-  if p.tok.tokType == tkInd:
-    pushInd(p.lex, p.tok.indent)
+  let oldInd = p.currInd
+  if realInd(p):
+    p.currInd = p.tok.indent
     getTok(p)
     wasIndented = true
   
-  while true: 
-    if p.tok.tokType == tkSad: getTok(p)
+  while true:
+    if sameInd(p): getTok(p)
     case p.tok.tokType
-    of tkOf: 
-      if inElif: break 
+    of tkOf:
+      if inElif: break
       b = newNodeP(nkOfBranch, p)
       exprList(p, tkColon, b)
-    of tkElif: 
+    of tkElif:
       inElif = true
       b = newNodeP(nkElifBranch, p)
       getTok(p)
       optInd(p, b)
       addSon(b, parseExpr(p))
       eat(p, tkColon)
-    of tkElse: 
+    of tkElse:
       b = newNodeP(nkElse, p)
       getTok(p)
       eat(p, tkColon)
-    else: break 
+    else: break
     skipComment(p, b)
     addSon(b, parseStmt(p))
     addSon(result, b)
     if b.kind == nkElse: break
   
   if wasIndented:
-    if p.tok.tokType != tkEof: eat(p, tkDed)
-    popInd(p.lex)
+    p.currInd = oldInd
     
-proc parseTry(p: var TParser): PNode = 
+proc parseTry(p: var TParser): PNode =
+  #| tryStmt = 'try' colcom stmt &('except'|'finally')
+  #|            ('except' exprList colcom stmt)*
+  #|            ('finally' colcom stmt)?
   result = newNodeP(nkTryStmt, p)
   getTok(p)
   eat(p, tkColon)
@@ -1069,12 +1170,12 @@ proc parseTry(p: var TParser): PNode =
     case p.tok.tokType
     of tkExcept: 
       b = newNodeP(nkExceptBranch, p)
-      qualifiedIdentListAux(p, tkColon, b)
+      exprList(p, tkColon, b)
     of tkFinally: 
       b = newNodeP(nkFinally, p)
       getTok(p)
       eat(p, tkColon)
-    else: break 
+    else: break
     skipComment(p, b)
     addSon(b, parseStmt(p))
     addSon(result, b)
@@ -1082,19 +1183,20 @@ proc parseTry(p: var TParser): PNode =
   if b == nil: parMessage(p, errTokenExpected, "except")
 
 proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode =
+  #| exceptBlock = 'except' colcom stmt
   result = newNodeP(kind, p)
   getTok(p)
   eat(p, tkColon)
   skipComment(p, result)
   addSon(result, parseStmt(p))
 
-proc parseFor(p: var TParser): PNode = 
+proc parseFor(p: var TParser): PNode =
+  #| forStmt = 'for' symbol (comma symbol)* 'in' expr colcom stmt
   result = newNodeP(nkForStmt, p)
   getTok(p)
-  optInd(p, result)
   var a = parseSymbol(p)
   addSon(result, a)
-  while p.tok.tokType == tkComma: 
+  while p.tok.tokType == tkComma:
     getTok(p)
     optInd(p, a)
     a = parseSymbol(p)
@@ -1106,28 +1208,27 @@ proc parseFor(p: var TParser): PNode =
   addSon(result, parseStmt(p))
 
 proc parseBlock(p: var TParser): PNode = 
+  #| blockStmt = 'block' symbol? colcom stmt
   result = newNodeP(nkBlockStmt, p)
   getTok(p)
-  optInd(p, result)
-  case p.tok.tokType
-  of tkEof, tkSad, tkDed, tkColon: addSon(result, ast.emptyNode)
+  if p.tok.tokType = tkColon: addSon(result, ast.emptyNode)
   else: addSon(result, parseSymbol(p))
   eat(p, tkColon)
   skipComment(p, result)
   addSon(result, parseStmt(p))
 
 proc parseStatic(p: var TParser): PNode =
+  #| staticStmt = 'static' colcom stmt
   result = newNodeP(nkStaticStmt, p)
   getTok(p)
-  optInd(p, result)
   eat(p, tkColon)
   skipComment(p, result)
   addSon(result, parseStmt(p))
   
-proc parseAsm(p: var TParser): PNode = 
+proc parseAsm(p: var TParser): PNode =
+  #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT)
   result = newNodeP(nkAsmStmt, p)
   getTok(p)
-  optInd(p, result)
   if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p))
   else: addSon(result, ast.emptyNode)
   case p.tok.tokType
@@ -1141,7 +1242,8 @@ proc parseAsm(p: var TParser): PNode =
     return 
   getTok(p)
 
-proc parseGenericParam(p: var TParser): PNode = 
+proc parseGenericParam(p: var TParser): PNode =
+  #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)?
   var a: PNode
   result = newNodeP(nkIdentDefs, p)
   while true: 
@@ -1168,6 +1270,8 @@ proc parseGenericParam(p: var TParser): PNode =
     addSon(result, ast.emptyNode)
 
 proc parseGenericParamList(p: var TParser): PNode = 
+  #| genericParamList = '[' optInd 
+  #|   (genericParam (comma/semicolon genericParam)*)? optPar ']'
   result = newNodeP(nkGenericParams, p)
   getTok(p)
   optInd(p, result)
@@ -1181,11 +1285,15 @@ proc parseGenericParamList(p: var TParser): PNode =
   eat(p, tkBracketRi)
 
 proc parsePattern(p: var TParser): PNode =
+  #| pattern = '{' stmt '}'
   eat(p, tkCurlyLe)
   result = parseStmt(p)
   eat(p, tkCurlyRi)
 
 proc parseRoutine(p: var TParser, kind: TNodeKind): PNode = 
+  #| indAndComment = (IND{>} COMMENT)? | COMMENT?
+  #| routine = optInd identVis pattern? genericParamList?
+  #|   paramListColon pragma? ('=' COMMENT? stmt)? indAndComment
   result = newNodeP(kind, p)
   getTok(p)
   optInd(p, result)
@@ -1205,52 +1313,52 @@ proc parseRoutine(p: var TParser, kind: TNodeKind): PNode =
     addSon(result, parseStmt(p))
   else: 
     addSon(result, ast.emptyNode)
-  indAndComment(p, result)    # XXX: document this in the grammar!
+  indAndComment(p, result)
   
 proc newCommentStmt(p: var TParser): PNode =
+  #| commentStmt = COMMENT
   result = newNodeP(nkCommentStmt, p)
   result.info.line = result.info.line - int16(1) - int16(p.tok.iNumber)
 
-type 
+type
   TDefParser = proc (p: var TParser): PNode {.nimcall.}
 
-proc parseSection(p: var TParser, kind: TNodeKind, 
-                  defparser: TDefParser): PNode = 
+proc parseSection(p: var TParser, kind: TNodeKind,
+                  defparser: TDefParser): PNode =
+  #| section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED)
   result = newNodeP(kind, p)
   getTok(p)
   skipComment(p, result)
   case p.tok.tokType
-  of tkInd: 
-    pushInd(p.lex, p.tok.indent)
-    getTok(p)
-    skipComment(p, result)
-    while true: 
-      case p.tok.tokType
-      of tkSad: 
-        getTok(p)
-      of tkSymbol, tkAccent: 
-        var a = defparser(p)
-        skipComment(p, a)
-        addSon(result, a)
-      of tkDed: 
-        getTok(p)
-        break 
-      of tkEof: 
-        break                 # BUGFIX
-      of tkComment: 
-        var a = newCommentStmt(p)
-        skipComment(p, a)
-        addSon(result, a)
-      else: 
-        parMessage(p, errIdentifierExpected, p.tok)
-        break 
-    popInd(p.lex)
+  of tkInd:
+    if not realInd(p): parMessage(p, errInvalidIndentation)
+    withInd(p):
+      getTok(p)
+      skipComment(p, result)
+      while true: 
+        case p.tok.tokType
+        of tkSad: 
+          getTok(p)
+        of tkSymbol, tkAccent: 
+          var a = defparser(p)
+          skipComment(p, a)
+          addSon(result, a)
+        of tkEof:
+          break
+        of tkComment: 
+          var a = newCommentStmt(p)
+          skipComment(p, a)
+          addSon(result, a)
+        else: 
+          parMessage(p, errIdentifierExpected, p.tok)
+          break
   of tkSymbol, tkAccent, tkParLe: 
     # tkParLe is allowed for ``var (x, y) = ...`` tuple parsing
     addSon(result, defparser(p))
   else: parMessage(p, errIdentifierExpected, p.tok)
   
-proc parseConstant(p: var TParser): PNode = 
+proc parseConstant(p: var TParser): PNode =
+  #| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment
   result = newNodeP(nkConstDef, p)
   addSon(result, identWithPragma(p))
   if p.tok.tokType == tkColon: 
@@ -1262,24 +1370,21 @@ proc parseConstant(p: var TParser): PNode =
   eat(p, tkEquals)
   optInd(p, result)
   addSon(result, parseExpr(p))
-  indAndComment(p, result)    # XXX: special extension!
+  indAndComment(p, result)
   
 proc parseEnum(p: var TParser): PNode = 
-  var a, b: PNode
+  #| enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+
   result = newNodeP(nkEnumTy, p)
-  a = nil
   getTok(p)
   addSon(result, ast.emptyNode)
   optInd(p, result)
-  while true: 
-    case p.tok.tokType
-    of tkEof, tkSad, tkDed: break 
-    else: a = parseSymbol(p)
+  while p.tok.tokType notin {tkEof, tkInd}: 
+    var a = parseSymbol(p)
     optInd(p, a)
     if p.tok.tokType == tkEquals: 
       getTok(p)
       optInd(p, a)
-      b = a
+      var b = a
       a = newNodeP(nkEnumFieldDef, p)
       addSon(a, b)
       addSon(a, parseExpr(p))
@@ -1293,6 +1398,9 @@ proc parseEnum(p: var TParser): PNode =
 
 proc parseObjectPart(p: var TParser): PNode
 proc parseObjectWhen(p: var TParser): PNode = 
+  #| objectWhen = 'when' expr colcom objectPart COMMENT?
+  #|             ('elif' expr colcom objectPart COMMENT?)*
+  #|             ('else' colcom objectPart COMMENT?)?
   result = newNodeP(nkRecWhen, p)
   while true: 
     getTok(p)                 # skip `when`, `elif`
@@ -1311,9 +1419,17 @@ proc parseObjectWhen(p: var TParser): PNode =
     eat(p, tkColon)
     skipComment(p, branch)
     addSon(branch, parseObjectPart(p))
+    # XXX no skipComment(p, branch) here?
     addSon(result, branch)
 
 proc parseObjectCase(p: var TParser): PNode = 
+  #| objectBranch = 'of' exprList colcom objectPart
+  #| objectBranches = objectBranch (IND{=} objectBranch)*
+  #|                       (IND{=} 'elif' expr colcom objectPart)*
+  #|                       (IND{=} 'else' colcom objectPart)?
+  #| objectCase = 'case' identWithPragma ':'? COMMENT?
+  #|             (IND{>} objectBranches
+  #|             | IND{=} objectBranches)
   result = newNodeP(nkRecCase, p)
   getTok(p)
   var a = newNodeP(nkIdentDefs, p)
@@ -1403,6 +1519,7 @@ proc parseObject(p: var TParser): PNode =
   addSon(result, parseObjectPart(p))
 
 proc parseDistinct(p: var TParser): PNode = 
+  #| distinct = 'distinct' optInd typeDesc
   result = newNodeP(nkDistinctTy, p)
   getTok(p)
   optInd(p, result)
@@ -1444,6 +1561,7 @@ proc parseVariable(p: var TParser): PNode =
   indAndComment(p, result)    # special extension!
   
 proc parseBind(p: var TParser, k: TNodeKind): PNode =
+  #| bindStmt = 'bind' optInd qualifiedIdent (comma qualifiedIdent)*
   result = newNodeP(k, p)
   getTok(p)
   optInd(p, result)
@@ -1452,15 +1570,17 @@ proc parseBind(p: var TParser, k: TNodeKind): PNode =
     addSon(result, a)
     if p.tok.tokType != tkComma: break
     getTok(p)
-    optInd(p, a)  
+    optInd(p, a)
   expectNl(p)
   
 proc parseStmtPragma(p: var TParser): PNode =
+  #| pragmaStmt = pragma (':' COMMENT? stmt)?
   result = parsePragma(p)
   if p.tok.tokType == tkColon:
     let a = result
     result = newNodeI(nkPragmaBlock, a.info)
     getTok(p)
+    skipComment(p, result)
     result.add a
     result.add parseStmt(p)
 
@@ -1470,8 +1590,8 @@ proc simpleStmt(p: var TParser): PNode =
   of tkRaise: result = parseReturnOrRaise(p, nkRaiseStmt)
   of tkYield: result = parseReturnOrRaise(p, nkYieldStmt)
   of tkDiscard: result = parseReturnOrRaise(p, nkDiscardStmt)
-  of tkBreak: result = parseBreakOrContinue(p, nkBreakStmt)
-  of tkContinue: result = parseBreakOrContinue(p, nkContinueStmt)
+  of tkBreak: result = parseReturnOrRaise(p, nkBreakStmt)
+  of tkContinue: result = parseReturnOrRaise(p, nkContinueStmt)
   of tkCurlyDotLe: result = parseStmtPragma(p)
   of tkImport: result = parseImport(p, nkImportStmt)
   of tkExport: result = parseImport(p, nkExportStmt)