# # # The Nim Compiler # (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # # This module implements the parser of the standard Nim syntax. # The parser strictly reflects the grammar ("doc/grammar.txt"); however # it uses several helper routines to keep the parser small. A special # efficient algorithm is used for the precedence levels. The parser here can # be seen as a refinement of the grammar, as it specifies how the AST is built # from the grammar and how comments belong to the AST. # In fact the grammar is generated from this file: when isMainModule: # Leave a note in grammar.txt that it is generated: #| # This file is generated by compiler/parser.nim. import pegs var outp = open("doc/grammar.txt", fmWrite) for line in lines("compiler/parser.nim"): if line =~ peg" \s* '#| ' {.*}": outp.write matches[0], "\L" outp.close import ".." / tools / grammar_nanny checkGrammarFile() import llstream, lexer, idents, strutils, ast, msgs, options, lineinfos, pathutils when defined(nimpretty): import layouter type Parser* = object # A Parser object represents a file that # is being parsed currInd: int # current indentation level firstTok: bool # Has the first token been read? hasProgress: bool # some while loop requires progress ensurance lex*: Lexer # The lexer that is used for parsing tok*: Token # The current token lineStartPrevious*: int lineNumberPrevious*: int bufposPrevious*: int inPragma*: int # Pragma level inSemiStmtList*: int emptyNode: PNode when defined(nimpretty): em*: Emitter SymbolMode = enum smNormal, smAllowNil, smAfterDot PrimaryMode = enum pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix proc parseAll*(p: var Parser): PNode proc closeParser*(p: var Parser) proc parseTopLevelStmt*(p: var Parser): PNode # helpers for the other parsers proc isOperator*(tok: Token): bool proc getTok*(p: var Parser) proc parMessage*(p: Parser, msg: TMsgKind, arg: string = "") proc skipComment*(p: var Parser, node: PNode) proc newNodeP*(kind: TNodeKind, p: Parser): PNode proc newIntNodeP*(kind: TNodeKind, intVal: BiggestInt, p: Parser): PNode proc newFloatNodeP*(kind: TNodeKind, floatVal: BiggestFloat, p: Parser): PNode proc newStrNodeP*(kind: TNodeKind, strVal: string, p: Parser): PNode proc newIdentNodeP*(ident: PIdent, p: Parser): PNode proc expectIdentOrKeyw*(p: Parser) proc expectIdent*(p: Parser) proc parLineInfo*(p: Parser): TLineInfo proc eat*(p: var Parser, tokType: TokType) proc skipInd*(p: var Parser) proc optPar*(p: var Parser) proc optInd*(p: var Parser, n: PNode) proc indAndComment*(p: var Parser, n: PNode, maybeMissEquals = false) proc setBaseFlags*(n: PNode, base: NumericalBase) proc parseSymbol*(p: var Parser, mode = smNormal): PNode proc parseTry(p: var Parser; isExpr: bool): PNode proc parseCase(p: var Parser): PNode proc parseStmtPragma(p: var Parser): PNode proc parsePragma(p: var Parser): PNode proc postExprBlocks(p: var Parser, x: PNode): PNode proc parseExprStmt(p: var Parser): PNode proc parseBlock(p: var Parser): PNode proc primary(p: var Parser, mode: PrimaryMode): PNode proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode # implementation template prettySection(body) = when defined(nimpretty): beginSection(p.em) body when defined(nimpretty): endSection(p.em) proc getTok(p: var Parser) = ## Get the next token from the parser's lexer, and store it in the parser's ## `tok` member. p.lineNumberPrevious = p.lex.lineNumber p.lineStartPrevious = p.lex.lineStart p.bufposPrevious = p.lex.bufpos rawGetTok(p.lex, p.tok) p.hasProgress = true when defined(nimpretty): emitTok(p.em, p.lex, p.tok) # skip the additional tokens that nimpretty needs but the parser has no # interest in: while p.tok.tokType == tkComment: rawGetTok(p.lex, p.tok) emitTok(p.em, p.lex, p.tok) proc openParser*(p: var Parser, fileIdx: FileIndex, inputStream: PLLStream, cache: IdentCache; config: ConfigRef) = ## Open a parser, using the given arguments to set up its internal state. ## initToken(p.tok) openLexer(p.lex, fileIdx, inputStream, cache, config) when defined(nimpretty): openEmitter(p.em, cache, config, fileIdx) getTok(p) # read the first token p.firstTok = true p.emptyNode = newNode(nkEmpty) proc openParser*(p: var Parser, filename: AbsoluteFile, inputStream: PLLStream, cache: IdentCache; config: ConfigRef) = openParser(p, fileInfoIdx(config, filename), inputStream, cache, config) proc closeParser(p: var Parser) = ## Close a parser, freeing up its resources. closeLexer(p.lex) when defined(nimpretty): closeEmitter(p.em) proc parMessage(p: Parser, msg: TMsgKind, arg = "") = ## Produce and emit the parser message `arg` to output. lexMessageTok(p.lex, msg, p.tok, arg) proc parMessage(p: Parser, msg: string, tok: Token) = ## Produce and emit a parser message to output about the token `tok` parMessage(p, errGenerated, msg % prettyTok(tok)) proc parMessage(p: Parser, arg: string) = ## Produce and emit the parser message `arg` to output. lexMessageTok(p.lex, errGenerated, p.tok, arg) template withInd(p, body: untyped) = let oldInd = p.currInd p.currInd = p.tok.indent body p.currInd = oldInd template newlineWasSplitting(p: var Parser) = when defined(nimpretty): layouter.newlineWasSplitting(p.em) template realInd(p): bool = p.tok.indent > p.currInd template sameInd(p): bool = p.tok.indent == p.currInd template sameOrNoInd(p): bool = p.tok.indent == p.currInd or p.tok.indent < 0 proc validInd(p: var Parser): bool {.inline.} = result = p.tok.indent < 0 or p.tok.indent > p.currInd proc rawSkipComment(p: var Parser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: var rhs = node.comment when defined(nimpretty): if p.tok.commentOffsetB > p.tok.commentOffsetA: rhs.add fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB) else: rhs.add p.tok.literal else: rhs.add p.tok.literal node.comment = move rhs else: parMessage(p, errInternal, "skipComment") getTok(p) proc skipComment(p: var Parser, node: PNode) = if p.tok.indent < 0: rawSkipComment(p, node) proc flexComment(p: var Parser, node: PNode) = if p.tok.indent < 0 or realInd(p): rawSkipComment(p, node) const errInvalidIndentation = "invalid indentation" errIdentifierExpected = "identifier expected, but got '$1'" errExprExpected = "expression expected, but found '$1'" proc skipInd(p: var Parser) = if p.tok.indent >= 0: if not realInd(p): parMessage(p, errInvalidIndentation) proc optPar(p: var Parser) = if p.tok.indent >= 0: if p.tok.indent < p.currInd: parMessage(p, errInvalidIndentation) proc optInd(p: var Parser, n: PNode) = skipComment(p, n) skipInd(p) proc getTokNoInd(p: var Parser) = getTok(p) if p.tok.indent >= 0: parMessage(p, errInvalidIndentation) proc expectIdentOrKeyw(p: Parser) = if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType): lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok)) proc expectIdent(p: Parser) = if p.tok.tokType != tkSymbol: lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok)) proc eat(p: var Parser, tokType: TokType) = ## Move the parser to the next token if the current token is of type ## `tokType`, otherwise error. if p.tok.tokType == tokType: getTok(p) else: lexMessage(p.lex, errGenerated, "expected: '" & $tokType & "', but got: '" & prettyTok(p.tok) & "'") proc parLineInfo(p: Parser): TLineInfo = ## Retrieve the line information associated with the parser's current state. result = getLineInfo(p.lex, p.tok) proc indAndComment(p: var Parser, n: PNode, maybeMissEquals = false) = if p.tok.indent > p.currInd: if p.tok.tokType == tkComment: rawSkipComment(p, n) elif maybeMissEquals: let col = p.bufposPrevious - p.lineStartPrevious var info = newLineInfo(p.lex.fileIdx, p.lineNumberPrevious, col) parMessage(p, "invalid indentation, maybe you forgot a '=' at $1 ?" % [p.lex.config$info]) else: parMessage(p, errInvalidIndentation) else: skipComment(p, n) proc newNodeP(kind: TNodeKind, p: Parser): PNode = result = newNodeI(kind, parLineInfo(p)) proc newIntNodeP(kind: TNodeKind, intVal: BiggestInt, p: Parser): PNode = result = newNodeP(kind, p) result.intVal = intVal proc newFloatNodeP(kind: TNodeKind, floatVal: BiggestFloat, p: Parser): PNode = result = newNodeP(kind, p) result.floatVal = floatVal proc newStrNodeP(kind: TNodeKind, strVal: string, p: Parser): PNode = result = newNodeP(kind, p) result.strVal = strVal proc newIdentNodeP(ident: PIdent, p: Parser): PNode = result = newNodeP(nkIdent, p) result.ident = ident proc parseExpr(p: var Parser): PNode proc parseStmt(p: var Parser): PNode proc parseTypeDesc(p: var Parser): PNode proc parseParamList(p: var Parser, retColon = true): PNode proc isSigilLike(tok: Token): bool {.inline.} = result = tok.tokType == tkOpr and tok.ident.s[0] == '@' proc isRightAssociative(tok: Token): bool {.inline.} = ## Determines whether the token is right assocative. result = tok.tokType == tkOpr and tok.ident.s[0] == '^' # or (tok.ident.s.len > 1 and tok.ident.s[^1] == '>') proc isUnary(tok: Token): bool = ## Check if the given token is a unary operator tok.tokType in {tkOpr, tkDotDot} and tok.strongSpaceB == 0 and tok.strongSpaceA > 0 proc checkBinary(p: Parser) {.inline.} = ## Check if the current parser token is a binary operator. # we don't check '..' here as that's too annoying if p.tok.tokType == tkOpr: if p.tok.strongSpaceB > 0 and p.tok.strongSpaceA == 0: parMessage(p, warnInconsistentSpacing, prettyTok(p.tok)) #| module = stmt ^* (';' / IND{=}) #| #| comma = ',' COMMENT? #| semicolon = ';' COMMENT? #| colon = ':' COMMENT? #| colcom = ':' COMMENT? #| #| operator = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 #| | 'or' | 'xor' | 'and' #| | 'is' | 'isnot' | 'in' | 'notin' | 'of' | 'as' | 'from' #| | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'static' | '..' #| #| prefixOperator = operator #| #| optInd = COMMENT? IND? #| optPar = (IND{>} | IND{=})? #| #| simpleExpr = arrowExpr (OP0 optInd arrowExpr)* pragma? #| arrowExpr = assignExpr (OP1 optInd assignExpr)* #| assignExpr = orExpr (OP2 optInd orExpr)* #| orExpr = andExpr (OP3 optInd andExpr)* #| andExpr = cmpExpr (OP4 optInd cmpExpr)* #| cmpExpr = sliceExpr (OP5 optInd sliceExpr)* #| sliceExpr = ampExpr (OP6 optInd ampExpr)* #| ampExpr = plusExpr (OP7 optInd plusExpr)* #| plusExpr = mulExpr (OP8 optInd mulExpr)* #| mulExpr = dollarExpr (OP9 optInd dollarExpr)* #| dollarExpr = primary (OP10 optInd primary)* proc isOperator(tok: Token): bool = #| operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 | #| 'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' | #| 'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor' tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, tkOr, tkXor} proc colcom(p: var Parser, n: PNode) = eat(p, tkColon) skipComment(p, n) const tkBuiltInMagics = {tkType, tkStatic, tkAddr} proc parseSymbol(p: var Parser, mode = smNormal): PNode = #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' #| | IDENT | KEYW case p.tok.tokType of tkSymbol: result = newIdentNodeP(p.tok.ident, p) getTok(p) of tokKeywordLow..tokKeywordHigh: if p.tok.tokType in tkBuiltInMagics or mode == smAfterDot: # for backwards compatibility these 2 are always valid: result = newIdentNodeP(p.tok.ident, p) getTok(p) elif p.tok.tokType == tkNil and mode == smAllowNil: result = newNodeP(nkNilLit, p) getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) result = p.emptyNode of tkAccent: result = newNodeP(nkAccQuoted, p) getTok(p) # progress guaranteed while true: case p.tok.tokType of tkAccent: if result.len == 0: parMessage(p, errIdentifierExpected, p.tok) break of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi: let lineinfo = parLineInfo(p) var accm = "" while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi}: accm.add($p.tok) getTok(p) let node = newNodeI(nkIdent, lineinfo) node.ident = p.lex.cache.getIdent(accm) result.add(node) of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCustomLit: result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p)) getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) break eat(p, tkAccent) else: parMessage(p, errIdentifierExpected, p.tok) # BUGFIX: We must consume a token here to prevent endless loops! # But: this really sucks for idetools and keywords, so we don't do it # if it is a keyword: #if not isKeyword(p.tok.tokType): getTok(p) result = p.emptyNode proc colonOrEquals(p: var Parser, a: PNode): PNode = if p.tok.tokType == tkColon: result = newNodeP(nkExprColonExpr, p) getTok(p) newlineWasSplitting(p) #optInd(p, result) result.add(a) result.add(parseExpr(p)) elif p.tok.tokType == tkEquals: result = newNodeP(nkExprEqExpr, p) getTok(p) #optInd(p, result) result.add(a) result.add(parseExpr(p)) else: result = a proc exprColonEqExpr(p: var Parser): PNode = #| exprColonEqExpr = expr (':'|'=' expr)? var a = parseExpr(p) if p.tok.tokType == tkDo: result = postExprBlocks(p, a) else: result = colonOrEquals(p, a) proc exprList(p: var Parser, endTok: TokType, result: PNode) = #| exprList = expr ^+ comma when defined(nimpretty): inc p.em.doIndentMore getTok(p) optInd(p, result) # progress guaranteed while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): var a = parseExpr(p) result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) when defined(nimpretty): dec p.em.doIndentMore proc exprColonEqExprListAux(p: var Parser, endTok: TokType, result: PNode) = assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi}) getTok(p) flexComment(p, result) optPar(p) # progress guaranteed while p.tok.tokType != endTok and p.tok.tokType != tkEof: var a = exprColonEqExpr(p) result.add(a) if p.tok.tokType != tkComma: break elif result.kind == nkPar: result.transitionSonsKind(nkTupleConstr) getTok(p) skipComment(p, a) optPar(p) eat(p, endTok) proc exprColonEqExprList(p: var Parser, kind: TNodeKind, endTok: TokType): PNode = #| exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? result = newNodeP(kind, p) exprColonEqExprListAux(p, endTok, result) proc dotExpr(p: var Parser, a: PNode): PNode = var info = p.parLineInfo getTok(p) result = newNodeI(nkDotExpr, info) optInd(p, result) result.add(a) result.add(parseSymbol(p, smAfterDot)) if p.tok.tokType == tkBracketLeColon and p.tok.strongSpaceA <= 0: var x = newNodeI(nkBracketExpr, p.parLineInfo) # rewrite 'x.y[:z]()' to 'y[z](x)' x.add result[1] exprList(p, tkBracketRi, x) eat(p, tkBracketRi) var y = newNodeI(nkCall, p.parLineInfo) y.add x y.add result[0] if p.tok.tokType == tkParLe and p.tok.strongSpaceA <= 0: exprColonEqExprListAux(p, tkParRi, y) result = y proc dotLikeExpr(p: var Parser, a: PNode): PNode = var info = p.parLineInfo result = newNodeI(nkInfix, info) optInd(p, result) var opNode = newIdentNodeP(p.tok.ident, p) getTok(p) result.add(opNode) result.add(a) result.add(parseSymbol(p, smAfterDot)) proc qualifiedIdent(p: var Parser): PNode = #| qualifiedIdent = symbol ('.' optInd symbol)? result = parseSymbol(p) if p.tok.tokType == tkDot: result = dotExpr(p, result) proc setOrTableConstr(p: var Parser): PNode = #| setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}' result = newNodeP(nkCurly, p) getTok(p) # skip '{' optInd(p, result) if p.tok.tokType == tkColon: getTok(p) # skip ':' result.transitionSonsKind(nkTableConstr) else: # progress guaranteed while p.tok.tokType notin {tkCurlyRi, tkEof}: var a = exprColonEqExpr(p) if a.kind == nkExprColonExpr: result.transitionSonsKind(nkTableConstr) result.add(a) if p.tok.tokType != tkComma: break getTok(p) skipComment(p, a) optPar(p) eat(p, tkCurlyRi) # skip '}' proc parseCast(p: var Parser): PNode = #| castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') / # ('(' optInd exprColonEqExpr optPar ')') result = newNodeP(nkCast, p) getTok(p) if p.tok.tokType == tkBracketLe: getTok(p) optInd(p, result) result.add(parseTypeDesc(p)) optPar(p) eat(p, tkBracketRi) eat(p, tkParLe) optInd(p, result) result.add(parseExpr(p)) else: result.add p.emptyNode eat(p, tkParLe) optInd(p, result) result.add(exprColonEqExpr(p)) optPar(p) eat(p, tkParRi) proc setBaseFlags(n: PNode, base: NumericalBase) = case base of base10: discard of base2: incl(n.flags, nfBase2) of base8: incl(n.flags, nfBase8) of base16: incl(n.flags, nfBase16) proc parseGStrLit(p: var Parser, a: PNode): PNode = case p.tok.tokType of tkGStrLit: result = newNodeP(nkCallStrLit, p) result.add(a) result.add(newStrNodeP(nkRStrLit, p.tok.literal, p)) getTok(p) of tkGTripleStrLit: result = newNodeP(nkCallStrLit, p) result.add(a) result.add(newStrNodeP(nkTripleStrLit, p.tok.literal, p)) getTok(p) else: result = a proc complexOrSimpleStmt(p: var Parser): PNode proc simpleExpr(p: var Parser, mode = pmNormal): PNode proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode proc semiStmtList(p: var Parser, result: PNode) = inc p.inSemiStmtList withInd(p): # Be lenient with the first stmt/expr let a = case p.tok.tokType of tkIf: parseIfOrWhenExpr(p, nkIfStmt) of tkWhen: parseIfOrWhenExpr(p, nkWhenStmt) else: complexOrSimpleStmt(p) result.add a while p.tok.tokType != tkEof: if p.tok.tokType == tkSemiColon: getTok(p) if p.tok.tokType == tkParRi: break elif not (sameInd(p) or realInd(p)): parMessage(p, errInvalidIndentation) let a = complexOrSimpleStmt(p) if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) getTok(p) else: result.add a dec p.inSemiStmtList result.transitionSonsKind(nkStmtListExpr) proc parsePar(p: var Parser): PNode = #| parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' #| | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' #| | 'when' | 'var' | 'mixin' #| par = '(' optInd #| ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';' #| | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';' #| | pragmaStmt #| | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? ) #| | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) #| optPar ')' # # unfortunately it's ambiguous: (expr: expr) vs (exprStmt); however a # leading ';' could be used to enforce a 'stmt' context ... result = newNodeP(nkPar, p) getTok(p) optInd(p, result) flexComment(p, result) if p.tok.tokType in {tkDiscard, tkInclude, tkIf, tkWhile, tkCase, tkTry, tkDefer, tkFinally, tkExcept, tkBlock, tkConst, tkLet, tkWhen, tkVar, tkFor, tkMixin}: # XXX 'bind' used to be an expression, so we exclude it here; # tests/reject/tbind2 fails otherwise. semiStmtList(p, result) elif p.tok.tokType == tkSemiColon: # '(;' enforces 'stmt' context: getTok(p) optInd(p, result) semiStmtList(p, result) elif p.tok.tokType == tkCurlyDotLe: result.add(parseStmtPragma(p)) elif p.tok.tokType == tkParRi: # Empty tuple '()' result.transitionSonsKind(nkTupleConstr) else: var a = simpleExpr(p) if p.tok.tokType == tkDo: result = postExprBlocks(p, a) elif p.tok.tokType == tkEquals: # special case: allow assignments let asgn = newNodeP(nkAsgn, p) getTok(p) optInd(p, result) let b = parseExpr(p) asgn.add a asgn.add b result.add(asgn) if p.tok.tokType == tkSemiColon: semiStmtList(p, result) elif p.tok.tokType == tkSemiColon: # stmt context: result.add(a) semiStmtList(p, result) else: a = colonOrEquals(p, a) if a.kind == nkExprColonExpr: result.transitionSonsKind(nkTupleConstr) result.add(a) if p.tok.tokType == tkComma: getTok(p) skipComment(p, a) # (1,) produces a tuple expression: result.transitionSonsKind(nkTupleConstr) # progress guaranteed while p.tok.tokType != tkParRi and p.tok.tokType != tkEof: var a = exprColonEqExpr(p) result.add(a) if p.tok.tokType != tkComma: break getTok(p) skipComment(p, a) optPar(p) eat(p, tkParRi) proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT #| | CHAR_LIT | CUSTOM_NUMERIC_LIT #| | NIL #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT #| identOrLiteral = generalizedLit | symbol | literal #| | par | arrayConstr | setOrTableConstr | tupleConstr #| | castExpr #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' case p.tok.tokType of tkSymbol, tkBuiltInMagics, tkOut: result = newIdentNodeP(p.tok.ident, p) getTok(p) result = parseGStrLit(p, result) of tkAccent: result = parseSymbol(p) # literals of tkIntLit: result = newIntNodeP(nkIntLit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkInt8Lit: result = newIntNodeP(nkInt8Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkInt16Lit: result = newIntNodeP(nkInt16Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkInt32Lit: result = newIntNodeP(nkInt32Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkInt64Lit: result = newIntNodeP(nkInt64Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkUIntLit: result = newIntNodeP(nkUIntLit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkUInt8Lit: result = newIntNodeP(nkUInt8Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkUInt16Lit: result = newIntNodeP(nkUInt16Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkUInt32Lit: result = newIntNodeP(nkUInt32Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkUInt64Lit: result = newIntNodeP(nkUInt64Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkFloatLit: result = newFloatNodeP(nkFloatLit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkFloat32Lit: result = newFloatNodeP(nkFloat32Lit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkFloat64Lit: result = newFloatNodeP(nkFloat64Lit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkFloat128Lit: result = newFloatNodeP(nkFloat128Lit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) of tkStrLit: result = newStrNodeP(nkStrLit, p.tok.literal, p) getTok(p) of tkRStrLit: result = newStrNodeP(nkRStrLit, p.tok.literal, p) getTok(p) of tkTripleStrLit: result = newStrNodeP(nkTripleStrLit, p.tok.literal, p) getTok(p) of tkCharLit: result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p) getTok(p) of tkCustomLit: let splitPos = p.tok.iNumber.int let str = newStrNodeP(nkRStrLit, p.tok.literal.substr(0, splitPos-1), p) let callee = newIdentNodeP(getIdent(p.lex.cache, p.tok.literal.substr(splitPos)), p) result = newNodeP(nkDotExpr, p) result.add str result.add callee getTok(p) of tkNil: result = newNodeP(nkNilLit, p) getTok(p) of tkParLe: # () constructor if mode in {pmTypeDesc, pmTypeDef}: result = exprColonEqExprList(p, nkPar, tkParRi) else: result = parsePar(p) of tkCurlyLe: # {} constructor result = setOrTableConstr(p) of tkBracketLe: # [] constructor result = exprColonEqExprList(p, nkBracket, tkBracketRi) of tkCast: result = parseCast(p) else: parMessage(p, errExprExpected, p.tok) getTok(p) # we must consume a token here to prevent endless loops! result = p.emptyNode proc namedParams(p: var Parser, callee: PNode, kind: TNodeKind, endTok: TokType): PNode = let a = callee result = newNodeP(kind, p) result.add(a) # progress guaranteed exprColonEqExprListAux(p, endTok, result) proc commandParam(p: var Parser, isFirstParam: var bool; mode: PrimaryMode): PNode = if mode == pmTypeDesc: result = simpleExpr(p, mode) else: result = parseExpr(p) if p.tok.tokType == tkDo: result = postExprBlocks(p, result) elif p.tok.tokType == tkEquals and not isFirstParam: let lhs = result result = newNodeP(nkExprEqExpr, p) getTok(p) result.add(lhs) result.add(parseExpr(p)) isFirstParam = false proc commandExpr(p: var Parser; r: PNode; mode: PrimaryMode): PNode = result = newNodeP(nkCommand, p) result.add(r) var isFirstParam = true # progress NOT guaranteed p.hasProgress = false result.add commandParam(p, isFirstParam, mode) proc isDotLike(tok: Token): bool = result = tok.tokType == tkOpr and tok.ident.s.len > 1 and tok.ident.s[0] == '.' and tok.ident.s[1] != '.' proc primarySuffix(p: var Parser, r: PNode, baseIndent: int, mode: PrimaryMode): PNode = #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' #| | '.' optInd symbol ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit? #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax result = r # progress guaranteed while p.tok.indent < 0 or (p.tok.tokType == tkDot and p.tok.indent >= baseIndent): case p.tok.tokType of tkParLe: # progress guaranteed if p.tok.strongSpaceA > 0: # inside type sections, expressions such as `ref (int, bar)` # are parsed as a nkCommand with a single tuple argument (nkPar) if mode == pmTypeDef: result = newNodeP(nkCommand, p) result.add r result.add primary(p, pmNormal) else: result = commandExpr(p, result, mode) break result = namedParams(p, result, nkCall, tkParRi) if result.len > 1 and result[1].kind == nkExprColonExpr: result.transitionSonsKind(nkObjConstr) of tkDot: # progress guaranteed result = dotExpr(p, result) result = parseGStrLit(p, result) of tkBracketLe: # progress guaranteed if p.tok.strongSpaceA > 0: result = commandExpr(p, result, mode) break result = namedParams(p, result, nkBracketExpr, tkBracketRi) of tkCurlyLe: # progress guaranteed if p.tok.strongSpaceA > 0: result = commandExpr(p, result, mode) break result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) of tkSymbol, tkAccent, tkIntLit..tkCustomLit, tkNil, tkCast, tkOpr, tkDotDot, tkVar, tkOut, tkStatic, tkType, tkEnum, tkTuple, tkObject, tkProc: # XXX: In type sections we allow the free application of the # command syntax, with the exception of expressions such as # `foo ref` or `foo ptr`. Unfortunately, these two are also # used as infix operators for the memory regions feature and # the current parsing rules don't play well here. let isDotLike2 = p.tok.isDotLike if isDotLike2 and p.lex.config.isDefined("nimPreviewDotLikeOps"): # synchronize with `tkDot` branch result = dotLikeExpr(p, result) result = parseGStrLit(p, result) else: if isDotLike2: parMessage(p, warnDotLikeOps, "dot-like operators will be parsed differently with `-d:nimPreviewDotLikeOps`") if p.inPragma == 0 and (isUnary(p.tok) or p.tok.tokType notin {tkOpr, tkDotDot}): # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet # solution, but pragmas.nim can't handle that result = commandExpr(p, result, mode) break else: break proc parseOperators(p: var Parser, headNode: PNode, limit: int, mode: PrimaryMode): PNode = result = headNode # expand while operators have priorities higher than 'limit' var opPrec = getPrecedence(p.tok) let modeB = if mode == pmTypeDef: pmTypeDesc else: mode # the operator itself must not start on a new line: # progress guaranteed while opPrec >= limit and p.tok.indent < 0 and not isUnary(p.tok): checkBinary(p) let leftAssoc = ord(not isRightAssociative(p.tok)) var a = newNodeP(nkInfix, p) var opNode = newIdentNodeP(p.tok.ident, p) # skip operator: getTok(p) flexComment(p, a) optPar(p) # read sub-expression with higher priority: var b = simpleExprAux(p, opPrec + leftAssoc, modeB) a.add(opNode) a.add(result) a.add(b) result = a opPrec = getPrecedence(p.tok) proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode = result = primary(p, mode) if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)) and mode == pmNormal: var pragmaExp = newNodeP(nkPragmaExpr, p) pragmaExp.add result pragmaExp.add p.parsePragma result = pragmaExp result = parseOperators(p, result, limit, mode) proc simpleExpr(p: var Parser, mode = pmNormal): PNode = when defined(nimpretty): inc p.em.doIndentMore result = simpleExprAux(p, -1, mode) when defined(nimpretty): dec p.em.doIndentMore proc parsePragma(p: var Parser): PNode = #| pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}') result = newNodeP(nkPragma, p) inc p.inPragma when defined(nimpretty): inc p.em.doIndentMore inc p.em.keepIndents getTok(p) optInd(p, result) while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}: p.hasProgress = false var a = exprColonEqExpr(p) if not p.hasProgress: break result.add(a) if p.tok.tokType == tkComma: getTok(p) skipComment(p, a) optPar(p) if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: when defined(nimpretty): if p.tok.tokType == tkCurlyRi: curlyRiWasPragma(p.em) getTok(p) else: parMessage(p, "expected '.}'") dec p.inPragma when defined(nimpretty): dec p.em.doIndentMore dec p.em.keepIndents proc identVis(p: var Parser; allowDot=false): PNode = #| identVis = symbol OPR? # postfix position #| identVisDot = symbol '.' optInd symbol OPR? var a = parseSymbol(p) if p.tok.tokType == tkOpr: when defined(nimpretty): starWasExportMarker(p.em) result = newNodeP(nkPostfix, p) result.add(newIdentNodeP(p.tok.ident, p)) result.add(a) getTok(p) elif p.tok.tokType == tkDot and allowDot: result = dotExpr(p, a) else: result = a proc identWithPragma(p: var Parser; allowDot=false): PNode = #| identWithPragma = identVis pragma? #| identWithPragmaDot = identVisDot pragma? var a = identVis(p, allowDot) if p.tok.tokType == tkCurlyDotLe: result = newNodeP(nkPragmaExpr, p) result.add(a) result.add(parsePragma(p)) else: result = a type DeclaredIdentFlag = enum withPragma, # identifier may have pragma withBothOptional # both ':' and '=' parts are optional withDot # allow 'var ident.ident = value' DeclaredIdentFlags = set[DeclaredIdentFlag] proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode = #| declColonEquals = identWithPragma (comma identWithPragma)* comma? #| (':' optInd typeDesc)? ('=' optInd expr)? #| identColonEquals = IDENT (comma IDENT)* comma? #| (':' optInd typeDesc)? ('=' optInd expr)?) var a: PNode result = newNodeP(nkIdentDefs, p) # progress guaranteed while true: case p.tok.tokType of tkSymbol, tkAccent: if withPragma in flags: a = identWithPragma(p, allowDot=withDot in flags) else: a = parseSymbol(p) if a.kind == nkEmpty: return else: break result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) if p.tok.tokType == tkColon: getTok(p) optInd(p, result) result.add(parseTypeDesc(p)) else: result.add(newNodeP(nkEmpty, p)) if p.tok.tokType != tkEquals and withBothOptional notin flags: parMessage(p, "':' or '=' expected, but got '$1'", p.tok) if p.tok.tokType == tkEquals: getTok(p) optInd(p, result) result.add(parseExpr(p)) else: result.add(newNodeP(nkEmpty, p)) proc parseTuple(p: var Parser, indentAllowed = false): PNode = #| tupleDecl = 'tuple' #| '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' | #| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? result = newNodeP(nkTupleTy, p) getTok(p) if p.tok.tokType == tkBracketLe: getTok(p) optInd(p, result) # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent}: var a = parseIdentColonEquals(p, {}) result.add(a) if p.tok.tokType notin {tkComma, tkSemiColon}: break when defined(nimpretty): commaWasSemicolon(p.em) getTok(p) skipComment(p, a) optPar(p) eat(p, tkBracketRi) elif indentAllowed: skipComment(p, result) if realInd(p): withInd(p): rawSkipComment(p, result) # progress guaranteed while true: case p.tok.tokType of tkSymbol, tkAccent: var a = parseIdentColonEquals(p, {}) if p.tok.indent < 0 or p.tok.indent >= p.currInd: rawSkipComment(p, a) result.add(a) of tkEof: break else: parMessage(p, errIdentifierExpected, p.tok) break if not sameInd(p): break elif p.tok.tokType == tkParLe: parMessage(p, errGenerated, "the syntax for tuple types is 'tuple[...]', not 'tuple(...)'") else: result = newNodeP(nkTupleClassTy, p) proc parseParamList(p: var Parser, retColon = true): PNode = #| paramList = '(' declColonEquals ^* (comma/semicolon) ')' #| paramListArrow = paramList? ('->' optInd typeDesc)? #| paramListColon = paramList? (':' optInd typeDesc)? var a: PNode result = newNodeP(nkFormalParams, p) result.add(p.emptyNode) # return type when defined(nimpretty): inc p.em.doIndentMore inc p.em.keepIndents let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0 if hasParLe: getTok(p) optInd(p, result) # progress guaranteed while true: case p.tok.tokType of tkSymbol, tkAccent: a = parseIdentColonEquals(p, {withBothOptional, withPragma}) of tkParRi: break of tkVar: parMessage(p, errGenerated, "the syntax is 'parameter: var T', not 'var parameter: T'") break else: parMessage(p, "expected closing ')'") break result.add(a) if p.tok.tokType notin {tkComma, tkSemiColon}: break when defined(nimpretty): commaWasSemicolon(p.em) getTok(p) skipComment(p, a) optPar(p) eat(p, tkParRi) let hasRet = if retColon: p.tok.tokType == tkColon else: p.tok.tokType == tkOpr and p.tok.ident.s == "->" if hasRet and p.tok.indent < 0: getTok(p) optInd(p, result) result[0] = parseTypeDesc(p) elif not retColon and not hasParLe: # Mark as "not there" in order to mark for deprecation in the semantic pass: result = p.emptyNode when defined(nimpretty): dec p.em.doIndentMore dec p.em.keepIndents proc optPragmas(p: var Parser): PNode = if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)): result = parsePragma(p) else: result = p.emptyNode proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = #| doBlock = 'do' paramListArrow pragma? colcom stmt let params = parseParamList(p, retColon=false) let pragmas = optPragmas(p) colcom(p, result) result = parseStmt(p) if params.kind != nkEmpty: result = newProcNode(nkDo, info, body = result, params = params, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? # either a proc type or a anonymous proc let info = parLineInfo(p) getTok(p) let hasSignature = p.tok.tokType in {tkParLe, tkColon} and p.tok.indent < 0 let params = parseParamList(p) let pragmas = optPragmas(p) if p.tok.tokType == tkEquals and isExpr: getTok(p) skipComment(p, result) result = newProcNode(kind, info, body = parseStmt(p), params = params, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) else: result = newNodeI(nkProcTy, info) if hasSignature: result.add(params) if kind == nkFuncDef: parMessage(p, "func keyword is not allowed in type descriptions, use proc with {.noSideEffect.} pragma instead") result.add(pragmas) proc isExprStart(p: Parser): bool = case p.tok.tokType of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor, tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr, tkTuple, tkObject, tkWhen, tkCase, tkOut: result = true else: result = false proc parseSymbolList(p: var Parser, result: PNode) = # progress guaranteed while true: var s = parseSymbol(p, smAllowNil) if s.kind == nkEmpty: break result.add(s) if p.tok.tokType != tkComma: break getTok(p) optInd(p, s) proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, mode: PrimaryMode): PNode = result = newNodeP(kind, p) getTok(p) if p.tok.indent != -1 and p.tok.indent <= p.currInd: return optInd(p, result) if not isOperator(p.tok) and isExprStart(p): result.add(primary(p, mode)) if kind == nkDistinctTy and p.tok.tokType == tkSymbol: # XXX document this feature! var nodeKind: TNodeKind if p.tok.ident.s == "with": nodeKind = nkWith elif p.tok.ident.s == "without": nodeKind = nkWithout else: return result getTok(p) let list = newNodeP(nodeKind, p) result.add list parseSymbolList(p, list) proc parseVarTuple(p: var Parser): PNode proc parseFor(p: var Parser): PNode = #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt #| forExpr = forStmt getTokNoInd(p) result = newNodeP(nkForStmt, p) if p.tok.tokType == tkParLe: result.add(parseVarTuple(p)) else: var a = identWithPragma(p) result.add(a) while p.tok.tokType == tkComma: getTok(p) optInd(p, a) if p.tok.tokType == tkParLe: result.add(parseVarTuple(p)) break a = identWithPragma(p) result.add(a) eat(p, tkIn) result.add(parseExpr(p)) colcom(p, result) result.add(parseStmt(p)) template nimprettyDontTouch(body) = when defined(nimpretty): inc p.em.keepIndents body when defined(nimpretty): dec p.em.keepIndents proc parseExpr(p: var Parser): PNode = #| expr = (blockExpr #| | ifExpr #| | whenExpr #| | caseStmt #| | forExpr #| | tryExpr) #| / simpleExpr case p.tok.tokType of tkBlock: nimprettyDontTouch: result = parseBlock(p) of tkIf: nimprettyDontTouch: result = parseIfOrWhenExpr(p, nkIfExpr) of tkFor: nimprettyDontTouch: result = parseFor(p) of tkWhen: nimprettyDontTouch: result = parseIfOrWhenExpr(p, nkWhenExpr) of tkCase: # Currently we think nimpretty is good enough with case expressions, # so it is allowed to touch them: #nimprettyDontTouch: result = parseCase(p) of tkTry: nimprettyDontTouch: result = parseTry(p, isExpr=true) else: result = simpleExpr(p) proc parseEnum(p: var Parser): PNode proc parseObject(p: var Parser): PNode proc parseTypeClass(p: var Parser): PNode proc primary(p: var Parser, mode: PrimaryMode): PNode = #| primary = operatorB primary primarySuffix* | #| tupleDecl | routineExpr | enumDecl #| objectDecl | conceptDecl | ('bind' primary) #| ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary #| / prefixOperator* identOrLiteral primarySuffix* if isOperator(p.tok): # Note 'sigil like' operators are currently not reflected in the grammar # and should be removed for Nim 2.0, I don't think anybody uses them. let isSigil = isSigilLike(p.tok) result = newNodeP(nkPrefix, p) var a = newIdentNodeP(p.tok.ident, p) result.add(a) getTok(p) optInd(p, a) if isSigil: #XXX prefix operators let baseInd = p.lex.currLineIndent result.add(primary(p, pmSkipSuffix)) result = primarySuffix(p, result, baseInd, mode) else: result.add(primary(p, pmNormal)) return case p.tok.tokType of tkTuple: result = parseTuple(p, mode == pmTypeDef) of tkProc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkLambda) of tkFunc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkFuncDef) of tkIterator: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}, nkLambda) if result.kind == nkLambda: result.transitionSonsKind(nkIteratorDef) else: result.transitionSonsKind(nkIteratorTy) of tkEnum: if mode == pmTypeDef: prettySection: result = parseEnum(p) else: result = newNodeP(nkEnumTy, p) getTok(p) of tkObject: if mode == pmTypeDef: prettySection: result = parseObject(p) else: result = newNodeP(nkObjectTy, p) getTok(p) of tkConcept: if mode == pmTypeDef: result = parseTypeClass(p) else: parMessage(p, "the 'concept' keyword is only valid in 'type' sections") of tkBind: result = newNodeP(nkBind, p) getTok(p) optInd(p, result) result.add(primary(p, pmNormal)) of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode) of tkOut: # I like this parser extension to be in 1.4 as it still might turn out # useful in the long run. result = parseTypeDescKAux(p, nkMutableTy, mode) of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode) of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode) of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode) else: let baseInd = p.lex.currLineIndent result = identOrLiteral(p, mode) if mode != pmSkipSuffix: result = primarySuffix(p, result, baseInd, mode) proc binaryNot(p: var Parser; a: PNode): PNode = if p.tok.tokType == tkNot: let notOpr = newIdentNodeP(p.tok.ident, p) getTok(p) optInd(p, notOpr) let b = parseExpr(p) result = newNodeP(nkInfix, p) result.add notOpr result.add a result.add b else: result = a proc parseTypeDesc(p: var Parser): PNode = #| typeDesc = simpleExpr ('not' expr)? newlineWasSplitting(p) result = simpleExpr(p, pmTypeDesc) result = binaryNot(p, result) proc parseTypeDefAux(p: var Parser): PNode = #| typeDefAux = simpleExpr ('not' expr)? result = simpleExpr(p, pmTypeDef) result = binaryNot(p, result) proc makeCall(n: PNode): PNode = ## Creates a call if the given node isn't already a call. if n.kind in nkCallKinds: result = n else: result = newNodeI(nkCall, n.info) result.add n proc postExprBlocks(p: var Parser, x: PNode): PNode = #| postExprBlocks = ':' stmt? ( IND{=} doBlock #| | IND{=} 'of' exprList ':' stmt #| | IND{=} 'elif' expr ':' stmt #| | IND{=} 'except' exprList ':' stmt #| | IND{=} 'finally' ':' stmt #| | IND{=} 'else' ':' stmt )* result = x if p.tok.indent >= 0: return var openingParams = p.emptyNode openingPragmas = p.emptyNode if p.tok.tokType == tkDo: getTok(p) openingParams = parseParamList(p, retColon=false) openingPragmas = optPragmas(p) if p.tok.tokType == tkColon: result = makeCall(result) getTok(p) skipComment(p, result) if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}: var stmtList = newNodeP(nkStmtList, p) stmtList.add parseStmt(p) # to keep backwards compatibility (see tests/vm/tstringnil) if stmtList[0].kind == nkStmtList: stmtList = stmtList[0] stmtList.flags.incl nfBlockArg if openingParams.kind != nkEmpty: result.add newProcNode(nkDo, stmtList.info, body = stmtList, params = openingParams, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = openingPragmas, exceptions = p.emptyNode) else: result.add stmtList while sameInd(p): var nextBlock: PNode let nextToken = p.tok.tokType if nextToken == tkDo: let info = parLineInfo(p) getTok(p) nextBlock = parseDoBlock(p, info) else: case nextToken of tkOf: nextBlock = newNodeP(nkOfBranch, p) exprList(p, tkColon, nextBlock) of tkElif: nextBlock = newNodeP(nkElifBranch, p) getTok(p) optInd(p, nextBlock) nextBlock.add parseExpr(p) of tkExcept: nextBlock = newNodeP(nkExceptBranch, p) exprList(p, tkColon, nextBlock) of tkFinally: nextBlock = newNodeP(nkFinally, p) getTok(p) of tkElse: nextBlock = newNodeP(nkElse, p) getTok(p) else: break eat(p, tkColon) nextBlock.add parseStmt(p) nextBlock.flags.incl nfBlockArg result.add nextBlock if nextBlock.kind in {nkElse, nkFinally}: break else: if openingParams.kind != nkEmpty: parMessage(p, "expected ':'") proc parseExprStmt(p: var Parser): PNode = #| exprStmt = simpleExpr #| (( '=' optInd expr colonBody? ) #| / ( expr ^+ comma #| postExprBlocks #| ))? var a = simpleExpr(p) if p.tok.tokType == tkEquals: result = newNodeP(nkAsgn, p) getTok(p) optInd(p, result) var b = parseExpr(p) b = postExprBlocks(p, b) result.add(a) result.add(b) else: # simpleExpr parsed 'p a' from 'p a, b'? var isFirstParam = false if p.tok.indent < 0 and p.tok.tokType == tkComma and a.kind == nkCommand: result = a while true: getTok(p) optInd(p, result) result.add(commandParam(p, isFirstParam, pmNormal)) if p.tok.tokType != tkComma: break elif p.tok.indent < 0 and isExprStart(p): result = newTreeI(nkCommand, a.info, a) while true: result.add(commandParam(p, isFirstParam, pmNormal)) if p.tok.tokType != tkComma: break getTok(p) optInd(p, result) else: result = a result = postExprBlocks(p, result) proc parseModuleName(p: var Parser, kind: TNodeKind): PNode = result = parseExpr(p) when false: # parseExpr already handles 'as' syntax ... if p.tok.tokType == tkAs and kind == nkImportStmt: let a = result result = newNodeP(nkImportAs, p) getTok(p) result.add(a) result.add(parseExpr(p)) proc parseImport(p: var Parser, kind: TNodeKind): PNode = #| importStmt = 'import' optInd expr #| ((comma expr)* #| / 'except' optInd (expr ^+ comma)) #| exportStmt = 'export' optInd expr #| ((comma expr)* #| / 'except' optInd (expr ^+ comma)) result = newNodeP(kind, p) getTok(p) # skip `import` or `export` optInd(p, result) var a = parseModuleName(p, kind) result.add(a) if p.tok.tokType in {tkComma, tkExcept}: if p.tok.tokType == tkExcept: result.transitionSonsKind(succ(kind)) getTok(p) optInd(p, result) while true: # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: p.hasProgress = false a = parseModuleName(p, kind) if a.kind == nkEmpty or not p.hasProgress: break result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) proc parseIncludeStmt(p: var Parser): PNode = #| includeStmt = 'include' optInd expr ^+ comma result = newNodeP(nkIncludeStmt, p) getTok(p) # skip `import` or `include` optInd(p, result) while true: # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: p.hasProgress = false var a = parseExpr(p) if a.kind == nkEmpty or not p.hasProgress: break result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) proc parseFromStmt(p: var Parser): PNode = #| fromStmt = 'from' expr 'import' optInd expr (comma expr)* result = newNodeP(nkFromStmt, p) getTok(p) # skip `from` optInd(p, result) var a = parseModuleName(p, nkImportStmt) result.add(a) #optInd(p, a); eat(p, tkImport) optInd(p, result) while true: # p.tok.tokType notin {tkEof, tkSad, tkDed}: p.hasProgress = false a = parseExpr(p) if a.kind == nkEmpty or not p.hasProgress: break result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = #| returnStmt = 'return' optInd expr? #| raiseStmt = 'raise' optInd expr? #| yieldStmt = 'yield' optInd expr? #| discardStmt = 'discard' optInd expr? #| breakStmt = 'break' optInd expr? #| continueStmt = 'continue' optInd expr? result = newNodeP(kind, p) getTok(p) if p.tok.tokType == tkComment: skipComment(p, result) result.add(p.emptyNode) elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p): # NL terminates: result.add(p.emptyNode) # nimpretty here! else: var e = parseExpr(p) e = postExprBlocks(p, e) result.add(e) proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = #| condStmt = expr colcom stmt COMMENT? #| (IND{=} 'elif' expr colcom stmt)* #| (IND{=} 'else' colcom stmt)? #| ifStmt = 'if' condStmt #| whenStmt = 'when' condStmt result = newNodeP(kind, p) while true: getTok(p) # skip `if`, `when`, `elif` var branch = newNodeP(nkElifBranch, p) optInd(p, branch) branch.add(parseExpr(p)) colcom(p, branch) branch.add(parseStmt(p)) skipComment(p, branch) result.add(branch) if p.tok.tokType != tkElif or not sameOrNoInd(p): break if p.tok.tokType == tkElse and sameOrNoInd(p): var branch = newNodeP(nkElse, p) eat(p, tkElse) colcom(p, branch) branch.add(parseStmt(p)) result.add(branch) proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = #| condExpr = expr colcom expr optInd #| ('elif' expr colcom expr optInd)* #| 'else' colcom expr #| ifExpr = 'if' condExpr #| whenExpr = 'when' condExpr result = newNodeP(kind, p) while true: getTok(p) # skip `if`, `when`, `elif` var branch = newNodeP(nkElifExpr, p) optInd(p, branch) branch.add(parseExpr(p)) colcom(p, branch) branch.add(parseStmt(p)) skipComment(p, branch) result.add(branch) if p.tok.tokType != tkElif: break if p.tok.tokType == tkElse: var branch = newNodeP(nkElseExpr, p) eat(p, tkElse) colcom(p, branch) branch.add(parseStmt(p)) result.add(branch) proc parseWhile(p: var Parser): PNode = #| whileStmt = 'while' expr colcom stmt result = newNodeP(nkWhileStmt, p) getTok(p) optInd(p, result) result.add(parseExpr(p)) colcom(p, result) result.add(parseStmt(p)) proc parseCase(p: var Parser): PNode = #| ofBranch = 'of' exprList colcom stmt #| ofBranches = ofBranch (IND{=} ofBranch)* #| (IND{=} 'elif' expr colcom stmt)* #| (IND{=} 'else' colcom stmt)? #| caseStmt = 'case' expr ':'? COMMENT? #| (IND{>} ofBranches DED #| | IND{=} ofBranches) var b: PNode inElif = false wasIndented = false result = newNodeP(nkCaseStmt, p) getTok(p) result.add(parseExpr(p)) if p.tok.tokType == tkColon: getTok(p) skipComment(p, result) let oldInd = p.currInd if realInd(p): p.currInd = p.tok.indent wasIndented = true while sameInd(p): case p.tok.tokType of tkOf: if inElif: break b = newNodeP(nkOfBranch, p) exprList(p, tkColon, b) of tkElif: inElif = true b = newNodeP(nkElifBranch, p) getTok(p) optInd(p, b) b.add(parseExpr(p)) of tkElse: b = newNodeP(nkElse, p) getTok(p) else: break colcom(p, b) b.add(parseStmt(p)) result.add(b) if b.kind == nkElse: break if wasIndented: p.currInd = oldInd proc parseTry(p: var Parser; isExpr: bool): PNode = #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') #| (IND{=}? 'except' exprList colcom stmt)* #| (IND{=}? 'finally' colcom stmt)? #| tryExpr = 'try' colcom stmt &(optInd 'except'|'finally') #| (optInd 'except' exprList colcom stmt)* #| (optInd 'finally' colcom stmt)? result = newNodeP(nkTryStmt, p) getTok(p) colcom(p, result) result.add(parseStmt(p)) var b: PNode = nil while sameOrNoInd(p) or isExpr: case p.tok.tokType of tkExcept: b = newNodeP(nkExceptBranch, p) exprList(p, tkColon, b) of tkFinally: b = newNodeP(nkFinally, p) getTok(p) else: break colcom(p, b) b.add(parseStmt(p)) result.add(b) if b == nil: parMessage(p, "expected 'except'") proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode = result = newNodeP(kind, p) getTok(p) colcom(p, result) result.add(parseStmt(p)) proc parseBlock(p: var Parser): PNode = #| blockStmt = 'block' symbol? colcom stmt #| blockExpr = 'block' symbol? colcom stmt result = newNodeP(nkBlockStmt, p) getTokNoInd(p) if p.tok.tokType == tkColon: result.add(p.emptyNode) else: result.add(parseSymbol(p)) colcom(p, result) result.add(parseStmt(p)) proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode = #| staticStmt = 'static' colcom stmt #| deferStmt = 'defer' colcom stmt result = newNodeP(k, p) getTok(p) colcom(p, result) result.add(parseStmt(p)) proc parseAsm(p: var Parser): PNode = #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT) result = newNodeP(nkAsmStmt, p) getTokNoInd(p) if p.tok.tokType == tkCurlyDotLe: result.add(parsePragma(p)) else: result.add(p.emptyNode) case p.tok.tokType of tkStrLit: result.add(newStrNodeP(nkStrLit, p.tok.literal, p)) of tkRStrLit: result.add(newStrNodeP(nkRStrLit, p.tok.literal, p)) of tkTripleStrLit: result.add(newStrNodeP(nkTripleStrLit, p.tok.literal, p)) else: parMessage(p, "the 'asm' statement takes a string literal") result.add(p.emptyNode) return getTok(p) proc parseGenericParam(p: var Parser): PNode = #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)? var a: PNode result = newNodeP(nkIdentDefs, p) # progress guaranteed while true: case p.tok.tokType of tkIn, tkOut: let x = p.lex.cache.getIdent(if p.tok.tokType == tkIn: "in" else: "out") a = newNodeP(nkPrefix, p) a.add newIdentNodeP(x, p) getTok(p) expectIdent(p) a.add(parseSymbol(p)) of tkSymbol, tkAccent: a = parseSymbol(p) if a.kind == nkEmpty: return else: break result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) if p.tok.tokType == tkColon: getTok(p) optInd(p, result) result.add(parseExpr(p)) else: result.add(p.emptyNode) if p.tok.tokType == tkEquals: getTok(p) optInd(p, result) result.add(parseExpr(p)) else: result.add(p.emptyNode) proc parseGenericParamList(p: var Parser): PNode = #| genericParamList = '[' optInd #| genericParam ^* (comma/semicolon) optPar ']' result = newNodeP(nkGenericParams, p) getTok(p) optInd(p, result) # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent, tkIn, tkOut}: var a = parseGenericParam(p) result.add(a) if p.tok.tokType notin {tkComma, tkSemiColon}: break when defined(nimpretty): commaWasSemicolon(p.em) getTok(p) skipComment(p, a) optPar(p) eat(p, tkBracketRi) proc parsePattern(p: var Parser): PNode = #| pattern = '{' stmt '}' eat(p, tkCurlyLe) result = parseStmt(p) eat(p, tkCurlyRi) proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = #| indAndComment = (IND{>} COMMENT)? | COMMENT? #| routine = optInd identVis pattern? genericParamList? #| paramListColon pragma? ('=' COMMENT? stmt)? indAndComment result = newNodeP(kind, p) getTok(p) optInd(p, result) result.add(identVis(p)) if p.tok.tokType == tkCurlyLe and p.validInd: result.add(p.parsePattern) else: result.add(p.emptyNode) if p.tok.tokType == tkBracketLe and p.validInd: result.add(p.parseGenericParamList) else: result.add(p.emptyNode) result.add(p.parseParamList) if p.tok.tokType == tkCurlyDotLe and p.validInd: result.add(p.parsePragma) else: result.add(p.emptyNode) # empty exception tracking: result.add(p.emptyNode) let maybeMissEquals = p.tok.tokType != tkEquals if (not maybeMissEquals) and p.validInd: getTok(p) skipComment(p, result) result.add(parseStmt(p)) else: result.add(p.emptyNode) indAndComment(p, result, maybeMissEquals) let body = result[^1] if body.kind == nkStmtList and body.len > 0 and body[0].comment.len > 0 and body[0].kind != nkCommentStmt: if result.comment.len == 0: # proc fn*(a: int): int = a ## foo # => moves comment `foo` to `fn` result.comment = body[0].comment body[0].comment = "" #else: # assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. # Yeah, that worked so well. There IS a bug in this logic, now what? proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT result = newNodeP(nkCommentStmt, p) result.comment = p.tok.literal getTok(p) proc parseSection(p: var Parser, kind: TNodeKind, defparser: proc (p: var Parser): PNode {.nimcall.}): PNode = #| section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED) result = newNodeP(kind, p) if kind != nkTypeSection: getTok(p) skipComment(p, result) if realInd(p): withInd(p): skipComment(p, result) # progress guaranteed while sameInd(p): case p.tok.tokType of tkSymbol, tkAccent, tkParLe: var a = defparser(p) skipComment(p, a) result.add(a) of tkComment: var a = newCommentStmt(p) result.add(a) else: parMessage(p, errIdentifierExpected, p.tok) break if result.len == 0: parMessage(p, errIdentifierExpected, p.tok) elif p.tok.tokType in {tkSymbol, tkAccent, tkParLe} and p.tok.indent < 0: # tkParLe is allowed for ``var (x, y) = ...`` tuple parsing result.add(defparser(p)) else: parMessage(p, errIdentifierExpected, p.tok) proc parseEnum(p: var Parser): PNode = #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ result = newNodeP(nkEnumTy, p) getTok(p) result.add(p.emptyNode) optInd(p, result) flexComment(p, result) # progress guaranteed while true: var a = parseSymbol(p) if a.kind == nkEmpty: return var symPragma = a var pragma: PNode if p.tok.tokType == tkCurlyDotLe: pragma = optPragmas(p) symPragma = newNodeP(nkPragmaExpr, p) symPragma.add(a) symPragma.add(pragma) # nimpretty support here if p.tok.indent >= 0 and p.tok.indent <= p.currInd: result.add(symPragma) break if p.tok.tokType == tkEquals and p.tok.indent < 0: getTok(p) optInd(p, symPragma) var b = symPragma symPragma = newNodeP(nkEnumFieldDef, p) symPragma.add(b) symPragma.add(parseExpr(p)) if p.tok.indent < 0 or p.tok.indent >= p.currInd: rawSkipComment(p, symPragma) if p.tok.tokType == tkComma and p.tok.indent < 0: getTok(p) rawSkipComment(p, symPragma) else: if p.tok.indent < 0 or p.tok.indent >= p.currInd: rawSkipComment(p, symPragma) result.add(symPragma) if p.tok.indent >= 0 and p.tok.indent <= p.currInd or p.tok.tokType == tkEof: break if result.len <= 1: parMessage(p, errIdentifierExpected, p.tok) proc parseObjectPart(p: var Parser): PNode proc parseObjectWhen(p: var Parser): PNode = #| objectWhen = 'when' expr colcom objectPart COMMENT? #| ('elif' expr colcom objectPart COMMENT?)* #| ('else' colcom objectPart COMMENT?)? result = newNodeP(nkRecWhen, p) # progress guaranteed while sameInd(p): getTok(p) # skip `when`, `elif` var branch = newNodeP(nkElifBranch, p) optInd(p, branch) branch.add(parseExpr(p)) colcom(p, branch) branch.add(parseObjectPart(p)) flexComment(p, branch) result.add(branch) if p.tok.tokType != tkElif: break if p.tok.tokType == tkElse and sameInd(p): var branch = newNodeP(nkElse, p) eat(p, tkElse) colcom(p, branch) branch.add(parseObjectPart(p)) flexComment(p, branch) result.add(branch) proc parseObjectCase(p: var Parser): PNode = #| objectBranch = 'of' exprList colcom objectPart #| objectBranches = objectBranch (IND{=} objectBranch)* #| (IND{=} 'elif' expr colcom objectPart)* #| (IND{=} 'else' colcom objectPart)? #| objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT? #| (IND{>} objectBranches DED #| | IND{=} objectBranches) result = newNodeP(nkRecCase, p) getTokNoInd(p) var a = newNodeP(nkIdentDefs, p) a.add(identWithPragma(p)) eat(p, tkColon) a.add(parseTypeDesc(p)) a.add(p.emptyNode) result.add(a) if p.tok.tokType == tkColon: getTok(p) flexComment(p, result) var wasIndented = false let oldInd = p.currInd if realInd(p): p.currInd = p.tok.indent wasIndented = true # progress guaranteed while sameInd(p): var b: PNode case p.tok.tokType of tkOf: b = newNodeP(nkOfBranch, p) exprList(p, tkColon, b) of tkElse: b = newNodeP(nkElse, p) getTok(p) else: break colcom(p, b) var fields = parseObjectPart(p) if fields.kind == nkEmpty: parMessage(p, errIdentifierExpected, p.tok) fields = newNodeP(nkNilLit, p) # don't break further semantic checking b.add(fields) result.add(b) if b.kind == nkElse: break if wasIndented: p.currInd = oldInd proc parseObjectPart(p: var Parser): PNode = #| objectPart = IND{>} objectPart^+IND{=} DED #| / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals if realInd(p): result = newNodeP(nkRecList, p) withInd(p): rawSkipComment(p, result) while sameInd(p): case p.tok.tokType of tkCase, tkWhen, tkSymbol, tkAccent, tkNil, tkDiscard: result.add(parseObjectPart(p)) else: parMessage(p, errIdentifierExpected, p.tok) break elif sameOrNoInd(p): case p.tok.tokType of tkWhen: result = parseObjectWhen(p) of tkCase: result = parseObjectCase(p) of tkSymbol, tkAccent: result = parseIdentColonEquals(p, {withPragma}) if p.tok.indent < 0 or p.tok.indent >= p.currInd: rawSkipComment(p, result) of tkNil, tkDiscard: result = newNodeP(nkNilLit, p) getTok(p) else: result = p.emptyNode else: result = p.emptyNode proc parseObject(p: var Parser): PNode = #| objectDecl = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart result = newNodeP(nkObjectTy, p) getTok(p) if p.tok.tokType == tkCurlyDotLe and p.validInd: # Deprecated since v0.20.0 parMessage(p, warnDeprecated, "type pragmas follow the type name; this form of writing pragmas is deprecated") result.add(parsePragma(p)) else: result.add(p.emptyNode) if p.tok.tokType == tkOf and p.tok.indent < 0: var a = newNodeP(nkOfInherit, p) getTok(p) a.add(parseTypeDesc(p)) result.add(a) else: result.add(p.emptyNode) if p.tok.tokType == tkComment: skipComment(p, result) # an initial IND{>} HAS to follow: if not realInd(p): result.add(p.emptyNode) else: result.add(parseObjectPart(p)) proc parseTypeClassParam(p: var Parser): PNode = let modifier = case p.tok.tokType of tkOut, tkVar: nkVarTy of tkPtr: nkPtrTy of tkRef: nkRefTy of tkStatic: nkStaticTy of tkType: nkTypeOfExpr else: nkEmpty if modifier != nkEmpty: result = newNodeP(modifier, p) getTok(p) result.add(p.parseSymbol) else: result = p.parseSymbol proc parseTypeClass(p: var Parser): PNode = #| conceptParam = ('var' | 'out')? symbol #| conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? #| &IND{>} stmt result = newNodeP(nkTypeClassTy, p) getTok(p) if p.tok.tokType == tkComment: skipComment(p, result) if p.tok.indent < 0: var args = newNodeP(nkArgList, p) result.add(args) args.add(p.parseTypeClassParam) while p.tok.tokType == tkComma: getTok(p) args.add(p.parseTypeClassParam) else: result.add(p.emptyNode) # see ast.isNewStyleConcept if p.tok.tokType == tkCurlyDotLe and p.validInd: result.add(parsePragma(p)) else: result.add(p.emptyNode) if p.tok.tokType == tkOf and p.tok.indent < 0: var a = newNodeP(nkOfInherit, p) getTok(p) # progress guaranteed while true: a.add(parseTypeDesc(p)) if p.tok.tokType != tkComma: break getTok(p) result.add(a) else: result.add(p.emptyNode) if p.tok.tokType == tkComment: skipComment(p, result) # an initial IND{>} HAS to follow: if not realInd(p): if result.isNewStyleConcept: parMessage(p, "routine expected, but found '$1' (empty new-styled concepts are not allowed)", p.tok) result.add(p.emptyNode) else: result.add(parseStmt(p)) proc parseTypeDef(p: var Parser): PNode = #| #| typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux #| indAndComment? / identVisDot genericParamList? pragma '=' optInd typeDefAux #| indAndComment? result = newNodeP(nkTypeDef, p) var identifier = identVis(p, allowDot=true) var identPragma = identifier var pragma: PNode var genericParam: PNode var noPragmaYet = true if p.tok.tokType == tkCurlyDotLe: pragma = optPragmas(p) identPragma = newNodeP(nkPragmaExpr, p) identPragma.add(identifier) identPragma.add(pragma) noPragmaYet = false if p.tok.tokType == tkBracketLe and p.validInd: if not noPragmaYet: # Deprecated since v0.20.0 parMessage(p, warnDeprecated, "pragma before generic parameter list is deprecated") genericParam = parseGenericParamList(p) else: genericParam = p.emptyNode if noPragmaYet: pragma = optPragmas(p) if pragma.kind != nkEmpty: identPragma = newNodeP(nkPragmaExpr, p) identPragma.add(identifier) identPragma.add(pragma) elif p.tok.tokType == tkCurlyDotLe: parMessage(p, errGenerated, "pragma already present") result.add(identPragma) result.add(genericParam) if p.tok.tokType == tkEquals: result.info = parLineInfo(p) getTok(p) optInd(p, result) result.add(parseTypeDefAux(p)) else: result.add(p.emptyNode) indAndComment(p, result) # special extension! proc parseVarTuple(p: var Parser): PNode = #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr result = newNodeP(nkVarTuple, p) getTok(p) # skip '(' optInd(p, result) # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent}: var a = identWithPragma(p, allowDot=true) result.add(a) if p.tok.tokType != tkComma: break getTok(p) skipComment(p, a) result.add(p.emptyNode) # no type desc optPar(p) eat(p, tkParRi) proc parseVariable(p: var Parser): PNode = #| colonBody = colcom stmt postExprBlocks? #| variable = (varTuple / identColonEquals) colonBody? indAndComment if p.tok.tokType == tkParLe: result = parseVarTuple(p) eat(p, tkEquals) optInd(p, result) result.add(parseExpr(p)) else: result = parseIdentColonEquals(p, {withPragma, withDot}) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) proc parseConstant(p: var Parser): PNode = #| constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment if p.tok.tokType == tkParLe: result = parseVarTuple(p) else: result = newNodeP(nkConstDef, p) result.add(identWithPragma(p)) if p.tok.tokType == tkColon: getTok(p) optInd(p, result) result.add(parseTypeDesc(p)) else: result.add(p.emptyNode) eat(p, tkEquals) optInd(p, result) #add(result, parseStmtListExpr(p)) result.add(parseExpr(p)) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) proc parseBind(p: var Parser, k: TNodeKind): PNode = #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma #| mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma result = newNodeP(k, p) getTok(p) optInd(p, result) # progress guaranteed while true: var a = qualifiedIdent(p) result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) proc parseStmtPragma(p: var Parser): PNode = #| pragmaStmt = pragma (':' COMMENT? stmt)? result = parsePragma(p) if p.tok.tokType == tkColon and p.tok.indent < 0: let a = result result = newNodeI(nkPragmaBlock, a.info) getTok(p) skipComment(p, result) result.add a result.add parseStmt(p) proc simpleStmt(p: var Parser): PNode = #| simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt #| | continueStmt | pragmaStmt | importStmt | exportStmt | fromStmt #| | includeStmt | commentStmt) / exprStmt) COMMENT? #| case p.tok.tokType of tkReturn: result = parseReturnOrRaise(p, nkReturnStmt) of tkRaise: result = parseReturnOrRaise(p, nkRaiseStmt) of tkYield: result = parseReturnOrRaise(p, nkYieldStmt) of tkDiscard: result = parseReturnOrRaise(p, nkDiscardStmt) of tkBreak: result = parseReturnOrRaise(p, nkBreakStmt) of tkContinue: result = parseReturnOrRaise(p, nkContinueStmt) of tkCurlyDotLe: result = parseStmtPragma(p) of tkImport: result = parseImport(p, nkImportStmt) of tkExport: result = parseImport(p, nkExportStmt) of tkFrom: result = parseFromStmt(p) of tkInclude: result = parseIncludeStmt(p) of tkComment: result = newCommentStmt(p) else: if isExprStart(p): result = parseExprStmt(p) else: result = p.emptyNode if result.kind notin {nkEmpty, nkCommentStmt}: skipComment(p, result) proc complexOrSimpleStmt(p: var Parser): PNode = #| complexOrSimpleStmt = (ifStmt | whenStmt | whileStmt #| | tryStmt | forStmt #| | blockStmt | staticStmt | deferStmt | asmStmt #| | 'proc' routine #| | 'method' routine #| | 'func' routine #| | 'iterator' routine #| | 'macro' routine #| | 'template' routine #| | 'converter' routine #| | 'type' section(typeDef) #| | 'const' section(constant) #| | ('let' | 'var' | 'using') section(variable) #| | bindStmt | mixinStmt) #| / simpleStmt case p.tok.tokType of tkIf: result = parseIfOrWhen(p, nkIfStmt) of tkWhile: result = parseWhile(p) of tkCase: result = parseCase(p) of tkTry: result = parseTry(p, isExpr=false) of tkFinally: result = parseExceptBlock(p, nkFinally) of tkExcept: result = parseExceptBlock(p, nkExceptBranch) of tkFor: result = parseFor(p) of tkBlock: result = parseBlock(p) of tkStatic: result = parseStaticOrDefer(p, nkStaticStmt) of tkDefer: result = parseStaticOrDefer(p, nkDefer) of tkAsm: result = parseAsm(p) of tkProc: result = parseRoutine(p, nkProcDef) of tkFunc: result = parseRoutine(p, nkFuncDef) of tkMethod: result = parseRoutine(p, nkMethodDef) of tkIterator: result = parseRoutine(p, nkIteratorDef) of tkMacro: result = parseRoutine(p, nkMacroDef) of tkTemplate: result = parseRoutine(p, nkTemplateDef) of tkConverter: result = parseRoutine(p, nkConverterDef) of tkType: getTok(p) if p.tok.tokType == tkParLe: getTok(p) result = newNodeP(nkTypeOfExpr, p) result.add(primary(p, pmTypeDesc)) eat(p, tkParRi) result = parseOperators(p, result, -1, pmNormal) else: result = parseSection(p, nkTypeSection, parseTypeDef) of tkConst: prettySection: result = parseSection(p, nkConstSection, parseConstant) of tkLet: prettySection: result = parseSection(p, nkLetSection, parseVariable) of tkVar: prettySection: result = parseSection(p, nkVarSection, parseVariable) of tkWhen: result = parseIfOrWhen(p, nkWhenStmt) of tkBind: result = parseBind(p, nkBindStmt) of tkMixin: result = parseBind(p, nkMixinStmt) of tkUsing: result = parseSection(p, nkUsingStmt, parseVariable) else: result = simpleStmt(p) proc parseStmt(p: var Parser): PNode = #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED) #| / simpleStmt ^+ ';' if p.tok.indent > p.currInd: # nimpretty support here result = newNodeP(nkStmtList, p) withInd(p): while true: if p.tok.indent == p.currInd: discard elif p.tok.tokType == tkSemiColon: getTok(p) if p.tok.indent < 0 or p.tok.indent == p.currInd: discard else: break else: if p.tok.indent > p.currInd and p.tok.tokType != tkDot: parMessage(p, errInvalidIndentation) break if p.tok.tokType in {tkCurlyRi, tkParRi, tkCurlyDotRi, tkBracketRi}: # XXX this ensures tnamedparamanonproc still compiles; # deprecate this syntax later break p.hasProgress = false if p.tok.tokType in {tkElse, tkElif}: break # Allow this too, see tests/parser/tifexprs let a = complexOrSimpleStmt(p) if a.kind == nkEmpty and not p.hasProgress: parMessage(p, errExprExpected, p.tok) break else: result.add a if not p.hasProgress and p.tok.tokType == tkEof: break else: # the case statement is only needed for better error messages: case p.tok.tokType of tkIf, tkWhile, tkCase, tkTry, tkFor, tkBlock, tkAsm, tkProc, tkFunc, tkIterator, tkMacro, tkType, tkConst, tkWhen, tkVar: parMessage(p, "nestable statement requires indentation") result = p.emptyNode else: if p.inSemiStmtList > 0: result = simpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) else: result = newNodeP(nkStmtList, p) while true: if p.tok.indent >= 0: parMessage(p, errInvalidIndentation) p.hasProgress = false let a = simpleStmt(p) let err = not p.hasProgress if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) result.add(a) if p.tok.tokType != tkSemiColon: break getTok(p) if err and p.tok.tokType == tkEof: break proc parseAll(p: var Parser): PNode = ## Parses the rest of the input stream held by the parser into a PNode. result = newNodeP(nkStmtList, p) while p.tok.tokType != tkEof: p.hasProgress = false var a = complexOrSimpleStmt(p) if a.kind != nkEmpty and p.hasProgress: result.add(a) else: parMessage(p, errExprExpected, p.tok) # bugfix: consume a token here to prevent an endless loop: getTok(p) if p.tok.indent != 0: parMessage(p, errInvalidIndentation) proc parseTopLevelStmt(p: var Parser): PNode = ## Implements an iterator which, when called repeatedly, returns the next ## top-level statement or emptyNode if end of stream. result = p.emptyNode # progress guaranteed while true: # nimpretty support here if p.tok.indent != 0: if p.firstTok and p.tok.indent < 0: discard elif p.tok.tokType != tkSemiColon: # special casing for better error messages: if p.tok.tokType == tkOpr and p.tok.ident.s == "*": parMessage(p, errGenerated, "invalid indentation; an export marker '*' follows the declared identifier") else: parMessage(p, errInvalidIndentation) p.firstTok = false case p.tok.tokType of tkSemiColon: getTok(p) if p.tok.indent <= 0: discard else: parMessage(p, errInvalidIndentation) p.firstTok = true of tkEof: break else: result = complexOrSimpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break proc parseString*(s: string; cache: IdentCache; config: ConfigRef; filename: string = ""; line: int = 0; errorHandler: ErrorHandler = nil): PNode = ## Parses a string into an AST, returning the top node. ## `filename` and `line`, although optional, provide info so that the ## compiler can generate correct error messages referring to the original ## source. var stream = llStreamOpen(s) stream.lineOffset = line var parser: Parser parser.lex.errorHandler = errorHandler openParser(parser, AbsoluteFile filename, stream, cache, config) result = parser.parseAll closeParser(parser)