diff options
Diffstat (limited to 'compiler/parser.nim')
| -rw-r--r-- | compiler/parser.nim | 2763 |
1 files changed, 1763 insertions, 1000 deletions
diff --git a/compiler/parser.nim b/compiler/parser.nim index 46294925d..747505097 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1,591 +1,799 @@ # # -# The Nimrod Compiler -# (c) Copyright 2013 Andreas Rumpf +# The Nim Compiler +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# This module implements the parser of the standard Nimrod syntax. +# This module implements the parser of the standard Nim syntax. # The parser strictly reflects the grammar ("doc/grammar.txt"); however # it uses several helper routines to keep the parser small. A special # efficient algorithm is used for the precedence levels. The parser here can # be seen as a refinement of the grammar, as it specifies how the AST is built -# from the grammar and how comments belong to the AST. +# from the grammar and how comments belong to the AST. # In fact the grammar is generated from this file: -when isMainModule: - import pegs - var outp = open("compiler/grammar.txt", fmWrite) - for line in lines("compiler/parser.nim"): - if line =~ peg" \s* '#| ' {.*}": - outp.writeln matches[0] - outp.close +when isMainModule or defined(nimTestGrammar): + # Leave a note in grammar.txt that it is generated: + #| # This file is generated by compiler/parser.nim. + import std/pegs + when defined(nimPreviewSlimSystem): + import std/syncio + + proc writeGrammarFile(x: string) = + var outp = open(x, fmWrite) + for line in lines("compiler/parser.nim"): + if line =~ peg" \s* '#| ' {.*}": + outp.write matches[0], "\L" + outp.close + + when defined(nimTestGrammar): + import std/os + from ../testament/lib/stdtest/specialpaths import buildDir + const newGrammarText = buildDir / "grammar.txt" + + if not dirExists(buildDir): + createDir(buildDir) + + writeGrammarFile(newGrammarText) + + proc checkSameGrammar*() = + doAssert sameFileContent(newGrammarText, "doc/grammar.txt"), + "execute 'nim r compiler/parser.nim' to keep grammar.txt up-to-date" + else: + writeGrammarFile("doc/grammar.txt") + import ".." / tools / grammar_nanny + checkGrammarFile() import - llstream, lexer, idents, strutils, ast, astalgo, msgs + llstream, lexer, idents, msgs, options, lineinfos, + pathutils + +when not defined(nimCustomAst): + import ast +else: + import plugins / customast + +import std/strutils + +when defined(nimpretty): + import layouter + +when defined(nimPreviewSlimSystem): + import std/assertions type - TParser*{.final.} = object # a TParser object represents a module that + Parser* = object # A Parser object represents a file that # is being parsed - currInd: int # current indentation - firstTok: bool - lex*: TLexer # the lexer that is used for parsing - tok*: TToken # the current token - -proc ParseAll*(p: var TParser): PNode -proc openParser*(p: var TParser, filename: string, inputstream: PLLStream) -proc closeParser*(p: var TParser) -proc parseTopLevelStmt*(p: var TParser): PNode - # implements an iterator. Returns the next top-level statement or - # emtyNode if end of stream. - -proc parseString*(s: string, filename: string = "", line: int = 0): PNode - # filename and line could be set optionally, when the string originates - # from a certain source file. This way, the compiler could generate - # correct error messages referring to the original source. - + currInd: int # current indentation level + firstTok: bool # Has the first token been read? + hasProgress: bool # some while loop requires progress ensurance + lex*: Lexer # The lexer that is used for parsing + tok*: Token # The current token + lineStartPrevious*: int + lineNumberPrevious*: int + bufposPrevious*: int + inPragma*: int # Pragma level + inSemiStmtList*: int + when not defined(nimCustomAst): + emptyNode: PNode + when defined(nimpretty): + em*: Emitter + + SymbolMode = enum + smNormal, smAllowNil, smAfterDot + + PrimaryMode = enum + pmNormal, pmTypeDesc, pmTypeDef, pmTrySimple + +when defined(nimCustomAst): + # For the `customast` version we cannot share nodes, not even empty nodes: + template emptyNode(p: Parser): PNode = newNode(nkEmpty) + # helpers for the other parsers -proc getPrecedence*(tok: TToken): int -proc isOperator*(tok: TToken): bool -proc getTok*(p: var TParser) -proc parMessage*(p: TParser, msg: TMsgKind, arg: string = "") -proc skipComment*(p: var TParser, node: PNode) -proc newNodeP*(kind: TNodeKind, p: TParser): PNode -proc newIntNodeP*(kind: TNodeKind, intVal: BiggestInt, p: TParser): PNode -proc newFloatNodeP*(kind: TNodeKind, floatVal: BiggestFloat, p: TParser): PNode -proc newStrNodeP*(kind: TNodeKind, strVal: string, p: TParser): PNode -proc newIdentNodeP*(ident: PIdent, p: TParser): PNode -proc expectIdentOrKeyw*(p: TParser) -proc ExpectIdent*(p: TParser) -proc parLineInfo*(p: TParser): TLineInfo -proc Eat*(p: var TParser, TokType: TTokType) -proc skipInd*(p: var TParser) -proc optPar*(p: var TParser) -proc optInd*(p: var TParser, n: PNode) -proc indAndComment*(p: var TParser, n: PNode) -proc setBaseFlags*(n: PNode, base: TNumericalBase) -proc parseSymbol*(p: var TParser): PNode -proc parseTry(p: var TParser): PNode -proc parseCase(p: var TParser): PNode +proc isOperator*(tok: Token): bool +proc getTok*(p: var Parser) +proc parMessage*(p: Parser, msg: TMsgKind, arg: string = "") +proc skipComment*(p: var Parser, node: PNode) +proc newNodeP*(kind: TNodeKind, p: Parser): PNode +proc newIntNodeP*(kind: TNodeKind, intVal: BiggestInt, p: Parser): PNode +proc newFloatNodeP*(kind: TNodeKind, floatVal: BiggestFloat, p: Parser): PNode +proc newStrNodeP*(kind: TNodeKind, strVal: sink string, p: Parser): PNode +proc newIdentNodeP*(ident: PIdent, p: Parser): PNode +proc expectIdentOrKeyw*(p: Parser) +proc expectIdent*(p: Parser) +proc parLineInfo*(p: Parser): TLineInfo +proc eat*(p: var Parser, tokType: TokType) +proc skipInd*(p: var Parser) +proc optPar*(p: var Parser) +proc optInd*(p: var Parser, n: PNode) +proc indAndComment*(p: var Parser, n: PNode, maybeMissEquals = false) +proc setBaseFlags*(n: PNode, base: NumericalBase) +proc parseSymbol*(p: var Parser, mode = smNormal): PNode +proc parseTry(p: var Parser; isExpr: bool): PNode +proc parseCase(p: var Parser): PNode +proc parseStmtPragma(p: var Parser): PNode +proc parsePragma(p: var Parser): PNode +proc postExprBlocks(p: var Parser, x: PNode): PNode +proc parseExprStmt(p: var Parser): PNode +proc parseBlock(p: var Parser): PNode +proc primary(p: var Parser, mode: PrimaryMode): PNode +proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode + # implementation -proc getTok(p: var TParser) = +template prettySection(body) = + when defined(nimpretty): beginSection(p.em) + body + when defined(nimpretty): endSection(p.em) + +proc getTok(p: var Parser) = + ## Get the next token from the parser's lexer, and store it in the parser's + ## `tok` member. + p.lineNumberPrevious = p.lex.lineNumber + p.lineStartPrevious = p.lex.lineStart + p.bufposPrevious = p.lex.bufpos rawGetTok(p.lex, p.tok) - -proc OpenParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream) = - initToken(p.tok) - OpenLexer(p.lex, fileIdx, inputstream) + p.hasProgress = true + when defined(nimpretty): + emitTok(p.em, p.lex, p.tok) + # skip the additional tokens that nimpretty needs but the parser has no + # interest in: + while p.tok.tokType == tkComment: + rawGetTok(p.lex, p.tok) + emitTok(p.em, p.lex, p.tok) + +proc openParser*(p: var Parser, fileIdx: FileIndex, inputStream: PLLStream, + cache: IdentCache; config: ConfigRef) = + ## Open a parser, using the given arguments to set up its internal state. + ## + reset(p.tok) + openLexer(p.lex, fileIdx, inputStream, cache, config) + when defined(nimpretty): + openEmitter(p.em, cache, config, fileIdx) getTok(p) # read the first token p.firstTok = true + when not defined(nimCustomAst): + p.emptyNode = newNode(nkEmpty) + +proc openParser*(p: var Parser, filename: AbsoluteFile, inputStream: PLLStream, + cache: IdentCache; config: ConfigRef) = + openParser(p, fileInfoIdx(config, filename), inputStream, cache, config) -proc OpenParser*(p: var TParser, filename: string, inputStream: PLLStream) = - openParser(p, filename.fileInfoIdx, inputStream) +proc closeParser*(p: var Parser) = + ## Close a parser, freeing up its resources. + closeLexer(p.lex) -proc CloseParser(p: var TParser) = - CloseLexer(p.lex) +proc parMessage(p: Parser, msg: TMsgKind, arg = "") = + ## Produce and emit the parser message `arg` to output. + lexMessageTok(p.lex, msg, p.tok, arg) -proc parMessage(p: TParser, msg: TMsgKind, arg: string = "") = - lexMessage(p.lex, msg, arg) +proc parMessage(p: Parser, msg: string, tok: Token) = + ## Produce and emit a parser message to output about the token `tok` + parMessage(p, errGenerated, msg % prettyTok(tok)) -proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) = - lexMessage(p.lex, msg, prettyTok(tok)) +proc parMessage(p: Parser, arg: string) = + ## Produce and emit the parser message `arg` to output. + lexMessageTok(p.lex, errGenerated, p.tok, arg) -template withInd(p: expr, body: stmt) {.immediate.} = +template withInd(p, body: untyped) = let oldInd = p.currInd p.currInd = p.tok.indent body p.currInd = oldInd +template newlineWasSplitting(p: var Parser) = + when defined(nimpretty): + layouter.newlineWasSplitting(p.em) + template realInd(p): bool = p.tok.indent > p.currInd template sameInd(p): bool = p.tok.indent == p.currInd template sameOrNoInd(p): bool = p.tok.indent == p.currInd or p.tok.indent < 0 -proc rawSkipComment(p: var TParser, node: PNode) = +proc validInd(p: var Parser): bool {.inline.} = + result = p.tok.indent < 0 or p.tok.indent > p.currInd + +proc rawSkipComment(p: var Parser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: - if node.comment == nil: node.comment = "" - add(node.comment, p.tok.literal) + var rhs = node.comment + when defined(nimpretty): + if p.tok.commentOffsetB > p.tok.commentOffsetA: + rhs.add fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB) + else: + rhs.add p.tok.literal + else: + rhs.add p.tok.literal + node.comment = move rhs else: parMessage(p, errInternal, "skipComment") getTok(p) -proc skipComment(p: var TParser, node: PNode) = +proc skipComment(p: var Parser, node: PNode) = if p.tok.indent < 0: rawSkipComment(p, node) -proc skipInd(p: var TParser) = +proc flexComment(p: var Parser, node: PNode) = + if p.tok.indent < 0 or realInd(p): rawSkipComment(p, node) + +const + errInvalidIndentation = "invalid indentation" + errIdentifierExpected = "identifier expected, but got '$1'" + errExprExpected = "expression expected, but found '$1'" + +proc skipInd(p: var Parser) = if p.tok.indent >= 0: if not realInd(p): parMessage(p, errInvalidIndentation) -proc optPar(p: var TParser) = +proc optPar(p: var Parser) = if p.tok.indent >= 0: if p.tok.indent < p.currInd: parMessage(p, errInvalidIndentation) -proc optInd(p: var TParser, n: PNode) = +proc optInd(p: var Parser, n: PNode) = skipComment(p, n) skipInd(p) -proc getTokNoInd(p: var TParser) = +proc getTokNoInd(p: var Parser) = getTok(p) if p.tok.indent >= 0: parMessage(p, errInvalidIndentation) -proc expectIdentOrKeyw(p: TParser) = +proc expectIdentOrKeyw(p: Parser) = if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType): - lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) - -proc ExpectIdent(p: TParser) = + lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok)) + +proc expectIdent(p: Parser) = if p.tok.tokType != tkSymbol: - lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) - -proc Eat(p: var TParser, TokType: TTokType) = - if p.tok.TokType == TokType: getTok(p) - else: lexMessage(p.lex, errTokenExpected, TokTypeToStr[tokType]) - -proc parLineInfo(p: TParser): TLineInfo = - result = getLineInfo(p.lex) - -proc indAndComment(p: var TParser, n: PNode) = + lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok)) + +proc eat(p: var Parser, tokType: TokType) = + ## Move the parser to the next token if the current token is of type + ## `tokType`, otherwise error. + if p.tok.tokType == tokType: + getTok(p) + else: + lexMessage(p.lex, errGenerated, + "expected: '" & $tokType & "', but got: '" & prettyTok(p.tok) & "'") + +proc parLineInfo(p: Parser): TLineInfo = + ## Retrieve the line information associated with the parser's current state. + result = getLineInfo(p.lex, p.tok) + +proc indAndComment(p: var Parser, n: PNode, maybeMissEquals = false) = if p.tok.indent > p.currInd: if p.tok.tokType == tkComment: rawSkipComment(p, n) + elif maybeMissEquals: + let col = p.bufposPrevious - p.lineStartPrevious + var info = newLineInfo(p.lex.fileIdx, p.lineNumberPrevious, col) + parMessage(p, "invalid indentation, maybe you forgot a '=' at $1 ?" % [p.lex.config$info]) else: parMessage(p, errInvalidIndentation) else: skipComment(p, n) - -proc newNodeP(kind: TNodeKind, p: TParser): PNode = - result = newNodeI(kind, getLineInfo(p.lex)) -proc newIntNodeP(kind: TNodeKind, intVal: BiggestInt, p: TParser): PNode = - result = newNodeP(kind, p) - result.intVal = intVal +proc newNodeP(kind: TNodeKind, p: Parser): PNode = + result = newNode(kind, parLineInfo(p)) -proc newFloatNodeP(kind: TNodeKind, floatVal: BiggestFloat, - p: TParser): PNode = - result = newNodeP(kind, p) - result.floatVal = floatVal +proc newIntNodeP(kind: TNodeKind, intVal: BiggestInt, p: Parser): PNode = + result = newAtom(kind, intVal, parLineInfo(p)) -proc newStrNodeP(kind: TNodeKind, strVal: string, p: TParser): PNode = - result = newNodeP(kind, p) - result.strVal = strVal - -proc newIdentNodeP(ident: PIdent, p: TParser): PNode = - result = newNodeP(nkIdent, p) - result.ident = ident - -proc parseExpr(p: var TParser): PNode -proc parseStmt(p: var TParser): PNode -proc parseTypeDesc(p: var TParser): PNode -proc parseDoBlocks(p: var TParser, call: PNode) -proc parseParamList(p: var TParser, retColon = true): PNode - -proc relevantOprChar(ident: PIdent): char {.inline.} = - result = ident.s[0] - var L = ident.s.len - if result == '\\' and L > 1: - result = ident.s[1] - -proc IsSigilLike(tok: TToken): bool {.inline.} = - result = tok.tokType == tkOpr and relevantOprChar(tok.ident) == '@' - -proc IsLeftAssociative(tok: TToken): bool {.inline.} = - result = tok.tokType != tkOpr or relevantOprChar(tok.ident) != '^' - -proc getPrecedence(tok: TToken): int = - case tok.tokType - of tkOpr: - let L = tok.ident.s.len - let relevantChar = relevantOprChar(tok.ident) - - template considerAsgn(value: expr) = - result = if tok.ident.s[L-1] == '=': 1 else: value - - case relevantChar - of '$', '^': considerAsgn(10) - of '*', '%', '/', '\\': considerAsgn(9) - of '~': result = 8 - of '+', '-', '|': considerAsgn(8) - of '&': considerAsgn(7) - of '=', '<', '>', '!': result = 5 - of '.': considerAsgn(6) - of '?': result = 2 - else: considerAsgn(2) - of tkDiv, tkMod, tkShl, tkShr: result = 9 - of tkIn, tkNotIn, tkIs, tkIsNot, tkNot, tkOf, tkAs: result = 5 - of tkDotDot: result = 6 - of tkAnd: result = 4 - of tkOr, tkXor: result = 3 - else: result = - 10 - -proc isOperator(tok: TToken): bool = - result = getPrecedence(tok) >= 0 - -#| module = stmt ^* (';' / IND{=}) +proc newFloatNodeP(kind: TNodeKind, floatVal: BiggestFloat, + p: Parser): PNode = + result = newAtom(kind, floatVal, parLineInfo(p)) + +proc newStrNodeP(kind: TNodeKind, strVal: sink string, p: Parser): PNode = + result = newAtom(kind, strVal, parLineInfo(p)) + +proc newIdentNodeP(ident: PIdent, p: Parser): PNode = + result = newAtom(ident, parLineInfo(p)) + +proc parseExpr(p: var Parser): PNode +proc parseStmt(p: var Parser): PNode +proc parseTypeDesc(p: var Parser, fullExpr = false): PNode +proc parseTypeDefValue(p: var Parser): PNode +proc parseParamList(p: var Parser, retColon = true): PNode + +proc isSigilLike(tok: Token): bool {.inline.} = + result = tok.tokType == tkOpr and tok.ident.s[0] == '@' + +proc isRightAssociative(tok: Token): bool {.inline.} = + ## Determines whether the token is right assocative. + result = tok.tokType == tkOpr and tok.ident.s[0] == '^' + # or (tok.ident.s.len > 1 and tok.ident.s[^1] == '>') + +proc isUnary(tok: Token): bool = + ## Check if the given token is a unary operator + tok.tokType in {tkOpr, tkDotDot} and + tok.spacing == {tsLeading} + +proc checkBinary(p: Parser) {.inline.} = + ## Check if the current parser token is a binary operator. + # we don't check '..' here as that's too annoying + if p.tok.tokType == tkOpr: + if p.tok.spacing == {tsTrailing}: + parMessage(p, warnInconsistentSpacing, prettyTok(p.tok)) + +#| module = complexOrSimpleStmt ^* (';' / IND{=}) #| #| comma = ',' COMMENT? #| semicolon = ';' COMMENT? #| colon = ':' COMMENT? #| colcom = ':' COMMENT? -#| +#| #| operator = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 #| | 'or' | 'xor' | 'and' -#| | 'is' | 'isnot' | 'in' | 'notin' | 'of' -#| | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'addr' | 'static' | '..' -#| +#| | 'is' | 'isnot' | 'in' | 'notin' | 'of' | 'as' | 'from' +#| | 'div' | 'mod' | 'shl' | 'shr' | 'not' | '..' +#| #| prefixOperator = operator -#| -#| optInd = COMMENT? +#| +#| optInd = COMMENT? IND? #| optPar = (IND{>} | IND{=})? -#| -#| simpleExpr = assignExpr (OP0 optInd assignExpr)* -#| assignExpr = orExpr (OP1 optInd orExpr)* -#| orExpr = andExpr (OP2 optInd andExpr)* -#| andExpr = cmpExpr (OP3 optInd cmpExpr)* -#| cmpExpr = sliceExpr (OP4 optInd sliceExpr)* -#| sliceExpr = ampExpr (OP5 optInd ampExpr)* -#| ampExpr = plusExpr (OP6 optInd plusExpr)* -#| plusExpr = mulExpr (OP7 optInd mulExpr)* -#| mulExpr = dollarExpr (OP8 optInd dollarExpr)* -#| dollarExpr = primary (OP9 optInd primary)* - -proc colcom(p: var TParser, n: PNode) = +#| +#| simpleExpr = arrowExpr (OP0 optInd arrowExpr)* pragma? +#| arrowExpr = assignExpr (OP1 optInd assignExpr)* +#| assignExpr = orExpr (OP2 optInd orExpr)* +#| orExpr = andExpr (OP3 optInd andExpr)* +#| andExpr = cmpExpr (OP4 optInd cmpExpr)* +#| cmpExpr = sliceExpr (OP5 optInd sliceExpr)* +#| sliceExpr = ampExpr (OP6 optInd ampExpr)* +#| ampExpr = plusExpr (OP7 optInd plusExpr)* +#| plusExpr = mulExpr (OP8 optInd mulExpr)* +#| mulExpr = dollarExpr (OP9 optInd dollarExpr)* +#| dollarExpr = primary (OP10 optInd primary)* + +proc isOperator(tok: Token): bool = + #| operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 | + #| 'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' | + #| 'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor' + tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, + tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, + tkOr, tkXor} + +proc colcom(p: var Parser, n: PNode) = eat(p, tkColon) skipComment(p, n) -proc parseSymbol(p: var TParser): PNode = - #| symbol = '`' (KEYW|IDENT|operator|'(' ')'|'[' ']'|'{' '}'|'='|literal)+ '`' - #| | IDENT +const tkBuiltInMagics = {tkType, tkStatic, tkAddr} + +template setEndInfo() = + when defined(nimsuggest): + result.endInfo = TLineInfo(fileIndex: p.lex.fileIdx, + line: p.lex.previousTokenEnd.line, + col: p.lex.previousTokenEnd.col) + +proc parseSymbol(p: var Parser, mode = smNormal): PNode = + #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' + #| | IDENT | 'addr' | 'type' | 'static' + #| symbolOrKeyword = symbol | KEYW case p.tok.tokType - of tkSymbol: + of tkSymbol: result = newIdentNodeP(p.tok.ident, p) getTok(p) - of tkAccent: + of tokKeywordLow..tokKeywordHigh: + if p.tok.tokType in tkBuiltInMagics or mode == smAfterDot: + # for backwards compatibility these 2 are always valid: + result = newIdentNodeP(p.tok.ident, p) + getTok(p) + elif p.tok.tokType == tkNil and mode == smAllowNil: + result = newNodeP(nkNilLit, p) + getTok(p) + else: + parMessage(p, errIdentifierExpected, p.tok) + result = p.emptyNode + of tkAccent: result = newNodeP(nkAccQuoted, p) getTok(p) + # progress guaranteed while true: case p.tok.tokType - of tkBracketLe: - add(result, newIdentNodeP(getIdent"[]", p)) - getTok(p) - eat(p, tkBracketRi) - of tkEquals: - add(result, newIdentNodeP(getIdent"=", p)) - getTok(p) - of tkParLe: - add(result, newIdentNodeP(getIdent"()", p)) - getTok(p) - eat(p, tkParRi) - of tkCurlyLe: - add(result, newIdentNodeP(getIdent"{}", p)) - getTok(p) - eat(p, tkCurlyRi) - of tokKeywordLow..tokKeywordHigh, tkSymbol, tkOpr, tkDotDot: - add(result, newIdentNodeP(p.tok.ident, p)) - getTok(p) - of tkIntLit..tkCharLit: - add(result, newIdentNodeP(getIdent(tokToStr(p.tok)), p)) + of tkAccent: + if not result.hasSon: + parMessage(p, errIdentifierExpected, p.tok) + break + of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi: + let lineinfo = parLineInfo(p) + var accm = "" + while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals, + tkParLe..tkParDotRi}: + accm.add($p.tok) + getTok(p) + let node = newAtom(p.lex.cache.getIdent(accm), lineinfo) + result.add(node) + of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCustomLit: + result.add(newIdentNodeP(p.lex.cache.getIdent($p.tok), p)) getTok(p) else: - if result.len == 0: - parMessage(p, errIdentifierExpected, p.tok) + parMessage(p, errIdentifierExpected, p.tok) break eat(p, tkAccent) else: parMessage(p, errIdentifierExpected, p.tok) - getTok(p) # BUGFIX: We must consume a token here to prevent endless loops! - result = ast.emptyNode - -proc indexExpr(p: var TParser): PNode = - #| indexExpr = expr - result = parseExpr(p) - -proc indexExprList(p: var TParser, first: PNode, k: TNodeKind, - endToken: TTokType): PNode = - #| indexExprList = indexExpr ^+ comma - result = newNodeP(k, p) - addSon(result, first) - getTok(p) - optInd(p, result) - while p.tok.tokType notin {endToken, tkEof}: - var a = indexExpr(p) - addSon(result, a) - if p.tok.tokType != tkComma: break + # BUGFIX: We must consume a token here to prevent endless loops! + # But: this really sucks for idetools and keywords, so we don't do it + # if it is a keyword: + #if not isKeyword(p.tok.tokType): getTok(p) + result = p.emptyNode + setEndInfo() + +proc equals(p: var Parser, a: PNode): PNode = + if p.tok.tokType == tkEquals: + result = newNodeP(nkExprEqExpr, p) getTok(p) - skipComment(p, a) - optPar(p) - eat(p, endToken) + #optInd(p, result) + result.add(a) + result.add(parseExpr(p)) + else: + result = a -proc colonOrEquals(p: var TParser, a: PNode): PNode = +proc colonOrEquals(p: var Parser, a: PNode): PNode = if p.tok.tokType == tkColon: result = newNodeP(nkExprColonExpr, p) getTok(p) + newlineWasSplitting(p) #optInd(p, result) - addSon(result, a) - addSon(result, parseExpr(p)) - elif p.tok.tokType == tkEquals: - result = newNodeP(nkExprEqExpr, p) - getTok(p) - #optInd(p, result) - addSon(result, a) - addSon(result, parseExpr(p)) + result.add(a) + result.add(parseExpr(p)) else: - result = a + result = equals(p, a) -proc exprColonEqExpr(p: var TParser): PNode = - #| exprColonEqExpr = expr (':'|'=' expr)? +proc exprColonEqExpr(p: var Parser): PNode = + #| exprColonEqExpr = expr ((':'|'=') expr + #| / doBlock extraPostExprBlock*)? var a = parseExpr(p) - result = colonOrEquals(p, a) + if p.tok.tokType == tkDo: + result = postExprBlocks(p, a) + else: + result = colonOrEquals(p, a) -proc exprList(p: var TParser, endTok: TTokType, result: PNode) = +proc exprEqExpr(p: var Parser): PNode = + #| exprEqExpr = expr ('=' expr + #| / doBlock extraPostExprBlock*)? + var a = parseExpr(p) + if p.tok.tokType == tkDo: + result = postExprBlocks(p, a) + else: + result = equals(p, a) + +proc exprList(p: var Parser, endTok: TokType, result: PNode) = #| exprList = expr ^+ comma + when defined(nimpretty): + inc p.em.doIndentMore getTok(p) optInd(p, result) - while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): - var a = parseExpr(p) - addSon(result, a) - if p.tok.tokType != tkComma: break + # progress guaranteed + var a = parseExpr(p) + result.add(a) + while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): + if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) - eat(p, endTok) - -proc dotExpr(p: var TParser, a: PNode): PNode = - #| dotExpr = expr '.' optInd ('type' | 'addr' | symbol) - var info = p.lex.getlineInfo + var a = parseExpr(p) + result.add(a) + when defined(nimpretty): + dec p.em.doIndentMore + +proc optionalExprList(p: var Parser, endTok: TokType, result: PNode) = + #| optionalExprList = expr ^* comma + when defined(nimpretty): + inc p.em.doIndentMore getTok(p) - optInd(p, a) - case p.tok.tokType - of tkType: - result = newNodeP(nkTypeOfExpr, p) - getTok(p) - addSon(result, a) - of tkAddr: - result = newNodeP(nkAddr, p) + optInd(p, result) + # progress guaranteed + while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): + var a = parseExpr(p) + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) - addSon(result, a) - else: - result = newNodeI(nkDotExpr, info) - addSon(result, a) - addSon(result, parseSymbol(p)) - -proc qualifiedIdent(p: var TParser): PNode = - #| qualifiedIdent = symbol ('.' optInd ('type' | 'addr' | symbol))? - result = parseSymbol(p) - if p.tok.tokType == tkDot: result = dotExpr(p, result) + optInd(p, a) + when defined(nimpretty): + dec p.em.doIndentMore -proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) = +proc exprColonEqExprListAux(p: var Parser, endTok: TokType, result: PNode) = assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi}) getTok(p) - optInd(p, result) + flexComment(p, result) + optPar(p) + # progress guaranteed while p.tok.tokType != endTok and p.tok.tokType != tkEof: var a = exprColonEqExpr(p) - addSon(result, a) - if p.tok.tokType != tkComma: break + result.add(a) + if p.tok.tokType != tkComma: break + elif result.kind == nkPar: + result.transitionSonsKind(nkTupleConstr) getTok(p) skipComment(p, a) optPar(p) eat(p, endTok) -proc exprColonEqExprList(p: var TParser, kind: TNodeKind, - endTok: TTokType): PNode = +proc exprColonEqExprList(p: var Parser, kind: TNodeKind, + endTok: TokType): PNode = #| exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? result = newNodeP(kind, p) exprColonEqExprListAux(p, endTok, result) -proc setOrTableConstr(p: var TParser): PNode = +proc dotExpr(p: var Parser, a: PNode): PNode = + var info = p.parLineInfo + getTok(p) + result = newNode(nkDotExpr, info) + optInd(p, result) + result.add(a) + result.add(parseSymbol(p, smAfterDot)) + if p.tok.tokType == tkBracketLeColon and tsLeading notin p.tok.spacing: + var x = newNode(nkBracketExpr, p.parLineInfo) + # rewrite 'x.y[:z]()' to 'y[z](x)' + x.add result.secondSon + exprList(p, tkBracketRi, x) + eat(p, tkBracketRi) + var y = newNode(nkCall, p.parLineInfo) + y.add x + y.add result.firstSon + if p.tok.tokType == tkParLe and tsLeading notin p.tok.spacing: + exprColonEqExprListAux(p, tkParRi, y) + result = y + +proc dotLikeExpr(p: var Parser, a: PNode): PNode = + var info = p.parLineInfo + result = newNode(nkInfix, info) + optInd(p, result) + var opNode = newIdentNodeP(p.tok.ident, p) + getTok(p) + result.add(opNode) + result.add(a) + result.add(parseSymbol(p, smAfterDot)) + +proc qualifiedIdent(p: var Parser): PNode = + #| qualifiedIdent = symbol ('.' optInd symbolOrKeyword)? + result = parseSymbol(p) + if p.tok.tokType == tkDot: result = dotExpr(p, result) + +proc setOrTableConstr(p: var Parser): PNode = #| setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}' result = newNodeP(nkCurly, p) getTok(p) # skip '{' optInd(p, result) if p.tok.tokType == tkColon: getTok(p) # skip ':' - result.kind = nkTableConstr + result.transitionSonsKind(nkTableConstr) else: + # progress guaranteed while p.tok.tokType notin {tkCurlyRi, tkEof}: var a = exprColonEqExpr(p) - if a.kind == nkExprColonExpr: result.kind = nkTableConstr - addSon(result, a) - if p.tok.tokType != tkComma: break + if a.kind == nkExprColonExpr: result.transitionSonsKind(nkTableConstr) + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) skipComment(p, a) optPar(p) eat(p, tkCurlyRi) # skip '}' -proc parseCast(p: var TParser): PNode = - #| castExpr = 'cast' '[' optInd typeDesc optPar ']' '(' optInd expr optPar ')' +proc parseCast(p: var Parser): PNode = + #| castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') / + # ('(' optInd exprColonEqExpr optPar ')') result = newNodeP(nkCast, p) getTok(p) - eat(p, tkBracketLe) - optInd(p, result) - addSon(result, parseTypeDesc(p)) - optPar(p) - eat(p, tkBracketRi) - eat(p, tkParLe) - optInd(p, result) - addSon(result, parseExpr(p)) + if p.tok.tokType == tkBracketLe: + getTok(p) + optInd(p, result) + result.add(parseTypeDesc(p)) + optPar(p) + eat(p, tkBracketRi) + eat(p, tkParLe) + optInd(p, result) + result.add(parseExpr(p)) + else: + result.add p.emptyNode + eat(p, tkParLe) + optInd(p, result) + result.add(exprColonEqExpr(p)) optPar(p) eat(p, tkParRi) + setEndInfo() + +template setNodeFlag(n: PNode; f: untyped) = + when defined(nimCustomAst): + discard + else: + incl n.flags, f -proc setBaseFlags(n: PNode, base: TNumericalBase) = +proc setBaseFlags(n: PNode, base: NumericalBase) = case base - of base10: nil - of base2: incl(n.flags, nfBase2) - of base8: incl(n.flags, nfBase8) - of base16: incl(n.flags, nfBase16) - -proc parseGStrLit(p: var TParser, a: PNode): PNode = + of base10: discard + of base2: setNodeFlag(n, nfBase2) + of base8: setNodeFlag(n, nfBase8) + of base16: setNodeFlag(n, nfBase16) + +proc parseGStrLit(p: var Parser, a: PNode): PNode = case p.tok.tokType - of tkGStrLit: + of tkGStrLit: result = newNodeP(nkCallStrLit, p) - addSon(result, a) - addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p)) + result.add(a) + result.add(newStrNodeP(nkRStrLit, p.tok.literal, p)) getTok(p) - of tkGTripleStrLit: + of tkGTripleStrLit: result = newNodeP(nkCallStrLit, p) - addSon(result, a) - addSon(result, newStrNodeP(nkTripleStrLit, p.tok.literal, p)) + result.add(a) + result.add(newStrNodeP(nkTripleStrLit, p.tok.literal, p)) getTok(p) else: result = a + setEndInfo() + +proc complexOrSimpleStmt(p: var Parser): PNode +proc simpleExpr(p: var Parser, mode = pmNormal): PNode +proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode + +proc semiStmtList(p: var Parser, result: PNode) = + inc p.inSemiStmtList + withInd(p): + # Be lenient with the first stmt/expr + let a = case p.tok.tokType + of tkIf: parseIfOrWhenExpr(p, nkIfStmt) + of tkWhen: parseIfOrWhenExpr(p, nkWhenStmt) + else: complexOrSimpleStmt(p) + result.add a -type - TPrimaryMode = enum pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix - -proc complexOrSimpleStmt(p: var TParser): PNode -proc simpleExpr(p: var TParser, mode = pmNormal): PNode - -proc semiStmtList(p: var TParser, result: PNode) = - result.add(complexOrSimpleStmt(p)) - while p.tok.tokType == tkSemicolon: - getTok(p) - optInd(p, result) - result.add(complexOrSimpleStmt(p)) - result.kind = nkStmtListExpr + while p.tok.tokType != tkEof: + if p.tok.tokType == tkSemiColon: + getTok(p) + if p.tok.tokType == tkParRi: + break + elif not (sameInd(p) or realInd(p)): + parMessage(p, errInvalidIndentation) + let a = complexOrSimpleStmt(p) + if a.kind == nkEmpty: + parMessage(p, errExprExpected, p.tok) + getTok(p) + else: + result.add a + dec p.inSemiStmtList + result.transitionSonsKind(nkStmtListExpr) -proc parsePar(p: var TParser): PNode = +proc parsePar(p: var Parser): PNode = #| parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' #| | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' #| | 'when' | 'var' | 'mixin' - #| par = '(' optInd (&parKeyw complexOrSimpleStmt ^+ ';' - #| | simpleExpr ('=' expr (';' complexOrSimpleStmt ^+ ';' )? )? - #| | (':' expr)? (',' (exprColonEqExpr comma?)*)? )? - #| optPar ')' + #| par = '(' optInd + #| ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';' + #| | ';' (ifExpr / complexOrSimpleStmt) ^+ ';' + #| | pragmaStmt + #| | simpleExpr ( (doBlock extraPostExprBlock*) + #| | ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? ) + #| | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) + #| optPar ')' # - # unfortunately it's ambiguous: (expr: expr) vs (exprStmt); however a + # unfortunately it's ambiguous: (expr: expr) vs (exprStmt); however a # leading ';' could be used to enforce a 'stmt' context ... result = newNodeP(nkPar, p) getTok(p) optInd(p, result) - if p.tok.tokType in {tkDiscard, tkInclude, tkIf, tkWhile, tkCase, - tkTry, tkFinally, tkExcept, tkFor, tkBlock, - tkConst, tkLet, tkWhen, tkVar, + flexComment(p, result) + if p.tok.tokType in {tkDiscard, tkInclude, tkIf, tkWhile, tkCase, + tkTry, tkDefer, tkFinally, tkExcept, tkBlock, + tkConst, tkLet, tkWhen, tkVar, tkFor, tkMixin}: # XXX 'bind' used to be an expression, so we exclude it here; # tests/reject/tbind2 fails otherwise. semiStmtList(p, result) - elif p.tok.tokType == tkSemicolon: + elif p.tok.tokType == tkSemiColon: # '(;' enforces 'stmt' context: getTok(p) optInd(p, result) semiStmtList(p, result) - elif p.tok.tokType != tkParRi: + elif p.tok.tokType == tkCurlyDotLe: + result.add(parseStmtPragma(p)) + elif p.tok.tokType == tkParRi: + # Empty tuple '()' + result.transitionSonsKind(nkTupleConstr) + else: var a = simpleExpr(p) - if p.tok.tokType == tkEquals: + if p.tok.tokType == tkDo: + result = postExprBlocks(p, a) + elif p.tok.tokType == tkEquals: # special case: allow assignments + let asgn = newNodeP(nkAsgn, p) getTok(p) optInd(p, result) let b = parseExpr(p) - let asgn = newNodeI(nkAsgn, a.info, 2) - asgn.sons[0] = a - asgn.sons[1] = b + asgn.add a + asgn.add b result.add(asgn) - elif p.tok.tokType == tkSemicolon: + if p.tok.tokType == tkSemiColon: + semiStmtList(p, result) + elif p.tok.tokType == tkSemiColon: # stmt context: result.add(a) semiStmtList(p, result) else: a = colonOrEquals(p, a) + if a.kind == nkExprColonExpr: + result.transitionSonsKind(nkTupleConstr) result.add(a) if p.tok.tokType == tkComma: getTok(p) skipComment(p, a) + # (1,) produces a tuple expression: + result.transitionSonsKind(nkTupleConstr) + # progress guaranteed while p.tok.tokType != tkParRi and p.tok.tokType != tkEof: var a = exprColonEqExpr(p) - addSon(result, a) - if p.tok.tokType != tkComma: break + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) skipComment(p, a) optPar(p) eat(p, tkParRi) - -proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode = + setEndInfo() + +proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = + #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT + #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT + #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT + #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT + #| | CHAR_LIT | CUSTOM_NUMERIC_LIT + #| | NIL #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT - #| identOrLiteral = generalizedLit | symbol - #| | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT - #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT - #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT - #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT - #| | CHAR_LIT - #| | NIL - #| | par | arrayConstr | setOrTableConstr + #| identOrLiteral = generalizedLit | symbol | literal + #| | par | arrayConstr | setOrTableConstr | tupleConstr #| | castExpr #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' case p.tok.tokType - of tkSymbol: + of tkSymbol, tkBuiltInMagics, tkOut: result = newIdentNodeP(p.tok.ident, p) getTok(p) result = parseGStrLit(p, result) - of tkAccent: + of tkAccent: result = parseSymbol(p) # literals of tkIntLit: result = newIntNodeP(nkIntLit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkInt8Lit: + of tkInt8Lit: result = newIntNodeP(nkInt8Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkInt16Lit: + of tkInt16Lit: result = newIntNodeP(nkInt16Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkInt32Lit: + of tkInt32Lit: result = newIntNodeP(nkInt32Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkInt64Lit: + of tkInt64Lit: result = newIntNodeP(nkInt64Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkUIntLit: + of tkUIntLit: result = newIntNodeP(nkUIntLit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkUInt8Lit: + of tkUInt8Lit: result = newIntNodeP(nkUInt8Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkUInt16Lit: + of tkUInt16Lit: result = newIntNodeP(nkUInt16Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkUInt32Lit: + of tkUInt32Lit: result = newIntNodeP(nkUInt32Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkUInt64Lit: + of tkUInt64Lit: result = newIntNodeP(nkUInt64Lit, p.tok.iNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkFloatLit: + of tkFloatLit: result = newFloatNodeP(nkFloatLit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkFloat32Lit: + of tkFloat32Lit: result = newFloatNodeP(nkFloat32Lit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkFloat64Lit: + of tkFloat64Lit: result = newFloatNodeP(nkFloat64Lit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) @@ -593,19 +801,27 @@ proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode = result = newFloatNodeP(nkFloat128Lit, p.tok.fNumber, p) setBaseFlags(result, p.tok.base) getTok(p) - of tkStrLit: + of tkStrLit: result = newStrNodeP(nkStrLit, p.tok.literal, p) getTok(p) - of tkRStrLit: + of tkRStrLit: result = newStrNodeP(nkRStrLit, p.tok.literal, p) getTok(p) - of tkTripleStrLit: + of tkTripleStrLit: result = newStrNodeP(nkTripleStrLit, p.tok.literal, p) getTok(p) - of tkCharLit: + of tkCharLit: result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p) getTok(p) - of tkNil: + of tkCustomLit: + let splitPos = p.tok.iNumber.int + let str = newStrNodeP(nkRStrLit, p.tok.literal.substr(0, splitPos-1), p) + let callee = newIdentNodeP(getIdent(p.lex.cache, p.tok.literal.substr(splitPos)), p) + result = newNodeP(nkDotExpr, p) + result.add str + result.add callee + getTok(p) + of tkNil: result = newNodeP(nkNilLit, p) getTok(p) of tkParLe: @@ -620,181 +836,270 @@ proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode = of tkBracketLe: # [] constructor result = exprColonEqExprList(p, nkBracket, tkBracketRi) - of tkCast: + of tkCast: result = parseCast(p) else: parMessage(p, errExprExpected, p.tok) - getTok(p) # we must consume a token here to prevend endless loops! - result = ast.emptyNode - -proc primarySuffix(p: var TParser, r: PNode): PNode = - #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? - #| | doBlocks - #| | '.' optInd ('type' | 'addr' | symbol) generalizedLit? - #| | '[' optInd indexExprList optPar ']' - #| | '{' optInd indexExprList optPar '}' + getTok(p) # we must consume a token here to prevent endless loops! + result = p.emptyNode + +proc namedParams(p: var Parser, callee: PNode, + kind: TNodeKind, endTok: TokType): PNode = + let a = callee + result = newNodeP(kind, p) + result.add(a) + # progress guaranteed + exprColonEqExprListAux(p, endTok, result) + +proc commandParam(p: var Parser, isFirstParam: var bool; mode: PrimaryMode): PNode = + if mode == pmTypeDesc: + result = simpleExpr(p, mode) + elif not isFirstParam: + result = exprEqExpr(p) + else: + result = parseExpr(p) + if p.tok.tokType == tkDo: + result = postExprBlocks(p, result) + isFirstParam = false + +proc commandExpr(p: var Parser; r: PNode; mode: PrimaryMode): PNode = + if mode == pmTrySimple: + result = r + else: + result = newNodeP(nkCommand, p) + result.add(r) + var isFirstParam = true + # progress NOT guaranteed + p.hasProgress = false + result.add commandParam(p, isFirstParam, mode) + +proc isDotLike(tok: Token): bool = + result = tok.tokType == tkOpr and tok.ident.s.len > 1 and + tok.ident.s[0] == '.' and tok.ident.s[1] != '.' + +proc primarySuffix(p: var Parser, r: PNode, + baseIndent: int, mode: PrimaryMode): PNode = + #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' + #| | '.' optInd symbolOrKeyword ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit? + #| | DOTLIKEOP optInd symbolOrKeyword generalizedLit? + #| | '[' optInd exprColonEqExprList optPar ']' + #| | '{' optInd exprColonEqExprList optPar '}' + # XXX strong spaces need to be reflected above result = r - while p.tok.indent < 0: + + # progress guaranteed + while p.tok.indent < 0 or + (p.tok.tokType == tkDot and p.tok.indent >= baseIndent): case p.tok.tokType - of tkParLe: - var a = result - result = newNodeP(nkCall, p) - addSon(result, a) - exprColonEqExprListAux(p, tkParRi, result) - if result.len > 1 and result.sons[1].kind == nkExprColonExpr: - result.kind = nkObjConstr - else: - parseDoBlocks(p, result) - of tkDo: - var a = result - result = newNodeP(nkCall, p) - addSon(result, a) - parseDoBlocks(p, result) + of tkParLe: + # progress guaranteed + if tsLeading in p.tok.spacing: + result = commandExpr(p, result, mode) + break + result = namedParams(p, result, nkCall, tkParRi) + if result.has2Sons and result.secondSon.kind == nkExprColonExpr: + result.transitionSonsKind(nkObjConstr) of tkDot: + # progress guaranteed result = dotExpr(p, result) result = parseGStrLit(p, result) - of tkBracketLe: - result = indexExprList(p, result, nkBracketExpr, tkBracketRi) + of tkBracketLe: + # progress guaranteed + if tsLeading in p.tok.spacing: + result = commandExpr(p, result, mode) + break + result = namedParams(p, result, nkBracketExpr, tkBracketRi) of tkCurlyLe: - result = indexExprList(p, result, nkCurlyExpr, tkCurlyRi) - else: break - -proc primary(p: var TParser, mode: TPrimaryMode): PNode + # progress guaranteed + if tsLeading in p.tok.spacing: + result = commandExpr(p, result, mode) + break + result = namedParams(p, result, nkCurlyExpr, tkCurlyRi) + of tkSymbol, tkAccent, tkIntLit..tkCustomLit, tkNil, tkCast, + tkOpr, tkDotDot, tkVar, tkOut, tkStatic, tkType, tkEnum, tkTuple, + tkObject, tkProc: + # XXX: In type sections we allow the free application of the + # command syntax, with the exception of expressions such as + # `foo ref` or `foo ptr`. Unfortunately, these two are also + # used as infix operators for the memory regions feature and + # the current parsing rules don't play well here. + let isDotLike2 = p.tok.isDotLike + if isDotLike2 and p.lex.config.isDefined("nimPreviewDotLikeOps"): + # synchronize with `tkDot` branch + result = dotLikeExpr(p, result) + result = parseGStrLit(p, result) + else: + if isDotLike2: + parMessage(p, warnDotLikeOps, "dot-like operators will be parsed differently with `-d:nimPreviewDotLikeOps`") + if p.inPragma == 0 and (isUnary(p.tok) or p.tok.tokType notin {tkOpr, tkDotDot}): + # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet + # solution, but pragmas.nim can't handle that + result = commandExpr(p, result, mode) + break + else: + break -proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode = - result = primary(p, mode) +proc parseOperators(p: var Parser, headNode: PNode, + limit: int, mode: PrimaryMode): PNode = + result = headNode # expand while operators have priorities higher than 'limit' var opPrec = getPrecedence(p.tok) let modeB = if mode == pmTypeDef: pmTypeDesc else: mode # the operator itself must not start on a new line: - while opPrec >= limit and p.tok.indent < 0: - var leftAssoc = ord(IsLeftAssociative(p.tok)) + # progress guaranteed + while opPrec >= limit and p.tok.indent < 0 and not isUnary(p.tok): + checkBinary(p) + let leftAssoc = ord(not isRightAssociative(p.tok)) var a = newNodeP(nkInfix, p) var opNode = newIdentNodeP(p.tok.ident, p) # skip operator: getTok(p) - optInd(p, opNode) + flexComment(p, a) + optPar(p) # read sub-expression with higher priority: var b = simpleExprAux(p, opPrec + leftAssoc, modeB) - addSon(a, opNode) - addSon(a, result) - addSon(a, b) + a.add(opNode) + a.add(result) + a.add(b) result = a opPrec = getPrecedence(p.tok) - -proc simpleExpr(p: var TParser, mode = pmNormal): PNode = + setEndInfo() + +proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode = + var mode = mode + result = primary(p, mode) + if mode == pmTrySimple: + mode = pmNormal + if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)) and + mode == pmNormal: + var pragmaExp = newNodeP(nkPragmaExpr, p) + pragmaExp.add result + pragmaExp.add p.parsePragma + result = pragmaExp + result = parseOperators(p, result, limit, mode) + +proc simpleExpr(p: var Parser, mode = pmNormal): PNode = + when defined(nimpretty): + inc p.em.doIndentMore result = simpleExprAux(p, -1, mode) + when defined(nimpretty): + dec p.em.doIndentMore -proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode = - #| condExpr = expr colcom expr optInd - #| ('elif' expr colcom expr optInd)* - #| 'else' colcom expr - #| ifExpr = 'if' condExpr - #| whenExpr = 'when' condExpr - result = newNodeP(kind, p) - while true: - getTok(p) # skip `if`, `elif` - var branch = newNodeP(nkElifExpr, p) - addSon(branch, parseExpr(p)) - colcom(p, branch) - addSon(branch, parseExpr(p)) - optInd(p, branch) - addSon(result, branch) - if p.tok.tokType != tkElif: break - var branch = newNodeP(nkElseExpr, p) - eat(p, tkElse) - colcom(p, branch) - addSon(branch, parseExpr(p)) - addSon(result, branch) - -proc parsePragma(p: var TParser): PNode = - #| pragma = '{.' optInd (exprColonExpr comma?)* optPar ('.}' | '}') +proc parsePragma(p: var Parser): PNode = + #| pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}') result = newNodeP(nkPragma, p) + inc p.inPragma + when defined(nimpretty): + inc p.em.doIndentMore + inc p.em.keepIndents getTok(p) optInd(p, result) while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}: + p.hasProgress = false var a = exprColonEqExpr(p) - addSon(result, a) + if not p.hasProgress: break + result.add(a) if p.tok.tokType == tkComma: getTok(p) skipComment(p, a) optPar(p) - if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: getTok(p) - else: parMessage(p, errTokenExpected, ".}") - -proc identVis(p: var TParser): PNode = - #| identVis = symbol opr? # postfix position + if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: + when defined(nimpretty): + if p.tok.tokType == tkCurlyRi: curlyRiWasPragma(p.em) + getTok(p) + else: + parMessage(p, "expected '.}'") + dec p.inPragma + when defined(nimpretty): + dec p.em.doIndentMore + dec p.em.keepIndents + setEndInfo() + +proc identVis(p: var Parser; allowDot=false): PNode = + #| identVis = symbol OPR? # postfix position + #| identVisDot = symbol '.' optInd symbolOrKeyword OPR? var a = parseSymbol(p) - if p.tok.tokType == tkOpr: + if p.tok.tokType == tkOpr: + when defined(nimpretty): + starWasExportMarker(p.em) result = newNodeP(nkPostfix, p) - addSon(result, newIdentNodeP(p.tok.ident, p)) - addSon(result, a) + result.add(newIdentNodeP(p.tok.ident, p)) + result.add(a) getTok(p) - else: + elif p.tok.tokType == tkDot and allowDot: + result = dotExpr(p, a) + else: result = a - -proc identWithPragma(p: var TParser): PNode = + +proc identWithPragma(p: var Parser; allowDot=false): PNode = #| identWithPragma = identVis pragma? - var a = identVis(p) - if p.tok.tokType == tkCurlyDotLe: + #| identWithPragmaDot = identVisDot pragma? + var a = identVis(p, allowDot) + if p.tok.tokType == tkCurlyDotLe: result = newNodeP(nkPragmaExpr, p) - addSon(result, a) - addSon(result, parsePragma(p)) - else: + result.add(a) + result.add(parsePragma(p)) + else: result = a type - TDeclaredIdentFlag = enum + DeclaredIdentFlag = enum withPragma, # identifier may have pragma withBothOptional # both ':' and '=' parts are optional - TDeclaredIdentFlags = set[TDeclaredIdentFlag] + withDot # allow 'var ident.ident = value' + DeclaredIdentFlags = set[DeclaredIdentFlag] -proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = +proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode = #| declColonEquals = identWithPragma (comma identWithPragma)* comma? - #| (':' optInd typeDesc)? ('=' optInd expr)? - #| identColonEquals = ident (comma ident)* comma? - #| (':' optInd typeDesc)? ('=' optInd expr)?) + #| (':' optInd typeDescExpr)? ('=' optInd expr)? + #| identColonEquals = IDENT (comma IDENT)* comma? + #| (':' optInd typeDescExpr)? ('=' optInd expr)?) var a: PNode result = newNodeP(nkIdentDefs, p) - while true: + # progress guaranteed + while true: case p.tok.tokType - of tkSymbol, tkAccent: - if withPragma in flags: a = identWithPragma(p) + of tkSymbol, tkAccent: + if withPragma in flags: a = identWithPragma(p, allowDot=withDot in flags) else: a = parseSymbol(p) - if a.kind == nkEmpty: return - else: break - addSon(result, a) - if p.tok.tokType != tkComma: break + if a.kind == nkEmpty: return + else: break + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) - if p.tok.tokType == tkColon: + if p.tok.tokType == tkColon: getTok(p) optInd(p, result) - addSon(result, parseTypeDesc(p)) - else: - addSon(result, ast.emptyNode) - if (p.tok.tokType != tkEquals) and not (withBothOptional in flags): - parMessage(p, errColonOrEqualsExpected, p.tok) - if p.tok.tokType == tkEquals: + result.add(parseTypeDesc(p, fullExpr = true)) + else: + result.add(newNodeP(nkEmpty, p)) + if p.tok.tokType != tkEquals and withBothOptional notin flags: + parMessage(p, "':' or '=' expected, but got '$1'", p.tok) + if p.tok.tokType == tkEquals: getTok(p) optInd(p, result) - addSon(result, parseExpr(p)) - else: - addSon(result, ast.emptyNode) - -proc parseTuple(p: var TParser, indentAllowed = false): PNode = - #| inlTupleDecl = 'tuple' - #| [' optInd (identColonEquals (comma/semicolon)?)* optPar ']' - #| extTupleDecl = 'tuple' - #| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? + result.add(parseExpr(p)) + else: + result.add(newNodeP(nkEmpty, p)) + setEndInfo() + +proc parseTuple(p: var Parser, indentAllowed = false): PNode = + #| tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' + #| tupleType = 'tuple' tupleTypeBracket + #| tupleDecl = 'tuple' (tupleTypeBracket / + #| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?) result = newNodeP(nkTupleTy, p) getTok(p) if p.tok.tokType == tkBracketLe: getTok(p) optInd(p, result) + # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent}: var a = parseIdentColonEquals(p, {}) - addSon(result, a) - if p.tok.tokType notin {tkComma, tkSemicolon}: break + result.add(a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + when defined(nimpretty): + commaWasSemicolon(p.em) getTok(p) skipComment(p, a) optPar(p) @@ -803,364 +1108,604 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode = skipComment(p, result) if realInd(p): withInd(p): - skipComment(p, result) + rawSkipComment(p, result) + # progress guaranteed while true: case p.tok.tokType of tkSymbol, tkAccent: var a = parseIdentColonEquals(p, {}) - skipComment(p, a) - addSon(result, a) + if p.tok.indent < 0 or p.tok.indent >= p.currInd: + rawSkipComment(p, a) + result.add(a) of tkEof: break else: parMessage(p, errIdentifierExpected, p.tok) break if not sameInd(p): break + elif p.tok.tokType == tkParLe: + parMessage(p, errGenerated, "the syntax for tuple types is 'tuple[...]', not 'tuple(...)'") + else: + result = newNodeP(nkTupleClassTy, p) + setEndInfo() -proc parseParamList(p: var TParser, retColon = true): PNode = +proc parseParamList(p: var Parser, retColon = true): PNode = #| paramList = '(' declColonEquals ^* (comma/semicolon) ')' #| paramListArrow = paramList? ('->' optInd typeDesc)? #| paramListColon = paramList? (':' optInd typeDesc)? var a: PNode result = newNodeP(nkFormalParams, p) - addSon(result, ast.emptyNode) # return type - if p.tok.tokType == tkParLe and p.tok.indent < 0: + result.add(p.emptyNode) # return type + when defined(nimpretty): + inc p.em.doIndentMore + inc p.em.keepIndents + let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0 + if hasParLe: getTok(p) optInd(p, result) + # progress guaranteed while true: case p.tok.tokType - of tkSymbol, tkAccent: + of tkSymbol, tkAccent: a = parseIdentColonEquals(p, {withBothOptional, withPragma}) - of tkParRi: - break - else: - parMessage(p, errTokenExpected, ")") - break - addSon(result, a) - if p.tok.tokType notin {tkComma, tkSemicolon}: break + of tkParRi: + break + of tkVar: + parMessage(p, errGenerated, "the syntax is 'parameter: var T', not 'var parameter: T'") + break + else: + parMessage(p, "expected closing ')'") + break + result.add(a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + when defined(nimpretty): + commaWasSemicolon(p.em) getTok(p) skipComment(p, a) optPar(p) eat(p, tkParRi) let hasRet = if retColon: p.tok.tokType == tkColon - else: p.tok.tokType == tkOpr and IdentEq(p.tok.ident, "->") + else: p.tok.tokType == tkOpr and p.tok.ident.s == "->" if hasRet and p.tok.indent < 0: getTok(p) optInd(p, result) - result.sons[0] = parseTypeDesc(p) - -proc optPragmas(p: var TParser): PNode = + result.replaceFirstSon parseTypeDesc(p) + elif not retColon and not hasParLe: + # Mark as "not there" in order to mark for deprecation in the semantic pass: + result = p.emptyNode + when defined(nimpretty): + dec p.em.doIndentMore + dec p.em.keepIndents + setEndInfo() + +proc optPragmas(p: var Parser): PNode = if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)): result = parsePragma(p) else: - result = ast.emptyNode + result = p.emptyNode -proc parseDoBlock(p: var TParser): PNode = - #| doBlock = 'do' paramListArrow pragmas? colcom stmt - let info = parLineInfo(p) - getTok(p) - let params = parseParamList(p, retColon=false) +proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = + #| doBlock = 'do' paramListArrow pragma? colcom stmt + result = nil + var params = parseParamList(p, retColon=false) let pragmas = optPragmas(p) - eat(p, tkColon) - skipComment(p, result) - result = newProcNode(nkDo, info, parseStmt(p), - params = params, - pragmas = pragmas) - -proc parseDoBlocks(p: var TParser, call: PNode) = - #| doBlocks = doBlock ^* IND{=} - if p.tok.tokType == tkDo: - addSon(call, parseDoBlock(p)) - while sameInd(p) and p.tok.tokType == tkDo: - addSon(call, parseDoBlock(p)) - -proc parseProcExpr(p: var TParser, isExpr: bool): PNode = - #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)? + colcom(p, result) + result = parseStmt(p) + if params.kind != nkEmpty or pragmas.kind != nkEmpty: + if params.kind == nkEmpty: + params = newNodeP(nkFormalParams, p) + params.add(p.emptyNode) # return type + result = newProcNode(nkDo, info, + body = result, params = params, name = p.emptyNode, pattern = p.emptyNode, + genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) + setEndInfo() + +proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = + #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? + #| routineType = ('proc' | 'iterator') paramListColon pragma? # either a proc type or a anonymous proc let info = parLineInfo(p) - getTok(p) let hasSignature = p.tok.tokType in {tkParLe, tkColon} and p.tok.indent < 0 let params = parseParamList(p) let pragmas = optPragmas(p) - if p.tok.tokType == tkEquals and isExpr: + if p.tok.tokType == tkEquals and isExpr: getTok(p) + result = newProcNode(kind, info, body = p.emptyNode, + params = params, name = p.emptyNode, pattern = p.emptyNode, + genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) skipComment(p, result) - result = newProcNode(nkLambda, info, parseStmt(p), - params = params, - pragmas = pragmas) + result.replaceSon bodyPos, parseStmt(p) else: - result = newNodeI(nkProcTy, info) - if hasSignature: - addSon(result, params) - addSon(result, pragmas) - -proc isExprStart(p: TParser): bool = + result = newNode(if kind == nkIteratorDef: nkIteratorTy else: nkProcTy, info) + if hasSignature or pragmas.kind != nkEmpty: + if hasSignature: + result.add(params) + else: # pragmas but no param list, implies typeclass with pragmas + result.add(p.emptyNode) + if kind == nkFuncDef: + parMessage(p, "func keyword is not allowed in type descriptions, use proc with {.noSideEffect.} pragma instead") + result.add(pragmas) + setEndInfo() + +proc isExprStart(p: Parser): bool = case p.tok.tokType - of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, - tkProc, tkIterator, tkBind, tkAddr, - tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, - tkTuple, tkObject, tkType, tkWhen, tkCase, tkShared: + of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor, + tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics, + tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr, + tkEnum, tkTuple, tkObject, tkWhen, tkCase, tkOut, tkTry, tkBlock: result = true else: result = false - -proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, - mode: TPrimaryMode): PNode = + +proc parseSymbolList(p: var Parser, result: PNode) = + # progress guaranteed + while true: + var s = parseSymbol(p, smAllowNil) + if s.kind == nkEmpty: break + result.add(s) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, s) + setEndInfo() + +proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, + mode: PrimaryMode): PNode = result = newNodeP(kind, p) getTok(p) + if p.tok.indent != -1 and p.tok.indent <= p.currInd: return optInd(p, result) + let isTypedef = mode == pmTypeDef and p.tok.tokType in {tkObject, tkTuple} if not isOperator(p.tok) and isExprStart(p): - addSon(result, primary(p, mode)) + if isTypedef: + result.add(parseTypeDefValue(p)) + else: + result.add(primary(p, mode)) + if kind == nkDistinctTy and p.tok.tokType == tkSymbol: + # XXX document this feature! + var nodeKind: TNodeKind + if p.tok.ident.s == "with": + nodeKind = nkWith + elif p.tok.ident.s == "without": + nodeKind = nkWithout + else: + return result + getTok(p) + let list = newNodeP(nodeKind, p) + result.add list + parseSymbolList(p, list) + if mode == pmTypeDef and not isTypedef: + result = parseOperators(p, result, -1, mode) + setEndInfo() + +proc parseVarTuple(p: var Parser): PNode + +proc parseFor(p: var Parser): PNode = + #| forStmt = 'for' ((varTuple / identWithPragma) ^+ comma) 'in' expr colcom stmt + #| forExpr = forStmt + getTokNoInd(p) + result = newNodeP(nkForStmt, p) + if p.tok.tokType == tkParLe: + result.add(parseVarTuple(p)) + else: + var a = identWithPragma(p) + result.add(a) + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, a) + if p.tok.tokType == tkParLe: + result.add(parseVarTuple(p)) + break + a = identWithPragma(p) + result.add(a) + eat(p, tkIn) + result.add(parseExpr(p)) + colcom(p, result) + result.add(parseStmt(p)) + setEndInfo() -proc parseExpr(p: var TParser): PNode = - #| expr = (ifExpr +template nimprettyDontTouch(body) = + when defined(nimpretty): + inc p.em.keepIndents + body + when defined(nimpretty): + dec p.em.keepIndents + +proc parseExpr(p: var Parser): PNode = + #| expr = (blockExpr + #| | ifExpr #| | whenExpr - #| | caseExpr - #| | tryStmt) + #| | caseStmt + #| | forExpr + #| | tryExpr) #| / simpleExpr - case p.tok.tokType: - of tkIf: result = parseIfExpr(p, nkIfExpr) - of tkWhen: result = parseIfExpr(p, nkWhenExpr) - of tkCase: result = parseCase(p) - of tkTry: result = parseTry(p) + case p.tok.tokType + of tkBlock: + nimprettyDontTouch: + result = parseBlock(p) + of tkIf: + nimprettyDontTouch: + result = parseIfOrWhenExpr(p, nkIfExpr) + of tkFor: + nimprettyDontTouch: + result = parseFor(p) + of tkWhen: + nimprettyDontTouch: + result = parseIfOrWhenExpr(p, nkWhenStmt) + of tkCase: + # Currently we think nimpretty is good enough with case expressions, + # so it is allowed to touch them: + #nimprettyDontTouch: + result = parseCase(p) + of tkTry: + nimprettyDontTouch: + result = parseTry(p, isExpr=true) else: result = simpleExpr(p) - -proc parseObject(p: var TParser): PNode -proc parseDistinct(p: var TParser): PNode -proc parseEnum(p: var TParser): PNode - -proc primary(p: var TParser, mode: TPrimaryMode): PNode = - #| typeKeyw = 'var' | 'ref' | 'ptr' | 'shared' | 'type' | 'tuple' - #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' - #| primary = typeKeyw typeDescK - #| / prefixOperator* identOrLiteral primarySuffix* - #| / 'addr' primary - #| / 'static' primary - #| / 'bind' primary + setEndInfo() + +proc parseEnum(p: var Parser): PNode +proc parseObject(p: var Parser): PNode +proc parseTypeClass(p: var Parser): PNode + +proc primary(p: var Parser, mode: PrimaryMode): PNode = + #| simplePrimary = SIGILLIKEOP? identOrLiteral primarySuffix* + #| commandStart = &('`'|IDENT|literal|'cast'|'addr'|'type'|'var'|'out'| + #| 'static'|'enum'|'tuple'|'object'|'proc') + #| primary = simplePrimary (commandStart expr (doBlock extraPostExprBlock*)?)? + #| / operatorB primary + #| / routineExpr + #| / rawTypeDesc + #| / prefixOperator primary + # XXX strong spaces need to be reflected in commandStart + # command part is handled in the primarySuffix proc + + # prefix operators: if isOperator(p.tok): - let isSigil = IsSigilLike(p.tok) + # Note 'sigil like' operators are currently not reflected in the grammar + # and should be removed for Nim 2.0, I don't think anybody uses them. + let isSigil = isSigilLike(p.tok) result = newNodeP(nkPrefix, p) var a = newIdentNodeP(p.tok.ident, p) - addSon(result, a) + result.add(a) getTok(p) optInd(p, a) - if isSigil: - #XXX prefix operators - addSon(result, primary(p, pmSkipSuffix)) - result = primarySuffix(p, result) + const identOrLiteralKinds = tkBuiltInMagics + {tkSymbol, tkAccent, tkNil, + tkIntLit..tkCustomLit, tkCast, tkOut, tkParLe, tkBracketLe, tkCurlyLe} + if isSigil and p.tok.tokType in identOrLiteralKinds: + let baseInd = p.lex.currLineIndent + result.add(identOrLiteral(p, mode)) + result = primarySuffix(p, result, baseInd, mode) else: - addSon(result, primary(p, pmNormal)) + result.add(primary(p, pmNormal)) return - - case p.tok.tokType: - of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode) - of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode) - of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode) - of tkShared: result = parseTypeDescKAux(p, nkSharedTy, mode) - of tkType: result = parseTypeDescKAux(p, nkTypeOfExpr, mode) - of tkTuple: result = parseTuple(p, mode == pmTypeDef) - of tkProc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}) + + case p.tok.tokType + of tkProc: + getTok(p) + result = parseProcExpr(p, mode != pmTypeDesc, nkLambda) + of tkFunc: + getTok(p) + result = parseProcExpr(p, mode != pmTypeDesc, nkFuncDef) of tkIterator: - if mode in {pmTypeDesc, pmTypeDef}: - result = parseProcExpr(p, false) - result.kind = nkIteratorTy - else: - # no anon iterators for now: - parMessage(p, errExprExpected, p.tok) - getTok(p) # we must consume a token here to prevend endless loops! - result = ast.emptyNode - of tkEnum: - if mode == pmTypeDef: - result = parseEnum(p) - else: - result = newNodeP(nkEnumTy, p) - getTok(p) - of tkObject: - if mode == pmTypeDef: - result = parseObject(p) - else: - result = newNodeP(nkObjectTy, p) - getTok(p) - of tkDistinct: - if mode == pmTypeDef: - result = parseDistinct(p) - else: - result = newNodeP(nkDistinctTy, p) - getTok(p) - of tkAddr: - result = newNodeP(nkAddr, p) - getTokNoInd(p) - addSon(result, primary(p, pmNormal)) - of tkStatic: - result = newNodeP(nkStaticExpr, p) - getTokNoInd(p) - addSon(result, primary(p, pmNormal)) + getTok(p) + result = parseProcExpr(p, mode != pmTypeDesc, nkIteratorDef) of tkBind: + # legacy syntax, no-op in current nim result = newNodeP(nkBind, p) getTok(p) optInd(p, result) - addSon(result, primary(p, pmNormal)) + result.add(primary(p, pmNormal)) + of tkTuple, tkEnum, tkObject, tkConcept, + tkVar, tkOut, tkRef, tkPtr, tkDistinct: + result = parseTypeDesc(p) else: + let baseInd = p.lex.currLineIndent result = identOrLiteral(p, mode) - if mode != pmSkipSuffix: - result = primarySuffix(p, result) + result = primarySuffix(p, result, baseInd, mode) -proc parseTypeDesc(p: var TParser): PNode = - #| typeDesc = simpleExpr - result = simpleExpr(p, pmTypeDesc) +proc binaryNot(p: var Parser; a: PNode): PNode = + if p.tok.tokType == tkNot and p.tok.indent < 0: + let notOpr = newIdentNodeP(p.tok.ident, p) + getTok(p) + optInd(p, notOpr) + let b = primary(p, pmTypeDesc) + result = newNodeP(nkInfix, p) + result.add notOpr + result.add a + result.add b + else: + result = a -proc parseTypeDefAux(p: var TParser): PNode = - #| typeDefAux = simpleExpr - result = simpleExpr(p, pmTypeDef) +proc parseTypeDesc(p: var Parser, fullExpr = false): PNode = + #| rawTypeDesc = (tupleType | routineType | 'enum' | 'object' | + #| ('var' | 'out' | 'ref' | 'ptr' | 'distinct') typeDesc?) + #| ('not' primary)? + #| typeDescExpr = (routineType / simpleExpr) ('not' primary)? + #| typeDesc = rawTypeDesc / typeDescExpr + newlineWasSplitting(p) + if fullExpr: + result = simpleExpr(p, pmTypeDesc) + else: + case p.tok.tokType + of tkTuple: + result = parseTuple(p, false) + of tkProc: + getTok(p) + result = parseProcExpr(p, false, nkLambda) + of tkIterator: + getTok(p) + result = parseProcExpr(p, false, nkIteratorDef) + of tkEnum: + result = newNodeP(nkEnumTy, p) + getTok(p) + of tkObject: + result = newNodeP(nkObjectTy, p) + getTok(p) + of tkConcept: + result = p.emptyNode + parMessage(p, "the 'concept' keyword is only valid in 'type' sections") + of tkVar: result = parseTypeDescKAux(p, nkVarTy, pmTypeDesc) + of tkOut: result = parseTypeDescKAux(p, nkOutTy, pmTypeDesc) + of tkRef: result = parseTypeDescKAux(p, nkRefTy, pmTypeDesc) + of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, pmTypeDesc) + of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, pmTypeDesc) + else: + result = simpleExpr(p, pmTypeDesc) + result = binaryNot(p, result) + setEndInfo() + +proc parseTypeDefValue(p: var Parser): PNode = + #| typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl | + #| ('ref' | 'ptr' | 'distinct') (tupleDecl | objectDecl)) + #| / (simpleExpr (exprEqExpr ^+ comma postExprBlocks?)?)) + #| ('not' primary)? + case p.tok.tokType + of tkTuple: result = parseTuple(p, true) + of tkRef: result = parseTypeDescKAux(p, nkRefTy, pmTypeDef) + of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, pmTypeDef) + of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, pmTypeDef) + of tkEnum: + prettySection: + result = parseEnum(p) + of tkObject: + prettySection: + result = parseObject(p) + of tkConcept: + result = parseTypeClass(p) + else: + result = simpleExpr(p, pmTypeDef) + if p.tok.tokType != tkNot: + if result.kind == nkCommand: + var isFirstParam = false + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, result) + result.add(commandParam(p, isFirstParam, pmTypeDef)) + result = postExprBlocks(p, result) + result = binaryNot(p, result) + setEndInfo() proc makeCall(n: PNode): PNode = + ## Creates a call if the given node isn't already a call. if n.kind in nkCallKinds: result = n else: - result = newNodeI(nkCall, n.info) + result = newNode(nkCall, n.info) result.add n -proc parseExprStmt(p: var TParser): PNode = - #| exprStmt = simpleExpr - #| (( '=' optInd expr ) - #| / ( expr ^+ comma - #| doBlocks - #| / ':' stmt? ( IND{=} 'of' exprList ':' stmt - #| | IND{=} 'elif' expr ':' stmt - #| | IND{=} 'except' exprList ':' stmt - #| | IND{=} 'else' ':' stmt )* - #| ))? - var a = simpleExpr(p) - if p.tok.tokType == tkEquals: +proc postExprBlocks(p: var Parser, x: PNode): PNode = + #| extraPostExprBlock = ( IND{=} doBlock + #| | IND{=} 'of' exprList ':' stmt + #| | IND{=} 'elif' expr ':' stmt + #| | IND{=} 'except' optionalExprList ':' stmt + #| | IND{=} 'finally' ':' stmt + #| | IND{=} 'else' ':' stmt ) + #| postExprBlocks = (doBlock / ':' (extraPostExprBlock / stmt)) extraPostExprBlock* + result = x + if p.tok.indent >= 0: return + + var + openingParams = p.emptyNode + openingPragmas = p.emptyNode + + if p.tok.tokType == tkDo: + getTok(p) + openingParams = parseParamList(p, retColon=false) + openingPragmas = optPragmas(p) + + if p.tok.tokType == tkColon: + result = makeCall(result) + getTok(p) + skipComment(p, result) + if not (p.tok.tokType in {tkOf, tkElif, tkElse, tkExcept, tkFinally} and sameInd(p)): + var stmtList = newNodeP(nkStmtList, p) + stmtList.add parseStmt(p) + # to keep backwards compatibility (see tests/vm/tstringnil) + if stmtList.firstSon.kind == nkStmtList: stmtList = stmtList.firstSon + + setNodeFlag stmtList, nfBlockArg + if openingParams.kind != nkEmpty or openingPragmas.kind != nkEmpty: + if openingParams.kind == nkEmpty: + openingParams = newNodeP(nkFormalParams, p) + openingParams.add(p.emptyNode) # return type + result.add newProcNode(nkDo, stmtList.info, body = stmtList, + params = openingParams, + name = p.emptyNode, pattern = p.emptyNode, + genericParams = p.emptyNode, + pragmas = openingPragmas, + exceptions = p.emptyNode) + else: + result.add stmtList + + while sameInd(p): + var nextBlock: PNode + let nextToken = p.tok.tokType + if nextToken == tkDo: + let info = parLineInfo(p) + getTok(p) + nextBlock = parseDoBlock(p, info) + else: + case nextToken + of tkOf: + nextBlock = newNodeP(nkOfBranch, p) + exprList(p, tkColon, nextBlock) + of tkElif: + nextBlock = newNodeP(nkElifBranch, p) + getTok(p) + optInd(p, nextBlock) + nextBlock.add parseExpr(p) + of tkExcept: + nextBlock = newNodeP(nkExceptBranch, p) + optionalExprList(p, tkColon, nextBlock) + of tkFinally: + nextBlock = newNodeP(nkFinally, p) + getTok(p) + of tkElse: + nextBlock = newNodeP(nkElse, p) + getTok(p) + else: break + eat(p, tkColon) + nextBlock.add parseStmt(p) + + setNodeFlag nextBlock, nfBlockArg + result.add nextBlock + + if nextBlock.kind in {nkElse, nkFinally}: break + else: + if openingParams.kind != nkEmpty: + parMessage(p, "expected ':'") + +proc parseExprStmt(p: var Parser): PNode = + #| exprStmt = simpleExpr postExprBlocks? + #| / simplePrimary (exprEqExpr ^+ comma) postExprBlocks? + #| / simpleExpr '=' optInd (expr postExprBlocks?) + var a = simpleExpr(p, pmTrySimple) + if p.tok.tokType == tkEquals: + result = newNodeP(nkAsgn, p) getTok(p) optInd(p, result) var b = parseExpr(p) - result = newNodeI(nkAsgn, a.info) - addSon(result, a) - addSon(result, b) + b = postExprBlocks(p, b) + result.add(a) + result.add(b) else: + var isFirstParam = false + # if an expression is starting here, a simplePrimary was parsed and + # this is the start of a command if p.tok.indent < 0 and isExprStart(p): - result = newNode(nkCommand, a.info, @[a]) + result = newTree(nkCommand, a.info, a) + let baseIndent = p.currInd while true: - var e = parseExpr(p) - addSon(result, e) - if p.tok.tokType != tkComma: break + result.add(commandParam(p, isFirstParam, pmNormal)) + if p.tok.tokType != tkComma or + (p.tok.indent >= 0 and p.tok.indent < baseIndent): + break getTok(p) optInd(p, result) else: result = a - if p.tok.tokType == tkDo and p.tok.indent < 0: - result = makeCall(result) - parseDoBlocks(p, result) - return result - if p.tok.tokType == tkColon and p.tok.indent < 0: - result = makeCall(result) + result = postExprBlocks(p, result) + setEndInfo() + +proc parseModuleName(p: var Parser, kind: TNodeKind): PNode = + result = parseExpr(p) + when false: + # parseExpr already handles 'as' syntax ... + if p.tok.tokType == tkAs and kind == nkImportStmt: + let a = result + result = newNodeP(nkImportAs, p) getTok(p) - skipComment(p, result) - if p.tok.TokType notin {tkOf, tkElif, tkElse, tkExcept}: - let body = parseStmt(p) - addSon(result, newProcNode(nkDo, body.info, body)) - while sameInd(p): - var b: PNode - case p.tok.tokType - of tkOf: - b = newNodeP(nkOfBranch, p) - exprList(p, tkColon, b) - of tkElif: - b = newNodeP(nkElifBranch, p) - getTok(p) - optInd(p, b) - addSon(b, parseExpr(p)) - eat(p, tkColon) - of tkExcept: - b = newNodeP(nkExceptBranch, p) - exprList(p, tkColon, b) - skipComment(p, b) - of tkElse: - b = newNodeP(nkElse, p) - getTok(p) - eat(p, tkColon) - else: break - addSon(b, parseStmt(p)) - addSon(result, b) - if b.kind == nkElse: break + result.add(a) + result.add(parseExpr(p)) + setEndInfo() -proc parseImport(p: var TParser, kind: TNodeKind): PNode = +proc parseImport(p: var Parser, kind: TNodeKind): PNode = #| importStmt = 'import' optInd expr #| ((comma expr)* #| / 'except' optInd (expr ^+ comma)) + #| exportStmt = 'export' optInd expr + #| ((comma expr)* + #| / 'except' optInd (expr ^+ comma)) result = newNodeP(kind, p) getTok(p) # skip `import` or `export` optInd(p, result) - var a = parseExpr(p) - addSon(result, a) + var a = parseModuleName(p, kind) + result.add(a) if p.tok.tokType in {tkComma, tkExcept}: if p.tok.tokType == tkExcept: - result.kind = succ(kind) + result.transitionSonsKind(succ(kind)) getTok(p) optInd(p, result) while true: # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: - a = parseExpr(p) - if a.kind == nkEmpty: break - addSon(result, a) - if p.tok.tokType != tkComma: break + p.hasProgress = false + a = parseModuleName(p, kind) + if a.kind == nkEmpty or not p.hasProgress: break + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() -proc parseIncludeStmt(p: var TParser): PNode = +proc parseIncludeStmt(p: var Parser): PNode = #| includeStmt = 'include' optInd expr ^+ comma result = newNodeP(nkIncludeStmt, p) getTok(p) # skip `import` or `include` optInd(p, result) while true: # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: + p.hasProgress = false var a = parseExpr(p) - if a.kind == nkEmpty: break - addSon(result, a) - if p.tok.tokType != tkComma: break + if a.kind == nkEmpty or not p.hasProgress: break + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() -proc parseFromStmt(p: var TParser): PNode = +proc parseFromStmt(p: var Parser): PNode = #| fromStmt = 'from' expr 'import' optInd expr (comma expr)* result = newNodeP(nkFromStmt, p) getTok(p) # skip `from` optInd(p, result) - var a = parseExpr(p) - addSon(result, a) #optInd(p, a); + var a = parseModuleName(p, nkImportStmt) + result.add(a) #optInd(p, a); eat(p, tkImport) optInd(p, result) while true: # p.tok.tokType notin {tkEof, tkSad, tkDed}: + p.hasProgress = false a = parseExpr(p) - if a.kind == nkEmpty: break - addSon(result, a) - if p.tok.tokType != tkComma: break + if a.kind == nkEmpty or not p.hasProgress: break + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() -proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode = +proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = #| returnStmt = 'return' optInd expr? #| raiseStmt = 'raise' optInd expr? #| yieldStmt = 'yield' optInd expr? #| discardStmt = 'discard' optInd expr? #| breakStmt = 'break' optInd expr? - #| continueStmt = 'break' optInd expr? + #| continueStmt = 'continue' optInd expr? result = newNodeP(kind, p) getTok(p) if p.tok.tokType == tkComment: skipComment(p, result) - addSon(result, ast.emptyNode) - elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or - p.tok.tokType == tkEof: + result.add(p.emptyNode) + elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p): # NL terminates: - addSon(result, ast.emptyNode) + result.add(p.emptyNode) + # nimpretty here! else: - addSon(result, parseExpr(p)) + var e = parseExpr(p) + e = postExprBlocks(p, e) + result.add(e) + setEndInfo() -proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode = +proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = #| condStmt = expr colcom stmt COMMENT? #| (IND{=} 'elif' expr colcom stmt)* #| (IND{=} 'else' colcom stmt)? @@ -1171,31 +1716,56 @@ proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode = getTok(p) # skip `if`, `when`, `elif` var branch = newNodeP(nkElifBranch, p) optInd(p, branch) - addSon(branch, parseExpr(p)) - eat(p, tkColon) - skipComment(p, branch) - addSon(branch, parseStmt(p)) + branch.add(parseExpr(p)) + colcom(p, branch) + branch.add(parseStmt(p)) skipComment(p, branch) - addSon(result, branch) + result.add(branch) if p.tok.tokType != tkElif or not sameOrNoInd(p): break if p.tok.tokType == tkElse and sameOrNoInd(p): var branch = newNodeP(nkElse, p) eat(p, tkElse) - eat(p, tkColon) + colcom(p, branch) + branch.add(parseStmt(p)) + result.add(branch) + setEndInfo() + +proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = + #| condExpr = expr colcom stmt optInd + #| ('elif' expr colcom stmt optInd)* + #| 'else' colcom stmt + #| ifExpr = 'if' condExpr + #| whenExpr = 'when' condExpr + result = newNodeP(kind, p) + while true: + getTok(p) # skip `if`, `when`, `elif` + var branch = newNodeP(nkElifExpr, p) + optInd(p, branch) + branch.add(parseExpr(p)) + colcom(p, branch) + branch.add(parseStmt(p)) skipComment(p, branch) - addSon(branch, parseStmt(p)) - addSon(result, branch) + result.add(branch) + if p.tok.tokType != tkElif: break + if p.tok.tokType == tkElse: + var branch = newNodeP(nkElseExpr, p) + eat(p, tkElse) + colcom(p, branch) + branch.add(parseStmt(p)) + result.add(branch) + setEndInfo() -proc parseWhile(p: var TParser): PNode = +proc parseWhile(p: var Parser): PNode = #| whileStmt = 'while' expr colcom stmt result = newNodeP(nkWhileStmt, p) getTok(p) optInd(p, result) - addSon(result, parseExpr(p)) + result.add(parseExpr(p)) colcom(p, result) - addSon(result, parseStmt(p)) + result.add(parseStmt(p)) + setEndInfo() -proc parseCase(p: var TParser): PNode = +proc parseCase(p: var Parser): PNode = #| ofBranch = 'of' exprList colcom stmt #| ofBranches = ofBranch (IND{=} ofBranch)* #| (IND{=} 'elif' expr colcom stmt)* @@ -1205,19 +1775,19 @@ proc parseCase(p: var TParser): PNode = #| | IND{=} ofBranches) var b: PNode - inElif= false + inElif = false wasIndented = false result = newNodeP(nkCaseStmt, p) getTok(p) - addSon(result, parseExpr(p)) + result.add(parseExpr(p)) if p.tok.tokType == tkColon: getTok(p) skipComment(p, result) - + let oldInd = p.currInd if realInd(p): p.currInd = p.tok.indent wasIndented = true - + while sameInd(p): case p.tok.tokType of tkOf: @@ -1229,349 +1799,366 @@ proc parseCase(p: var TParser): PNode = b = newNodeP(nkElifBranch, p) getTok(p) optInd(p, b) - addSon(b, parseExpr(p)) - eat(p, tkColon) + b.add(parseExpr(p)) of tkElse: b = newNodeP(nkElse, p) getTok(p) - eat(p, tkColon) else: break - skipComment(p, b) - addSon(b, parseStmt(p)) - addSon(result, b) + colcom(p, b) + b.add(parseStmt(p)) + result.add(b) if b.kind == nkElse: break - + if wasIndented: p.currInd = oldInd - -proc parseTry(p: var TParser): PNode = + setEndInfo() + +proc parseTry(p: var Parser; isExpr: bool): PNode = #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') - #| (IND{=}? 'except' exprList colcom stmt)* + #| (IND{=}? 'except' optionalExprList colcom stmt)* #| (IND{=}? 'finally' colcom stmt)? + #| tryExpr = 'try' colcom stmt &(optInd 'except'|'finally') + #| (optInd 'except' optionalExprList colcom stmt)* + #| (optInd 'finally' colcom stmt)? result = newNodeP(nkTryStmt, p) + let parentIndent = p.currInd # isExpr getTok(p) - eat(p, tkColon) - skipComment(p, result) - addSon(result, parseStmt(p)) + colcom(p, result) + result.add(parseStmt(p)) var b: PNode = nil - while sameOrNoInd(p): + + while sameOrNoInd(p) or (isExpr and parentIndent <= p.tok.indent): case p.tok.tokType - of tkExcept: + of tkExcept: b = newNodeP(nkExceptBranch, p) - exprList(p, tkColon, b) - of tkFinally: + optionalExprList(p, tkColon, b) + of tkFinally: b = newNodeP(nkFinally, p) - getTokNoInd(p) - eat(p, tkColon) + getTok(p) else: break - skipComment(p, b) - addSon(b, parseStmt(p)) - addSon(result, b) - if b.kind == nkFinally: break - if b == nil: parMessage(p, errTokenExpected, "except") - -proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode = - #| exceptBlock = 'except' colcom stmt - result = newNodeP(kind, p) - getTokNoInd(p) - colcom(p, result) - addSon(result, parseStmt(p)) + colcom(p, b) + b.add(parseStmt(p)) + result.add(b) + if b == nil: parMessage(p, "expected 'except'") + setEndInfo() -proc parseFor(p: var TParser): PNode = - #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt - result = newNodeP(nkForStmt, p) - getTokNoInd(p) - var a = identWithPragma(p) - addSon(result, a) - while p.tok.tokType == tkComma: - getTok(p) - optInd(p, a) - a = identWithPragma(p) - addSon(result, a) - eat(p, tkIn) - addSon(result, parseExpr(p)) +proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode = + result = newNodeP(kind, p) + getTok(p) colcom(p, result) - addSon(result, parseStmt(p)) + result.add(parseStmt(p)) + setEndInfo() -proc parseBlock(p: var TParser): PNode = +proc parseBlock(p: var Parser): PNode = #| blockStmt = 'block' symbol? colcom stmt + #| blockExpr = 'block' symbol? colcom stmt result = newNodeP(nkBlockStmt, p) getTokNoInd(p) - if p.tok.tokType == tkColon: addSon(result, ast.emptyNode) - else: addSon(result, parseSymbol(p)) + if p.tok.tokType == tkColon: result.add(p.emptyNode) + else: result.add(parseSymbol(p)) colcom(p, result) - addSon(result, parseStmt(p)) + result.add(parseStmt(p)) + setEndInfo() -proc parseStatic(p: var TParser): PNode = +proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode = #| staticStmt = 'static' colcom stmt - result = newNodeP(nkStaticStmt, p) - getTokNoInd(p) + #| deferStmt = 'defer' colcom stmt + result = newNodeP(k, p) + getTok(p) colcom(p, result) - addSon(result, parseStmt(p)) - -proc parseAsm(p: var TParser): PNode = - #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT) + result.add(parseStmt(p)) + setEndInfo() + +proc parseAsm(p: var Parser): PNode = + #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT) result = newNodeP(nkAsmStmt, p) getTokNoInd(p) - if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p)) - else: addSon(result, ast.emptyNode) + if p.tok.tokType == tkCurlyDotLe: result.add(parsePragma(p)) + else: result.add(p.emptyNode) case p.tok.tokType - of tkStrLit: addSon(result, newStrNodeP(nkStrLit, p.tok.literal, p)) - of tkRStrLit: addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p)) - of tkTripleStrLit: addSon(result, - newStrNodeP(nkTripleStrLit, p.tok.literal, p)) - else: - parMessage(p, errStringLiteralExpected) - addSon(result, ast.emptyNode) - return + of tkStrLit: result.add(newStrNodeP(nkStrLit, p.tok.literal, p)) + of tkRStrLit: result.add(newStrNodeP(nkRStrLit, p.tok.literal, p)) + of tkTripleStrLit: result.add(newStrNodeP(nkTripleStrLit, p.tok.literal, p)) + else: + parMessage(p, "the 'asm' statement takes a string literal") + result.add(p.emptyNode) + return getTok(p) + setEndInfo() -proc parseGenericParam(p: var TParser): PNode = +proc parseGenericParam(p: var Parser): PNode = #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)? var a: PNode result = newNodeP(nkIdentDefs, p) - while true: + # progress guaranteed + while true: case p.tok.tokType - of tkSymbol, tkAccent: + of tkIn, tkOut: + let x = p.lex.cache.getIdent(if p.tok.tokType == tkIn: "in" else: "out") + a = newNodeP(nkPrefix, p) + a.add newIdentNodeP(x, p) + getTok(p) + expectIdent(p) + a.add(parseSymbol(p)) + of tkSymbol, tkAccent: a = parseSymbol(p) - if a.kind == nkEmpty: return - else: break - addSon(result, a) - if p.tok.tokType != tkComma: break + if a.kind == nkEmpty: return + else: break + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) - if p.tok.tokType == tkColon: + if p.tok.tokType == tkColon: getTok(p) optInd(p, result) - addSon(result, parseExpr(p)) - else: - addSon(result, ast.emptyNode) - if p.tok.tokType == tkEquals: + result.add(parseExpr(p)) + else: + result.add(p.emptyNode) + if p.tok.tokType == tkEquals: getTok(p) optInd(p, result) - addSon(result, parseExpr(p)) - else: - addSon(result, ast.emptyNode) + result.add(parseExpr(p)) + else: + result.add(p.emptyNode) + setEndInfo() -proc parseGenericParamList(p: var TParser): PNode = +proc parseGenericParamList(p: var Parser): PNode = #| genericParamList = '[' optInd #| genericParam ^* (comma/semicolon) optPar ']' result = newNodeP(nkGenericParams, p) getTok(p) optInd(p, result) - while p.tok.tokType in {tkSymbol, tkAccent}: + # progress guaranteed + while p.tok.tokType in {tkSymbol, tkAccent, tkIn, tkOut}: var a = parseGenericParam(p) - addSon(result, a) - if p.tok.tokType notin {tkComma, tkSemicolon}: break + result.add(a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + when defined(nimpretty): + commaWasSemicolon(p.em) getTok(p) skipComment(p, a) optPar(p) eat(p, tkBracketRi) + setEndInfo() -proc parsePattern(p: var TParser): PNode = +proc parsePattern(p: var Parser): PNode = #| pattern = '{' stmt '}' eat(p, tkCurlyLe) result = parseStmt(p) eat(p, tkCurlyRi) + setEndInfo() -proc validInd(p: var TParser): bool = - result = p.tok.indent < 0 or p.tok.indent > p.currInd - -proc parseRoutine(p: var TParser, kind: TNodeKind): PNode = +proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = #| indAndComment = (IND{>} COMMENT)? | COMMENT? #| routine = optInd identVis pattern? genericParamList? #| paramListColon pragma? ('=' COMMENT? stmt)? indAndComment result = newNodeP(kind, p) getTok(p) optInd(p, result) - addSon(result, identVis(p)) - if p.tok.tokType == tkCurlyLe and p.validInd: addSon(result, p.parsePattern) - else: addSon(result, ast.emptyNode) + if kind in {nkProcDef, nkLambda, nkIteratorDef, nkFuncDef} and + p.tok.tokType notin {tkSymbol, tokKeywordLow..tokKeywordHigh, tkAccent}: + # no name; lambda or proc type + # in every context that we can parse a routine, we can also parse these + result = parseProcExpr(p, true, if kind == nkProcDef: nkLambda else: kind) + return + result.add(identVis(p)) + if p.tok.tokType == tkCurlyLe and p.validInd: result.add(p.parsePattern) + else: result.add(p.emptyNode) if p.tok.tokType == tkBracketLe and p.validInd: result.add(p.parseGenericParamList) else: - addSon(result, ast.emptyNode) - addSon(result, p.parseParamList) - if p.tok.tokType == tkCurlyDotLe and p.validInd: addSon(result, p.parsePragma) - else: addSon(result, ast.emptyNode) + result.add(p.emptyNode) + result.add(p.parseParamList) + if p.tok.tokType == tkCurlyDotLe and p.validInd: result.add(p.parsePragma) + else: result.add(p.emptyNode) # empty exception tracking: - addSon(result, ast.emptyNode) - if p.tok.tokType == tkEquals and p.validInd: + result.add(p.emptyNode) + let maybeMissEquals = p.tok.tokType != tkEquals + if (not maybeMissEquals) and p.validInd: getTok(p) skipComment(p, result) - addSon(result, parseStmt(p)) + result.add(parseStmt(p)) else: - addSon(result, ast.emptyNode) - indAndComment(p, result) - -proc newCommentStmt(p: var TParser): PNode = + result.add(p.emptyNode) + indAndComment(p, result, maybeMissEquals) + let body = result.lastSon + if body.kind == nkStmtList and body.hasSon and body.firstSon.comment.len > 0 and body.firstSon.kind != nkCommentStmt: + if result.comment.len == 0: + # proc fn*(a: int): int = a ## foo + # => moves comment `foo` to `fn` + result.comment = body.firstSon.comment + body.firstSon.comment = "" + #else: + # assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. + # Yeah, that worked so well. There IS a bug in this logic, now what? + setEndInfo() + +proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT result = newNodeP(nkCommentStmt, p) - result.info.line = result.info.line - int16(1) - int16(p.tok.iNumber) result.comment = p.tok.literal getTok(p) -type - TDefParser = proc (p: var TParser): PNode {.nimcall.} - -proc parseSection(p: var TParser, kind: TNodeKind, - defparser: TDefParser): PNode = - #| section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED) +proc parseSection(p: var Parser, kind: TNodeKind, + defparser: proc (p: var Parser): PNode {.nimcall.}): PNode = + #| section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED) result = newNodeP(kind, p) - getTok(p) + if kind != nkTypeSection: getTok(p) skipComment(p, result) if realInd(p): withInd(p): skipComment(p, result) + # progress guaranteed while sameInd(p): case p.tok.tokType - of tkSymbol, tkAccent: + of tkSymbol, tkAccent, tkParLe: var a = defparser(p) skipComment(p, a) - addSon(result, a) - of tkComment: + result.add(a) + of tkComment: var a = newCommentStmt(p) - addSon(result, a) - else: + result.add(a) + else: parMessage(p, errIdentifierExpected, p.tok) break - if result.len == 0: parMessage(p, errIdentifierExpected, p.tok) + if not result.hasSon: parMessage(p, errIdentifierExpected, p.tok) elif p.tok.tokType in {tkSymbol, tkAccent, tkParLe} and p.tok.indent < 0: # tkParLe is allowed for ``var (x, y) = ...`` tuple parsing - addSon(result, defparser(p)) - else: + result.add(defparser(p)) + else: parMessage(p, errIdentifierExpected, p.tok) - -proc parseConstant(p: var TParser): PNode = - #| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment - result = newNodeP(nkConstDef, p) - addSon(result, identWithPragma(p)) - if p.tok.tokType == tkColon: - getTok(p) - optInd(p, result) - addSon(result, parseTypeDesc(p)) - else: - addSon(result, ast.emptyNode) - eat(p, tkEquals) - optInd(p, result) - addSon(result, parseExpr(p)) - indAndComment(p, result) - -proc parseEnum(p: var TParser): PNode = - #| enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+ + setEndInfo() + +proc parseEnum(p: var Parser): PNode = + #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ result = newNodeP(nkEnumTy, p) getTok(p) - addSon(result, ast.emptyNode) + result.add(p.emptyNode) optInd(p, result) + flexComment(p, result) + # progress guaranteed while true: var a = parseSymbol(p) + if a.kind == nkEmpty: return + + var symPragma = a + var pragma: PNode + if (p.tok.indent < 0 or p.tok.indent >= p.currInd) and p.tok.tokType == tkCurlyDotLe: + pragma = optPragmas(p) + symPragma = newNodeP(nkPragmaExpr, p) + symPragma.add(a) + symPragma.add(pragma) + # nimpretty support here if p.tok.indent >= 0 and p.tok.indent <= p.currInd: - add(result, a) + result.add(symPragma) break - if p.tok.tokType == tkEquals and p.tok.indent < 0: + + if p.tok.tokType == tkEquals and p.tok.indent < 0: getTok(p) - optInd(p, a) - var b = a - a = newNodeP(nkEnumFieldDef, p) - addSon(a, b) - addSon(a, parseExpr(p)) - skipComment(p, a) + optInd(p, symPragma) + var b = symPragma + symPragma = newNodeP(nkEnumFieldDef, p) + symPragma.add(b) + symPragma.add(parseExpr(p)) + if p.tok.indent < 0 or p.tok.indent >= p.currInd: + rawSkipComment(p, symPragma) if p.tok.tokType == tkComma and p.tok.indent < 0: getTok(p) - rawSkipComment(p, a) + rawSkipComment(p, symPragma) else: - skipComment(p, a) - addSon(result, a) + if p.tok.indent < 0 or p.tok.indent >= p.currInd: + rawSkipComment(p, symPragma) + result.add(symPragma) if p.tok.indent >= 0 and p.tok.indent <= p.currInd or p.tok.tokType == tkEof: break - if result.len <= 1: - lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) + if not result.has2Sons: + parMessage(p, errIdentifierExpected, p.tok) + setEndInfo() -proc parseObjectPart(p: var TParser): PNode -proc parseObjectWhen(p: var TParser): PNode = +proc parseObjectPart(p: var Parser): PNode +proc parseObjectWhen(p: var Parser): PNode = #| objectWhen = 'when' expr colcom objectPart COMMENT? #| ('elif' expr colcom objectPart COMMENT?)* #| ('else' colcom objectPart COMMENT?)? result = newNodeP(nkRecWhen, p) - while sameInd(p): + # progress guaranteed + while sameInd(p): getTok(p) # skip `when`, `elif` var branch = newNodeP(nkElifBranch, p) optInd(p, branch) - addSon(branch, parseExpr(p)) + branch.add(parseExpr(p)) colcom(p, branch) - addSon(branch, parseObjectPart(p)) - skipComment(p, branch) - addSon(result, branch) + branch.add(parseObjectPart(p)) + flexComment(p, branch) + result.add(branch) if p.tok.tokType != tkElif: break if p.tok.tokType == tkElse and sameInd(p): var branch = newNodeP(nkElse, p) eat(p, tkElse) colcom(p, branch) - addSon(branch, parseObjectPart(p)) - skipComment(p, branch) - addSon(result, branch) + branch.add(parseObjectPart(p)) + flexComment(p, branch) + result.add(branch) + setEndInfo() -proc parseObjectCase(p: var TParser): PNode = +proc parseObjectCase(p: var Parser): PNode = #| objectBranch = 'of' exprList colcom objectPart #| objectBranches = objectBranch (IND{=} objectBranch)* #| (IND{=} 'elif' expr colcom objectPart)* #| (IND{=} 'else' colcom objectPart)? - #| objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT? + #| objectCase = 'case' declColonEquals ':'? COMMENT? #| (IND{>} objectBranches DED #| | IND{=} objectBranches) result = newNodeP(nkRecCase, p) getTokNoInd(p) - var a = newNodeP(nkIdentDefs, p) - addSon(a, identWithPragma(p)) - eat(p, tkColon) - addSon(a, parseTypeDesc(p)) - addSon(a, ast.emptyNode) - addSon(result, a) + var a = parseIdentColonEquals(p, {withPragma}) + result.add(a) if p.tok.tokType == tkColon: getTok(p) - skipComment(p, result) + flexComment(p, result) var wasIndented = false let oldInd = p.currInd if realInd(p): p.currInd = p.tok.indent wasIndented = true + # progress guaranteed while sameInd(p): var b: PNode case p.tok.tokType - of tkOf: + of tkOf: b = newNodeP(nkOfBranch, p) exprList(p, tkColon, b) - of tkElse: + of tkElse: b = newNodeP(nkElse, p) getTok(p) - eat(p, tkColon) - else: break - skipComment(p, b) + else: break + colcom(p, b) var fields = parseObjectPart(p) if fields.kind == nkEmpty: parMessage(p, errIdentifierExpected, p.tok) fields = newNodeP(nkNilLit, p) # don't break further semantic checking - addSon(b, fields) - addSon(result, b) + b.add(fields) + result.add(b) if b.kind == nkElse: break if wasIndented: p.currInd = oldInd - -proc parseObjectPart(p: var TParser): PNode = + setEndInfo() + +proc parseObjectPart(p: var Parser): PNode = #| objectPart = IND{>} objectPart^+IND{=} DED - #| / objectWhen / objectCase / 'nil' / declColonEquals + #| / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals if realInd(p): result = newNodeP(nkRecList, p) withInd(p): rawSkipComment(p, result) while sameInd(p): case p.tok.tokType - of tkCase, tkWhen, tkSymbol, tkAccent, tkNil: - addSon(result, parseObjectPart(p)) + of tkCase, tkWhen, tkSymbol, tkAccent, tkNil, tkDiscard: + result.add(parseObjectPart(p)) else: parMessage(p, errIdentifierExpected, p.tok) break - else: + elif sameOrNoInd(p): case p.tok.tokType of tkWhen: result = parseObjectWhen(p) @@ -1579,110 +2166,225 @@ proc parseObjectPart(p: var TParser): PNode = result = parseObjectCase(p) of tkSymbol, tkAccent: result = parseIdentColonEquals(p, {withPragma}) - skipComment(p, result) - of tkNil: + if p.tok.indent < 0 or p.tok.indent >= p.currInd: + rawSkipComment(p, result) + of tkNil, tkDiscard: result = newNodeP(nkNilLit, p) getTok(p) else: - result = ast.emptyNode - -proc parseObject(p: var TParser): PNode = - #| object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart + result = p.emptyNode + else: + result = p.emptyNode + setEndInfo() + +proc parseObject(p: var Parser): PNode = + #| objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart result = newNodeP(nkObjectTy, p) getTok(p) - if p.tok.tokType == tkCurlyDotLe and p.validInd: - addSon(result, parsePragma(p)) - else: - addSon(result, ast.emptyNode) + result.add(p.emptyNode) # compatibility with old pragma node if p.tok.tokType == tkOf and p.tok.indent < 0: var a = newNodeP(nkOfInherit, p) getTok(p) - addSon(a, parseTypeDesc(p)) - addSon(result, a) - else: - addSon(result, ast.emptyNode) + a.add(parseTypeDesc(p)) + result.add(a) + else: + result.add(p.emptyNode) if p.tok.tokType == tkComment: skipComment(p, result) # an initial IND{>} HAS to follow: if not realInd(p): - addSon(result, emptyNode) - return - addSon(result, parseObjectPart(p)) + result.add(p.emptyNode) + else: + result.add(parseObjectPart(p)) + setEndInfo() -proc parseDistinct(p: var TParser): PNode = - #| distinct = 'distinct' optInd typeDesc - result = newNodeP(nkDistinctTy, p) +proc parseTypeClassParam(p: var Parser): PNode = + let modifier = + case p.tok.tokType + of tkVar: nkVarTy + of tkOut: nkOutTy + of tkPtr: nkPtrTy + of tkRef: nkRefTy + of tkStatic: nkStaticTy + of tkType: nkTypeOfExpr + else: nkEmpty + + if modifier != nkEmpty: + result = newNodeP(modifier, p) + getTok(p) + result.add(p.parseSymbol) + else: + result = p.parseSymbol + setEndInfo() + +proc parseTypeClass(p: var Parser): PNode = + #| conceptParam = ('var' | 'out' | 'ptr' | 'ref' | 'static' | 'type')? symbol + #| conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? + #| &IND{>} stmt + result = newNodeP(nkTypeClassTy, p) getTok(p) - optInd(p, result) - addSon(result, parseTypeDesc(p)) + if p.tok.tokType == tkComment: + skipComment(p, result) + + if p.tok.indent < 0: + var args = newNodeP(nkArgList, p) + result.add(args) + args.add(p.parseTypeClassParam) + while p.tok.tokType == tkComma: + getTok(p) + args.add(p.parseTypeClassParam) + else: + result.add(p.emptyNode) # see ast.isNewStyleConcept + if p.tok.tokType == tkCurlyDotLe and p.validInd: + result.add(parsePragma(p)) + else: + result.add(p.emptyNode) + if p.tok.tokType == tkOf and p.tok.indent < 0: + var a = newNodeP(nkOfInherit, p) + getTok(p) + # progress guaranteed + while true: + a.add(parseTypeDesc(p)) + if p.tok.tokType != tkComma: break + getTok(p) + result.add(a) + else: + result.add(p.emptyNode) + if p.tok.tokType == tkComment: + skipComment(p, result) + # an initial IND{>} HAS to follow: + if not realInd(p): + if result.isNewStyleConcept: + parMessage(p, "routine expected, but found '$1' (empty new-styled concepts are not allowed)", p.tok) + result.add(p.emptyNode) + else: + result.add(parseStmt(p)) + setEndInfo() -proc parseTypeDef(p: var TParser): PNode = - #| typeDef = identWithPragma genericParamList? '=' optInd typeDefAux +proc parseTypeDef(p: var Parser): PNode = + #| + #| typeDef = identVisDot genericParamList? pragma '=' optInd typeDefValue #| indAndComment? result = newNodeP(nkTypeDef, p) - addSon(result, identWithPragma(p)) + var identifier = identVis(p, allowDot=true) + var identPragma = identifier + var pragma: PNode + var genericParam: PNode + if p.tok.tokType == tkBracketLe and p.validInd: - addSon(result, parseGenericParamList(p)) + genericParam = parseGenericParamList(p) else: - addSon(result, ast.emptyNode) + genericParam = p.emptyNode + + pragma = optPragmas(p) + if pragma.kind != nkEmpty: + identPragma = newNodeP(nkPragmaExpr, p) + identPragma.add(identifier) + identPragma.add(pragma) + + result.add(identPragma) + result.add(genericParam) + if p.tok.tokType == tkEquals: + result.info = parLineInfo(p) getTok(p) optInd(p, result) - addSon(result, parseTypeDefAux(p)) + result.add(parseTypeDefValue(p)) else: - addSon(result, ast.emptyNode) + result.add(p.emptyNode) indAndComment(p, result) # special extension! - -proc parseVarTuple(p: var TParser): PNode = - #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr + setEndInfo() + +proc parseVarTuple(p: var Parser): PNode = + #| varTupleLhs = '(' optInd (identWithPragma / varTupleLhs) ^+ comma optPar ')' (':' optInd typeDescExpr)? + #| varTuple = varTupleLhs '=' optInd expr result = newNodeP(nkVarTuple, p) getTok(p) # skip '(' optInd(p, result) - while p.tok.tokType in {tkSymbol, tkAccent}: - var a = identWithPragma(p) - addSon(result, a) - if p.tok.tokType != tkComma: break + # progress guaranteed + while p.tok.tokType in {tkSymbol, tkAccent, tkParLe}: + var a: PNode + if p.tok.tokType == tkParLe: + a = parseVarTuple(p) + a.add(p.emptyNode) + else: + a = identWithPragma(p, allowDot=true) + result.add(a) + if p.tok.tokType != tkComma: break getTok(p) skipComment(p, a) - addSon(result, ast.emptyNode) # no type desc optPar(p) eat(p, tkParRi) - eat(p, tkEquals) - optInd(p, result) - addSon(result, parseExpr(p)) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + result.add(parseTypeDesc(p, fullExpr = true)) + else: + result.add(p.emptyNode) # no type desc + setEndInfo() + +proc parseVariable(p: var Parser): PNode = + #| colonBody = colcom stmt postExprBlocks? + #| variable = (varTuple / identColonEquals) colonBody? indAndComment + if p.tok.tokType == tkParLe: + result = parseVarTuple(p) + eat(p, tkEquals) + optInd(p, result) + result.add(parseExpr(p)) + else: result = parseIdentColonEquals(p, {withPragma, withDot}) + result.setLastSon postExprBlocks(p, result.lastSon) + indAndComment(p, result) + setEndInfo() -proc parseVariable(p: var TParser): PNode = - #| variable = (varTuple / identColonEquals) indAndComment +proc parseConstant(p: var Parser): PNode = + #| constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment if p.tok.tokType == tkParLe: result = parseVarTuple(p) - else: result = parseIdentColonEquals(p, {withPragma}) + else: + result = newNodeP(nkConstDef, p) + result.add(identWithPragma(p)) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + result.add(parseTypeDesc(p)) + else: + result.add(p.emptyNode) + eat(p, tkEquals) + optInd(p, result) + #add(result, parseStmtListExpr(p)) + let a = parseExpr(p) + result.add postExprBlocks(p, a) indAndComment(p, result) - -proc parseBind(p: var TParser, k: TNodeKind): PNode = + setEndInfo() + +proc parseBind(p: var Parser, k: TNodeKind): PNode = #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma #| mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma result = newNodeP(k, p) getTok(p) optInd(p, result) + # progress guaranteed while true: var a = qualifiedIdent(p) - addSon(result, a) + result.add(a) if p.tok.tokType != tkComma: break getTok(p) optInd(p, a) #expectNl(p) - -proc parseStmtPragma(p: var TParser): PNode = + setEndInfo() + +proc parseStmtPragma(p: var Parser): PNode = #| pragmaStmt = pragma (':' COMMENT? stmt)? result = parsePragma(p) if p.tok.tokType == tkColon and p.tok.indent < 0: let a = result - result = newNodeI(nkPragmaBlock, a.info) + result = newNode(nkPragmaBlock, a.info) getTok(p) skipComment(p, result) result.add a result.add parseStmt(p) + setEndInfo() -proc simpleStmt(p: var TParser): PNode = +proc simpleStmt(p: var Parser): PNode = #| simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt #| | continueStmt | pragmaStmt | importStmt | exportStmt | fromStmt #| | includeStmt | commentStmt) / exprStmt) COMMENT? @@ -1702,126 +2404,187 @@ proc simpleStmt(p: var TParser): PNode = of tkComment: result = newCommentStmt(p) else: if isExprStart(p): result = parseExprStmt(p) - else: result = ast.emptyNode + else: result = p.emptyNode if result.kind notin {nkEmpty, nkCommentStmt}: skipComment(p, result) - -proc complexOrSimpleStmt(p: var TParser): PNode = + +proc complexOrSimpleStmt(p: var Parser): PNode = #| complexOrSimpleStmt = (ifStmt | whenStmt | whileStmt - #| | tryStmt | finallyStmt | exceptStmt | forStmt - #| | blockStmt | staticStmt | asmStmt + #| | tryStmt | forStmt + #| | blockStmt | staticStmt | deferStmt | asmStmt #| | 'proc' routine #| | 'method' routine + #| | 'func' routine #| | 'iterator' routine #| | 'macro' routine #| | 'template' routine #| | 'converter' routine #| | 'type' section(typeDef) #| | 'const' section(constant) - #| | ('let' | 'var') section(variable) + #| | ('let' | 'var' | 'using') section(variable) #| | bindStmt | mixinStmt) #| / simpleStmt case p.tok.tokType of tkIf: result = parseIfOrWhen(p, nkIfStmt) of tkWhile: result = parseWhile(p) of tkCase: result = parseCase(p) - of tkTry: result = parseTry(p) + of tkTry: result = parseTry(p, isExpr=false) of tkFinally: result = parseExceptBlock(p, nkFinally) of tkExcept: result = parseExceptBlock(p, nkExceptBranch) of tkFor: result = parseFor(p) of tkBlock: result = parseBlock(p) - of tkStatic: result = parseStatic(p) + of tkStatic: result = parseStaticOrDefer(p, nkStaticStmt) + of tkDefer: result = parseStaticOrDefer(p, nkDefer) of tkAsm: result = parseAsm(p) of tkProc: result = parseRoutine(p, nkProcDef) + of tkFunc: result = parseRoutine(p, nkFuncDef) of tkMethod: result = parseRoutine(p, nkMethodDef) of tkIterator: result = parseRoutine(p, nkIteratorDef) of tkMacro: result = parseRoutine(p, nkMacroDef) of tkTemplate: result = parseRoutine(p, nkTemplateDef) of tkConverter: result = parseRoutine(p, nkConverterDef) - of tkType: result = parseSection(p, nkTypeSection, parseTypeDef) - of tkConst: result = parseSection(p, nkConstSection, parseConstant) - of tkLet: result = parseSection(p, nkLetSection, parseVariable) + of tkType: + getTok(p) + if p.tok.tokType == tkParLe: + getTok(p) + result = newNodeP(nkTypeOfExpr, p) + result.add(primary(p, pmTypeDesc)) + eat(p, tkParRi) + result = parseOperators(p, result, -1, pmNormal) + else: + result = parseSection(p, nkTypeSection, parseTypeDef) + of tkConst: + prettySection: + result = parseSection(p, nkConstSection, parseConstant) + of tkLet: + prettySection: + result = parseSection(p, nkLetSection, parseVariable) + of tkVar: + prettySection: + result = parseSection(p, nkVarSection, parseVariable) of tkWhen: result = parseIfOrWhen(p, nkWhenStmt) - of tkVar: result = parseSection(p, nkVarSection, parseVariable) of tkBind: result = parseBind(p, nkBindStmt) of tkMixin: result = parseBind(p, nkMixinStmt) + of tkUsing: result = parseSection(p, nkUsingStmt, parseVariable) else: result = simpleStmt(p) - -proc parseStmt(p: var TParser): PNode = + +proc parseStmt(p: var Parser): PNode = #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED) #| / simpleStmt ^+ ';' if p.tok.indent > p.currInd: + # nimpretty support here result = newNodeP(nkStmtList, p) withInd(p): while true: if p.tok.indent == p.currInd: - nil - elif p.tok.tokType == tkSemicolon: - while p.tok.tokType == tkSemicolon: getTok(p) + discard + elif p.tok.tokType == tkSemiColon: + getTok(p) + if p.tok.indent < 0 or p.tok.indent == p.currInd: discard + else: break else: - if p.tok.indent > p.currInd: + if p.tok.indent > p.currInd and p.tok.tokType != tkDot: parMessage(p, errInvalidIndentation) break - if p.tok.toktype in {tkCurlyRi, tkParRi, tkCurlyDotRi, tkBracketRi}: + if p.tok.tokType in {tkCurlyRi, tkParRi, tkCurlyDotRi, tkBracketRi}: # XXX this ensures tnamedparamanonproc still compiles; # deprecate this syntax later break - var a = complexOrSimpleStmt(p) - if a.kind != nkEmpty: - addSon(result, a) - else: + p.hasProgress = false + if p.tok.tokType in {tkElse, tkElif}: + break # Allow this too, see tests/parser/tifexprs + + let a = complexOrSimpleStmt(p) + if a.kind == nkEmpty and not p.hasProgress: parMessage(p, errExprExpected, p.tok) - getTok(p) + break + else: + result.add a + + if not p.hasProgress and p.tok.tokType == tkEof: break else: # the case statement is only needed for better error messages: case p.tok.tokType - of tkIf, tkWhile, tkCase, tkTry, tkFor, tkBlock, tkAsm, tkProc, tkIterator, - tkMacro, tkType, tkConst, tkWhen, tkVar: - parMessage(p, errComplexStmtRequiresInd) - result = ast.emptyNode + of tkIf, tkWhile, tkCase, tkTry, tkFor, tkBlock, tkAsm, tkProc, tkFunc, + tkIterator, tkMacro, tkType, tkConst, tkWhen, tkVar: + parMessage(p, "nestable statement requires indentation") + result = p.emptyNode else: - result = newNodeP(nkStmtList, p) - while true: - if p.tok.indent >= 0: parMessage(p, errInvalidIndentation) - let a = simpleStmt(p) - if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) - result.add(a) - if p.tok.tokType != tkSemicolon: break - getTok(p) - -proc parseAll(p: var TParser): PNode = - result = newNodeP(nkStmtList, p) - while p.tok.tokType != tkEof: - var a = complexOrSimpleStmt(p) - if a.kind != nkEmpty: - addSon(result, a) - else: - parMessage(p, errExprExpected, p.tok) - # bugfix: consume a token here to prevent an endless loop: - getTok(p) - if p.tok.indent != 0: - parMessage(p, errInvalidIndentation) - -proc parseTopLevelStmt(p: var TParser): PNode = - result = ast.emptyNode + if p.inSemiStmtList > 0: + result = simpleStmt(p) + if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + else: + result = newNodeP(nkStmtList, p) + while true: + if p.tok.indent >= 0: + parMessage(p, errInvalidIndentation) + p.hasProgress = false + let a = simpleStmt(p) + let err = not p.hasProgress + if a.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + result.add(a) + if p.tok.tokType != tkSemiColon: break + getTok(p) + if err and p.tok.tokType == tkEof: break + setEndInfo() + +proc checkFirstLineIndentation*(p: var Parser) = + if p.tok.indent != 0 and tsLeading in p.tok.spacing: + parMessage(p, errInvalidIndentation) + +proc parseTopLevelStmt*(p: var Parser): PNode = + ## Implements an iterator which, when called repeatedly, returns the next + ## top-level statement or emptyNode if end of stream. + result = p.emptyNode + # progress guaranteed while true: - if p.tok.indent != 0: - if p.firstTok and p.tok.indent < 0: nil - else: parMessage(p, errInvalidIndentation) + # nimpretty support here + if p.tok.indent != 0: + if p.firstTok and p.tok.indent < 0: discard + elif p.tok.tokType != tkSemiColon: + # special casing for better error messages: + if p.tok.tokType == tkOpr and p.tok.ident.s == "*": + parMessage(p, errGenerated, + "invalid indentation; an export marker '*' follows the declared identifier") + else: + parMessage(p, errInvalidIndentation) p.firstTok = false case p.tok.tokType - of tkSemicolon: getTok(p) + of tkSemiColon: + getTok(p) + if p.tok.indent <= 0: discard + else: parMessage(p, errInvalidIndentation) + p.firstTok = true of tkEof: break else: result = complexOrSimpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break + setEndInfo() -proc parseString(s: string, filename: string = "", line: int = 0): PNode = - var stream = LLStreamOpen(s) +proc parseAll*(p: var Parser): PNode = + ## Parses the rest of the input stream held by the parser into a PNode. + result = newNodeP(nkStmtList, p) + while true: + let nextStmt = p.parseTopLevelStmt() + if nextStmt.kind == nkEmpty: + break + result &= nextStmt + setEndInfo() + +proc parseString*(s: string; cache: IdentCache; config: ConfigRef; + filename: string = ""; line: int = 0; + errorHandler: ErrorHandler = nil): PNode = + ## Parses a string into an AST, returning the top node. + ## `filename` and `line`, although optional, provide info so that the + ## compiler can generate correct error messages referring to the original + ## source. + var stream = llStreamOpen(s) stream.lineOffset = line - var parser: TParser - OpenParser(parser, filename, stream) + var p = Parser() + p.lex.errorHandler = errorHandler + openParser(p, AbsoluteFile filename, stream, cache, config) - result = parser.parseAll - CloseParser(parser) + result = p.parseAll + closeParser(p) + setEndInfo() |