diff options
Diffstat (limited to 'compiler/lexer.nim')
| -rw-r--r-- | compiler/lexer.nim | 11 |
1 files changed, 8 insertions, 3 deletions
diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 8b201431e..0a96ed0ba 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -735,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) = if c == '\226' and buf[pos+1] == '\128' and buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode - if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars: + if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or + isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos: lexMessage(L, errInvalidToken, "–") break inc(pos, magicIdentSeparatorRuneByteWidth) @@ -747,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = h = h !& ord(c) inc(pos) of '_': - if buf[pos+1] notin SymChars: + if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1): lexMessage(L, errInvalidToken, "_") break inc(pos) @@ -1056,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = inc(L.bufpos) of '_': inc(L.bufpos) - if L.buf[L.bufpos] notin SymChars: + if L.buf[L.bufpos] notin SymChars+{'_'} and not + isMagicIdentSeparatorRune(L.buf, L.bufpos): tok.tokType = tkSymbol tok.ident = getIdent("_") else: @@ -1077,6 +1079,9 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = tok.tokType = tkCharLit of '0'..'9': getNumber(L, tok) + let c = L.buf[L.bufpos] + if c in SymChars+{'_'}: + lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') else: if c in OpChars: getOperator(L, tok) |