diff options
| author | Andreas Rumpf <rumpf_a@web.de> | 2016-07-12 01:05:52 +0200 |
|---|---|---|
| committer | Andreas Rumpf <rumpf_a@web.de> | 2016-07-12 01:05:52 +0200 |
| commit | 56f99f56ac20647017199c0e947a67ae9a806e81 (patch) | |
| tree | e4db6124f01b35fb36425898d80aa6fa5a6e09b2 | |
| parent | 7a018007a442c74602fc6819ff5eb30697cbbf49 (diff) | |
| download | Nim-56f99f56ac20647017199c0e947a67ae9a806e81.tar.gz | |
fixes edge cases in the lexer
| -rw-r--r-- | compiler/lexer.nim | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 8b201431e..c6b11443d 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -735,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) = if c == '\226' and buf[pos+1] == '\128' and buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode - if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars: + if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or + isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos: lexMessage(L, errInvalidToken, "–") break inc(pos, magicIdentSeparatorRuneByteWidth) @@ -747,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = h = h !& ord(c) inc(pos) of '_': - if buf[pos+1] notin SymChars: + if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1): lexMessage(L, errInvalidToken, "_") break inc(pos) @@ -1056,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = inc(L.bufpos) of '_': inc(L.bufpos) - if L.buf[L.bufpos] notin SymChars: + if L.buf[L.bufpos] notin SymChars+{'_'} and not + isMagicIdentSeparatorRune(L.buf, L.bufpos): tok.tokType = tkSymbol tok.ident = getIdent("_") else: |