summary refs log tree commit diff stats
diff options
context:
space:
mode:
authorAndreas Rumpf <rumpf_a@web.de>2016-07-12 01:05:52 +0200
committerAndreas Rumpf <rumpf_a@web.de>2016-07-12 01:05:52 +0200
commit56f99f56ac20647017199c0e947a67ae9a806e81 (patch)
treee4db6124f01b35fb36425898d80aa6fa5a6e09b2
parent7a018007a442c74602fc6819ff5eb30697cbbf49 (diff)
downloadNim-56f99f56ac20647017199c0e947a67ae9a806e81.tar.gz
fixes edge cases in the lexer
-rw-r--r--compiler/lexer.nim8
1 files changed, 5 insertions, 3 deletions
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index 8b201431e..c6b11443d 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -735,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       if  c == '\226' and
           buf[pos+1] == '\128' and
           buf[pos+2] == '\147':  # It's a 'magic separator' en-dash Unicode
-        if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars:
+        if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or
+            isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos:
           lexMessage(L, errInvalidToken, "–")
           break
         inc(pos, magicIdentSeparatorRuneByteWidth)
@@ -747,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       h = h !& ord(c)
       inc(pos)
     of '_':
-      if buf[pos+1] notin SymChars:
+      if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1):
         lexMessage(L, errInvalidToken, "_")
         break
       inc(pos)
@@ -1056,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
       inc(L.bufpos)
     of '_':
       inc(L.bufpos)
-      if L.buf[L.bufpos] notin SymChars:
+      if L.buf[L.bufpos] notin SymChars+{'_'} and not
+          isMagicIdentSeparatorRune(L.buf, L.bufpos):
         tok.tokType = tkSymbol
         tok.ident = getIdent("_")
       else: