#
#
# The Nim Compiler
# (c) Copyright 2015 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# This scanner is handwritten for efficiency. I used an elegant buffering
# scheme which I have not seen anywhere else:
# We guarantee that a whole line is in the buffer. Thus only when scanning
# the \n or \r character we have to check wether we need to read in the next
# chunk. (\n or \r already need special handling for incrementing the line
# counter; choosing both \n and \r allows the scanner to properly read Unix,
# DOS or Macintosh text files, even when it is not the native format.
import
hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
wordrecg, lineinfos
const
MaxLineLength* = 80 # lines longer than this lead to a warning
numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'}
SymChars*: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
'|', '=', '%', '&', '$', '@', '~', ':'}
# don't forget to update the 'highlite' module if these charsets should change
type
TTokType* = enum
tkInvalid, tkEof, # order is important here!
tkSymbol, # keywords:
tkAddr, tkAnd, tkAs, tkAsm,
tkBind, tkBlock, tkBreak, tkCase, tkCast,
tkConcept, tkConst, tkContinue, tkConverter,
tkDefer, tkDiscard, tkDistinct, tkDiv, tkDo,
tkElif, tkElse, tkEnd, tkEnum, tkExcept, tkExport,
tkFinally, tkFor, tkFrom, tkFunc,
tkIf, tkImport, tkIn, tkInclude, tkInterface,
tkIs, tkIsnot, tkIterator,
tkLet,
tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin,
tkObject, tkOf, tkOr, tkOut,
tkProc, tkPtr, tkRaise, tkRef, tkReturn,
tkShl, tkShr, tkStatic,
tkTemplate,
tkTry, tkTuple, tkType, tkUsing,
tkVar, tkWhen, tkWhile, tkXor,
tkYield, # end of keywords
tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit,
tkUIntLit, tkUInt8Lit, tkUInt16Lit, tkUInt32Lit, tkUInt64Lit,
tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit,
tkStrLit, tkRStrLit, tkTripleStrLit,
tkGStrLit, tkGTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
tkBracketRi, tkCurlyLe, tkCurlyRi,
tkBracketDotLe, tkBracketDotRi, # [. and .]
tkCurlyDotLe, tkCurlyDotRi, # {. and .}
tkParDotLe, tkParDotRi, # (. and .)
tkComma, tkSemiColon,
tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
tkOpr, tkComment, tkAccent,
tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
TTokTypes* = set[TTokType]
const
weakTokens = {tkComma, tkSemiColon, tkColon,
tkParRi, tkParDotRi, tkBracketRi, tkBracketDotRi,
tkCurlyRi} # \
# tokens that should not be considered for previousToken
tokKeywordLow* = succ(tkSymbol)
tokKeywordHigh* = pred(tkIntLit)
TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
"tkSymbol",
"addr", "and", "as", "asm",
"bind", "block", "break", "case", "cast",
"concept", "const", "continue", "converter",
"defer", "discard", "distinct", "div", "do",
"elif", "else", "end", "enum", "except", "export",
"finally", "for", "from", "func", "if",
"import", "in", "include", "interface", "is", "isnot", "iterator",
"let",
"macro", "method", "mixin", "mod",
"nil", "not", "notin", "o