summary refs log tree commit diff stats
path: root/lib/posix/posix.nim
Commit message (Expand)AuthorAgeFilesLines
* lib/posix - Dropped 'T' from typespdw2015-06-041-456/+482
* Fixed SIGPIPE on MacOSyglukhov2015-04-081-1/+4
* fixes https://github.com/Araq/Nim/issues/2116Jakob Oesterling2015-04-011-1/+2
* Fixed Tdirent definition for OS X and BSD.Reimer Behrends2015-03-221-1/+1
* fixes #2116Araq2015-03-221-4/+5
* Add some posix dirent documentationdef2015-02-241-11/+12
* Speed up walkDir significantlydef2015-02-241-0/+13
* Always use 0x40 for POSIX_SPAWN_USEVFORKdef2015-02-171-6/+4
* fixes #2116Araq2015-02-131-11/+4
* Fixes parallel build on FreeBSDmodk2015-01-261-8/+8
* Add workaround for TCC to make POSIX_SPAWN_USEVFORK availabledef2015-01-061-1/+6
* ignore signal SIGPIPE on DarwinAndre2014-12-301-2/+2
* Merge branch 'devel' into bigbreakAraq2014-11-031-42/+0
|\
| * Merge pull request #1448 from def-/posix-mathAndreas Rumpf2014-11-031-42/+0
| |\
| | * Move floating point rounding and exceptions handling to mathdef2014-08-061-45/+0
| | * Add -lm for fesetround and fegetrounddef2014-08-051-0/+3
* | | Fixed FD_SET casingErik O'Leary2014-10-101-1/+1
* | | 'nimfix' improvements; FdSet is TFdSet againAraq2014-09-081-8/+8
* | | more tests greenAraq2014-08-311-16/+16
* | | fixed posix.nimAraq2014-08-301-1/+2
* | | posix.nim compiles againAraq2014-08-291-34/+34
* | | Nimrod renamed to NimAraq2014-08-281-3/+3
* | | big renameAraq2014-08-281-76/+81
* | | big renameAraq2014-08-281-2/+5
|/ /
* | Merge pull request #1405 from boydgreenfield/add_mmap_map_populateAndreas Rumpf2014-08-121-0/+10
|\ \ | |/ |/|
| * Only try to import MAP_POPULATE on Linux, define flag as 0 otherwise.Nick Greenfield2014-07-231-2/+10
| * Add automatic MAP_POPULATE flag for opening read_only (MAP_PRIVATE) and share...Nick Greenfield2014-07-231-0/+2
* | Add "struct" to POSIX flockdef2014-07-311-1/+1
|/
* Fixes missing MSG_NOSIGNAL on Mac OS X.Dominik Picheta2014-07-141-2/+11
* Added define check for openBSD around fmtmsg.h stuff, OpenBSD does not actual...Charlie Barto2014-05-041-2/+2
* New concurrency model: next stepsAraq2014-04-191-0/+2
* Merge pull request #927 from gradha/pr_adds_posix_timegmAndreas Rumpf2014-03-031-0/+1
|\
| * Adds posix.timegm(), brother of posix.mktime().Grzegorz Adam Hankiewicz2014-02-151-0/+1
* | Merge pull request #917 from zielmicha/vforkAndreas Rumpf2014-02-231-0/+1
|\ \
| * | osproc: make failed execv an exception (when using fork or clone)Michał Zieliński2014-02-151-0/+1
| |/
* / Implemented selector support for asyncio2.Dominik Picheta2014-02-221-1/+1
|/
* Adds utimes() importc to posix module.Grzegorz Adam Hankiewicz2014-02-021-0/+12
* Merge branch 'osproc' into develMichał Zieliński2014-01-201-446/+445
|\
| * Introduce poEvalCommand, poUsePath, fix remaining quoting issues.Michał Zieliński2013-12-211-2/+1
* | case consistency for linuxAraq2013-12-291-234/+234
|/
* Made TSocketHandle distinct and fixed problems it caused for SSL.Dominik Picheta2013-10-231-22/+26
* Added a TSocketHandle type to winlean.nim and posix.nimClay Sweetser2013-10-151-0/+3
* some fixes for HaikuAraq2013-09-011-45/+49
* Removes executable bit for text files.Grzegorz Adam Hankiewicz2013-03-161-0/+0
* Many fixes for asynchronous sockets. Asyncio should now work well with buffer...Dominik Picheta2012-07-221-1/+1
* changed integer promotion rules; added math.fmodAraq2012-06-281-1/+1
* produce errors on proc types with implicit empty param lists.Zahary Karadjov2012-04-201-2/+2
* implemented incompleteStruct pragma; embedded debugger works with posix moduleAraq2012-01-051-3/+12
* year 2012 for most copyright headersAraq2012-01-021-1/+1
* attempt to fix tunidecode test; GC cares for seq->openArray conversionsAraq2011-11-211-3/+5
='#n661'>661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
#
#
#           The Nim Compiler
#        (c) Copyright 2015 Andreas Rumpf
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

# This scanner is handwritten for efficiency. I used an elegant buffering
# scheme which I have not seen anywhere else:
# We guarantee that a whole line is in the buffer. Thus only when scanning
# the \n or \r character we have to check wether we need to read in the next
# chunk. (\n or \r already need special handling for incrementing the line
# counter; choosing both \n and \r allows the scanner to properly read Unix,
# DOS or Macintosh text files, even when it is not the native format.

import
  hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
  wordrecg

const
  MaxLineLength* = 80         # lines longer than this lead to a warning
  numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'}
  SymChars*: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
  SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
  OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
    '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}

# don't forget to update the 'highlite' module if these charsets should change

type
  TTokType* = enum
    tkInvalid, tkEof,         # order is important here!
    tkSymbol, # keywords:
    tkAddr, tkAnd, tkAs, tkAsm,
    tkBind, tkBlock, tkBreak, tkCase, tkCast,
    tkConcept, tkConst, tkContinue, tkConverter,
    tkDefer, tkDiscard, tkDistinct, tkDiv, tkDo,
    tkElif, tkElse, tkEnd, tkEnum, tkExcept, tkExport,
    tkFinally, tkFor, tkFrom, tkFunc,
    tkIf, tkImport, tkIn, tkInclude, tkInterface,
    tkIs, tkIsnot, tkIterator,
    tkLet,
    tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin,
    tkObject, tkOf, tkOr, tkOut,
    tkProc, tkPtr, tkRaise, tkRef, tkReturn,
    tkShl, tkShr, tkStatic,
    tkTemplate,
    tkTry, tkTuple, tkType, tkUsing,
    tkVar, tkWhen, tkWhile, tkXor,
    tkYield, # end of keywords
    tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit,
    tkUIntLit, tkUInt8Lit, tkUInt16Lit, tkUInt32Lit, tkUInt64Lit,
    tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit,
    tkStrLit, tkRStrLit, tkTripleStrLit,
    tkGStrLit, tkGTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
    tkBracketRi, tkCurlyLe, tkCurlyRi,
    tkBracketDotLe, tkBracketDotRi, # [. and  .]
    tkCurlyDotLe, tkCurlyDotRi, # {.  and  .}
    tkParDotLe, tkParDotRi,   # (. and .)
    tkComma, tkSemiColon,
    tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
    tkOpr, tkComment, tkAccent,
    tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr

  TTokTypes* = set[TTokType]

const
  weakTokens = {tkComma, tkSemiColon, tkColon,
                tkParRi, tkParDotRi, tkBracketRi, tkBracketDotRi,
                tkCurlyRi} # \
    # tokens that should not be considered for previousToken
  tokKeywordLow* = succ(tkSymbol)
  tokKeywordHigh* = pred(tkIntLit)
  TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
    "tkSymbol",
    "addr", "and", "as", "asm",
    "bind", "block", "break", "case", "cast",
    "concept", "const", "continue", "converter",
    "defer", "discard", "distinct", "div", "do",
    "elif", "else", "end", "enum", "except", "export",
    "finally", "for", "from", "func", "if",
    "import", "in", "include", "interface", "is", "isnot", "iterator",
    "let",
    "macro", "method", "mixin", "mod",
    "nil", "not", "notin", "object", "of", "or",
    "out", "proc", "ptr", "raise", "ref", "return",
    "shl", "shr", "static",
    "template",
    "try", "tuple", "type", "using",
    "var", "when", "while", "xor",
    "yield",
    "tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
    "tkUIntLit", "tkUInt8Lit", "tkUInt16Lit", "tkUInt32Lit", "tkUInt64Lit",
    "tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkFloat128Lit",
    "tkStrLit", "tkRStrLit",
    "tkTripleStrLit", "tkGStrLit", "tkGTripleStrLit", "tkCharLit", "(",
    ")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)",
    ",", ";",
    ":", "::", "=", ".", "..", "[:",
    "tkOpr", "tkComment", "`",
    "tkSpaces", "tkInfixOpr",
    "tkPrefixOpr", "tkPostfixOpr"]

type
  TNumericalBase* = enum
    base10,                   # base10 is listed as the first element,
                              # so that it is the correct default value
    base2, base8, base16

  CursorPosition* {.pure.} = enum ## XXX remove this again
    None, InToken, BeforeToken, AfterToken

  TToken* = object            # a Nim token
    tokType*: TTokType        # the type of the token
    indent*: int              # the indentation; != -1 if the token has been
                              # preceded with indentation
    ident*: PIdent            # the parsed identifier
    iNumber*: BiggestInt      # the parsed integer literal
    fNumber*: BiggestFloat    # the parsed floating point literal
    base*: TNumericalBase     # the numerical base; only valid for int
                              # or float literals
    strongSpaceA*: int8       # leading spaces of an operator
    strongSpaceB*: int8       # trailing spaces of an operator
    literal*: string          # the parsed (string) literal; and
                              # documentation comments are here too
    line*, col*: int
    when defined(nimpretty):
      offsetA*, offsetB*: int   # used for pretty printing so that literals
                                # like 0b01 or  r"\L" are unaffected
      commentOffsetA*, commentOffsetB*: int

  TErrorHandler* = proc (info: TLineInfo; msg: TMsgKind; arg: string)
  TLexer* = object of TBaseLexer
    fileIdx*: int32
    indentAhead*: int         # if > 0 an indendation has already been read
                              # this is needed because scanning comments
                              # needs so much look-ahead
    currLineIndent*: int
    strongSpaces*, allowTabs*: bool
    cursor*: CursorPosition
    errorHandler*: TErrorHandler
    cache*: IdentCache
    when defined(nimsuggest):
      previousToken: TLineInfo

when defined(nimpretty):
  var
    gIndentationWidth*: int

var gLinesCompiled*: int  # all lines that have been compiled

proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
  result = newLineInfo(L.fileIdx, tok.line, tok.col)
  when defined(nimpretty):
    result.offsetA = tok.offsetA
    result.offsetB = tok.offsetB
    result.commentOffsetA = tok.commentOffsetA
    result.commentOffsetB = tok.commentOffsetB

proc isKeyword*(kind: TTokType): bool =
  result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)

template ones(n): untyped = ((1 shl n)-1) # for utf-8 conversion

proc isNimIdentifier*(s: string): bool =
  if s[0] in SymStartChars:
    var i = 1
    var sLen = s.len
    while i < sLen:
      if s[i] == '_':
        inc(i)
      if s[i] notin SymChars: return
      inc(i)
    result = true

proc tokToStr*(tok: TToken): string =
  case tok.tokType
  of tkIntLit..tkInt64Lit: result = $tok.iNumber
  of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
  of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
  of tkParLe..tkColon, tkEof, tkAccent:
    result = TokTypeToStr[tok.tokType]
  else:
    if tok.ident != nil:
      result = tok.ident.s
    else:
      result = ""

proc prettyTok*(tok: TToken): string =
  if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
  else: result = tokToStr(tok)

proc printTok*(tok: TToken) =
  msgWriteln($tok.line & ":" & $tok.col & "\t" &
      TokTypeToStr[tok.tokType] & " " & tokToStr(tok))

proc initToken*(L: var TToken) =
  L.tokType = tkInvalid
  L.iNumber = 0
  L.indent = 0
  L.strongSpaceA = 0
  L.literal = ""
  L.fNumber = 0.0
  L.base = base10
  L.ident = nil
  when defined(nimpretty):
    L.commentOffsetA = 0
    L.commentOffsetB = 0

proc fillToken(L: var TToken) =
  L.tokType = tkInvalid
  L.iNumber = 0
  L.indent = 0
  L.strongSpaceA = 0
  setLen(L.literal, 0)
  L.fNumber = 0.0
  L.base = base10
  L.ident = nil
  when defined(nimpretty):
    L.commentOffsetA = 0
    L.commentOffsetB = 0

proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream;
                 cache: IdentCache) =
  openBaseLexer(lex, inputstream)
  lex.fileIdx = fileidx
  lex.indentAhead = - 1
  lex.currLineIndent = 0
  inc(lex.lineNumber, inputstream.lineOffset)
  lex.cache = cache
  when defined(nimsuggest):
    lex.previousToken.fileIndex = fileIdx

proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream;
                cache: IdentCache) =
  openLexer(lex, filename.fileInfoIdx, inputstream, cache)

proc closeLexer*(lex: var TLexer) =
  inc(gLinesCompiled, lex.lineNumber)
  closeBaseLexer(lex)

proc getLineInfo(L: TLexer): TLineInfo =
  result = newLineInfo(L.fileIdx, L.lineNumber, getColNumber(L, L.bufpos))

proc dispMessage(L: TLexer; info: TLineInfo; msg: TMsgKind; arg: string) =
  if L.errorHandler.isNil:
    msgs.message(info, msg, arg)
  else:
    L.errorHandler(info, msg, arg)

proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
  L.dispMessage(getLineInfo(L), msg, arg)

proc lexMessageTok*(L: TLexer, msg: TMsgKind, tok: TToken, arg = "") =
  var info = newLineInfo(L.fileIdx, tok.line, tok.col)
  L.dispMessage(info, msg, arg)

proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
  var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
  L.dispMessage(info, msg, arg)

proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
  result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)

template tokenBegin(tok, pos) {.dirty.} =
  when defined(nimsuggest):
    var colA = getColNumber(L, pos)
  when defined(nimpretty):
    tok.offsetA = L.offsetBase + pos

template tokenEnd(tok, pos) {.dirty.} =
  when defined(nimsuggest):
    let colB = getColNumber(L, pos)+1
    if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
        L.lineNumber == gTrackPos.line and gIdeCmd in {ideSug, ideCon}:
      L.cursor = CursorPosition.InToken
      gTrackPos.col = colA.int16
    colA = 0
  when defined(nimpretty):
    tok.offsetB = L.offsetBase + pos

template tokenEndIgnore(tok, pos) =
  when defined(nimsuggest):
    let colB = getColNumber(L, pos)
    if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
        L.lineNumber == gTrackPos.line and gIdeCmd in {ideSug, ideCon}:
      gTrackPos.fileIndex = trackPosInvalidFileIdx
      gTrackPos.line = -1
    colA = 0
  when defined(nimpretty):
    tok.offsetB = L.offsetBase + pos

template tokenEndPrevious(tok, pos) =
  when defined(nimsuggest):
    # when we detect the cursor in whitespace, we attach the track position
    # to the token that came before that, but only if we haven't detected
    # the cursor in a string literal or comment:
    let colB = getColNumber(L, pos)
    if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
        L.lineNumber == gTrackPos.line and gIdeCmd in {ideSug, ideCon}:
      L.cursor = CursorPosition.BeforeToken
      gTrackPos = L.previousToken
      gTrackPosAttached = true
    colA = 0
  when defined(nimpretty):
    tok.offsetB = L.offsetBase + pos

{.push overflowChecks: off.}
# We need to parse the largest uint literal without overflow checks
proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
  var i = start
  if s[i] in {'0'..'9'}:
    b = 0
    while s[i] in {'0'..'9'}:
      b = b * 10 + (ord(s[i]) - ord('0'))
      inc(i)
      while s[i] == '_': inc(i) # underscores are allowed and ignored
    result = i - start
{.pop.} # overflowChecks


template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
  add(t.literal, replacementChar)
  inc(L.bufpos)

template eatChar(L: var TLexer, t: var TToken) =
  add(t.literal, L.buf[L.bufpos])
  inc(L.bufpos)

proc getNumber(L: var TLexer, result: var TToken) =
  proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
    var pos = L.bufpos              # use registers for pos, buf
    var buf = L.buf
    while true:
      if buf[pos] in chars:
        add(tok.literal, buf[pos])
        inc(pos)
      else:
        break
      if buf[pos] == '_':
        if buf[pos+1] notin chars:
          lexMessage(L, errInvalidToken, "_")
          break
        add(tok.literal, '_')
        inc(pos)
    L.bufpos = pos

  proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
    var pos = L.bufpos              # use registers for pos, buf
    var buf = L.buf
    while buf[pos] in chars:
      add(tok.literal, buf[pos])
      inc(pos)
    L.bufpos = pos

  proc lexMessageLitNum(L: var TLexer, msg: TMsgKind, startpos: int) =
    # Used to get slightly human friendlier err messages.
    # Note: the erroneous 'O' char in the character set is intentional
    const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
      'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
    var msgPos = L.bufpos
    var t: TToken
    t.literal = ""
    L.bufpos = startpos # Use L.bufpos as pos because of matchChars
    matchChars(L, t, literalishChars)
    # We must verify +/- specifically so that we're not past the literal
    if  L.buf[L.bufpos] in {'+', '-'} and
        L.buf[L.bufpos - 1] in {'e', 'E'}:
      add(t.literal, L.buf[L.bufpos])
      inc(L.bufpos)
      matchChars(L, t, literalishChars)
    if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
      inc(L.bufpos)
      add(t.literal, L.buf[L.bufpos])
      matchChars(L, t, {'0'..'9'})
    L.bufpos = msgPos
    lexMessage(L, msg, t.literal)

  var
    startpos, endpos: int
    xi: BiggestInt
    isBase10 = true
  const
    baseCodeChars = {'X', 'x', 'o', 'c', 'C', 'b', 'B'}
    literalishChars = baseCodeChars + {'A'..'F', 'a'..'f', '0'..'9', '_', '\''}
    floatTypes = {tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit}
  result.tokType = tkIntLit   # int literal until we know better
  result.literal = ""
  result.base = base10
  startpos = L.bufpos
  tokenBegin(result, startPos)

  # First stage: find out base, make verifications, build token literal string
  if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'O'}:
    isBase10 = false
    eatChar(L, result, '0')
    case L.buf[L.bufpos]
    of 'O':
      lexMessageLitNum(L, errInvalidNumberOctalCode, startpos)
    of 'x', 'X':
      eatChar(L, result, 'x')
      matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'})
    of 'o', 'c', 'C':
      eatChar(L, result, 'c')
      matchUnderscoreChars(L, result, {'0'..'7'})
    of 'b', 'B':
      eatChar(L, result, 'b')
      matchUnderscoreChars(L, result, {'0'..'1'})
    else:
      internalError(getLineInfo(L), "getNumber")
  else:
    matchUnderscoreChars(L, result, {'0'..'9'})
    if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
      result.tokType = tkFloatLit
      eatChar(L, result, '.')
      matchUnderscoreChars(L, result, {'0'..'9'})
    if L.buf[L.bufpos] in {'e', 'E'}:
      result.tokType = tkFloatLit
      eatChar(L, result, 'e')
      if L.buf[L.bufpos] in {'+', '-'}:
        eatChar(L, result)
      matchUnderscoreChars(L, result, {'0'..'9'})
  endpos = L.bufpos

  # Second stage, find out if there's a datatype suffix and handle it
  var postPos = endpos
  if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
    if L.buf[postPos] == '\'':
      inc(postPos)

    case L.buf[postPos]
    of 'f', 'F':
      inc(postPos)
      if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
        result.tokType = tkFloat32Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
        result.tokType = tkFloat64Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '1') and
           (L.buf[postPos + 1] == '2') and
           (L.buf[postPos + 2] == '8'):
        result.tokType = tkFloat128Lit
        inc(postPos, 3)
      else:   # "f" alone defaults to float32
        result.tokType = tkFloat32Lit
    of 'd', 'D':  # ad hoc convenience shortcut for f64
      inc(postPos)
      result.tokType = tkFloat64Lit
    of 'i', 'I':
      inc(postPos)
      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
        result.tokType = tkInt64Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
        result.tokType = tkInt32Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
        result.tokType = tkInt16Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '8'):
        result.tokType = tkInt8Lit
        inc(postPos)
      else:
        lexMessageLitNum(L, errInvalidNumber, startpos)
    of 'u', 'U':
      inc(postPos)
      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
        result.tokType = tkUInt64Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
        result.tokType = tkUInt32Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
        result.tokType = tkUInt16Lit
        inc(postPos, 2)
      elif (L.buf[postPos] == '8'):
        result.tokType = tkUInt8Lit
        inc(postPos)
      else:
        result.tokType = tkUIntLit
    else:
      lexMessageLitNum(L, errInvalidNumber, startpos)

  # Is there still a literalish char awaiting? Then it's an error!
  if  L.buf[postPos] in literalishChars or
     (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
    lexMessageLitNum(L, errInvalidNumber, startpos)

  # Third stage, extract actual number
  L.bufpos = startpos            # restore position
  var pos: int = startpos
  try:
    if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
      inc(pos, 2)
      xi = 0                  # it is a base prefix

      case L.buf[pos - 1]
      of 'b', 'B':
        result.base = base2
        while pos < endpos:
          if L.buf[pos] != '_':
            xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
          inc(pos)
      of 'o', 'c', 'C':
        result.base = base8
        while pos < endpos:
          if L.buf[pos] != '_':
            xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
          inc(pos)
      of 'x', 'X':
        result.base = base16
        while pos < endpos:
          case L.buf[pos]
          of '_':
            inc(pos)
          of '0'..'9':
            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
            inc(pos)
          of 'a'..'f':
            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('a') + 10)
            inc(pos)
          of 'A'..'F':
            xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
            inc(pos)
          else:
            break
      else:
        internalError(getLineInfo(L), "getNumber")

      case result.tokType
      of tkIntLit, tkInt64Lit: result.iNumber = xi
      of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
      of tkInt16Lit: result.iNumber = BiggestInt(int16(toU16(int(xi))))
      of tkInt32Lit: result.iNumber = BiggestInt(int32(toU32(int64(xi))))
      of tkUIntLit, tkUInt64Lit: result.iNumber = xi
      of tkUInt8Lit: result.iNumber = BiggestInt(uint8(toU8(int(xi))))
      of tkUInt16Lit: result.iNumber = BiggestInt(uint16(toU16(int(xi))))
      of tkUInt32Lit: result.iNumber = BiggestInt(uint32(toU32(int64(xi))))
      of tkFloat32Lit:
        result.fNumber = (cast[PFloat32](addr(xi)))[]
        # note: this code is endian neutral!
        # XXX: Test this on big endian machine!
      of tkFloat64Lit, tkFloatLit:
        result.fNumber = (cast[PFloat64](addr(xi)))[]
      else: internalError(getLineInfo(L), "getNumber")

      # Bounds checks. Non decimal literals are allowed to overflow the range of
      # the datatype as long as their pattern don't overflow _bitwise_, hence
      # below checks of signed sizes against uint*.high is deliberate:
      # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
      if result.tokType notin floatTypes:
        let outOfRange = case result.tokType:
        of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
        of tkInt8Lit: (xi > BiggestInt(uint8.high))
        of tkInt16Lit: (xi > BiggestInt(uint16.high))
        of tkInt32Lit: (xi > BiggestInt(uint32.high))
        else: false

        if outOfRange:
          #echo "out of range num: ", result.iNumber, " vs ", xi
          lexMessageLitNum(L, errNumberOutOfRange, startpos)

    else:
      case result.tokType
      of floatTypes:
        result.fNumber = parseFloat(result.literal)
      of tkUint64Lit:
        xi = 0
        let len = unsafeParseUInt(result.literal, xi)
        if len != result.literal.len or len == 0:
          raise newException(ValueError, "invalid integer: " & $xi)
        result.iNumber = xi
      else:
        result.iNumber = parseBiggestInt(result.literal)

      # Explicit bounds checks
      let outOfRange = case result.tokType:
      of tkInt8Lit: (result.iNumber < int8.low or result.iNumber > int8.high)
      of tkUInt8Lit: (result.iNumber < BiggestInt(uint8.low) or
                      result.iNumber > BiggestInt(uint8.high))
      of tkInt16Lit: (result.iNumber < int16.low or result.iNumber > int16.high)
      of tkUInt16Lit: (result.iNumber < BiggestInt(uint16.low) or
                      result.iNumber > BiggestInt(uint16.high))
      of tkInt32Lit: (result.iNumber < int32.low or result.iNumber > int32.high)
      of tkUInt32Lit: (result.iNumber < BiggestInt(uint32.low) or
                      result.iNumber > BiggestInt(uint32.high))
      else: false

      if outOfRange: lexMessageLitNum(L, errNumberOutOfRange, startpos)

    # Promote int literal to int64? Not always necessary, but more consistent
    if result.tokType == tkIntLit:
      if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
        result.tokType = tkInt64Lit

  except ValueError:
    lexMessageLitNum(L, errInvalidNumber, startpos)
  except OverflowError, RangeError:
    lexMessageLitNum(L, errNumberOutOfRange, startpos)
  tokenEnd(result, postPos-1)
  L.bufpos = postPos

proc handleHexChar(L: var TLexer, xi: var int) =
  case L.buf[L.bufpos]
  of '0'..'9':
    xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('0'))
    inc(L.bufpos)
  of 'a'..'f':
    xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('a') + 10)
    inc(L.bufpos)
  of 'A'..'F':
    xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('A') + 10)
    inc(L.bufpos)
  else: discard

proc handleDecChars(L: var TLexer, xi: var int) =
  while L.buf[L.bufpos] in {'0'..'9'}:
    xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
    inc(L.bufpos)

proc getEscapedChar(L: var TLexer, tok: var TToken) =
  inc(L.bufpos)               # skip '\'
  case L.buf[L.bufpos]
  of 'n', 'N':
    if gOldNewlines:
      if tok.tokType == tkCharLit: lexMessage(L, errNnotAllowedInCharacter)
      add(tok.literal, tnl)
    else:
      add(tok.literal, '\L')
    inc(L.bufpos)
  of 'p', 'P':
    if tok.tokType == tkCharLit:
      lexMessage(L, errGenerated, "\\p not allowed in character literal")
    add(tok.literal, tnl)
    inc(L.bufpos)
  of 'r', 'R', 'c', 'C':
    add(tok.literal, CR)
    inc(L.bufpos)
  of 'l', 'L':
    add(tok.literal, LF)
    inc(L.bufpos)
  of 'f', 'F':
    add(tok.literal, FF)
    inc(L.bufpos)
  of 'e', 'E':
    add(tok.literal, ESC)
    inc(L.bufpos)
  of 'a', 'A':
    add(tok.literal, BEL)
    inc(L.bufpos)
  of 'b', 'B':
    add(tok.literal, BACKSPACE)
    inc(L.bufpos)
  of 'v', 'V':
    add(tok.literal, VT)
    inc(L.bufpos)
  of 't', 'T':
    add(tok.literal, '\t')
    inc(L.bufpos)
  of '\'', '\"':
    add(tok.literal, L.buf[L.bufpos])
    inc(L.bufpos)
  of '\\':
    add(tok.literal, '\\')
    inc(L.bufpos)
  of 'x', 'X', 'u', 'U':
    var tp = L.buf[L.bufpos]
    inc(L.bufpos)
    var xi = 0
    handleHexChar(L, xi)
    handleHexChar(L, xi)
    if tp in {'u', 'U'}:
      handleHexChar(L, xi)
      handleHexChar(L, xi)
      # inlined toUTF-8 to avoid unicode and strutils dependencies.
      if xi <=% 127:
        add(tok.literal, xi.char )
      elif xi <=% 0x07FF:
        add(tok.literal, ((xi shr 6) or 0b110_00000).char )
        add(tok.literal, ((xi and ones(6)) or 0b10_0000_00).char )
      elif xi <=% 0xFFFF:
        add(tok.literal, (xi shr 12 or 0b1110_0000).char )
        add(tok.literal, (xi shr 6 and ones(6) or 0b10_0000_00).char )
        add(tok.literal, (xi and ones(6) or 0b10_0000_00).char )
      else: # value is 0xFFFF
        add(tok.literal, "\xef\xbf\xbf" )
    else:
      add(tok.literal, chr(xi))
  of '0'..'9':
    if matchTwoChars(L, '0', {'0'..'9'}):
      lexMessage(L, warnOctalEscape)
    var xi = 0
    handleDecChars(L, xi)
    if (xi <= 255): add(tok.literal, chr(xi))
    else: lexMessage(L, errInvalidCharacterConstant)
  else: lexMessage(L, errInvalidCharacterConstant)

proc newString(s: cstring, len: int): string =
  ## XXX, how come there is no support for this?
  result = newString(len)
  for i in 0 ..< len:
    result[i] = s[i]

proc handleCRLF(L: var TLexer, pos: int): int =
  template registerLine =
    let col = L.getColNumber(pos)

    if col > MaxLineLength:
      lexMessagePos(L, hintLineTooLong, pos)

    if optEmbedOrigSrc in gGlobalOptions:
      let lineStart = cast[ByteAddress](L.buf) + L.lineStart
      let line = newString(cast[cstring](lineStart), col)
      addSourceLine(L.fileIdx, line)

  case L.buf[pos]
  of CR:
    registerLine()
    result = nimlexbase.handleCR(L, pos)
  of LF:
    registerLine()
    result = nimlexbase.handleLF(L, pos)
  else: result = pos

proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
  var pos = L.bufpos
  var buf = L.buf                 # put `buf` in a register
  var line = L.lineNumber         # save linenumber for better error message
  tokenBegin(tok, pos)
  inc pos # skip "
  if buf[pos] == '\"' and buf[pos+1] == '\"':
    tok.tokType = tkTripleStrLit # long string literal:
    inc(pos, 2)               # skip ""
    # skip leading newline:
    if buf[pos] in {' ', '\t'}:
      var newpos = pos+1
      while buf[newpos] in {' ', '\t'}: inc newpos
      if buf[newpos] in {CR, LF}: pos = newpos
    pos = handleCRLF(L, pos)
    buf = L.buf
    while true:
      case buf[pos]
      of '\"':
        if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
            buf[pos+3] != '\"':
          tokenEndIgnore(tok, pos+2)
          L.bufpos = pos + 3 # skip the three """
          break
        add(tok.literal, '\"')
        inc(pos)
      of CR, LF:
        tokenEndIgnore(tok, pos)
        pos = handleCRLF(L, pos)
        buf = L.buf
        add(tok.literal, "\n")
      of nimlexbase.EndOfFile:
        tokenEndIgnore(tok, pos)
        var line2 = L.lineNumber
        L.lineNumber = line
        lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
        L.lineNumber = line2
        L.bufpos = pos
        break
      else:
        add(tok.literal, buf[pos])
        inc(pos)
  else:
    # ordinary string literal
    if rawMode: tok.tokType = tkRStrLit
    else: tok.tokType = tkStrLit
    while true:
      var c = buf[pos]
      if c == '\"':
        if rawMode and buf[pos+1] == '\"':
          inc(pos, 2)
          add(tok.literal, '"')
        else:
          tokenEndIgnore(tok, pos)
          inc(pos) # skip '"'
          break
      elif c in {CR, LF, nimlexbase.EndOfFile}:
        tokenEndIgnore(tok, pos)
        lexMessage(L, errClosingQuoteExpected)
        break
      elif (c == '\\') and not rawMode:
        L.bufpos = pos
        getEscapedChar(L, tok)
        pos = L.bufpos
      else:
        add(tok.literal, c)
        inc(pos)
    L.bufpos = pos

proc getCharacter(L: var TLexer, tok: var TToken) =
  tokenBegin(tok, L.bufpos)
  inc(L.bufpos)               # skip '
  var c = L.buf[L.bufpos]
  case c
  of '\0'..pred(' '), '\'': lexMessage(L, errInvalidCharacterConstant)
  of '\\': getEscapedChar(L, tok)
  else:
    tok.literal = $c
    inc(L.bufpos)
  if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote)
  tokenEndIgnore(tok, L.bufpos)
  inc(L.bufpos)               # skip '

proc getSymbol(L: var TLexer, tok: var TToken) =
  var h: Hash = 0
  var pos = L.bufpos
  var buf = L.buf
  tokenBegin(tok, pos)
  while true:
    var c = buf[pos]
    case c
    of 'a'..'z', '0'..'9', '\x80'..'\xFF':
      h = h !& ord(c)
      inc(pos)
    of 'A'..'Z':
      c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
      h = h !& ord(c)
      inc(pos)
    of '_':
      if buf[pos+1] notin SymChars:
        lexMessage(L, errInvalidToken, "_")
        break
      inc(pos)
    else: break
  tokenEnd(tok, pos-1)
  h = !$h
  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
  L.bufpos = pos
  if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
      (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
    tok.tokType = tkSymbol
  else:
    tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))

proc endOperator(L: var TLexer, tok: var TToken, pos: int,
                 hash: Hash) {.inline.} =
  var h = !$hash
  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
  if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
  else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon))
  L.bufpos = pos

proc getOperator(L: var TLexer, tok: var TToken) =
  var pos = L.bufpos
  var buf = L.buf
  tokenBegin(tok, pos)
  var h: Hash = 0
  while true:
    var c = buf[pos]
    if c notin OpChars: break
    h = h !& ord(c)
    inc(pos)
  endOperator(L, tok, pos, h)
  tokenEnd(tok, pos-1)
  # advance pos but don't store it in L.bufpos so the next token (which might
  # be an operator too) gets the preceding spaces:
  tok.strongSpaceB = 0
  while buf[pos] == ' ':
    inc pos
    inc tok.strongSpaceB
  if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
    tok.strongSpaceB = -1

proc newlineFollows*(L: var TLexer): bool =
  var pos = L.bufpos
  var buf = L.buf
  while true:
    case buf[pos]
    of ' ', '\t':
      inc(pos)
    of CR, LF:
      result = true
      break
    of '#':
      inc(pos)
      if buf[pos] == '#': inc(pos)
      if buf[pos] != '[': return true
    else:
      break

proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
                          isDoc: bool) =
  var pos = start
  var buf = L.buf
  var toStrip = 0
  tokenBegin(tok, pos)
  # detect the amount of indentation:
  if isDoc:
    toStrip = getColNumber(L, pos)
    while buf[pos] == ' ': inc pos
    if buf[pos] in {CR, LF}:
      pos = handleCRLF(L, pos)
      buf = L.buf
      toStrip = 0
      while buf[pos] == ' ':
        inc pos
        inc toStrip
  var nesting = 0
  while true:
    case buf[pos]
    of '#':
      if isDoc:
        if buf[pos+1] == '#' and buf[pos+2] == '[':
          inc nesting
        tok.literal.add '#'
      elif buf[pos+1] == '[':
        inc nesting
      inc pos
    of ']':
      if isDoc:
        if buf[pos+1] == '#' and buf[pos+2] == '#':
          if nesting == 0:
            tokenEndIgnore(tok, pos+2)
            inc(pos, 3)
            break
          dec nesting
        tok.literal.add ']'
      elif buf[pos+1] == '#':
        if nesting == 0:
          tokenEndIgnore(tok, pos+1)
          inc(pos, 2)
          break
        dec nesting
      inc pos
    of CR, LF:
      tokenEndIgnore(tok, pos)
      pos = handleCRLF(L, pos)
      buf = L.buf
      # strip leading whitespace:
      when defined(nimpretty): tok.literal.add "\L"
      if isDoc:
        when not defined(nimpretty): tok.literal.add "\n"
        inc tok.iNumber
        var c = toStrip
        while buf[pos] == ' ' and c > 0:
          inc pos
          dec c
    of nimlexbase.EndOfFile:
      tokenEndIgnore(tok, pos)
      lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
      break
    else:
      if isDoc or defined(nimpretty): tok.literal.add buf[pos]
      inc(pos)
  L.bufpos = pos

proc scanComment(L: var TLexer, tok: var TToken) =
  var pos = L.bufpos
  var buf = L.buf
  tok.tokType = tkComment
  # iNumber contains the number of '\n' in the token
  tok.iNumber = 0
  assert buf[pos+1] == '#'
  if buf[pos+2] == '[':
    skipMultiLineComment(L, tok, pos+3, true)
    return
  tokenBegin(tok, pos)
  inc(pos, 2)

  var toStrip = 0
  while buf[pos] == ' ':
    inc pos
    inc toStrip

  while true:
    var lastBackslash = -1
    while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
      if buf[pos] == '\\': lastBackslash = pos+1
      add(tok.literal, buf[pos])
      inc(pos)
    tokenEndIgnore(tok, pos)
    pos = handleCRLF(L, pos)
    buf = L.buf
    var indent = 0
    while buf[pos] == ' ':
      inc(pos)
      inc(indent)

    if buf[pos] == '#' and buf[pos+1] == '#':
      tok.literal.add "\n"
      inc(pos, 2)
      var c = toStrip
      while buf[pos] == ' ' and c > 0:
        inc pos
        dec c
      inc tok.iNumber
    else:
      if buf[pos] > ' ':
        L.indentAhead = indent
      tokenEndIgnore(tok, pos)
      break
  L.bufpos = pos

proc skip(L: var TLexer, tok: var TToken) =
  var pos = L.bufpos
  var buf = L.buf
  tokenBegin(tok, pos)
  tok.strongSpaceA = 0
  when defined(nimpretty):
    var hasComment = false
    tok.commentOffsetA = L.offsetBase + pos
    tok.commentOffsetB = tok.commentOffsetA
  while true:
    case buf[pos]
    of ' ':
      inc(pos)
      inc(tok.strongSpaceA)
    of '\t':
      if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
      inc(pos)
    of CR, LF:
      tokenEndPrevious(tok, pos)
      pos = handleCRLF(L, pos)
      buf = L.buf
      var indent = 0
      while true:
        if buf[pos] == ' ':
          inc(pos)
          inc(indent)
        elif buf[pos] == '#' and buf[pos+1] == '[':
          when defined(nimpretty): hasComment = true
          skipMultiLineComment(L, tok, pos+2, false)
          pos = L.bufpos
          buf = L.buf
        else:
          break
      tok.strongSpaceA = 0
      if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'):
        tok.indent = indent
        L.currLineIndent = indent
        break
    of '#':
      # do not skip documentation comment:
      if buf[pos+1] == '#': break
      when defined(nimpretty): hasComment = true
      if buf[pos+1] == '[':
        skipMultiLineComment(L, tok, pos+2, false)
        pos = L.bufpos
        buf = L.buf
      else:
        tokenBegin(tok, pos)
        while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
        tokenEndIgnore(tok, pos+1)
        when defined(nimpretty):
          tok.commentOffsetB = L.offsetBase + pos + 1
    else:
      break                   # EndOfFile also leaves the loop
  tokenEndPrevious(tok, pos-1)
  L.bufpos = pos
  when defined(nimpretty):
    if hasComment:
      tok.commentOffsetB = L.offsetBase + pos
      tok.tokType = tkComment
    if gIndentationWidth <= 0:
      gIndentationWidth = tok.indent

proc rawGetTok*(L: var TLexer, tok: var TToken) =
  template atTokenEnd() {.dirty.} =
    when defined(nimsuggest):
      # we attach the cursor to the last *strong* token
      if tok.tokType notin weakTokens:
        L.previousToken.line = tok.line.int16
        L.previousToken.col = tok.col.int16

  when defined(nimsuggest):
    L.cursor = CursorPosition.None
  fillToken(tok)
  if L.indentAhead >= 0:
    tok.indent = L.indentAhead
    L.currLineIndent = L.indentAhead
    L.indentAhead = -1
  else:
    tok.indent = -1
  skip(L, tok)
  when defined(nimpretty):
    if tok.tokType == tkComment: return
  var c = L.buf[L.bufpos]
  tok.line = L.lineNumber
  tok.col = getColNumber(L, L.bufpos)
  if c in SymStartChars - {'r', 'R'}:
    getSymbol(L, tok)
  else:
    case c
    of '#':
      scanComment(L, tok)
    of '*':
      # '*:' is unfortunately a special case, because it is two tokens in
      # 'var v*: int'.
      if L.buf[L.bufpos+1] == ':' and L.buf[L.bufpos+2] notin OpChars:
        var h = 0 !& ord('*')
        endOperator(L, tok, L.bufpos+1, h)
      else:
        getOperator(L, tok)
    of ',':
      tok.tokType = tkComma
      inc(L.bufpos)
    of 'r', 'R':
      if L.buf[L.bufpos + 1] == '\"':
        inc(L.bufpos)
        getString(L, tok, true)
      else:
        getSymbol(L, tok)
    of '(':
      inc(L.bufpos)
      if L.buf[L.bufpos] == '.' and L.buf[L.bufpos+1] != '.':
        tok.tokType = tkParDotLe
        inc(L.bufpos)
      else:
        tok.tokType = tkParLe
        when defined(nimsuggest):
          if L.fileIdx == gTrackPos.fileIndex and tok.col < gTrackPos.col and
                    tok.line == gTrackPos.line and gIdeCmd == ideCon:
            gTrackPos.col = tok.col.int16
    of ')':
      tok.tokType = tkParRi
      inc(L.bufpos)
    of '[':
      inc(L.bufpos)
      if L.buf[L.bufpos] == '.' and L.buf[L.bufpos+1] != '.':
        tok.tokType = tkBracketDotLe
        inc(L.bufpos)
      elif L.buf[L.bufpos] == ':':
        tok.tokType = tkBracketLeColon
        inc(L.bufpos)
      else:
        tok.tokType = tkBracketLe
    of ']':
      tok.tokType = tkBracketRi
      inc(L.bufpos)
    of '.':
      when defined(nimsuggest):
        if L.fileIdx == gTrackPos.fileIndex and tok.col+1 == gTrackPos.col and
            tok.line == gTrackPos.line and gIdeCmd == ideSug:
          tok.tokType = tkDot
          L.cursor = CursorPosition.InToken
          gTrackPos.col = tok.col.int16
          inc(L.bufpos)
          atTokenEnd()
          return
      if L.buf[L.bufpos+1] == ']':
        tok.tokType = tkBracketDotRi
        inc(L.bufpos, 2)
      elif L.buf[L.bufpos+1] == '}':
        tok.tokType = tkCurlyDotRi
        inc(L.bufpos, 2)
      elif L.buf[L.bufpos+1] == ')':
        tok.tokType = tkParDotRi
        inc(L.bufpos, 2)
      else:
        getOperator(L, tok)
    of '{':
      inc(L.bufpos)
      if L.buf[L.bufpos] == '.' and L.buf[L.bufpos+1] != '.':
        tok.tokType = tkCurlyDotLe
        inc(L.bufpos)
      else:
        tok.tokType = tkCurlyLe
    of '}':
      tok.tokType = tkCurlyRi
      inc(L.bufpos)
    of ';':
      tok.tokType = tkSemiColon
      inc(L.bufpos)
    of '`':
      tok.tokType = tkAccent
      inc(L.bufpos)
    of '_':
      inc(L.bufpos)
      if L.buf[L.bufpos] notin SymChars+{'_'}:
        tok.tokType = tkSymbol
        tok.ident = L.cache.getIdent("_")
      else:
        tok.literal = $c
        tok.tokType = tkInvalid
        lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')
    of '\"':
      # check for extended raw string literal:
      var rawMode = L.bufpos > 0 and L.buf[L.bufpos-1] in SymChars
      getString(L, tok, rawMode)
      if rawMode:
        # tkRStrLit -> tkGStrLit
        # tkTripleStrLit -> tkGTripleStrLit
        inc(tok.tokType, 2)
    of '\'':
      tok.tokType = tkCharLit
      getCharacter(L, tok)
      tok.tokType = tkCharLit
    of '0'..'9':
      getNumber(L, tok)
      let c = L.buf[L.bufpos]
      if c in SymChars+{'_'}:
        lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')
    else:
      if c in OpChars:
        getOperator(L, tok)
      elif c == nimlexbase.EndOfFile:
        tok.tokType = tkEof
        tok.indent = 0
      else:
        tok.literal = $c
        tok.tokType = tkInvalid
        lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')
        inc(L.bufpos)
  atTokenEnd()