diff options
author | Andreas Rumpf <andreas@andreas-desktop> | 2010-01-16 23:16:29 +0100 |
---|---|---|
committer | Andreas Rumpf <andreas@andreas-desktop> | 2010-01-16 23:16:29 +0100 |
commit | 6c5693e6339f08af2de2af3578ac048187625a23 (patch) | |
tree | ecb54196f8bad4d00a0dfb26373f87e0c8bf0da1 | |
parent | 0bcdab83953075526c0298e025c8feec75b19d90 (diff) | |
download | Nim-6c5693e6339f08af2de2af3578ac048187625a23.tar.gz |
implemented extended quoting rules
-rwxr-xr-x | doc/manual.txt | 27 | ||||
-rwxr-xr-x | doc/nimrodc.txt | 6 | ||||
-rwxr-xr-x | lib/pure/md5.nim | 2 | ||||
-rwxr-xr-x | rod/highlite.nim | 76 | ||||
-rwxr-xr-x | rod/lexbase.nim | 12 | ||||
-rwxr-xr-x | rod/pnimsyn.nim | 3 | ||||
-rwxr-xr-x | rod/rnimsyn.nim | 395 | ||||
-rwxr-xr-x | rod/scanner.nim | 169 | ||||
-rwxr-xr-x | tests/tassert.nim | 8 | ||||
-rwxr-xr-x | tests/tinvalidnewseq.nim | 20 | ||||
-rwxr-xr-x | tests/tstrlits.nim | 14 |
11 files changed, 317 insertions, 415 deletions
diff --git a/doc/manual.txt b/doc/manual.txt index 8a1d9110c..d2bfde138 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -215,7 +215,15 @@ String literals can also be delimited by three double quotes Literals in this form may run for several lines, may contain ``"`` and do not interpret any escape sequences. For convenience, when the opening ``"""`` is immediately followed by a newline, -the newline is not included in the string. +the newline is not included in the string. The ending of the string literal is +defined by the pattern ``"""[^"]``, so this: + +.. code-block:: nimrod + """"long string within quotes"""" + +Produces:: + + "long string within quotes" Raw string literals @@ -230,6 +238,19 @@ convenient for regular expressions or Windows paths: var f = openFile(r"C:\texts\text.txt") # a raw string, so ``\t`` is no tab +To produce a single ``"`` within a raw string literal, it has to be doubled: + +.. code-block:: nimrod + + r"a""b" + +Produces:: + + a"b + +``r""""`` is not possible with this notation, because the three leading +quotes introduce a triple quoted string literal. + Generalized raw string literals ------------------------------- @@ -291,8 +312,8 @@ prefix), binary (prefix ``0b``), octal (prefix ``0o``) and hexadecimal There exists a literal for each numerical type that is defined. The suffix starting with an apostrophe ('\'') is called a -`type suffix`:idx:. Literals without a type prefix are of the type ``int``, -unless the literal contains a dot or an ``E`` in which case it is of +`type suffix`:idx:. Literals without a type suffix are of the type ``int``, +unless the literal contains a dot or ``E|e`` in which case it is of type ``float``. The type suffixes are: diff --git a/doc/nimrodc.txt b/doc/nimrodc.txt index d35ea68c8..79ce06ad1 100755 --- a/doc/nimrodc.txt +++ b/doc/nimrodc.txt @@ -42,12 +42,16 @@ Configuration file The default configuration file is ``nimrod.cfg``. The ``nimrod`` executable looks for it in the following directories (in this order): -1. ``/home/$user/.config/nimrod.cfg`` (UNIX) or ``$APPDATA/nimrod.cfg`` (Windows) +1. ``/home/$user/.config/nimrod.cfg`` (UNIX) or ``%APPDATA%/nimrod.cfg`` (Windows) 2. ``$nimrod/config/nimrod.cfg`` (UNIX, Windows) 3. ``/etc/nimrod.cfg`` (UNIX) The search stops as soon as a configuration file has been found. The reading of ``nimrod.cfg`` can be suppressed by the ``--skipCfg`` command line option. + +**Note:** The *project file name* is the name of the ``.nim`` file that is +passed as a command line argument to the compiler. + Configuration settings can be overwritten in a project specific configuration file that is read automatically. This specific file has to be in the same directory as the project and be of the same name, except diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index d9bb92949..e75f80b4c 100755 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2009 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/rod/highlite.nim b/rod/highlite.nim index 765cfafd8..c2fc95da8 100755 --- a/rod/highlite.nim +++ b/rod/highlite.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2008 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -57,13 +57,12 @@ proc getSourceLanguage(name: string): TSourceLanguage = result = langNone proc initGeneralTokenizer(g: var TGeneralTokenizer, buf: string) = - var pos: int g.buf = cstring(buf) g.kind = low(TTokenClass) g.start = 0 g.length = 0 g.state = low(TTokenClass) - pos = 0 # skip initial whitespace: + var pos = 0 # skip initial whitespace: while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) g.pos = pos @@ -71,8 +70,7 @@ proc deinitGeneralTokenizer(g: var TGeneralTokenizer) = nil proc nimGetKeyword(id: string): TTokenClass = - var i: PIdent - i = getIdent(id) + var i = getIdent(id) if (i.id >= ord(tokKeywordLow) - ord(tkSymbol)) and (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): result = gtKeyword @@ -80,8 +78,7 @@ proc nimGetKeyword(id: string): TTokenClass = result = gtIdentifier proc nimNumberPostfix(g: var TGeneralTokenizer, position: int): int = - var pos: int - pos = position + var pos = position if g.buf[pos] == '\'': inc(pos) case g.buf[pos] @@ -99,10 +96,8 @@ proc nimNumberPostfix(g: var TGeneralTokenizer, position: int): int = result = pos proc nimNumber(g: var TGeneralTokenizer, position: int): int = - const - decChars = {'0'..'9', '_'} - var pos: int - pos = position + const decChars = {'0'..'9', '_'} + var pos = position g.kind = gtDecNumber while g.buf[pos] in decChars: inc(pos) if g.buf[pos] == '.': @@ -121,10 +116,7 @@ proc nimNextToken(g: var TGeneralTokenizer) = hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'} octChars = {'0'..'7', '_'} binChars = {'0'..'1', '_'} - var - pos: int - id: string - pos = g.pos + var pos = g.pos g.start = g.pos if g.state == gtStringLit: g.kind = gtStringLit @@ -161,7 +153,7 @@ proc nimNextToken(g: var TGeneralTokenizer) = g.kind = gtComment while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': - id = "" + var id = "" while g.buf[pos] in scanner.SymChars + {'_'}: add(id, g.buf[pos]) inc(pos) @@ -175,14 +167,17 @@ proc nimNextToken(g: var TGeneralTokenizer) = break of '\"': inc(pos) - if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'): + if g.buf[pos] == '\"' and g.buf[pos+1] == '\"' and + g.buf[pos+2] != '\"': inc(pos, 2) break else: inc(pos) else: g.kind = gtRawData inc(pos) - while not (g.buf[pos] in {'\0', '\"', '\x0A', '\x0D'}): inc(pos) + while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): + if g.buf[pos] == '"' and g.buf[pos+1] != '"': break + inc(pos) if g.buf[pos] == '\"': inc(pos) else: g.kind = nimGetKeyword(id) @@ -228,7 +223,8 @@ proc nimNextToken(g: var TGeneralTokenizer) = break of '\"': inc(pos) - if (g.buf[pos] == '\"') and (g.buf[pos + 1] == '\"'): + if g.buf[pos] == '\"' and g.buf[pos+1] == '\"' and + g.buf[pos+2] != '\"': inc(pos, 2) break else: inc(pos) @@ -263,10 +259,8 @@ proc nimNextToken(g: var TGeneralTokenizer) = g.pos = pos proc generalNumber(g: var TGeneralTokenizer, position: int): int = - const - decChars = {'0'..'9'} - var pos: int - pos = position + const decChars = {'0'..'9'} + var pos = position g.kind = gtDecNumber while g.buf[pos] in decChars: inc(pos) if g.buf[pos] == '.': @@ -284,12 +278,9 @@ proc generalStrLit(g: var TGeneralTokenizer, position: int): int = const decChars = {'0'..'9'} hexChars = {'0'..'9', 'A'..'F', 'a'..'f'} - var - pos: int - c: Char - pos = position + var pos = position g.kind = gtStringLit - c = g.buf[pos] + var c = g.buf[pos] inc(pos) # skip " or ' while true: case g.buf[pos] @@ -316,12 +307,11 @@ proc generalStrLit(g: var TGeneralTokenizer, position: int): int = result = pos proc isKeyword(x: openarray[string], y: string): int = - var a, b, mid, c: int - a = 0 - b = len(x) - 1 + var a = 0 + var b = len(x) - 1 while a <= b: - mid = (a + b) div 2 - c = cmp(x[mid], y) + var mid = (a + b) div 2 + var c = cmp(x[mid], y) if c < 0: a = mid + 1 elif c > 0: @@ -331,12 +321,11 @@ proc isKeyword(x: openarray[string], y: string): int = result = - 1 proc isKeywordIgnoreCase(x: openarray[string], y: string): int = - var a, b, mid, c: int - a = 0 - b = len(x) - 1 + var a = 0 + var b = len(x) - 1 while a <= b: - mid = (a + b) div 2 - c = cmpIgnoreCase(x[mid], y) + var mid = (a + b) div 2 + var c = cmpIgnoreCase(x[mid], y) if c < 0: a = mid + 1 elif c > 0: @@ -357,10 +346,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string], octChars = {'0'..'7'} binChars = {'0'..'1'} symChars = {'A'..'Z', 'a'..'z', '0'..'9', '_', '\x80'..'\xFF'} - var - pos, nested: int - id: string - pos = g.pos + var pos = g.pos g.start = g.pos if g.state == gtStringLit: g.kind = gtStringLit @@ -400,7 +386,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string], while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) elif g.buf[pos] == '*': g.kind = gtLongComment - nested = 0 + var nested = 0 inc(pos) while true: case g.buf[pos] @@ -426,7 +412,7 @@ proc clikeNextToken(g: var TGeneralTokenizer, keywords: openarray[string], else: g.kind = gtOperator of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': - id = "" + var id = "" while g.buf[pos] in SymChars: add(id, g.buf[pos]) inc(pos) @@ -542,4 +528,4 @@ proc getNextToken(g: var TGeneralTokenizer, lang: TSourceLanguage) = of langC: cNextToken(g) of langJava: javaNextToken(g) else: InternalError("getNextToken") - \ No newline at end of file + diff --git a/rod/lexbase.nim b/rod/lexbase.nim index 0c16e986f..4f946b35f 100755 --- a/rod/lexbase.nim +++ b/rod/lexbase.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2008 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -83,7 +83,8 @@ proc FillBuffer(L: var TBaseLexer) = toCopy = L.BufLen - L.sentinel - 1 assert(toCopy >= 0) if toCopy > 0: - MoveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) # "moveMem" handles overlapping regions + MoveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) + # "moveMem" handles overlapping regions charsRead = LLStreamRead(L.stream, addr(L.buf[toCopy]), (L.sentinel + 1) * chrSize) div chrSize s = toCopy + charsRead @@ -142,7 +143,7 @@ proc skip_UTF_8_BOM(L: var TBaseLexer) = inc(L.bufpos, 3) inc(L.lineStart, 3) -proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen: int = 8192) = +proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) = assert(bufLen > 0) L.bufpos = 0 L.bufLen = bufLen @@ -158,13 +159,12 @@ proc getColNumber(L: TBaseLexer, pos: int): int = result = abs(pos - L.lineStart) proc getCurrentLine(L: TBaseLexer, marker: bool = true): string = - var i: int result = "" - i = L.lineStart + var i = L.lineStart while not (L.buf[i] in {CR, LF, EndOfFile}): add(result, L.buf[i]) inc(i) result = result & "\n" if marker: result = result & RepeatChar(getColNumber(L, L.bufpos)) & '^' & "\n" - \ No newline at end of file + diff --git a/rod/pnimsyn.nim b/rod/pnimsyn.nim index d8846dff1..766e01671 100755 --- a/rod/pnimsyn.nim +++ b/rod/pnimsyn.nim @@ -113,9 +113,8 @@ proc parLineInfo(p: TParser): TLineInfo = result = getLineInfo(p.lex^) proc indAndComment(p: var TParser, n: PNode) = - var info: TLineInfo if p.tok.tokType == tkInd: - info = parLineInfo(p) + var info = parLineInfo(p) getTok(p) if p.tok.tokType == tkComment: skipComment(p, n) else: liMessage(info, errInvalidIndentation) diff --git a/rod/rnimsyn.nim b/rod/rnimsyn.nim index c7f8ea11f..6a0de3182 100755 --- a/rod/rnimsyn.nim +++ b/rod/rnimsyn.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2009 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -51,8 +51,8 @@ const LineCommentColumn = 30 proc InitSrcGen(g: var TSrcGen, renderFlags: TRenderFlags) = - g.comStack = @ [] - g.tokens = @ [] + g.comStack = @[] + g.tokens = @[] g.indent = 0 g.lineLen = 0 g.pos = 0 @@ -62,8 +62,7 @@ proc InitSrcGen(g: var TSrcGen, renderFlags: TRenderFlags) = g.pendingNL = - 1 proc addTok(g: var TSrcGen, kind: TTokType, s: string) = - var length: int - length = len(g.tokens) + var length = len(g.tokens) setlen(g.tokens, length + 1) g.tokens[length].kind = kind g.tokens[length].length = int16(len(s)) @@ -128,15 +127,11 @@ proc makeNimString(s: string): string = add(result, '\"') proc putComment(g: var TSrcGen, s: string) = - var - i, j, ind, comIndent: int - isCode: bool - com: string - i = 0 - comIndent = 1 - isCode = (len(s) >= 2) and (s[0 + 1] != ' ') - ind = g.lineLen - com = "" + var i = 0 + var comIndent = 1 + var isCode = (len(s) >= 2) and (s[0 + 1] != ' ') + var ind = g.lineLen + var com = "" while true: case s[i] of '\0': @@ -167,7 +162,7 @@ proc putComment(g: var TSrcGen, s: string) = # we may break the comment into a multi-line comment if the line # gets too long: # compute length of the following word: - j = i + var j = i while s[j] > ' ': inc(j) if not isCode and (g.lineLen + (j - i) > MaxLineLen): put(g, tkComment, com) @@ -181,10 +176,9 @@ proc putComment(g: var TSrcGen, s: string) = optNL(g) proc maxLineLength(s: string): int = - var i, linelen: int result = 0 - i = 0 - lineLen = 0 + var i = 0 + var lineLen = 0 while true: case s[i] of '\0': @@ -203,12 +197,9 @@ proc maxLineLength(s: string): int = inc(i) proc putRawStr(g: var TSrcGen, kind: TTokType, s: string) = - var - i, hi: int - str: string - i = 0 - hi = len(s) + 0 - 1 - str = "" + var i = 0 + var hi = len(s) + 0 - 1 + var str = "" while i <= hi: case s[i] of '\x0D': @@ -237,8 +228,7 @@ proc containsNL(s: string): bool = result = false proc pushCom(g: var TSrcGen, n: PNode) = - var length: int - length = len(g.comStack) + var length = len(g.comStack) setlen(g.comStack, length + 1) g.comStack[length] = n @@ -258,15 +248,15 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool = (renderDocComments in g.flags) and startsWith(n.comment, "##") proc gcom(g: var TSrcGen, n: PNode) = - var ml: int assert(n != nil) if shouldRenderComment(g, n): if (g.pendingNL < 0) and (len(g.buf) > 0) and (g.buf[len(g.buf)] != ' '): - put(g, tkSpaces, Space) # Before long comments we cannot make sure that a newline is generated, - # because this might be wrong. But it is no problem in practice. + put(g, tkSpaces, Space) + # Before long comments we cannot make sure that a newline is generated, + # because this might be wrong. But it is no problem in practice. if (g.pendingNL < 0) and (len(g.buf) > 0) and (g.lineLen < LineCommentColumn): - ml = maxLineLength(n.comment) + var ml = maxLineLength(n.comment) if ml + LineCommentColumn <= maxLineLen: put(g, tkSpaces, repeatChar(LineCommentColumn - g.lineLen)) putComment(g, n.comment) #assert(g.comStack[high(g.comStack)] = n); @@ -285,30 +275,18 @@ proc litAux(n: PNode, x: biggestInt, size: int): string = proc atom(n: PNode): string = var f: float32 case n.kind - of nkEmpty: - result = "" - of nkIdent: - result = n.ident.s - of nkSym: - result = n.sym.name.s - of nkStrLit: - result = makeNimString(n.strVal) - of nkRStrLit: - result = "r\"" & n.strVal & '\"' - of nkTripleStrLit: - result = "\"\"\"" & n.strVal & "\"\"\"" - of nkCharLit: - result = '\'' & toNimChar(chr(int(n.intVal))) & '\'' - of nkIntLit: - result = litAux(n, n.intVal, 4) - of nkInt8Lit: - result = litAux(n, n.intVal, 1) & "\'i8" - of nkInt16Lit: - result = litAux(n, n.intVal, 2) & "\'i16" - of nkInt32Lit: - result = litAux(n, n.intVal, 4) & "\'i32" - of nkInt64Lit: - result = litAux(n, n.intVal, 8) & "\'i64" + of nkEmpty: result = "" + of nkIdent: result = n.ident.s + of nkSym: result = n.sym.name.s + of nkStrLit: result = makeNimString(n.strVal) + of nkRStrLit: result = "r\"" & replace(n.strVal, "\"", "\"\"") & '\"' + of nkTripleStrLit: result = "\"\"\"" & n.strVal & "\"\"\"" + of nkCharLit: result = '\'' & toNimChar(chr(int(n.intVal))) & '\'' + of nkIntLit: result = litAux(n, n.intVal, 4) + of nkInt8Lit: result = litAux(n, n.intVal, 1) & "\'i8" + of nkInt16Lit: result = litAux(n, n.intVal, 2) & "\'i16" + of nkInt32Lit: result = litAux(n, n.intVal, 4) & "\'i32" + of nkInt64Lit: result = litAux(n, n.intVal, 8) & "\'i64" of nkFloatLit: if n.flags * {nfBase2, nfBase8, nfBase16} == {}: result = $(n.floatVal) else: result = litAux(n, (cast[PInt64](addr(n.floatVal)))^ , 8) @@ -323,8 +301,7 @@ proc atom(n: PNode): string = result = $(n.floatVal) & "\'f64" else: result = litAux(n, (cast[PInt64](addr(n.floatVal)))^ , 8) & "\'f64" - of nkNilLit: - result = "nil" + of nkNilLit: result = "nil" of nkType: if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s else: result = "[type node]" @@ -346,142 +323,87 @@ proc lsons(n: PNode, start: int = 0, theEnd: int = - 1): int = proc lsub(n: PNode): int = # computes the length of a tree - var L: int - if n == nil: - return 0 - if n.comment != nil: - return maxLineLen + 1 + if n == nil: return 0 + if n.comment != nil: return maxLineLen + 1 case n.kind of nkTripleStrLit: if containsNL(n.strVal): result = maxLineLen + 1 else: result = len(atom(n)) of nkEmpty..pred(nkTripleStrLit), succ(nkTripleStrLit)..nkNilLit: result = len(atom(n)) - of nkCall, nkBracketExpr, nkConv: - result = lsub(n.sons[0]) + lcomma(n, 1) + 2 - of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: - result = lsub(n.sons[1]) - of nkCast: - result = lsub(n.sons[0]) + lsub(n.sons[1]) + len("cast[]()") - of nkAddr: - result = lsub(n.sons[0]) + len("addr()") - of nkHiddenAddr, nkHiddenDeref: - result = lsub(n.sons[0]) - of nkCommand: - result = lsub(n.sons[0]) + lcomma(n, 1) + 1 - of nkExprEqExpr, nkAsgn, nkFastAsgn: - result = lsons(n) + 3 - of nkPar, nkCurly, nkBracket: - result = lcomma(n) + 2 - of nkSymChoice: - result = lsons(n) + len("()") + sonsLen(n) - 1 - of nkTupleTy: - result = lcomma(n) + len("tuple[]") - of nkDotExpr: - result = lsons(n) + 1 - of nkBind: - result = lsons(n) + len("bind_") - of nkCheckedFieldExpr: - result = lsub(n.sons[0]) - of nkLambda: - result = lsons(n) + len("lambda__=_") + of nkCall, nkBracketExpr, nkConv: result = lsub(n.sons[0]) + lcomma(n, 1) + 2 + of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: result = lsub(n[1]) + of nkCast: result = lsub(n.sons[0]) + lsub(n.sons[1]) + len("cast[]()") + of nkAddr: result = lsub(n.sons[0]) + len("addr()") + of nkHiddenAddr, nkHiddenDeref: result = lsub(n.sons[0]) + of nkCommand: result = lsub(n.sons[0]) + lcomma(n, 1) + 1 + of nkExprEqExpr, nkAsgn, nkFastAsgn: result = lsons(n) + 3 + of nkPar, nkCurly, nkBracket: result = lcomma(n) + 2 + of nkSymChoice: result = lsons(n) + len("()") + sonsLen(n) - 1 + of nkTupleTy: result = lcomma(n) + len("tuple[]") + of nkDotExpr: result = lsons(n) + 1 + of nkBind: result = lsons(n) + len("bind_") + of nkCheckedFieldExpr: result = lsub(n.sons[0]) + of nkLambda: result = lsons(n) + len("lambda__=_") of nkConstDef, nkIdentDefs: result = lcomma(n, 0, - 3) - L = sonsLen(n) + var L = sonsLen(n) if n.sons[L - 2] != nil: result = result + lsub(n.sons[L - 2]) + 2 if n.sons[L - 1] != nil: result = result + lsub(n.sons[L - 1]) + 3 - of nkVarTuple: - result = lcomma(n, 0, - 3) + len("() = ") + lsub(lastSon(n)) - of nkChckRangeF: - result = len("chckRangeF") + 2 + lcomma(n) - of nkChckRange64: - result = len("chckRange64") + 2 + lcomma(n) - of nkChckRange: - result = len("chckRange") + 2 + lcomma(n) + of nkVarTuple: result = lcomma(n, 0, - 3) + len("() = ") + lsub(lastSon(n)) + of nkChckRangeF: result = len("chckRangeF") + 2 + lcomma(n) + of nkChckRange64: result = len("chckRange64") + 2 + lcomma(n) + of nkChckRange: result = len("chckRange") + 2 + lcomma(n) of nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString, nkPassAsOpenArray: result = 2 if sonsLen(n) >= 1: result = result + lsub(n.sons[0]) result = result + lcomma(n, 1) - of nkExprColonExpr: - result = lsons(n) + 2 - of nkInfix: - result = lsons(n) + 2 - of nkPrefix: - result = lsons(n) + 1 - of nkPostfix: - result = lsons(n) - of nkCallStrLit: - result = lsons(n) - of nkPragmaExpr: - result = lsub(n.sons[0]) + lcomma(n, 1) - of nkRange: - result = lsons(n) + 2 - of nkDerefExpr: - result = lsub(n.sons[0]) + 2 - of nkAccQuoted: - result = lsub(n.sons[0]) + 2 + of nkExprColonExpr: result = lsons(n) + 2 + of nkInfix: result = lsons(n) + 2 + of nkPrefix: result = lsons(n) + 1 + of nkPostfix: result = lsons(n) + of nkCallStrLit: result = lsons(n) + of nkPragmaExpr: result = lsub(n.sons[0]) + lcomma(n, 1) + of nkRange: result = lsons(n) + 2 + of nkDerefExpr: result = lsub(n.sons[0]) + 2 + of nkAccQuoted: result = lsub(n.sons[0]) + 2 of nkIfExpr: result = lsub(n.sons[0].sons[0]) + lsub(n.sons[0].sons[1]) + lsons(n, 1) + len("if_:_") - of nkElifExpr: - result = lsons(n) + len("_elif_:_") - of nkElseExpr: - result = lsub(n.sons[0]) + len("_else:_") # type descriptions - of nkTypeOfExpr: - result = lsub(n.sons[0]) + len("type_") - of nkRefTy: - result = lsub(n.sons[0]) + len("ref_") - of nkPtrTy: - result = lsub(n.sons[0]) + len("ptr_") - of nkVarTy: - result = lsub(n.sons[0]) + len("var_") - of nkDistinctTy: - result = lsub(n.sons[0]) + len("Distinct_") - of nkTypeDef: - result = lsons(n) + 3 - of nkOfInherit: - result = lsub(n.sons[0]) + len("of_") - of nkProcTy: - result = lsons(n) + len("proc_") - of nkEnumTy: - result = lsub(n.sons[0]) + lcomma(n, 1) + len("enum_") - of nkEnumFieldDef: - result = lsons(n) + 3 + of nkElifExpr: result = lsons(n) + len("_elif_:_") + of nkElseExpr: result = lsub(n.sons[0]) + len("_else:_") # type descriptions + of nkTypeOfExpr: result = lsub(n.sons[0]) + len("type_") + of nkRefTy: result = lsub(n.sons[0]) + len("ref_") + of nkPtrTy: result = lsub(n.sons[0]) + len("ptr_") + of nkVarTy: result = lsub(n.sons[0]) + len("var_") + of nkDistinctTy: result = lsub(n.sons[0]) + len("Distinct_") + of nkTypeDef: result = lsons(n) + 3 + of nkOfInherit: result = lsub(n.sons[0]) + len("of_") + of nkProcTy: result = lsons(n) + len("proc_") + of nkEnumTy: result = lsub(n.sons[0]) + lcomma(n, 1) + len("enum_") + of nkEnumFieldDef: result = lsons(n) + 3 of nkVarSection: if sonsLen(n) > 1: result = maxLineLen + 1 else: result = lsons(n) + len("var_") - of nkReturnStmt: - result = lsub(n.sons[0]) + len("return_") - of nkRaiseStmt: - result = lsub(n.sons[0]) + len("raise_") - of nkYieldStmt: - result = lsub(n.sons[0]) + len("yield_") - of nkDiscardStmt: - result = lsub(n.sons[0]) + len("discard_") - of nkBreakStmt: - result = lsub(n.sons[0]) + len("break_") - of nkContinueStmt: - result = lsub(n.sons[0]) + len("continue_") - of nkPragma: - result = lcomma(n) + 4 - of nkCommentStmt: - result = len(n.comment) - of nkOfBranch: - result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("of_:_") - of nkElifBranch: - result = lsons(n) + len("elif_:_") - of nkElse: - result = lsub(n.sons[0]) + len("else:_") - of nkFinally: - result = lsub(n.sons[0]) + len("finally:_") - of nkGenericParams: - result = lcomma(n) + 2 + of nkReturnStmt: result = lsub(n.sons[0]) + len("return_") + of nkRaiseStmt: result = lsub(n.sons[0]) + len("raise_") + of nkYieldStmt: result = lsub(n.sons[0]) + len("yield_") + of nkDiscardStmt: result = lsub(n.sons[0]) + len("discard_") + of nkBreakStmt: result = lsub(n.sons[0]) + len("break_") + of nkContinueStmt: result = lsub(n.sons[0]) + len("continue_") + of nkPragma: result = lcomma(n) + 4 + of nkCommentStmt: result = len(n.comment) + of nkOfBranch: result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("of_:_") + of nkElifBranch: result = lsons(n) + len("elif_:_") + of nkElse: result = lsub(n.sons[0]) + len("else:_") + of nkFinally: result = lsub(n.sons[0]) + len("finally:_") + of nkGenericParams: result = lcomma(n) + 2 of nkFormalParams: result = lcomma(n, 1) + 2 if n.sons[0] != nil: result = result + lsub(n.sons[0]) + 2 of nkExceptBranch: - result = lcomma(n, 0, - 2) + lsub(lastSon(n)) + len("except_:_") + result = lcomma(n, 0, -2) + lsub(lastSon(n)) + len("except_:_") else: result = maxLineLen + 1 proc fits(g: TSrcGen, x: int): bool = @@ -509,15 +431,12 @@ proc gsub(g: var TSrcGen, n: PNode) = proc hasCom(n: PNode): bool = result = false if n == nil: return - if n.comment != nil: - return true + if n.comment != nil: return true case n.kind - of nkEmpty..nkNilLit: - nil + of nkEmpty..nkNilLit: nil else: for i in countup(0, sonsLen(n) - 1): - if hasCom(n.sons[i]): - return true + if hasCom(n.sons[i]): return true proc putWithSpace(g: var TSrcGen, kind: TTokType, s: string) = put(g, kind, s) @@ -525,12 +444,9 @@ proc putWithSpace(g: var TSrcGen, kind: TTokType, s: string) = proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0, theEnd: int = - 1) = - var - sublen: int - c: bool for i in countup(start, sonsLen(n) + theEnd): - c = i < sonsLen(n) + theEnd - sublen = lsub(n.sons[i]) + ord(c) + var c = i < sonsLen(n) + theEnd + var sublen = lsub(n.sons[i]) + ord(c) if not fits(g, sublen) and (ind + sublen < maxLineLen): optNL(g, ind) gsub(g, n.sons[i]) if c: @@ -550,19 +466,17 @@ proc gcomma(g: var TSrcGen, n: PNode, c: TContext, start: int = 0, gcommaAux(g, n, ind, start, theEnd) proc gcomma(g: var TSrcGen, n: PNode, start: int = 0, theEnd: int = - 1) = - var ind: int - ind = g.lineLen + var ind = g.lineLen if ind > maxLineLen div 2: ind = g.indent + longIndentWid gcommaAux(g, n, ind, start, theEnd) proc gsons(g: var TSrcGen, n: PNode, c: TContext, start: int = 0, theEnd: int = - 1) = - for i in countup(start, sonsLen(n) + theEnd): - gsub(g, n.sons[i], c) + for i in countup(start, sonsLen(n) + theEnd): gsub(g, n.sons[i], c) -proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType, k: string) = - if sonsLen(n) == 0: - return # empty var sections are possible +proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType, + k: string) = + if sonsLen(n) == 0: return # empty var sections are possible putWithSpace(g, kind, k) gcoms(g) indentNL(g) @@ -600,7 +514,6 @@ proc gstmts(g: var TSrcGen, n: PNode, c: TContext) = proc gif(g: var TSrcGen, n: PNode) = var c: TContext - length: int gsub(g, n.sons[0].sons[0]) initContext(c) putWithSpace(g, tkColon, ":") @@ -608,7 +521,7 @@ proc gif(g: var TSrcGen, n: PNode) = incl(c.flags, rfLongMode) gcoms(g) # a good place for comments gstmts(g, n.sons[0].sons[1], c) - length = sonsLen(n) + var length = sonsLen(n) for i in countup(1, length - 1): optNL(g) gsub(g, n.sons[i], c) @@ -636,10 +549,8 @@ proc gtry(g: var TSrcGen, n: PNode) = gsons(g, n, c, 1) proc gfor(g: var TSrcGen, n: PNode) = - var - c: TContext - length: int - length = sonsLen(n) + var c: TContext + var length = sonsLen(n) putWithSpace(g, tkFor, "for") initContext(c) if longMode(n) or @@ -665,18 +576,16 @@ proc gmacro(g: var TSrcGen, n: PNode) = gsons(g, n, c, 1) proc gcase(g: var TSrcGen, n: PNode) = - var - c: TContext - length, last: int + var c: TContext initContext(c) - length = sonsLen(n) - if n.sons[length - 1].kind == nkElse: last = - 2 - else: last = - 1 + var length = sonsLen(n) + var last = if n.sons[length-1].kind == nkElse: -2 else: -1 if longMode(n, 0, last): incl(c.flags, rfLongMode) putWithSpace(g, tkCase, "case") gsub(g, n.sons[0]) gcoms(g) optNL(g) + last = 0 gsons(g, n, c, 1, last) if last == - 2: initContext(c) @@ -722,10 +631,8 @@ proc gasm(g: var TSrcGen, n: PNode) = gsub(g, n.sons[1]) proc gident(g: var TSrcGen, n: PNode) = - var - s: string - t: TTokType - s = atom(n) + var t: TTokType + var s = atom(n) if (s[0] in scanner.SymChars): if (n.kind == nkIdent): if (n.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or @@ -747,36 +654,21 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = if n == nil: return if n.comment != nil: pushCom(g, n) case n.kind # atoms: - of nkTripleStrLit: - putRawStr(g, tkTripleStrLit, n.strVal) - of nkEmpty, nkType: - put(g, tkInvalid, atom(n)) - of nkSym, nkIdent: - gident(g, n) - of nkIntLit: - put(g, tkIntLit, atom(n)) - of nkInt8Lit: - put(g, tkInt8Lit, atom(n)) - of nkInt16Lit: - put(g, tkInt16Lit, atom(n)) - of nkInt32Lit: - put(g, tkInt32Lit, atom(n)) - of nkInt64Lit: - put(g, tkInt64Lit, atom(n)) - of nkFloatLit: - put(g, tkFloatLit, atom(n)) - of nkFloat32Lit: - put(g, tkFloat32Lit, atom(n)) - of nkFloat64Lit: - put(g, tkFloat64Lit, atom(n)) - of nkStrLit: - put(g, tkStrLit, atom(n)) - of nkRStrLit: - put(g, tkRStrLit, atom(n)) - of nkCharLit: - put(g, tkCharLit, atom(n)) - of nkNilLit: - put(g, tkNil, atom(n)) # complex expressions + of nkTripleStrLit: putRawStr(g, tkTripleStrLit, n.strVal) + of nkEmpty, nkType: put(g, tkInvalid, atom(n)) + of nkSym, nkIdent: gident(g, n) + of nkIntLit: put(g, tkIntLit, atom(n)) + of nkInt8Lit: put(g, tkInt8Lit, atom(n)) + of nkInt16Lit: put(g, tkInt16Lit, atom(n)) + of nkInt32Lit: put(g, tkInt32Lit, atom(n)) + of nkInt64Lit: put(g, tkInt64Lit, atom(n)) + of nkFloatLit: put(g, tkFloatLit, atom(n)) + of nkFloat32Lit: put(g, tkFloat32Lit, atom(n)) + of nkFloat64Lit: put(g, tkFloat64Lit, atom(n)) + of nkStrLit: put(g, tkStrLit, atom(n)) + of nkRStrLit: put(g, tkRStrLit, atom(n)) + of nkCharLit: put(g, tkCharLit, atom(n)) + of nkNilLit: put(g, tkNil, atom(n)) # complex expressions of nkCall, nkConv, nkDotCall: if sonsLen(n) >= 1: gsub(g, n.sons[0]) put(g, tkParLe, "(") @@ -785,11 +677,10 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = of nkCallStrLit: gsub(g, n.sons[0]) if n.sons[1].kind == nkRStrLit: - put(g, tkRStrLit, '\"' & n.sons[1].strVal & '\"') + put(g, tkRStrLit, '\"' & replace(n[1].strVal, "\"", "\"\"") & '\"') else: gsub(g, n.sons[0]) - of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: - gsub(g, n.sons[0]) + of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv: gsub(g, n.sons[0]) of nkCast: put(g, tkCast, "cast") put(g, tkBracketLe, "[") @@ -878,7 +769,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gsub(g, n.sons[codePos]) of nkConstDef, nkIdentDefs: gcomma(g, n, 0, - 3) - L = sonsLen(n) + var L = sonsLen(n) if n.sons[L - 2] != nil: putWithSpace(g, tkColon, ":") gsub(g, n.sons[L - 2]) @@ -919,8 +810,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gsub(g, n.sons[1]) of nkDerefExpr: gsub(g, n.sons[0]) - putWithSpace(g, tkHat, "^") # unfortunately this requires a space, because ^. would be - # only one operator + putWithSpace(g, tkHat, "^") + # unfortunately this requires a space, because ^. would be only one operator of nkAccQuoted: put(g, tkAccent, "`") gsub(g, n.sons[0]) @@ -996,28 +887,20 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = put(g, tkSpaces, Space) putWithSpace(g, tkEquals, "=") gsub(g, n.sons[1]) - of nkStmtList, nkStmtListExpr: - gstmts(g, n, emptyContext) + of nkStmtList, nkStmtListExpr: gstmts(g, n, emptyContext) of nkIfStmt: putWithSpace(g, tkIf, "if") gif(g, n) of nkWhenStmt, nkRecWhen: putWithSpace(g, tkWhen, "when") gif(g, n) - of nkWhileStmt: - gwhile(g, n) - of nkCaseStmt, nkRecCase: - gcase(g, n) - of nkMacroStmt: - gmacro(g, n) - of nkTryStmt: - gtry(g, n) - of nkForStmt: - gfor(g, n) - of nkBlockStmt, nkBlockExpr: - gblock(g, n) - of nkAsmStmt: - gasm(g, n) + of nkWhileStmt: gwhile(g, n) + of nkCaseStmt, nkRecCase: gcase(g, n) + of nkMacroStmt: gmacro(g, n) + of nkTryStmt: gtry(g, n) + of nkForStmt: gfor(g, n) + of nkBlockStmt, nkBlockExpr: gblock(g, n) + of nkAsmStmt: gasm(g, n) of nkProcDef: putWithSpace(g, tkProc, "proc") gproc(g, n) @@ -1170,8 +1053,7 @@ proc renderModule(n: PNode, filename: string, renderFlags: TRenderFlags = {}) = if n.sons[i] != nil: case n.sons[i].kind of nkTypeSection, nkConstSection, nkVarSection, nkCommentStmt: putNL(g) - else: - nil + else: nil gcoms(g) if open(f, filename, fmWrite): write(f, g.buf) @@ -1182,10 +1064,9 @@ proc initTokRender(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) = gsub(r, n) proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) = - var length: int if r.idx < len(r.tokens): kind = r.tokens[r.idx].kind - length = r.tokens[r.idx].length + var length = r.tokens[r.idx].length literal = copy(r.buf, r.pos + 0, r.pos + 0 + length - 1) inc(r.pos, length) inc(r.idx) diff --git a/rod/scanner.nim b/rod/scanner.nim index 9382c7621..8ee7d44b4 100755 --- a/rod/scanner.nim +++ b/rod/scanner.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2009 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -52,15 +52,18 @@ type tkImplies, tkImport, tkIn, tkInclude, tkIs, tkIsnot, tkIterator, tkLambda, tkMacro, tkMethod, tkMod, tkNil, tkNot, tkNotin, tkObject, tkOf, tkOr, tkOut, tkProc, tkPtr, tkRaise, tkRef, tkReturn, tkShl, tkShr, tkTemplate, - tkTry, tkTuple, tkType, tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor, tkYield, #[[[end]]] + tkTry, tkTuple, tkType, tkVar, tkWhen, tkWhile, tkWith, tkWithout, tkXor, + tkYield, #[[[end]]] tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit, tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkStrLit, tkRStrLit, tkTripleStrLit, tkCallRStrLit, tkCallTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe, - tkBracketRi, tkCurlyLe, tkCurlyRi, tkBracketDotLe, tkBracketDotRi, # [. and .] + tkBracketRi, tkCurlyLe, tkCurlyRi, + tkBracketDotLe, tkBracketDotRi, # [. and .] tkCurlyDotLe, tkCurlyDotRi, # {. and .} tkParDotLe, tkParDotRi, # (. and .) tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot, tkHat, tkOpr, - tkComment, tkAccent, tkInd, tkSad, tkDed, # pseudo token types used by the source renderers: + tkComment, tkAccent, tkInd, tkSad, + tkDed, # pseudo token types used by the source renderers: tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr TTokTypes* = set[TTokType] @@ -70,16 +73,18 @@ const tokOperators*: TTokTypes = {tkOpr, tkSymbol, tkBracketLe, tkBracketRi, tkIn, tkIs, tkIsNot, tkEquals, tkDot, tkHat, tkNot, tkAnd, tkOr, tkXor, tkShl, tkShr, tkDiv, tkMod, tkNotIn} - TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]", "tkSymbol", #[[[cog - #cog.out(strings) - #]]] + TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]", + "tkSymbol", #[[[cog + #cog.out(strings) + #]]] "addr", "and", "as", "asm", "bind", "block", "break", "case", "cast", "const", "continue", "converter", "discard", "distinct", "div", "elif", "else", "end", "enum", "except", "finally", "for", "from", "generic", "if", "implies", "import", "in", "include", "is", "isnot", "iterator", "lambda", "macro", "method", "mod", "nil", "not", "notin", "object", "of", "or", "out", "proc", "ptr", "raise", "ref", "return", "shl", "shr", "template", - "try", "tuple", "type", "var", "when", "while", "with", "without", "xor", "yield", #[[[end]]] + "try", "tuple", "type", "var", "when", "while", "with", "without", "xor", + "yield", #[[[end]]] "tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit", "tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkStrLit", "tkRStrLit", "tkTripleStrLit", "tkCallRStrLit", "tkCallTripleStrLit", "tkCharLit", "(", @@ -213,17 +218,13 @@ proc lexMessage(L: TLexer, msg: TMsgKind, arg: string = "") = msgs.liMessage(getLineInfo(L), msg, arg) proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg: string = "") = - var info: TLineInfo - info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart) + var info = newLineInfo(L.filename, L.linenumber, pos - L.lineStart) msgs.liMessage(info, msg, arg) proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: TCharSet) = # matches ([chars]_)* - var - pos: int - buf: cstring - pos = L.bufpos # use registers for pos, buf - buf = L.buf + var pos = L.bufpos # use registers for pos, buf + var buf = L.buf while true: if buf[pos] in chars: add(tok.literal, buf[pos]) @@ -252,11 +253,12 @@ proc GetNumber(L: var TLexer): TToken = result.tokType = tkIntLit # int literal until we know better result.literal = "" result.base = base10 # BUGFIX - pos = L.bufpos # make sure the literal is correct for error messages: + pos = L.bufpos # make sure the literal is correct for error messages: matchUnderscoreChars(L, result, {'A'..'Z', 'a'..'z', '0'..'9'}) if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}): add(result.literal, '.') - inc(L.bufpos) #matchUnderscoreChars(L, result, ['A'..'Z', 'a'..'z', '0'..'9']) + inc(L.bufpos) + #matchUnderscoreChars(L, result, ['A'..'Z', 'a'..'z', '0'..'9']) matchUnderscoreChars(L, result, {'0'..'9'}) if L.buf[L.bufpos] in {'e', 'E'}: add(result.literal, 'e') @@ -355,19 +357,15 @@ proc GetNumber(L: var TLexer): TToken = else: break else: InternalError(getLineInfo(L), "getNumber") case result.tokType - of tkIntLit, tkInt64Lit: - result.iNumber = xi - of tkInt8Lit: - result.iNumber = biggestInt(int8(toU8(int(xi)))) - of tkInt16Lit: - result.iNumber = biggestInt(toU16(int(xi))) - of tkInt32Lit: - result.iNumber = biggestInt(toU32(xi)) + of tkIntLit, tkInt64Lit: result.iNumber = xi + of tkInt8Lit: result.iNumber = biggestInt(int8(toU8(int(xi)))) + of tkInt16Lit: result.iNumber = biggestInt(toU16(int(xi))) + of tkInt32Lit: result.iNumber = biggestInt(toU32(xi)) of tkFloat32Lit: - result.fNumber = (cast[PFloat32](addr(xi)))^ # note: this code is endian neutral! - # XXX: Test this on big endian machine! - of tkFloat64Lit: - result.fNumber = (cast[PFloat64](addr(xi)))^ + result.fNumber = (cast[PFloat32](addr(xi)))^ + # note: this code is endian neutral! + # XXX: Test this on big endian machine! + of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))^ else: InternalError(getLineInfo(L), "getNumber") elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or (result.tokType == tkFloat64Lit): @@ -380,12 +378,9 @@ proc GetNumber(L: var TLexer): TToken = result.tokType = tkInt64Lit elif result.tokType != tkInt64Lit: lexMessage(L, errInvalidNumber, result.literal) - except EInvalidValue: - lexMessage(L, errInvalidNumber, result.literal) - except EOverflow: - lexMessage(L, errNumberOutOfRange, result.literal) - except EOutOfRange: - lexMessage(L, errNumberOutOfRange, result.literal) + except EInvalidValue: lexMessage(L, errInvalidNumber, result.literal) + except EOverflow: lexMessage(L, errNumberOutOfRange, result.literal) + except EOutOfRange: lexMessage(L, errNumberOutOfRange, result.literal) L.bufpos = endpos proc handleHexChar(L: var TLexer, xi: var int) = @@ -472,23 +467,22 @@ proc HandleCRLF(L: var TLexer, pos: int): int = else: result = pos proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = - var - line, line2, pos: int - c: Char - buf: cstring - pos = L.bufPos + 1 # skip " - buf = L.buf # put `buf` in a register - line = L.linenumber # save linenumber for better error message - if (buf[pos] == '\"') and (buf[pos + 1] == '\"'): + var pos = L.bufPos + 1 # skip " + var buf = L.buf # put `buf` in a register + var line = L.linenumber # save linenumber for better error message + if buf[pos] == '\"' and buf[pos+1] == '\"': tok.tokType = tkTripleStrLit # long string literal: inc(pos, 2) # skip "" - # skip leading newline: + # skip leading newline: pos = HandleCRLF(L, pos) buf = L.buf while true: case buf[pos] of '\"': - if (buf[pos + 1] == '\"') and (buf[pos + 2] == '\"'): break + if buf[pos+1] == '\"' and buf[pos+2] == '\"' and + buf[pos+3] != '\"': + L.bufpos = pos + 3 # skip the three """ + break add(tok.literal, '\"') Inc(pos) of CR, LF: @@ -496,7 +490,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = buf = L.buf tok.literal = tok.literal & tnl of lexbase.EndOfFile: - line2 = L.linenumber + var line2 = L.linenumber L.LineNumber = line lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart) L.LineNumber = line2 @@ -504,20 +498,23 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = else: add(tok.literal, buf[pos]) Inc(pos) - L.bufpos = pos + 3 # skip the three """ else: # ordinary string literal if rawMode: tok.tokType = tkRStrLit else: tok.tokType = tkStrLit while true: - c = buf[pos] + var c = buf[pos] if c == '\"': - inc(pos) # skip '"' - break - if c in {CR, LF, lexbase.EndOfFile}: + if rawMode and buf[pos+1] == '\"': + inc(pos, 2) + add(tok.literal, '"') + else: + inc(pos) # skip '"' + break + elif c in {CR, LF, lexbase.EndOfFile}: lexMessage(L, errClosingQuoteExpected) break - if (c == '\\') and not rawMode: + elif (c == '\\') and not rawMode: L.bufPos = pos getEscapedChar(L, tok) pos = L.bufPos @@ -527,29 +524,23 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) = L.bufpos = pos proc getCharacter(L: var TLexer, tok: var TToken) = - var c: Char Inc(L.bufpos) # skip ' - c = L.buf[L.bufpos] + var c = L.buf[L.bufpos] case c of '\0'..Pred(' '), '\'': lexMessage(L, errInvalidCharacterConstant) of '\\': getEscapedChar(L, tok) else: - tok.literal = c & "" + tok.literal = $c Inc(L.bufpos) if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote) inc(L.bufpos) # skip ' proc getSymbol(L: var TLexer, tok: var TToken) = - var - pos: int - c: Char - buf: cstring - h: THash # hashing algorithm inlined - h = 0 - pos = L.bufpos - buf = L.buf + var h: THash = 0 + var pos = L.bufpos + var buf = L.buf while true: - c = buf[pos] + var c = buf[pos] case c of 'a'..'z', '0'..'9', '\x80'..'\xFF': h = h +% Ord(c) @@ -560,8 +551,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = h = h +% Ord(c) h = h +% h shl 10 h = h xor (h shr 6) - of '_': - nil + of '_': nil else: break Inc(pos) h = h +% h shl 3 @@ -580,16 +570,11 @@ proc getSymbol(L: var TLexer, tok: var TToken) = else: tok.tokType = tkCallTripleStrLit proc getOperator(L: var TLexer, tok: var TToken) = - var - pos: int - c: Char - buf: cstring - h: THash # hashing algorithm inlined - pos = L.bufpos - buf = L.buf - h = 0 + var pos = L.bufpos + var buf = L.buf + var h: THash = 0 while true: - c = buf[pos] + var c = buf[pos] if c in OpChars: h = h +% Ord(c) h = h +% h shl 10 @@ -606,9 +591,8 @@ proc getOperator(L: var TLexer, tok: var TToken) = L.bufpos = pos proc handleIndentation(L: var TLexer, tok: var TToken, indent: int) = - var i: int tok.indent = indent - i = high(L.indentStack) + var i = high(L.indentStack) if indent > L.indentStack[i]: tok.tokType = tkInd elif indent == L.indentStack[i]: @@ -625,22 +609,19 @@ proc handleIndentation(L: var TLexer, tok: var TToken, indent: int) = lexMessage(L, errInvalidIndentation) proc scanComment(L: var TLexer, tok: var TToken) = - var - buf: cstring - pos, col: int - indent: int - pos = L.bufpos - buf = L.buf # a comment ends if the next line does not start with the # on the same - # column after only whitespace + var pos = L.bufpos + var buf = L.buf + # a comment ends if the next line does not start with the # on the same + # column after only whitespace tok.tokType = tkComment - col = getColNumber(L, pos) + var col = getColNumber(L, pos) while true: while not (buf[pos] in {CR, LF, lexbase.EndOfFile}): add(tok.literal, buf[pos]) inc(pos) pos = handleCRLF(L, pos) buf = L.buf - indent = 0 + var indent = 0 while buf[pos] == ' ': inc(pos) inc(indent) @@ -654,11 +635,8 @@ proc scanComment(L: var TLexer, tok: var TToken) = L.bufpos = pos proc skip(L: var TLexer, tok: var TToken) = - var - buf: cstring - indent, pos: int - pos = L.bufpos - buf = L.buf + var pos = L.bufpos + var buf = L.buf while true: case buf[pos] of ' ': @@ -669,7 +647,7 @@ proc skip(L: var TLexer, tok: var TToken) = of CR, LF: pos = HandleCRLF(L, pos) buf = L.buf - indent = 0 + var indent = 0 while buf[pos] == ' ': Inc(pos) Inc(indent) @@ -681,7 +659,6 @@ proc skip(L: var TLexer, tok: var TToken) = L.bufpos = pos proc rawGetTok(L: var TLexer, tok: var TToken) = - var c: Char fillToken(tok) if L.dedent > 0: dec(L.dedent) @@ -691,10 +668,10 @@ proc rawGetTok(L: var TLexer, tok: var TToken) = else: tok.tokType = tkDed return - skip(L, tok) # skip - # got an documentation comment or tkIndent, return that: + skip(L, tok) + # got an documentation comment or tkIndent, return that: if tok.toktype != tkInvalid: return - c = L.buf[L.bufpos] + var c = L.buf[L.bufpos] if c in SymStartChars - {'r', 'R', 'l'}: getSymbol(L, tok) elif c in {'0'..'9'}: diff --git a/tests/tassert.nim b/tests/tassert.nim index 1eff23502..9fd18e9bd 100755 --- a/tests/tassert.nim +++ b/tests/tassert.nim @@ -7,10 +7,10 @@ proc callC() = callA() try: callC() except EAssertionFailed: - write(stdout, "assertion failure!\n") + write(stdout, "assertion failure!") except: - write(stdout, "unknown exception!\n") + write(stdout, "unknown exception!") finally: - system.write(stdout, "this shall be always written\n") + system.write(stdout, "this shall be always written") -assert(false) +assert(false) #OUT assertion failure!this shall be always written diff --git a/tests/tinvalidnewseq.nim b/tests/tinvalidnewseq.nim new file mode 100755 index 000000000..e2a3736c3 --- /dev/null +++ b/tests/tinvalidnewseq.nim @@ -0,0 +1,20 @@ +import regexprs, strutils + +type + TURL = tuple[protocol, subdomain, domain, port: string, path: seq[string]] + +proc parseURL(url: string): TURL = + #([a-zA-Z]+://)?(\w+?\.)?(\w+)(\.\w+)(:[0-9]+)?(/.+)? + var pattern: string = r"([a-zA-Z]+://)?(\w+?\.)?(\w+)(\.\w+)(:[0-9]+)?(/.+)?" + var m: array[0..6, string] #Array with the matches + newSeq(m, 7) #ERROR + discard regexprs.match(url, pattern, m) + + result = (protocol: m[1], subdomain: m[2], domain: m[3] & m[4], + port: m[5], path: m[6].split('/')) + +var r: TUrl + +r = parseUrl(r"http://google.com/search?var=bleahdhsad") +echo(r.domain) + diff --git a/tests/tstrlits.nim b/tests/tstrlits.nim new file mode 100755 index 000000000..48ae08212 --- /dev/null +++ b/tests/tstrlits.nim @@ -0,0 +1,14 @@ +# Test the new different string literals + +const + tripleEmpty = """"long string"""""""" # "long string """"" + + rawQuote = r"a""" + + raw = r"abc""def" + +stdout.write(rawQuote) +stdout.write(tripleEmpty) +stdout.write(raw) +#OUT a""long string"""""abc"def + |