diff options
39 files changed, 1973 insertions, 390 deletions
diff --git a/config/nimrod.cfg b/config/nimrod.cfg index 22dc35f5a..122980348 100755 --- a/config/nimrod.cfg +++ b/config/nimrod.cfg @@ -57,6 +57,7 @@ hint[LineTooLong]=off @if unix: @if not bsd: gcc.options.linker = "-ldl" + tcc.options.linker = "-ldl" @end @end diff --git a/doc/manual.txt b/doc/manual.txt index 9a6060264..5038e292b 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2542,10 +2542,10 @@ cannot be inherited from. Pure pragma ----------- The `pure`:idx: pragma serves two completely different purposes: -1) To mark a procedure that Nimrod should not generate any exit statements like +1. To mark a procedure that Nimrod should not generate any exit statements like ``return result;`` in the generated code. This is useful for procs that only consist of an assembler statement. -2) To mark an object type so that its type field should be omitted. This is +2. To mark an object type so that its type field should be omitted. This is necessary for binary compatibility with other compiled languages. @@ -2795,7 +2795,7 @@ a dynamic library. The pragma then has no argument and has to be used in conjunction with the ``exportc`` pragma: .. code-block:: Nimrod - proc exportme(): int {.cdecl, export, dynlib.} + proc exportme(): int {.cdecl, exportc, dynlib.} This is only useful if the program is compiled as a dynamic library via the ``--app:lib`` command line option. diff --git a/doc/nimrodc.txt b/doc/nimrodc.txt index bd185ca4c..7fada6584 100755 --- a/doc/nimrodc.txt +++ b/doc/nimrodc.txt @@ -17,7 +17,7 @@ Introduction This document describes the usage of the *Nimrod compiler* on the different supported platforms. It is not a definition of the Nimrod -programming language (therefore is the `manual <manual>`_). +programming language (therefore is the `manual <manual.html>`_). Nimrod is free software; it is licensed under the `GNU General Public License <gpl.html>`_. diff --git a/doc/pegdocs.txt b/doc/pegdocs.txt index 922b998c7..6dde73111 100755 --- a/doc/pegdocs.txt +++ b/doc/pegdocs.txt @@ -98,6 +98,11 @@ macro meaning ``\i`` ignore case for matching; use this at the start of the PEG ``\y`` ignore style for matching; use this at the start of the PEG ``\ident`` a standard ASCII identifier: ``[a-zA-Z_][a-zA-Z_0-9]*`` +``\letter`` any Unicode letter +``\upper`` any Unicode uppercase letter +``\lower`` any Unicode lowercase letter +``\title`` any Unicode title letter +``\white`` any Unicode whitespace character ============== ============================================================ A backslash followed by a letter is a built-in macro, otherwise it diff --git a/doc/tut2.txt b/doc/tut2.txt index a139fb5de..bd9769af3 100755 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -75,9 +75,9 @@ Inheritance is done with the ``object of`` syntax. Multiple inheritance is currently not supported. If an object type has no suitable ancestor, ``TObject`` can be used as its ancestor, but this is only a convention. -**Note**: Aggregation (*has-a* relation) is often preferable to inheritance +**Note**: Composition (*has-a* relation) is often preferable to inheritance (*is-a* relation) for simple code reuse. Since objects are value types in -Nimrod, aggregation is as efficient as inheritance. +Nimrod, composition is as efficient as inheritance. Mutually recursive types @@ -487,7 +487,7 @@ The example shows a generic binary tree. Depending on context, the brackets are used either to introduce type parameters or to instantiate a generic proc, iterator or type. As the example shows, generics work with overloading: the best match of ``add`` is used. The built-in ``add`` procedure for sequences -is not hidden and used in the ``preorder`` iterator. +is not hidden and is used in the ``preorder`` iterator. Templates diff --git a/examples/httpserver2.nim b/examples/httpserver2.nim new file mode 100644 index 000000000..0604e6a83 --- /dev/null +++ b/examples/httpserver2.nim @@ -0,0 +1,245 @@ +import strutils, os, osproc, strtabs, streams, sockets + +const + wwwNL* = "\r\L" + ServerSig = "Server: httpserver.nim/1.0.0" & wwwNL + +type + TRequestMethod = enum reqGet, reqPost + TServer* = object ## contains the current server state + socket: TSocket + job: seq[TJob] + TJob* = object + client: TSocket + process: PProcess + +# --------------- output messages -------------------------------------------- + +proc sendTextContentType(client: TSocket) = + send(client, "Content-type: text/html" & wwwNL) + send(client, wwwNL) + +proc badRequest(client: TSocket) = + # Inform the client that a request it has made has a problem. + send(client, "HTTP/1.0 400 BAD REQUEST" & wwwNL) + sendTextContentType(client) + send(client, "<p>Your browser sent a bad request, " & + "such as a POST without a Content-Length.</p>" & wwwNL) + + +proc cannotExec(client: TSocket) = + send(client, "HTTP/1.0 500 Internal Server Error" & wwwNL) + sendTextContentType(client) + send(client, "<P>Error prohibited CGI execution.</p>" & wwwNL) + + +proc headers(client: TSocket, filename: string) = + # XXX could use filename to determine file type + send(client, "HTTP/1.0 200 OK" & wwwNL) + send(client, ServerSig) + sendTextContentType(client) + +proc notFound(client: TSocket, path: string) = + send(client, "HTTP/1.0 404 NOT FOUND" & wwwNL) + send(client, ServerSig) + sendTextContentType(client) + send(client, "<html><title>Not Found</title>" & wwwNL) + send(client, "<body><p>The server could not fulfill" & wwwNL) + send(client, "your request because the resource <b>" & path & "</b>" & wwwNL) + send(client, "is unavailable or nonexistent.</p>" & wwwNL) + send(client, "</body></html>" & wwwNL) + + +proc unimplemented(client: TSocket) = + send(client, "HTTP/1.0 501 Method Not Implemented" & wwwNL) + send(client, ServerSig) + sendTextContentType(client) + send(client, "<html><head><title>Method Not Implemented" & + "</title></head>" & + "<body><p>HTTP request method not supported.</p>" & + "</body></HTML>" & wwwNL) + + +# ----------------- file serving --------------------------------------------- + +proc discardHeaders(client: TSocket) = skip(client) + +proc serveFile(client: TSocket, filename: string) = + discardHeaders(client) + + var f: TFile + if open(f, filename): + headers(client, filename) + const bufSize = 8000 # != 8K might be good for memory manager + var buf = alloc(bufsize) + while True: + var bytesread = readBuffer(f, buf, bufsize) + if bytesread > 0: + var byteswritten = send(client, buf, bytesread) + if bytesread != bytesWritten: + dealloc(buf) + close(f) + OSError() + if bytesread != bufSize: break + dealloc(buf) + close(f) + client.close() + else: + notFound(client, filename) + +# ------------------ CGI execution ------------------------------------------- + +proc executeCgi(server: var TServer, client: TSocket, path, query: string, + meth: TRequestMethod) = + var env = newStringTable(modeCaseInsensitive) + var contentLength = -1 + case meth + of reqGet: + discardHeaders(client) + + env["REQUEST_METHOD"] = "GET" + env["QUERY_STRING"] = query + of reqPost: + var buf = "" + var dataAvail = true + while dataAvail: + dataAvail = recvLine(client, buf) + if buf.len == 0: + break + var L = toLower(buf) + if L.startsWith("content-length:"): + var i = len("content-length:") + while L[i] in Whitespace: inc(i) + contentLength = parseInt(copy(L, i)) + + if contentLength < 0: + badRequest(client) + return + + env["REQUEST_METHOD"] = "POST" + env["CONTENT_LENGTH"] = $contentLength + + send(client, "HTTP/1.0 200 OK" & wwwNL) + + var process = startProcess(command=path, env=env) + + var job: TJob + job.process = process + job.client = client + server.job.add(job) + + if meth == reqPost: + # get from client and post to CGI program: + var buf = alloc(contentLength) + if recv(client, buf, contentLength) != contentLength: + dealloc(buf) + OSError() + var inp = process.inputStream + inp.writeData(inp, buf, contentLength) + dealloc(buf) + +proc animate(server: var TServer) = + # checks list of jobs, removes finished ones (pretty sloppy by seq copying) + var active_jobs: seq[TJob] = @[] + for i in 0..server.job.len-1: + var job = server.job[i] + if running(job.process): + active_jobs.add(job) + else: + # read process output stream and send it to client + var outp = job.process.outputStream + while true: + var line = outp.readstr(1024) + if line.len == 0: + break + else: + try: + send(job.client, line) + except: + echo("send failed, client diconnected") + close(job.client) + + server.job = active_jobs + +# --------------- Server Setup ----------------------------------------------- + +proc acceptRequest(server: var TServer, client: TSocket) = + var cgi = false + var query = "" + var buf = "" + discard recvLine(client, buf) + var path = "" + var data = buf.split() + var meth = reqGet + var q = find(data[1], '?') + + # extract path + if q >= 0: + # strip "?..." from path, this may be found in both POST and GET + path = data[1].copy(0, q-1) + else: + path = data[1] + # path starts with "/", by adding "." in front of it we serve files from cwd + path = "." & path + + echo("accept: " & path) + + if cmpIgnoreCase(data[0], "GET") == 0: + if q >= 0: + cgi = true + query = data[1].copy(q+1) + elif cmpIgnoreCase(data[0], "POST") == 0: + cgi = true + meth = reqPost + else: + unimplemented(client) + + if path[path.len-1] == '/' or existsDir(path): + path = path / "index.html" + + if not ExistsFile(path): + discardHeaders(client) + notFound(client, path) + client.close() + else: + when defined(Windows): + var ext = splitFile(path).ext.toLower + if ext == ".exe" or ext == ".cgi": + # XXX: extract interpreter information here? + cgi = true + else: + if {fpUserExec, fpGroupExec, fpOthersExec} * path.getFilePermissions != {}: + cgi = true + if not cgi: + serveFile(client, path) + else: + executeCgi(server, client, path, query, meth) + + + +when isMainModule: + var port = 80 + + var server: TServer + server.job = @[] + server.socket = socket(AF_INET) + if server.socket == InvalidSocket: OSError() + server.socket.bindAddr(port=TPort(port)) + listen(server.socket) + echo("server up on port " & $port) + + while true: + # check for new new connection & handle it + var list: seq[TSocket] = @[server.socket] + if select(list, 10) > 0: + var client = accept(server.socket) + try: + acceptRequest(server, client) + except: + echo("failed to accept client request") + + # pooling events + animate(server) + # some slack for CPU + sleep(10) + server.socket.close() diff --git a/lib/nimbase.h b/lib/nimbase.h index 983bb112d..0251364d1 100755 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -22,6 +22,10 @@ __TINYC__ #ifndef NIMBASE_H #define NIMBASE_H +#if defined(__GNUC__) +# define _GNU_SOURCE 1 +#endif + #if !defined(__TINYC__) # include <math.h> #else @@ -29,7 +33,6 @@ __TINYC__ # define GCC_MAJOR 4 # define __GNUC_MINOR__ 4 # define __GNUC_PATCHLEVEL__ 5 */ - # define __DECLSPEC_SUPPORTED 1 #endif diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index 643c67bec..af222caba 100755 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## This module implements helper procs for CGI applictions. Example: +## This module implements helper procs for CGI applications. Example: ## ## .. code-block:: Nimrod ## @@ -29,7 +29,7 @@ ## writeln(stdout, "your password: " & myData["password"]) ## writeln(stdout, "</body></html>") -import strutils, os, strtabs +import strutils, os, strtabs, cookies proc URLencode*(s: string): string = ## Encodes a value to be HTTP safe: This means that characters in the set @@ -355,32 +355,16 @@ proc setCookie*(name, value: string) = write(stdout, "Set-Cookie: ", name, "=", value, "\n") var - cookies: PStringTable = nil - -proc parseCookies(s: string): PStringTable = - result = newStringTable(modeCaseInsensitive) - var i = 0 - while true: - while s[i] == ' ' or s[i] == '\t': inc(i) - var keystart = i - while s[i] != '=' and s[i] != '\0': inc(i) - var keyend = i-1 - if s[i] == '\0': break - inc(i) # skip '=' - var valstart = i - while s[i] != ';' and s[i] != '\0': inc(i) - result[copy(s, keystart, keyend)] = copy(s, valstart, i-1) - if s[i] == '\0': break - inc(i) # skip ';' + gcookies: PStringTable = nil proc getCookie*(name: string): string = ## Gets a cookie. If no cookie of `name` exists, "" is returned. - if cookies == nil: cookies = parseCookies(getHttpCookie()) - result = cookies[name] + if gcookies == nil: gcookies = parseCookies(getHttpCookie()) + result = gcookies[name] proc existsCookie*(name: string): bool = ## Checks if a cookie of `name` exists. - if cookies == nil: cookies = parseCookies(getHttpCookie()) - result = hasKey(cookies, name) + if gcookies == nil: gcookies = parseCookies(getHttpCookie()) + result = hasKey(gcookies, name) diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim new file mode 100644 index 000000000..eed6c7512 --- /dev/null +++ b/lib/pure/cookies.nim @@ -0,0 +1,30 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2010 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements helper procs for parsing Cookies. + +import strtabs + +proc parseCookies*(s: string): PStringTable = + ## parses cookies into a string table. + result = newStringTable(modeCaseInsensitive) + var i = 0 + while true: + while s[i] == ' ' or s[i] == '\t': inc(i) + var keystart = i + while s[i] != '=' and s[i] != '\0': inc(i) + var keyend = i-1 + if s[i] == '\0': break + inc(i) # skip '=' + var valstart = i + while s[i] != ';' and s[i] != '\0': inc(i) + result[copy(s, keystart, keyend)] = copy(s, valstart, i-1) + if s[i] == '\0': break + inc(i) # skip ';' + diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index bff1b381d..cb36ea541 100755 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -136,9 +136,12 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) = if meth == reqPost: # get from client and post to CGI program: var buf = alloc(contentLength) - if recv(client, buf, contentLength) != contentLength: OSError() + if recv(client, buf, contentLength) != contentLength: + dealloc(buf) + OSError() var inp = process.inputStream inp.writeData(inp, buf, contentLength) + dealloc(buf) var outp = process.outputStream while running(process) or not outp.atEnd(outp): @@ -153,10 +156,21 @@ proc acceptRequest(client: TSocket) = var query = "" var buf = "" discard recvLine(client, buf) + var path = "" var data = buf.split() var meth = reqGet + + var q = find(data[1], '?') + + # extract path + if q >= 0: + # strip "?..." from path, this may be found in both POST and GET + path = "." & data[1].copy(0, q-1) + else: + path = "." & data[1] + # path starts with "/", by adding "." in front of it we serve files from cwd + if cmpIgnoreCase(data[0], "GET") == 0: - var q = find(data[1], '?') if q >= 0: cgi = true query = data[1].copy(q+1) @@ -166,7 +180,6 @@ proc acceptRequest(client: TSocket) = else: unimplemented(client) - var path = data[1] if path[path.len-1] == '/' or existsDir(path): path = path / "index.html" @@ -221,7 +234,7 @@ proc next*(s: var TServer) = var buf = "" discard recvLine(s.client, buf) var data = buf.split() - if cmpIgnoreCase(data[0], "GET") == 0: + if cmpIgnoreCase(data[0], "GET") == 0 or cmpIgnoreCase(data[0], "POST") == 0: var q = find(data[1], '?') if q >= 0: s.query = data[1].copy(q+1) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 8e3dd4bdb..c9f71b64b 100755 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -159,6 +159,7 @@ when not defined(ECMAScript): ## same as ``sqrt(x*x + y*y)``. proc sinh*(x: float): float {.importc: "sinh", header: "<math.h>".} + proc sin*(x: float): float {.importc: "sin", header: "<math.h>".} proc tan*(x: float): float {.importc: "tan", header: "<math.h>".} proc tanh*(x: float): float {.importc: "tanh", header: "<math.h>".} proc pow*(x, y: float): float {.importc: "pow", header: "<math.h>".} @@ -209,6 +210,7 @@ else: proc cosh*(x: float): float = return (exp(x)+exp(-x))*0.5 proc hypot*(x, y: float): float = return sqrt(x*x + y*y) proc sinh*(x: float): float = return (exp(x)-exp(-x))*0.5 + proc sin*(x: float): float {.importc: "Math.sin", nodecl.} proc tan*(x: float): float {.importc: "Math.tan", nodecl.} proc tanh*(x: float): float = var y = exp(2.0*x) @@ -226,21 +228,27 @@ proc push*(s: var TRunningStat, x: float) = inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 if s.n == 1: + s.min = x + s.max = x s.oldM = x s.mean = x s.oldS = 0.0 else: + if s.min > x: s.min = x + if s.max < x: s.max = x s.mean = s.oldM + (x - s.oldM)/toFloat(s.n) s.newS = s.oldS + (x - s.oldM)*(x - s.mean) # set up for next iteration: s.oldM = s.mean s.oldS = s.newS - s.sum = s.sum + x - if s.min > x: s.min = x - if s.max < x: s.max = x - + +proc push*(s: var TRunningStat, x: int) = + ## pushes a value `x` for processing. `x` is simply converted to ``float`` + ## and the other push operation is called. + push(s, toFloat(x)) + proc variance*(s: TRunningStat): float = ## computes the current variance of `s` if s.n > 1: result = s.newS / (toFloat(s.n - 1)) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 0a373125d..175713a0c 100755 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -37,6 +37,11 @@ type pkAny, ## any character (.) pkAnyRune, ## any Unicode character (_) pkNewLine, ## CR-LF, LF, CR + pkLetter, ## Unicode letter + pkLower, ## Unicode lower case letter + pkUpper, ## Unicode upper case letter + pkTitle, ## Unicode title character + pkWhitespace, ## Unicode whitespace character pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, @@ -71,7 +76,7 @@ type rule: TNode ## the rule that the symbol refers to TNode {.final.} = object case kind: TPegKind - of pkEmpty, pkAny, pkAnyRune, pkGreedyAny, pkNewLine: nil + of pkEmpty..pkWhitespace: nil of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle: term: string of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] @@ -196,6 +201,7 @@ proc `@`*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsSearch".} = proc `@@`*(a: TPeg): TPeg {.noSideEffect, rtl, extern: "npgegsCapturedSearch".} = + ## constructs a "captured search" for the PEG `a` result.kind = pkCapturedSearch result.sons = @[a] @@ -237,6 +243,27 @@ proc newLine*: TPeg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result.kind = pkNewline +proc UnicodeLetter*: TPeg {.inline.} = + ## constructs the PEG ``\letter`` which matches any Unicode letter. + result.kind = pkLetter + +proc UnicodeLower*: TPeg {.inline.} = + ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. + result.kind = pkLower + +proc UnicodeUpper*: TPeg {.inline.} = + ## constructs the PEG ``\upper`` which matches any Unicode lowercase letter. + result.kind = pkUpper + +proc UnicodeTitle*: TPeg {.inline.} = + ## constructs the PEG ``\title`` which matches any Unicode title letter. + result.kind = pkTitle + +proc UnicodeWhitespace*: TPeg {.inline.} = + ## constructs the PEG ``\white`` which matches any Unicode + ## whitespace character. + result.kind = pkWhitespace + proc capture*(a: TPeg): TPeg {.nosideEffect, rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result.kind = pkCapture @@ -267,8 +294,8 @@ proc spaceCost(n: TPeg): int = case n.kind of pkEmpty: nil of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, - pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, pkAny, pkAnyRune, - pkNewLine, pkGreedyAny: + pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, + pkAny..pkWhitespace, pkGreedyAny: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -379,6 +406,12 @@ proc toStrAux(r: TPeg, res: var string) = of pkEmpty: add(res, "()") of pkAny: add(res, '.') of pkAnyRune: add(res, '_') + of pkLetter: add(res, "\\letter") + of pkLower: add(res, "\\lower") + of pkUpper: add(res, "\\upper") + of pkTitle: add(res, "\\title") + of pkWhitespace: add(res, "\\white") + of pkNewline: add(res, "\\n") of pkTerminal: add(res, singleQuoteEsc(r.term)) of pkTerminalIgnoreCase: @@ -460,10 +493,15 @@ proc `$` *(r: TPeg): string {.nosideEffect, rtl, extern: "npegsToString".} = # --------------------- core engine ------------------------------------------- type - TMatchClosure {.final.} = object + TCaptures* {.final.} = object ## contains the captured substrings. matches: array[0..maxSubpatterns-1, tuple[first, last: int]] ml: int +proc bounds*(c: TCaptures, + i: range[0..maxSubpatterns-1]): tuple[first, last: int] = + ## returns the bounds ``[first..last]`` of the `i`'th capture. + result = c.matches[i] + when not useUnicode: type TRune = char @@ -472,9 +510,17 @@ when not useUnicode: inc(i) template runeLenAt(s, i: expr): expr = 1 -proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = - ## this implements a simple PEG interpreter. Thanks to superoperators it - ## has competitive performance nevertheless. + proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z','A'..'Z'} + proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + proc isTitle(a: char): bool {.inline.} = return false + proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + +proc rawMatch*(s: string, p: TPeg, start: int, c: var TCaptures): int {. + nosideEffect, rtl, extern: "npegs$1".} = + ## low-level matching proc that implements the PEG interpreter. Use this + ## for maximum efficiency (every other PEG operation ends up calling this + ## proc). ## Returns -1 if it does not match, else the length of the match case p.kind of pkEmpty: result = 0 # match of length 0 @@ -486,6 +532,51 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = result = runeLenAt(s, start) else: result = -1 + of pkLetter: + if s[start] != '\0': + var a: TRune + result = start + fastRuneAt(s, result, a) + if isAlpha(a): dec(result, start) + else: result = -1 + else: + result = -1 + of pkLower: + if s[start] != '\0': + var a: TRune + result = start + fastRuneAt(s, result, a) + if isLower(a): dec(result, start) + else: result = -1 + else: + result = -1 + of pkUpper: + if s[start] != '\0': + var a: TRune + result = start + fastRuneAt(s, result, a) + if isUpper(a): dec(result, start) + else: result = -1 + else: + result = -1 + of pkTitle: + if s[start] != '\0': + var a: TRune + result = start + fastRuneAt(s, result, a) + if isTitle(a): dec(result, start) + else: result = -1 + else: + result = -1 + of pkWhitespace: + if s[start] != '\0': + var a: TRune + result = start + fastRuneAt(s, result, a) + if isWhitespace(a): dec(result, start) + else: result = -1 + else: + result = -1 of pkGreedyAny: result = len(s) - start of pkNewLine: @@ -537,14 +628,14 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = of pkNonTerminal: var oldMl = c.ml when false: echo "enter: ", p.nt.name - result = m(s, p.nt.rule, start, c) + result = rawMatch(s, p.nt.rule, start, c) when false: echo "leave: ", p.nt.name if result < 0: c.ml = oldMl of pkSequence: var oldMl = c.ml result = 0 for i in 0..high(p.sons): - var x = m(s, p.sons[i], start+result, c) + var x = rawMatch(s, p.sons[i], start+result, c) if x < 0: c.ml = oldMl result = -1 @@ -553,14 +644,14 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = of pkOrderedChoice: var oldMl = c.ml for i in 0..high(p.sons): - result = m(s, p.sons[i], start, c) + result = rawMatch(s, p.sons[i], start, c) if result >= 0: break c.ml = oldMl of pkSearch: var oldMl = c.ml result = 0 while start+result < s.len: - var x = m(s, p.sons[0], start+result, c) + var x = rawMatch(s, p.sons[0], start+result, c) if x >= 0: inc(result, x) return @@ -572,7 +663,7 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = inc(c.ml) result = 0 while start+result < s.len: - var x = m(s, p.sons[0], start+result, c) + var x = rawMatch(s, p.sons[0], start+result, c) if x >= 0: if idx < maxSubpatterns: c.matches[idx] = (start, start+result-1) @@ -585,7 +676,7 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = of pkGreedyRep: result = 0 while true: - var x = m(s, p.sons[0], start+result, c) + var x = rawMatch(s, p.sons[0], start+result, c) # if x == 0, we have an endless loop; so the correct behaviour would be # not to break. But endless loops can be easily introduced: # ``(comment / \w*)*`` is such an example. Breaking for x == 0 does the @@ -600,15 +691,15 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = result = 0 while contains(p.charChoice^, s[start+result]): inc(result) of pkOption: - result = max(0, m(s, p.sons[0], start, c)) + result = max(0, rawMatch(s, p.sons[0], start, c)) of pkAndPredicate: var oldMl = c.ml - result = m(s, p.sons[0], start, c) + result = rawMatch(s, p.sons[0], start, c) if result >= 0: result = 0 # do not consume anything else: c.ml = oldMl of pkNotPredicate: var oldMl = c.ml - result = m(s, p.sons[0], start, c) + result = rawMatch(s, p.sons[0], start, c) if result < 0: result = 0 else: c.ml = oldMl @@ -616,7 +707,7 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = of pkCapture: var idx = c.ml # reserve a slot for the subpattern inc(c.ml) - result = m(s, p.sons[0], start, c) + result = rawMatch(s, p.sons[0], start, c) if result >= 0: if idx < maxSubpatterns: c.matches[idx] = (start, start+result-1) @@ -629,7 +720,7 @@ proc m(s: string, p: TPeg, start: int, c: var TMatchClosure): int = var n: TPeg n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) n.term = s.copy(a, b) - result = m(s, n, start, c) + result = rawMatch(s, n, start, c) of pkRule, pkList: assert false proc match*(s: string, pattern: TPeg, matches: var openarray[string], @@ -638,8 +729,8 @@ proc match*(s: string, pattern: TPeg, matches: var openarray[string], ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. - var c: TMatchClosure - result = m(s, pattern, start, c) == len(s) -start + var c: TCaptures + result = rawMatch(s, pattern, start, c) == len(s) -start if result: for i in 0..c.ml-1: matches[i] = copy(s, c.matches[i][0], c.matches[i][1]) @@ -647,8 +738,8 @@ proc match*(s: string, pattern: TPeg, matches: var openarray[string], proc match*(s: string, pattern: TPeg, start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. - var c: TMatchClosure - result = m(s, pattern, start, c) == len(s)-start + var c: TCaptures + result = rawMatch(s, pattern, start, c) == len(s)-start proc matchLen*(s: string, pattern: TPeg, matches: var openarray[string], start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = @@ -656,8 +747,8 @@ proc matchLen*(s: string, pattern: TPeg, matches: var openarray[string], ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains ## that does not belong to the match. - var c: TMatchClosure - result = m(s, pattern, start, c) + var c: TCaptures + result = rawMatch(s, pattern, start, c) if result >= 0: for i in 0..c.ml-1: matches[i] = copy(s, c.matches[i][0], c.matches[i][1]) @@ -668,8 +759,8 @@ proc matchLen*(s: string, pattern: TPeg, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains ## that does not belong to the match. - var c: TMatchClosure - result = m(s, pattern, start, c) + var c: TCaptures + result = rawMatch(s, pattern, start, c) proc find*(s: string, pattern: TPeg, matches: var openarray[string], start = 0): int {.nosideEffect, rtl, extern: "npegs$1Capture".} = @@ -681,6 +772,18 @@ proc find*(s: string, pattern: TPeg, matches: var openarray[string], return -1 # could also use the pattern here: (!P .)* P +proc findBounds*(s: string, pattern: TPeg, matches: var openarray[string], + start = 0): tuple[first, last: int] {. + nosideEffect, rtl, extern: "npegs$1Capture".} = + ## returns the starting position and end position of ``pattern`` in ``s`` + ## and the captured + ## substrings in the array ``matches``. If it does not match, nothing + ## is written into ``matches`` and (-1,0) is returned. + for i in start .. s.len-1: + var L = matchLen(s, pattern, matches, i) + if L >= 0: return (i, i+L-1) + return (-1, 0) + proc find*(s: string, pattern: TPeg, start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not @@ -1351,6 +1454,11 @@ proc primary(p: var TPegParser): TPeg = of "a": result = charset({'a'..'z', 'A'..'Z'}) of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'}) of "ident": result = pegs.ident + of "letter": result = UnicodeLetter() + of "upper": result = UnicodeUpper() + of "lower": result = UnicodeLower() + of "title": result = UnicodeTitle() + of "white": result = UnicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) getTok(p) of tkEscaped: @@ -1439,9 +1547,12 @@ proc rawParse(p: var TPegParser): TPeg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(input: string, filename = "pattern", line = 1, col = 0): TPeg = +proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): TPeg = + ## constructs a TPeg object from `pattern`. `filename`, `line`, `col` are + ## used for error messages, but they only provide start offsets. `parsePeg` + ## keeps track of line and column numbers within `pattern`. var p: TPegParser - init(TPegLexer(p), input, filename, line, col) + init(TPegLexer(p), pattern, filename, line, col) p.tok.kind = tkInvalid p.tok.modifier = modNone p.tok.literal = "" @@ -1505,9 +1616,9 @@ when isMainModule: expr.rule = sequence(capture(ident), *sequence( nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) - var c: TMatchClosure + var c: TCaptures var s = "a+b + c +d+e+f" - assert m(s, expr.rule, 0, c) == len(s) + assert rawMatch(s, expr.rule, 0, c) == len(s) var a = "" for i in 0..c.ml-1: a.add(copy(s, c.matches[i][0], c.matches[i][1])) @@ -1559,4 +1670,10 @@ when isMainModule: else: assert false - + assert match("eine übersicht und außerdem", peg"(\letter \white*)+") + # ß is not a lower cased letter?! + assert match("eine übersicht und auerdem", peg"(\lower \white*)+") + assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+") + assert(not match("456678", peg"(\letter)+")) + + diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 85628db78..add41afd6 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -9,7 +9,7 @@ ## This module implements a simple portable type-safe sockets layer. -import os +import os, parseutils when defined(Windows): import winlean @@ -146,18 +146,66 @@ proc listen*(socket: TSocket, attempts = 5) = ## listens to socket. if listen(cint(socket), cint(attempts)) < 0'i32: OSError() -proc bindAddr*(socket: TSocket, port = TPort(0)) = - ## binds a port number to a socket. +proc invalidIp4(s: string) {.noreturn, noinline.} = + raise newException(EInvalidValue, "invalid ip4 address: " & s) + +proc parseIp4*(s: string): int32 = + ## parses an IP version 4 in dotted decimal form like "a.b.c.d". + ## Raises EInvalidValue in case of an error. + var a, b, c, d: int + var i = 0 + var j = parseInt(s, a, i) + if j <= 0: invalidIp4(s) + inc(i, j) + if s[i] == '.': inc(i) + else: invalidIp4(s) + j = parseInt(s, b, i) + if j <= 0: invalidIp4(s) + inc(i, j) + if s[i] == '.': inc(i) + else: invalidIp4(s) + j = parseInt(s, c, i) + if j <= 0: invalidIp4(s) + inc(i, j) + if s[i] == '.': inc(i) + else: invalidIp4(s) + j = parseInt(s, d, i) + if j <= 0: invalidIp4(s) + inc(i, j) + if s[i] != '\0': invalidIp4(s) + result = int32(a shl 24 or b shl 16 or c shl 8 or d) + +proc bindAddr*(socket: TSocket, port = TPort(0), address = "") = + ## binds an address/port number to a socket. + ## Use address string in dotted decimal form like "a.b.c.d" + ## or leave "" for any address. var name: Tsockaddr_in when defined(Windows): name.sin_family = int16(ord(AF_INET)) else: name.sin_family = posix.AF_INET name.sin_port = sockets.htons(int16(port)) - name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) + if address == "": + name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) + else: + name.sin_addr.s_addr = parseIp4(address) if bindSocket(cint(socket), cast[ptr TSockAddr](addr(name)), sizeof(name)) < 0'i32: OSError() + +when false: + proc bindAddr*(socket: TSocket, port = TPort(0)) = + ## binds a port number to a socket. + var name: Tsockaddr_in + when defined(Windows): + name.sin_family = int16(ord(AF_INET)) + else: + name.sin_family = posix.AF_INET + name.sin_port = sockets.htons(int16(port)) + name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) + if bindSocket(cint(socket), cast[ptr TSockAddr](addr(name)), + sizeof(name)) < 0'i32: + OSError() proc getSockName*(socket: TSocket): TPort = ## returns the socket's associated port number. @@ -410,6 +458,33 @@ proc send*(socket: TSocket, data: string) = if send(socket, cstring(data), data.len) != data.len: OSError() when defined(Windows): + const + SOCKET_ERROR = -1 + IOCPARM_MASK = 127 + IOC_IN = int(-2147483648) + FIONBIO = int(IOC_IN or ((sizeof(int) and IOCPARM_MASK) shl 16) or + (102 shl 8) or 126) + + proc ioctlsocket(s: TWinSocket, cmd: clong, + argptr: ptr clong): cint {. + stdcall, importc:"ioctlsocket", dynlib: "ws2_32.dll".} + +proc setBlocking*(s: TSocket, blocking: bool) = + ## sets blocking mode on socket + when defined(Windows): + var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking + if SOCKET_ERROR == ioctlsocket(TWinSocket(s), FIONBIO, addr(mode)): + OSError() + else: # BSD sockets + var x: int = fcntl(cint(s), F_GETFL, 0) + if x == -1: + OSError() + else: + var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK + if fcntl(cint(s), F_SETFL, mode) == -1: + OSError() + +when defined(Windows): var wsa: TWSADATA if WSAStartup(0x0101'i16, wsa) != 0: OSError() diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f5adf1abb..f6de035a8 100755 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -449,8 +449,83 @@ proc repeatChar*(count: int, c: Char = ' '): string {.noSideEffect, ## Returns a string of length `count` consisting only of ## the character `c`. result = newString(count) - for i in 0..count-1: - result[i] = c + for i in 0..count-1: result[i] = c + +proc align*(s: string, count: int): string {. + noSideEffect, rtl, extern: "nsuAlignString".} = + ## Aligns a string `s` with spaces, so that is of length `count`. Spaces are + ## added before `s` resulting in right alignment. If ``s.len >= count``, no + ## spaces are added and `s` is returned unchanged. + if s.len < count: + result = newString(count) + var spaces = count - s.len + for i in 0..spaces-1: result[i] = ' ' + for i in spaces..count-1: result[i] = s[i-spaces] + else: + result = s + +iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[ + token: string, isSep: bool] = + ## Tokenizes the string `s` into substrings. + ## + ## Substrings are separated by a substring containing only `seps`. + ## Examples: + ## + ## .. code-block:: nimrod + ## for word in tokenize(" this is an example "): + ## writeln(stdout, word) + ## + ## Results in: + ## + ## .. code-block:: nimrod + ## (" ", true) + ## ("this", false) + ## (" ", true) + ## ("is", false) + ## (" ", true) + ## ("an", false) + ## (" ", true) + ## ("example", false) + ## (" ", true) + var i = 0 + while true: + var j = i + var isSep = s[j] in seps + while j < s.len and (s[j] in seps) == isSep: inc(j) + if j > i: + yield (copy(s, i, j-1), isSep) + else: + break + i = j + +proc wordWrap*(s: string, maxLineWidth = 80, + splitLongWords = true, + seps: set[char] = whitespace, + newLine = "\n"): string {. + noSideEffect, rtl, extern: "nsuWordWrap".} = + ## word wraps `s`. + result = "" + var SpaceLeft = maxLineWidth + for word, isSep in tokenize(s, seps): + if len(word) > SpaceLeft: + if splitLongWords and len(word) > maxLineWidth: + result.add(copy(word, 0, spaceLeft-1)) + var w = spaceLeft+1 + var wordLeft = len(word) - spaceLeft + while wordLeft > 0: + result.add(newLine) + var L = min(maxLineWidth, wordLeft) + SpaceLeft = maxLineWidth - L + result.add(copy(word, w, w+L-1)) + inc(w, L) + dec(wordLeft, L) + else: + SpaceLeft = maxLineWidth - len(Word) + result.add(newLine) + result.add(word) + else: + SpaceLeft = SpaceLeft - len(Word) + result.add(word) proc startsWith*(s, prefix: string): bool {.noSideEffect, rtl, extern: "nsuStartsWith".} = @@ -739,7 +814,7 @@ proc validEmailAddress*(s: string): bool {.noSideEffect, rtl, extern: "nsuValidEmailAddress".} = ## returns true if `s` seems to be a valid e-mail address. ## The checking also uses a domain list. - ## Note: This will be moved into another module soon. + ## Note: This will be moved to another module soon. const chars = Letters + Digits + {'!','#','$','%','&', '\'','*','+','/','=','?','^','_','`','{','}','|','~','-','.'} @@ -862,3 +937,12 @@ proc editDistance*(a, b: string): int {.noSideEffect, #dealloc(row) {.pop.} + +when isMainModule: + assert align("abc", 4) == " abc" + assert align("a", 0) == "a" + assert align("1232", 6) == " 1232" + echo wordWrap(""" this is a long text -- muchlongerthan10chars and here + it goes""", 10, false) + + diff --git a/lib/system.nim b/lib/system.nim index dad8d2d79..262f0926d 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -664,6 +664,9 @@ proc `&` * (x, y: string): string {. proc `&` * (x: char, y: string): string {. magic: "ConStrStr", noSideEffect, merge.} ## is the `concatenation operator`. It concatenates `x` and `y`. + +# implementation note: These must all have the same magic value "ConStrStr" so +# that the merge optimization works properly. proc add*(x: var string, y: char) {.magic: "AppendStrCh", noSideEffect.} proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.} @@ -1099,6 +1102,15 @@ iterator items*(a: cstring): char {.inline.} = yield a[i] inc(i) +iterator enumerate*[TContainer, TItem](a: TContainer): tuple[ + index: int, item: TItem] {.inline.} = + ## iterates over each item of `a` via `items` and yields an additional + ## counter/index starting from 0. + var j = 0 + for it in items(a): + yield (j, a) + inc j + proc isNil*[T](x: seq[T]): bool {.noSideEffect, magic: "IsNil".} proc isNil*[T](x: ref T): bool {.noSideEffect, magic: "IsNil".} proc isNil*(x: string): bool {.noSideEffect, magic: "IsNil".} @@ -1108,20 +1120,20 @@ proc isNil*(x: cstring): bool {.noSideEffect, magic: "IsNil".} ## Fast check whether `x` is nil. This is sometimes more efficient than ## ``== nil``. -proc `&` *[T](x, y: openArray[T]): seq[T] {.noSideEffect.} = +proc `&` *[T](x, y: seq[T]): seq[T] {.noSideEffect.} = newSeq(result, x.len + y.len) for i in 0..x.len-1: result[i] = x[i] for i in 0..y.len-1: result[i+x.len] = y[i] -proc `&` *[T](x: openArray[T], y: T): seq[T] {.noSideEffect.} = +proc `&` *[T](x: seq[T], y: T): seq[T] {.noSideEffect.} = newSeq(result, x.len + 1) for i in 0..x.len-1: result[i] = x[i] result[x.len] = y -proc `&` *[T](x: T, y: openArray[T]): seq[T] {.noSideEffect.} = +proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = newSeq(result, y.len + 1) for i in 0..y.len-1: result[i] = y[i] diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 938583bf7..9beeb659e 100755 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -62,10 +62,7 @@ proc popCurrentException {.compilerRtl, inl.} = # some platforms have native support for stack traces: const nativeStackTrace = (defined(macosx) or defined(linux)) and - not nimrodStackTrace and false - -# `nativeStackTrace` does not work for me --> deactivated for now. Maybe for -# the next release version. + not nimrodStackTrace when nativeStacktrace: type diff --git a/lib/system/gc.nim b/lib/system/gc.nim index aca3705cd..72ae84096 100755 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -76,17 +76,17 @@ var # This is wasteful but safe. This is a lock against recursive garbage # collection, not a lock for threads! -proc lock(gch: var TGcHeap) {.inline.} = +proc aquire(gch: var TGcHeap) {.inline.} = when hasThreadSupport: if isMultiThreaded: - Lock(gch.zctLock) - lock(gch.cycleRootsLock) + aquire(gch.zctLock) + aquire(gch.cycleRootsLock) -proc unlock(gch: var TGcHeap) {.inline.} = +proc release(gch: var TGcHeap) {.inline.} = when hasThreadSupport: if isMultiThreaded: - unlock(gch.zctLock) - unlock(gch.cycleRootsLock) + release(gch.zctLock) + release(gch.cycleRootsLock) proc addZCT(s: var TCellSeq, c: PCell) {.noinline.} = if (c.refcount and rcZct) == 0: @@ -205,18 +205,18 @@ proc prepareDealloc(cell: PCell) = proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! when hasThreadSupport: - if isMultiThreaded: Lock(gch.cycleRootsLock) + if isMultiThreaded: Aquire(gch.cycleRootsLock) incl(gch.cycleRoots, c) when hasThreadSupport: - if isMultiThreaded: Unlock(gch.cycleRootsLock) + if isMultiThreaded: Release(gch.cycleRootsLock) proc rtlAddZCT(c: PCell) {.rtl, inl.} = # we MUST access gch as a global here, because this crosses DLL boundaries! when hasThreadSupport: - if isMultiThreaded: Lock(gch.zctLock) + if isMultiThreaded: Aquire(gch.zctLock) addZCT(gch.zct, c) when hasThreadSupport: - if isMultiThreaded: Unlock(gch.zctLock) + if isMultiThreaded: Release(gch.zctLock) proc decRef(c: PCell) {.inline.} = when stressGC: @@ -333,7 +333,7 @@ proc checkCollection {.inline.} = proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = # generates a new object and sets its reference counter to 0 - lock(gch) + aquire(gch) assert(typ.kind in {tyRef, tyString, tySequence}) checkCollection() var res = cast[PCell](rawAlloc(allocator, size + sizeof(TCell))) @@ -362,7 +362,7 @@ proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = add(gch.zct, res) when logGC: writeCell("new cell", res) gcTrace(res, csAllocated) - unlock(gch) + release(gch) result = cellToUsr(res) proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = @@ -372,7 +372,7 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = cast[PGenericSeq](result).space = len proc growObj(old: pointer, newsize: int): pointer {.rtl.} = - lock(gch) + aquire(gch) checkCollection() var ol = usrToCell(old) assert(ol.typ != nil) @@ -410,7 +410,7 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} = else: assert(ol.typ != nil) zeroMem(ol, sizeof(TCell)) - unlock(gch) + release(gch) result = cellToUsr(res) # ---------------- cycle collector ------------------------------------------- @@ -679,12 +679,12 @@ when not defined(useNimRtl): # set to the max value to suppress the cycle detector proc GC_fullCollect() = - lock(gch) + aquire(gch) var oldThreshold = cycleThreshold cycleThreshold = 0 # forces cycle collection collectCT(gch) cycleThreshold = oldThreshold - unlock(gch) + release(gch) proc GC_getStatistics(): string = GC_disable() diff --git a/lib/system/systhread.nim b/lib/system/systhread.nim index af001985e..a124fa92f 100755 --- a/lib/system/systhread.nim +++ b/lib/system/systhread.nim @@ -48,6 +48,7 @@ proc atomicDec(memLoc: var int, x: int): int = when defined(Windows): type THandle = int + TSysThread = THandle TSysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi DebugInfo: pointer LockCount: int32 @@ -58,9 +59,9 @@ when defined(Windows): proc InitLock(L: var TSysLock) {.stdcall, dynlib: "kernel32", importc: "InitializeCriticalSection".} - proc Lock(L: var TSysLock) {.stdcall, + proc Aquire(L: var TSysLock) {.stdcall, dynlib: "kernel32", importc: "EnterCriticalSection".} - proc Unlock(L: var TSysLock) {.stdcall, + proc Release(L: var TSysLock) {.stdcall, dynlib: "kernel32", importc: "LeaveCriticalSection".} proc CreateThread(lpThreadAttributes: Pointer, dwStackSize: int32, @@ -76,20 +77,38 @@ else: proc InitLock(L: var TSysLock, attr: pointer = nil) {. importc: "pthread_mutex_init", header: "<pthread.h>".} - proc Lock(L: var TSysLock) {. + proc Aquire(L: var TSysLock) {. importc: "pthread_mutex_lock", header: "<pthread.h>".} - proc Unlock(L: var TSysLock) {. + proc Release(L: var TSysLock) {. importc: "pthread_mutex_unlock", header: "<pthread.h>".} type - TThread* {.final, pure.} = object - id: int - next: ptr TThread + TThread* = TSysThread + TLock* = TSysLock TThreadFunc* = proc (closure: pointer) {.cdecl.} -proc createThread*(t: var TThread, fn: TThreadFunc) = +#DWORD WINAPI SuspendThread( +# __in HANDLE hThread +#); +#DWORD WINAPI ResumeThread( +# __in HANDLE hThread +#); +#DWORD WINAPI ThreadProc( +# __in LPVOID lpParameter +#); + +proc createThread*(t: var TThread, fn: TThreadFunc, closure: pointer) = + when defined(windows): + nil + else: + nil + #pthread_create( + +proc joinThread*(t: TThread) = nil + +#proc pthread_exit(void *value_ptr) proc destroyThread*(t: var TThread) = nil diff --git a/rod/ast.nim b/rod/ast.nim index a2d35044e..4b16078fc 100755 --- a/rod/ast.nim +++ b/rod/ast.nim @@ -537,8 +537,8 @@ const var gId*: int -proc getID*(): int -proc setID*(id: int) +proc getID*(): int {.inline.} +proc setID*(id: int) {.inline.} proc IDsynchronizationPoint*(idRange: int) # creator procs: @@ -568,10 +568,10 @@ proc copyStrTable*(dest: var TStrTable, src: TStrTable) proc copyTable*(dest: var TTable, src: TTable) proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet) proc copyIdTable*(dest: var TIdTable, src: TIdTable) -proc sonsLen*(n: PNode): int -proc sonsLen*(n: PType): int -proc lastSon*(n: PNode): PNode -proc lastSon*(n: PType): PType +proc sonsLen*(n: PNode): int {.inline.} +proc sonsLen*(n: PType): int {.inline.} +proc lastSon*(n: PNode): PNode {.inline.} +proc lastSon*(n: PType): PType {.inline.} proc newSons*(father: PNode, length: int) proc newSons*(father: PType, length: int) proc addSon*(father, son: PNode) @@ -903,6 +903,21 @@ proc copyNode(src: PNode): PNode = of nkStrLit..nkTripleStrLit: result.strVal = src.strVal else: nil +proc shallowCopy*(src: PNode): PNode = + # does not copy its sons, but provides space for them: + if src == nil: return nil + result = newNode(src.kind) + result.info = src.info + result.typ = src.typ + result.flags = src.flags * PersistentNodeFlags + case src.Kind + of nkCharLit..nkInt64Lit: result.intVal = src.intVal + of nkFloatLit, nkFloat32Lit, nkFloat64Lit: result.floatVal = src.floatVal + of nkSym: result.sym = src.sym + of nkIdent: result.ident = src.ident + of nkStrLit..nkTripleStrLit: result.strVal = src.strVal + else: newSons(result, sonsLen(src)) + proc copyTree(src: PNode): PNode = # copy a whole syntax tree; performs deep copying if src == nil: @@ -920,7 +935,8 @@ proc copyTree(src: PNode): PNode = else: result.sons = nil newSons(result, sonsLen(src)) - for i in countup(0, sonsLen(src) - 1): result.sons[i] = copyTree(src.sons[i]) + for i in countup(0, sonsLen(src) - 1): + result.sons[i] = copyTree(src.sons[i]) proc lastSon(n: PNode): PNode = result = n.sons[sonsLen(n) - 1] @@ -939,9 +955,9 @@ proc hasSubnodeWith(n: PNode, kind: TNodeKind): bool = of nkEmpty..nkNilLit: result = n.kind == kind else: for i in countup(0, sonsLen(n) - 1): - if (n.sons[i] != nil) and (n.sons[i].kind == kind) or - hasSubnodeWith(n.sons[i], kind): - return true + if n.sons[i] != nil: + if (n.sons[i].kind == kind) or hasSubnodeWith(n.sons[i], kind): + return true result = false proc replaceSons(n: PNode, oldKind, newKind: TNodeKind) = @@ -986,11 +1002,11 @@ proc getStrOrChar*(a: PNode): string = internalError(a.info, "getStrOrChar") result = "" -proc mustRehash(length, counter: int): bool = +proc mustRehash(length, counter: int): bool {.inline.} = assert(length > counter) result = (length * 2 < counter * 3) or (length - counter < 4) -proc nextTry(h, maxHash: THash): THash = +proc nextTry(h, maxHash: THash): THash {.inline.} = result = ((5 * h) + 1) and maxHash # For any initial h in range(maxHash), repeating that maxHash times # generates each int in range(maxHash) exactly once (see any text on diff --git a/rod/astalgo.nim b/rod/astalgo.nim index 596531d0c..894af5b05 100755 --- a/rod/astalgo.nim +++ b/rod/astalgo.nim @@ -767,15 +767,13 @@ proc IdNodeTableRawInsert(data: var TIdNodePairSeq, key: PIdObj, val: PNode) = data[h].val = val proc IdNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) = - var - index: int - n: TIdNodePairSeq - index = IdNodeTableRawGet(t, key) + var index = IdNodeTableRawGet(t, key) if index >= 0: assert(t.data[index].key != nil) t.data[index].val = val else: if mustRehash(len(t.data), t.counter): + var n: TIdNodePairSeq newSeq(n, len(t.data) * growthFactor) for i in countup(0, high(t.data)): if t.data[i].key != nil: diff --git a/rod/c2nim/clex.nim b/rod/c2nim/clex.nim index 7b4fa73fc..8136ad998 100755 --- a/rod/c2nim/clex.nim +++ b/rod/c2nim/clex.nim @@ -259,7 +259,7 @@ proc getNumber2(L: var TLexer, tok: var TToken) = L.bufpos = pos proc getNumber8(L: var TLexer, tok: var TToken) = - var pos = L.bufpos + 2 # skip 0b + var pos = L.bufpos + 1 # skip 0 tok.base = base8 var xi: biggestInt = 0 var bits = 0 diff --git a/rod/cgmeth.nim b/rod/cgmeth.nim index 05118f78a..7b3e5a75f 100755 --- a/rod/cgmeth.nim +++ b/rod/cgmeth.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2009 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -201,4 +201,4 @@ proc generateMethodDispatchers(): PNode = sortBucket(gMethods[bucket], relevantCols) addSon(result, newSymNode(genDispatcher(gMethods[bucket], relevantCols))) -gMethods = @ [] \ No newline at end of file +gMethods = @[] diff --git a/rod/docgen.nim b/rod/docgen.nim index 9da191d8d..7e141d633 100755 --- a/rod/docgen.nim +++ b/rod/docgen.nim @@ -315,17 +315,13 @@ proc getRstName(n: PNode): PRstNode = result = nil proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = - var - r: TSrcGen - kind: TTokType - literal: string - name, result, comm: PRope if not isVisible(nameNode): return - name = toRope(getName(nameNode)) - result = nil - literal = "" - kind = tkEof - comm = genRecComment(d, n) # call this here for the side-effect! + var name = toRope(getName(nameNode)) + var result: PRope = nil + var literal = "" + var kind = tkEof + var comm = genRecComment(d, n) # call this here for the side-effect! + var r: TSrcGen initTokRender(r, n, {renderNoPragmas, renderNoBody, renderNoComments, renderDocComments}) while true: @@ -630,10 +626,10 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope = result = renderAux(d, n, disp("<ul class=\"simple\">$1</ul>\n", "\\begin{itemize}$1\\end{itemize}\n")) of rnBulletItem, rnEnumItem: - result = renderAux(d, n, disp("<li>$1</li>" & "\n", "\\item $1" & "\n")) + result = renderAux(d, n, disp("<li>$1</li>\n", "\\item $1\n")) of rnEnumList: - result = renderAux(d, n, disp("<ol class=\"simple\">$1</ol>" & "\n", - "\\begin{enumerate}$1\\end{enumerate}" & "\n")) + result = renderAux(d, n, disp("<ol class=\"simple\">$1</ol>\n", + "\\begin{enumerate}$1\\end{enumerate}\n")) of rnDefList: result = renderAux(d, n, disp("<dl class=\"docutils\">$1</dl>\n", "\\begin{description}$1\\end{description}\n")) @@ -756,6 +752,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope = of rnTitle: d.meta[metaTitle] = renderRstToOut(d, n.sons[0]) else: InternalError("renderRstToOut") +proc checkForFalse(n: PNode): bool = + result = n.kind == nkIdent and IdentEq(n.ident, "false") + proc generateDoc(d: PDoc, n: PNode) = if n == nil: return case n.kind @@ -782,12 +781,13 @@ proc generateDoc(d: PDoc, n: PNode) = for i in countup(0, sonsLen(n) - 1): generateDoc(d, n.sons[i]) of nkWhenStmt: # generate documentation for the first branch only: - generateDoc(d, lastSon(n.sons[0])) + if not checkForFalse(n.sons[0].sons[0]): + generateDoc(d, lastSon(n.sons[0])) else: nil proc genSection(d: PDoc, kind: TSymKind) = if d.section[kind] == nil: return - var title = toRope(copy($kind, 0 + 2) & 's') + var title = toRope(copy($kind, 2) & 's') d.section[kind] = ropeFormatNamedVars(getConfigVar("doc.section"), [ "sectionid", "sectionTitle", "sectionTitleID", "content"], [ toRope(ord(kind)), title, toRope(ord(kind) + 50), d.section[kind]]) diff --git a/rod/ecmasgen.nim b/rod/ecmasgen.nim index c57ee3879..62cb5b781 100755 --- a/rod/ecmasgen.nim +++ b/rod/ecmasgen.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2009 Andreas Rumpf +# (c) Copyright 2010 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -389,12 +389,12 @@ const # magic checked op; magic unchecked op; checked op; unchecked op ["", "", "Math.floor($1)", "Math.floor($1)"], # ToBiggestInt ["nimCharToStr", "nimCharToStr", "nimCharToStr($1)", "nimCharToStr($1)"], ["nimBoolToStr", "nimBoolToStr", "nimBoolToStr($1)", "nimBoolToStr($1)"], [ - "cstrToNimStr", "cstrToNimStr", "cstrToNimStr(($1)+\"\")", - "cstrToNimStr(($1)+\"\")"], ["cstrToNimStr", "cstrToNimStr", - "cstrToNimStr(($1)+\"\")", - "cstrToNimStr(($1)+\"\")"], ["cstrToNimStr", - "cstrToNimStr", "cstrToNimStr(($1)+\"\")", "cstrToNimStr(($1)+\"\")"], - ["cstrToNimStr", "cstrToNimStr", "cstrToNimStr($1)", "cstrToNimStr($1)"], + "cstrToNimstr", "cstrToNimstr", "cstrToNimstr(($1)+\"\")", + "cstrToNimstr(($1)+\"\")"], ["cstrToNimstr", "cstrToNimstr", + "cstrToNimstr(($1)+\"\")", + "cstrToNimstr(($1)+\"\")"], ["cstrToNimstr", + "cstrToNimstr", "cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")"], + ["cstrToNimstr", "cstrToNimstr", "cstrToNimstr($1)", "cstrToNimstr($1)"], ["", "", "$1", "$1"]] proc binaryExpr(p: var TProc, n: PNode, r: var TCompRes, magic, frmt: string) = diff --git a/rod/extccomp.nim b/rod/extccomp.nim index f6e74f8a8..0f36d0815 100644 --- a/rod/extccomp.nim +++ b/rod/extccomp.nim @@ -10,7 +10,7 @@ # module for calling the different external C compilers # some things are read in from the configuration file -import +import lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc type diff --git a/rod/idents.nim b/rod/idents.nim index 03d155169..13be258ba 100755 --- a/rod/idents.nim +++ b/rod/idents.nim @@ -16,7 +16,7 @@ import type TIdObj* = object of TObject - id*: int # unique id; use this for comparisons and not the pointers + id*: int # unique id; use this for comparisons and not the pointers PIdObj* = ref TIdObj PIdent* = ref TIdent @@ -48,8 +48,8 @@ proc cmpIgnoreStyle(a, b: cstring, blen: int): int = result = 1 while j < blen: while a[i] == '_': inc(i) - while b[j] == '_': - inc(j) # tolower inlined: + while b[j] == '_': inc(j) + # tolower inlined: aa = a[i] bb = b[j] if (aa >= 'A') and (aa <= 'Z'): aa = chr(ord(aa) + (ord('a') - ord('A'))) @@ -129,4 +129,4 @@ proc getIdent(identifier: cstring, length: int, h: THash): PIdent = result.id = - wordCounter else: result.id = id # writeln('new word ', result.s); - \ No newline at end of file + diff --git a/rod/sigmatch.nim b/rod/sigmatch.nim index a33265f40..6a91550e6 100755 --- a/rod/sigmatch.nim +++ b/rod/sigmatch.nim @@ -14,20 +14,20 @@ type TCandidateState = enum csEmpty, csMatch, csNoMatch TCandidate{.final.} = object - exactMatches*: int - subtypeMatches*: int - intConvMatches*: int # conversions to int are not as expensive - convMatches*: int - genericMatches*: int - state*: TCandidateState - callee*: PType # may not be nil! - calleeSym*: PSym # may be nil - call*: PNode # modified call - bindings*: TIdTable # maps sym-ids to types - baseTypeMatch*: bool # needed for conversions from T to openarray[T] - # for example + exactMatches: int + subtypeMatches: int + intConvMatches: int # conversions to int are not as expensive + convMatches: int + genericMatches: int + state: TCandidateState + callee: PType # may not be nil! + calleeSym: PSym # may be nil + call: PNode # modified call + bindings: TIdTable # maps sym-ids to types + baseTypeMatch: bool # needed for conversions from T to openarray[T] + # for example - TTypeRelation = enum # order is important! + TTypeRelation = enum # order is important! isNone, isConvertible, isIntConv, isSubtype, isGeneric, isEqual proc initCandidateAux(c: var TCandidate, callee: PType) {.inline.} = @@ -89,7 +89,7 @@ proc writeMatches(c: TCandidate) = Writeln(stdout, "generic matches: " & $(c.genericMatches)) proc getNotFoundError(c: PContext, n: PNode): string = - # Gives a detailed error message; this is seperated from semDirectCall, + # Gives a detailed error message; this is separated from semDirectCall, # as semDirectCall is already pretty slow (and we need this information only # in case of an error). result = msgKindToString(errTypeMismatch) @@ -516,7 +516,7 @@ proc ParamTypesMatch(c: PContext, m: var TCandidate, f, a: PType, x.calleeSym = m.calleeSym y.calleeSym = m.calleeSym z.calleeSym = m.calleeSym - var best = - 1 + var best = -1 for i in countup(0, sonsLen(arg) - 1): # iterators are not first class yet, so ignore them if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}: diff --git a/rod/transf.nim b/rod/transf.nim index 16c279c80..88d9ff984 100755 --- a/rod/transf.nim +++ b/rod/transf.nim @@ -13,7 +13,7 @@ # * inlines iterators # * inlines constants # * performes contant folding -# * introduces nkHiddenDeref, nkHiddenSubConv, etc. +# * converts "continue" to "break" # * introduces method dispatchers import @@ -27,22 +27,59 @@ proc transfPass*(): TPass # implementation type + PTransNode* = distinct PNode + PTransCon = ref TTransCon TTransCon{.final.} = object # part of TContext; stackable - mapping*: TIdNodeTable # mapping from symbols to nodes - owner*: PSym # current owner - forStmt*: PNode # current for stmt - next*: PTransCon # for stacking + mapping: TIdNodeTable # mapping from symbols to nodes + owner: PSym # current owner + forStmt: PNode # current for stmt + forLoopBody: PTransNode # transformed for loop body + yieldStmts: int # we count the number of yield statements, + # because we need to introduce new variables + # if we encounter the 2nd yield statement + next: PTransCon # for stacking TTransfContext = object of passes.TPassContext - module*: PSym - transCon*: PTransCon # top of a TransCon stack + module: PSym + transCon: PTransCon # top of a TransCon stack + inlining: int # > 0 if we are in inlining context (copy vars) + blocksyms: seq[PSym] PTransf = ref TTransfContext -proc newTransCon(): PTransCon = +proc newTransNode(a: PNode): PTransNode {.inline.} = + result = PTransNode(shallowCopy(a)) + +proc newTransNode(kind: TNodeKind, info: TLineInfo, + sons: int): PTransNode {.inline.} = + var x = newNodeI(kind, info) + newSeq(x.sons, sons) + result = x.PTransNode + +proc newTransNode(kind: TNodeKind, n: PNode, + sons: int): PTransNode {.inline.} = + var x = newNodeIT(kind, n.info, n.typ) + newSeq(x.sons, sons) + x.typ = n.typ + result = x.PTransNode + +proc `[]=`(a: PTransNode, i: int, x: PTransNode) {.inline.} = + var n = PNode(a) + n.sons[i] = PNode(x) + +proc `[]`(a: PTransNode, i: int): PTransNode {.inline.} = + var n = PNode(a) + result = n.sons[i].PTransNode + +proc add(a, b: PTransNode) {.inline.} = addSon(PNode(a), PNode(b)) +proc len(a: PTransNode): int {.inline.} = result = sonsLen(a.PNode) + +proc newTransCon(owner: PSym): PTransCon = + assert owner != nil new(result) initIdNodeTable(result.mapping) + result.owner = owner proc pushTransCon(c: PTransf, t: PTransCon) = t.next = c.transCon @@ -62,7 +99,12 @@ proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PSym = result.typ = skipTypes(typ, {tyGenericInst}) incl(result.flags, sfFromGeneric) -proc transform(c: PTransf, n: PNode): PNode +proc transform(c: PTransf, n: PNode): PTransNode + +proc transformSons(c: PTransf, n: PNode): PTransNode = + result = newTransNode(n) + for i in countup(0, sonsLen(n)-1): + result[i] = transform(c, n.sons[i]) # Transforming iterators into non-inlined versions is pretty hard, but # unavoidable for not bloating the code too much. If we had direct access to @@ -112,12 +154,12 @@ proc transform(c: PTransf, n: PNode): PNode # label1: inc(c.i) # -proc newAsgnStmt(c: PTransf, le, ri: PNode): PNode = - result = newNodeI(nkFastAsgn, ri.info) - addSon(result, le) - addSon(result, ri) +proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = + result = newTransNode(nkFastAsgn, PNode(ri).info, 2) + result[0] = PTransNode(le) + result[1] = ri -proc transformSym(c: PTransf, n: PNode): PNode = +proc transformSymAux(c: PTransf, n: PNode): PNode = var b: PNode if (n.kind != nkSym): internalError(n.info, "transformSym") var tc = c.transCon @@ -128,12 +170,10 @@ proc transformSym(c: PTransf, n: PNode): PNode = b = newSymNode(b.sym) b.info = n.info else: - b = n #writeln('transformSym', n.sym.id : 5); + b = n while tc != nil: result = IdNodeTableGet(tc.mapping, b.sym) - if result != nil: - return #write('not found in: '); - #writeIdNodeTable(tc.mapping); + if result != nil: return tc = tc.next result = b case b.sym.kind @@ -145,33 +185,33 @@ proc transformSym(c: PTransf, n: PNode): PNode = else: nil -proc transformContinueAux(c: PTransf, n: PNode, labl: PSym, counter: var int) = +proc transformSym(c: PTransf, n: PNode): PTransNode = + result = PTransNode(transformSymAux(c, n)) + +proc hasContinue(n: PNode): bool = if n == nil: return case n.kind - of nkEmpty..nkNilLit, nkForStmt, nkWhileStmt: - nil - of nkContinueStmt: - n.kind = nkBreakStmt - addSon(n, newSymNode(labl)) - inc(counter) + of nkEmpty..nkNilLit, nkForStmt, nkWhileStmt: nil + of nkContinueStmt: result = true else: for i in countup(0, sonsLen(n) - 1): - transformContinueAux(c, n.sons[i], labl, counter) - -proc transformContinue(c: PTransf, n: PNode): PNode = - # we transform the continue statement into a block statement - result = n - for i in countup(0, sonsLen(n) - 1): result.sons[i] = transform(c, n.sons[i]) - var counter = 0 - var labl = newSym(skLabel, nil, getCurrOwner(c)) - labl.name = getIdent(genPrefix & $(labl.id)) - labl.info = result.info - transformContinueAux(c, result, labl, counter) - if counter > 0: - var x = newNodeI(nkBlockStmt, result.info) - addSon(x, newSymNode(labl)) - addSon(x, result) - result = x + if hasContinue(n.sons[i]): return true + +proc transformLoopBody(c: PTransf, n: PNode): PTransNode = + # XXX BUG: What if it contains "continue" and "break"? "break" needs + # an explicit label too, but not the same! + if hasContinue(n): + var labl = newSym(skLabel, nil, getCurrOwner(c)) + labl.name = getIdent(genPrefix & $labl.id) + labl.info = n.info + c.blockSyms.add(labl) + + result = newTransNode(nkBlockStmt, n.info, 2) + result[0] = newSymNode(labl).PTransNode + result[1] = transform(c, n) + discard c.blockSyms.pop() + else: + result = transform(c, n) proc skipConv(n: PNode): PNode = case n.kind @@ -189,67 +229,69 @@ proc newTupleAccess(tup: PNode, i: int): PNode = lit.intVal = i addSon(result, lit) -proc unpackTuple(c: PTransf, n, father: PNode) = +proc unpackTuple(c: PTransf, n: PNode, father: PTransNode) = # XXX: BUG: what if `n` is an expression with side-effects? for i in countup(0, sonsLen(c.transCon.forStmt) - 3): - addSon(father, newAsgnStmt(c, c.transCon.forStmt.sons[i], - transform(c, newTupleAccess(n, i)))) + add(father, newAsgnStmt(c, c.transCon.forStmt.sons[i], + transform(c, newTupleAccess(n, i)))) -proc transformYield(c: PTransf, n: PNode): PNode = - result = newNodeI(nkStmtList, n.info) +proc transformYield(c: PTransf, n: PNode): PTransNode = + result = newTransNode(nkStmtList, n.info, 0) var e = n.sons[0] if skipTypes(e.typ, {tyGenericInst}).kind == tyTuple: e = skipConv(e) if e.kind == nkPar: for i in countup(0, sonsLen(e) - 1): - addSon(result, newAsgnStmt(c, c.transCon.forStmt.sons[i], - transform(c, copyTree(e.sons[i])))) + add(result, newAsgnStmt(c, c.transCon.forStmt.sons[i], + transform(c, e.sons[i]))) else: unpackTuple(c, e, result) else: - e = transform(c, copyTree(e)) - addSon(result, newAsgnStmt(c, c.transCon.forStmt.sons[0], e)) - addSon(result, transform(c, lastSon(c.transCon.forStmt))) + var x = transform(c, e) + add(result, newAsgnStmt(c, c.transCon.forStmt.sons[0], x)) + + inc(c.transCon.yieldStmts) + if c.transCon.yieldStmts <= 1: + # common case + add(result, c.transCon.forLoopBody) + else: + # we need to transform again to introduce new local variables: + add(result, transform(c, c.transCon.forLoopBody.pnode)) -proc inlineIter(c: PTransf, n: PNode): PNode = - result = n - if n == nil: return - case n.kind - of nkEmpty..nkNilLit: - result = transform(c, copyTree(n)) - of nkYieldStmt: - result = transformYield(c, n) - of nkVarSection: - result = copyTree(n) - for i in countup(0, sonsLen(result) - 1): - var it = result.sons[i] - if it.kind == nkCommentStmt: continue - if it.kind == nkIdentDefs: - if (it.sons[0].kind != nkSym): InternalError(it.info, "inlineIter") - var newVar = copySym(it.sons[0].sym) - incl(newVar.flags, sfFromGeneric) - # fixes a strange bug for rodgen: - #include(it.sons[0].sym.flags, sfFromGeneric); +proc transformVarSection(c: PTransf, v: PNode): PTransNode = + result = newTransNode(v) + for i in countup(0, sonsLen(v)-1): + var it = v.sons[i] + if it.kind == nkCommentStmt: + result[i] = PTransNode(it) + elif it.kind == nkIdentDefs: + if (it.sons[0].kind != nkSym): + InternalError(it.info, "transformVarSection") + var newVar = copySym(it.sons[0].sym) + incl(newVar.flags, sfFromGeneric) + # fixes a strange bug for rodgen: + #include(it.sons[0].sym.flags, sfFromGeneric); + newVar.owner = getCurrOwner(c) + IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) + var defs = newTransNode(nkIdentDefs, it.info, 3) + defs[0] = newSymNode(newVar).PTransNode + defs[1] = it.sons[1].PTransNode + defs[2] = transform(c, it.sons[2]) + result[i] = defs + else: + if it.kind != nkVarTuple: + InternalError(it.info, "transformVarSection: not nkVarTuple") + var L = sonsLen(it) + var defs = newTransNode(it.kind, it.info, L) + for j in countup(0, L-3): + var newVar = copySym(it.sons[j].sym) + incl(newVar.flags, sfFromGeneric) newVar.owner = getCurrOwner(c) - IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) - it.sons[0] = newSymNode(newVar) - it.sons[2] = transform(c, it.sons[2]) - else: - if it.kind != nkVarTuple: - InternalError(it.info, "inlineIter: not nkVarTuple") - var L = sonsLen(it) - for j in countup(0, L - 3): - var newVar = copySym(it.sons[j].sym) - incl(newVar.flags, sfFromGeneric) - newVar.owner = getCurrOwner(c) - IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) - it.sons[j] = newSymNode(newVar) - assert(it.sons[L - 2] == nil) - it.sons[L - 1] = transform(c, it.sons[L - 1]) - else: - result = copyNode(n) - for i in countup(0, sonsLen(n) - 1): addSon(result, inlineIter(c, n.sons[i])) - result = transform(c, result) + IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) + defs[j] = newSymNode(newVar).PTransNode + assert(it.sons[L-2] == nil) + defs[L-1] = transform(c, it.sons[L-1]) + result[i] = defs proc addVar(father, v: PNode) = var vpart = newNodeI(nkIdentDefs, v.info) @@ -258,98 +300,124 @@ proc addVar(father, v: PNode) = addSon(vpart, nil) addSon(father, vpart) -proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PNode = +proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode = case n.sons[0].kind of nkObjUpConv, nkObjDownConv, nkPassAsOpenArray, nkChckRange, nkChckRangeF, nkChckRange64: var m = n.sons[0].sons[0] if (m.kind == a) or (m.kind == b): # addr ( nkPassAsOpenArray ( deref ( x ) ) ) --> nkPassAsOpenArray(x) - n.sons[0].sons[0] = m.sons[0] - return transform(c, n.sons[0]) + var x = copyTree(n) + x.sons[0].sons[0] = m.sons[0] + result = transform(c, x.sons[0]) + + #result = newTransNode(n.sons[0]) + #result[0] = transform(c, m.sons[0]) + else: + result = transformSons(c, n) of nkHiddenStdConv, nkHiddenSubConv, nkConv: var m = n.sons[0].sons[1] if (m.kind == a) or (m.kind == b): # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x) - n.sons[0].sons[1] = m.sons[0] - return transform(c, n.sons[0]) + + var x = copyTree(n) + x.sons[0].sons[1] = m.sons[0] + result = transform(c, x.sons[0]) + + #result = newTransNode(n.sons[0]) + #result[1] = transform(c, m.sons[0]) + + #if skipTypes(n.sons[0].typ, abstractVar).kind == tyOpenArray: + # debug(result.pnode) + # liMessage(n.info, warnUser, + # "nkPassAsOpenArray introduced here " & renderTree(n)) + else: + result = transformSons(c, n) else: if (n.sons[0].kind == a) or (n.sons[0].kind == b): # addr ( deref ( x )) --> x - return transform(c, n.sons[0].sons[0]) - n.sons[0] = transform(c, n.sons[0]) - result = n + result = transform(c, n.sons[0].sons[0]) + else: + result = transformSons(c, n) -proc transformConv(c: PTransf, n: PNode): PNode = - n.sons[1] = transform(c, n.sons[1]) - result = n # numeric types need range checks: +proc transformConv(c: PTransf, n: PNode): PTransNode = + # numeric types need range checks: var dest = skipTypes(n.typ, abstractVarRange) var source = skipTypes(n.sons[1].typ, abstractVarRange) case dest.kind of tyInt..tyInt64, tyEnum, tyChar, tyBool: if not isOrdinalType(source): # XXX int64 -> float conversion? - result = n + result = transformSons(c, n) elif firstOrd(dest) <= firstOrd(source) and lastOrd(source) <= lastOrd(dest): # BUGFIX: simply leave n as it is; we need a nkConv node, # but no range check: - result = n + result = transformSons(c, n) else: # generate a range check: if (dest.kind == tyInt64) or (source.kind == tyInt64): - result = newNodeIT(nkChckRange64, n.info, n.typ) + result = newTransNode(nkChckRange64, n, 3) else: - result = newNodeIT(nkChckRange, n.info, n.typ) + result = newTransNode(nkChckRange, n, 3) dest = skipTypes(n.typ, abstractVar) - addSon(result, n.sons[1]) - addSon(result, newIntTypeNode(nkIntLit, firstOrd(dest), source)) - addSon(result, newIntTypeNode(nkIntLit, lastOrd(dest), source)) + result[0] = transform(c, n.sons[1]) + result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), source).PTransNode + result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), source).PTransNode of tyFloat..tyFloat128: if skipTypes(n.typ, abstractVar).kind == tyRange: - result = newNodeIT(nkChckRangeF, n.info, n.typ) + result = newTransNode(nkChckRangeF, n, 3) dest = skipTypes(n.typ, abstractVar) - addSon(result, n.sons[1]) - addSon(result, copyTree(dest.n.sons[0])) - addSon(result, copyTree(dest.n.sons[1])) + result[0] = transform(c, n.sons[1]) + result[1] = copyTree(dest.n.sons[0]).PTransNode + result[2] = copyTree(dest.n.sons[1]).PTransNode + else: + result = transformSons(c, n) of tyOpenArray: - result = newNodeIT(nkPassAsOpenArray, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkPassAsOpenArray, n, 1) + result[0] = transform(c, n.sons[1]) of tyCString: if source.kind == tyString: - result = newNodeIT(nkStringToCString, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkStringToCString, n, 1) + result[0] = transform(c, n.sons[1]) + else: + result = transformSons(c, n) of tyString: if source.kind == tyCString: - result = newNodeIT(nkCStringToString, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkCStringToString, n, 1) + result[0] = transform(c, n.sons[1]) + else: + result = transformSons(c, n) of tyRef, tyPtr: dest = skipTypes(dest, abstractPtrs) source = skipTypes(source, abstractPtrs) if source.kind == tyObject: var diff = inheritanceDiff(dest, source) if diff < 0: - result = newNodeIT(nkObjUpConv, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkObjUpConv, n, 1) + result[0] = transform(c, n.sons[1]) elif diff > 0: - result = newNodeIT(nkObjDownConv, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkObjDownConv, n, 1) + result[0] = transform(c, n.sons[1]) else: - result = n.sons[1] + result = transform(c, n.sons[1]) + else: + result = transformSons(c, n) of tyObject: var diff = inheritanceDiff(dest, source) if diff < 0: - result = newNodeIT(nkObjUpConv, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkObjUpConv, n, 1) + result[0] = transform(c, n.sons[1]) elif diff > 0: - result = newNodeIT(nkObjDownConv, n.info, n.typ) - addSon(result, n.sons[1]) + result = newTransNode(nkObjDownConv, n, 1) + result[0] = transform(c, n.sons[1]) else: - result = n.sons[1] + result = transform(c, n.sons[1]) of tyGenericParam, tyOrdinal: - result = n.sons[1] # happens sometimes for generated assignments, etc. + result = transform(c, n.sons[1]) + # happens sometimes for generated assignments, etc. else: - nil + result = transformSons(c, n) proc skipPassAsOpenArray(n: PNode): PNode = result = n @@ -377,32 +445,31 @@ proc putArgInto(arg: PNode, formal: PType): TPutArgInto = if skipTypes(formal, abstractInst).kind == tyVar: result = paVarAsgn else: result = paFastAsgn -proc transformFor(c: PTransf, n: PNode): PNode = +proc transformFor(c: PTransf, n: PNode): PTransNode = # generate access statements for the parameters (unless they are constant) # put mapping from formal parameters to actual parameters - if (n.kind != nkForStmt): InternalError(n.info, "transformFor") - result = newNodeI(nkStmtList, n.info) + if n.kind != nkForStmt: InternalError(n.info, "transformFor") + result = newTransNode(nkStmtList, n.info, 0) var length = sonsLen(n) - n.sons[length - 1] = transformContinue(c, n.sons[length - 1]) + var loopBody = transformLoopBody(c, n.sons[length-1]) var v = newNodeI(nkVarSection, n.info) for i in countup(0, length - 3): addVar(v, copyTree(n.sons[i])) # declare new vars - addSon(result, v) - var newC = newTransCon() + add(result, v.ptransNode) var call = n.sons[length - 2] if (call.kind != nkCall) or (call.sons[0].kind != nkSym): InternalError(call.info, "transformFor") - newC.owner = call.sons[0].sym + + var newC = newTransCon(call.sons[0].sym) newC.forStmt = n + newC.forLoopBody = loopBody if (newC.owner.kind != skIterator): InternalError(call.info, "transformFor") # generate access statements for the parameters (unless they are constant) pushTransCon(c, newC) for i in countup(1, sonsLen(call) - 1): - var arg = skipPassAsOpenArray(transform(c, call.sons[i])) + var arg = skipPassAsOpenArray(transform(c, call.sons[i]).pnode) var formal = skipTypes(newC.owner.typ, abstractInst).n.sons[i].sym - #if IdentEq(newc.Owner.name, 'items') then - # liMessage(arg.info, warnUser, 'items: ' + nodeKindToStr[arg.kind]); case putArgInto(arg, formal.typ) of paDirectMapping: IdNodeTablePut(newC.mapping, formal, arg) @@ -410,14 +477,16 @@ proc transformFor(c: PTransf, n: PNode): PNode = # generate a temporary and produce an assignment statement: var temp = newTemp(c, formal.typ, formal.info) addVar(v, newSymNode(temp)) - addSon(result, newAsgnStmt(c, newSymNode(temp), arg)) + add(result, newAsgnStmt(c, newSymNode(temp), arg.ptransNode)) IdNodeTablePut(newC.mapping, formal, newSymNode(temp)) of paVarAsgn: assert(skipTypes(formal.typ, abstractInst).kind == tyVar) InternalError(arg.info, "not implemented: pass to var parameter") var body = newC.owner.ast.sons[codePos] pushInfoContext(n.info) - addSon(result, inlineIter(c, body)) + inc(c.inlining) + add(result, transform(c, body)) + dec(c.inlining) popInfoContext() popTransCon(c) @@ -486,63 +555,63 @@ proc transformLambda(c: PTransf, n: PNode): PNode = # all variables that are accessed should be accessed by the new closure # parameter: if sonsLen(closure) > 0: - var newC = newTransCon() + var newC = newTransCon(c.transCon.owner) for i in countup(0, sonsLen(closure) - 1): IdNodeTablePut(newC.mapping, closure.sons[i].sym, indirectAccess(param, closure.sons[i].sym)) pushTransCon(c, newC) - n.sons[codePos] = transform(c, n.sons[codePos]) + n.sons[codePos] = transform(c, n.sons[codePos]).pnode popTransCon(c) -proc transformCase(c: PTransf, n: PNode): PNode = +proc transformCase(c: PTransf, n: PNode): PTransNode = # removes `elif` branches of a case stmt # adds ``else: nil`` if needed for the code generator - var length = sonsLen(n) - var i = length - 1 - if n.sons[i].kind == nkElse: dec(i) - if n.sons[i].kind == nkElifBranch: - while n.sons[i].kind == nkElifBranch: dec(i) - if (n.sons[i].kind != nkOfBranch): - InternalError(n.sons[i].info, "transformCase") - var ifs = newNodeI(nkIfStmt, n.sons[i + 1].info) - var elsen = newNodeI(nkElse, ifs.info) - for j in countup(i + 1, length - 1): addSon(ifs, n.sons[j]) - setlen(n.sons, i + 2) - addSon(elsen, ifs) - n.sons[i + 1] = elsen - elif (n.sons[length - 1].kind != nkElse) and - not (skipTypes(n.sons[0].Typ, abstractVarRange).Kind in - {tyInt..tyInt64, tyChar, tyEnum}): - #MessageOut(renderTree(n)); - var elsen = newNodeI(nkElse, n.info) - addSon(elsen, newNodeI(nkNilLit, n.info)) - addSon(n, elsen) - result = n - for j in countup(0, sonsLen(n) - 1): result.sons[j] = transform(c, n.sons[j]) + result = newTransNode(nkCaseStmt, n, 0) + var ifs = PTransNode(nil) + for i in 0 .. sonsLen(n)-1: + var it = n.sons[i] + var e = transform(c, it) + case it.kind + of nkElifBranch: + if ifs.pnode == nil: + ifs = newTransNode(nkIfStmt, it.info, 0) + ifs.add(e) + of nkElse: + if ifs.pnode == nil: result.add(e) + else: ifs.add(e) + else: + result.add(e) + if ifs.pnode != nil: + var elseBranch = newTransNode(nkElse, n.info, 1) + elseBranch[0] = ifs + result.add(elseBranch) + elif result.Pnode.lastSon.kind != nkElse and not ( + skipTypes(n.sons[0].Typ, abstractVarRange).Kind in + {tyInt..tyInt64, tyChar, tyEnum}): + # fix a stupid code gen bug by normalizing: + var elseBranch = newTransNode(nkElse, n.info, 1) + elseBranch[0] = newTransNode(nkNilLit, n.info, 0) + add(result, elseBranch) -proc transformArrayAccess(c: PTransf, n: PNode): PNode = - result = copyTree(n) - result.sons[0] = skipConv(result.sons[0]) - result.sons[1] = skipConv(result.sons[1]) - for i in countup(0, sonsLen(result) - 1): - result.sons[i] = transform(c, result.sons[i]) +proc transformArrayAccess(c: PTransf, n: PNode): PTransNode = + result = newTransNode(n) + result[0] = transform(c, skipConv(n.sons[0])) + result[1] = transform(c, skipConv(n.sons[1])) proc getMergeOp(n: PNode): PSym = - result = nil case n.kind of nkCall, nkHiddenCallConv, nkCommand, nkInfix, nkPrefix, nkPostfix, nkCallStrLit: if (n.sons[0].Kind == nkSym) and (n.sons[0].sym.kind == skProc) and (sfMerge in n.sons[0].sym.flags): result = n.sons[0].sym - else: - nil + else: nil proc flattenTreeAux(d, a: PNode, op: PSym) = var op2 = getMergeOp(a) if op2 != nil and (op2.id == op.id or op.magic != mNone and op2.magic == op.magic): - for i in countup(1, sonsLen(a) - 1): flattenTreeAux(d, a.sons[i], op) + for i in countup(1, sonsLen(a)-1): flattenTreeAux(d, a.sons[i], op) else: addSon(d, copyTree(a)) @@ -555,57 +624,60 @@ proc flattenTree(root: PNode): PNode = else: result = root -proc transformCall(c: PTransf, n: PNode): PNode = - result = flattenTree(n) - for i in countup(0, sonsLen(result) - 1): - result.sons[i] = transform(c, result.sons[i]) - var op = getMergeOp(result) - if (op != nil) and (op.magic != mNone) and (sonsLen(result) >= 3): - var m = result - result = newNodeIT(nkCall, m.info, m.typ) - addSon(result, copyTree(m.sons[0])) +proc transformCall(c: PTransf, n: PNode): PTransNode = + var n = flattenTree(n) + var op = getMergeOp(n) + if (op != nil) and (op.magic != mNone) and (sonsLen(n) >= 3): + result = newTransNode(nkCall, n, 0) + add(result, transform(c, n.sons[0])) var j = 1 - while j < sonsLen(m): - var a = m.sons[j] + while j < sonsLen(n): + var a = n.sons[j] inc(j) if isConstExpr(a): - while (j < sonsLen(m)) and isConstExpr(m.sons[j]): - a = evalOp(op.magic, m, a, m.sons[j], nil) + while (j < sonsLen(n)) and isConstExpr(n.sons[j]): + a = evalOp(op.magic, n, a, n.sons[j], nil) inc(j) - addSon(result, a) - if sonsLen(result) == 2: result = result.sons[1] - elif (result.sons[0].kind == nkSym) and - (result.sons[0].sym.kind == skMethod): + add(result, transform(c, a)) + if len(result) == 2: result = result[1] + elif (n.sons[0].kind == nkSym) and (n.sons[0].sym.kind == skMethod): # use the dispatcher for the call: - result = methodCall(result) + result = methodCall(transformSons(c, n).pnode).ptransNode + else: + result = transformSons(c, n) -proc transform(c: PTransf, n: PNode): PNode = - result = n +proc transform(c: PTransf, n: PNode): PTransNode = if n == nil: return case n.kind of nkSym: return transformSym(c, n) of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: - # nothing to be done for leaves + # nothing to be done for leaves: + result = PTransNode(n) of nkBracketExpr: result = transformArrayAccess(c, n) of nkLambda: - result = transformLambda(c, n) + when false: result = transformLambda(c, n) of nkForStmt: result = transformFor(c, n) of nkCaseStmt: result = transformCase(c, n) of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef: if n.sons[genericParamsPos] == nil: - n.sons[codePos] = transform(c, n.sons[codePos]) + n.sons[codePos] = PNode(transform(c, n.sons[codePos])) if n.kind == nkMethodDef: methodDef(n.sons[namePos].sym) + result = PTransNode(n) + of nkContinueStmt: + result = PTransNode(newNode(nkBreakStmt)) + var labl = c.blockSyms[c.blockSyms.high] + add(result, PTransNode(newSymNode(labl))) of nkWhileStmt: - if (sonsLen(n) != 2): InternalError(n.info, "transform") - n.sons[0] = transform(c, n.sons[0]) - n.sons[1] = transformContinue(c, n.sons[1]) + result = newTransNode(n) + result[0] = transform(c, n.sons[0]) + result[1] = transformLoopBody(c, n.sons[1]) of nkCall, nkHiddenCallConv, nkCommand, nkInfix, nkPrefix, nkPostfix, nkCallStrLit: - result = transformCall(c, result) + result = transformCall(c, n) of nkAddr, nkHiddenAddr: result = transformAddrDeref(c, n, nkDerefExpr, nkHiddenDeref) of nkDerefExpr, nkHiddenDeref: @@ -613,26 +685,42 @@ proc transform(c: PTransf, n: PNode): PNode = of nkHiddenStdConv, nkHiddenSubConv, nkConv: result = transformConv(c, n) of nkDiscardStmt: - for i in countup(0, sonsLen(n) - 1): result.sons[i] = transform(c, n.sons[i]) - if isConstExpr(result.sons[0]): result = newNode(nkCommentStmt) + result = transformSons(c, n) + if isConstExpr(PNode(result).sons[0]): + # ensure that e.g. discard "some comment" gets optimized away completely: + result = PTransNode(newNode(nkCommentStmt)) of nkCommentStmt, nkTemplateDef: - return + return n.ptransNode of nkConstSection: # do not replace ``const c = 3`` with ``const 3 = 3`` - return - else: - for i in countup(0, sonsLen(n) - 1): result.sons[i] = transform(c, n.sons[i]) - var cnst = getConstExpr(c.module, result) + return n.ptransNode + of nkVarSection: + if c.inlining > 0: + # we need to copy the variables for multiple yield statements: + result = transformVarSection(c, n) + else: + result = transformSons(c, n) + of nkYieldStmt: + if c.inlining > 0: + result = transformYield(c, n) + else: + result = transformSons(c, n) + else: + result = transformSons(c, n) + var cnst = getConstExpr(c.module, PNode(result)) if cnst != nil: - result = cnst # do not miss an optimization - + result = PTransNode(cnst) # do not miss an optimization + proc processTransf(context: PPassContext, n: PNode): PNode = var c = PTransf(context) - result = transform(c, n) + pushTransCon(c, newTransCon(getCurrOwner(c))) + result = PNode(transform(c, n)) + popTransCon(c) proc openTransf(module: PSym, filename: string): PPassContext = var n: PTransf new(n) + n.blocksyms = @[] n.module = module result = n diff --git a/tests/accept/compile/toop.nim b/tests/accept/compile/toop.nim new file mode 100644 index 000000000..d103c6304 --- /dev/null +++ b/tests/accept/compile/toop.nim @@ -0,0 +1,18 @@ + +type + TA = object + x, y: int + + TB = object of TA + z: int + + TC = object of TB + whatever: string + +proc p(a: var TA) = nil +proc p(b: var TB) = nil + +var c: TC + +p(c) + diff --git a/tests/accept/compile/tquicksort.nim b/tests/accept/compile/tquicksort.nim index 421564ecd..6706a185e 100755 --- a/tests/accept/compile/tquicksort.nim +++ b/tests/accept/compile/tquicksort.nim @@ -9,7 +9,9 @@ proc QuickSort(list: seq[int]): seq[int] = left.add(list[i]) elif list[i] > pivot: right.add(list[i]) - result = QuickSort(left) & pivot & QuickSort(right) + result = QuickSort(left) & + pivot & + QuickSort(right) proc echoSeq(a: seq[int]) = for i in low(a)..high(a): diff --git a/tests/accept/run/spec.csv b/tests/accept/run/spec.csv index 68954cf48..75e85cd14 100755 --- a/tests/accept/run/spec.csv +++ b/tests/accept/run/spec.csv @@ -28,12 +28,13 @@ tfloat1.nim;Error: unhandled exception: FPU operation caused an overflow [EFloat tfloat2.nim;Error: unhandled exception: FPU operation caused a NaN result [EFloatInvalidOp] tformat.nim;Hi Andreas! How do you feel, Rumpf? thintoff.nim;0 -tinit.nim;Hallo from module! Hallo from main module! +tinit.nim;Hello from module! Hello from main module! tints.nim;Success tisopr.nim;falsetrue titer2.nim;123 titer3.nim;1231 titer5.nim;abcxyz +titer6.nim;000 tlenopenarray.nim;1 tlowhigh.nim;10 tmatrix.nim;111 diff --git a/tests/accept/run/titer6.nim b/tests/accept/run/titer6.nim new file mode 100644 index 000000000..8a1d9cf1b --- /dev/null +++ b/tests/accept/run/titer6.nim @@ -0,0 +1,31 @@ +# Test iterator with more than 1 yield statement + +import strutils + +iterator tokenize2(s: string, seps: set[char] = Whitespace): tuple[ + token: string, isSep: bool] = + var i = 0 + while i < s.len: + var j = i + if s[j] in seps: + while j < s.len and s[j] in seps: inc(j) + if j > i: + yield (copy(s, i, j-1), true) + else: + while j < s.len and s[j] notin seps: inc(j) + if j > i: + yield (copy(s, i, j-1), false) + i = j + +for word, isSep in tokenize2("ta da", whiteSpace): + var titer2TestVar = 0 + stdout.write(titer2TestVar) + +proc wordWrap2(s: string, maxLineWidth = 80, + splitLongWords = true, + seps: set[char] = whitespace, + newLine = "\n"): string = + result = "" + for word, isSep in tokenize2(s, seps): + var w = 0 + diff --git a/tests/gc/talloc.nim b/tests/gc/talloc.nim new file mode 100755 index 000000000..79a842415 --- /dev/null +++ b/tests/gc/talloc.nim @@ -0,0 +1,637 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2010 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# Low level allocator for Nimrod. Has been designed to support the GC. +# TODO: +# - eliminate "used" field +# - make searching for block O(1) + +const + debugGC = false # we wish to debug the GC... + logGC = false + traceGC = false # extensive debugging + reallyDealloc = true # for debugging purposes this can be set to false + cycleGC = true # (de)activate the cycle GC + stressGC = false + reallyOsDealloc = true + coalescRight = true + coalescLeft = true + overwriteFree = false + +# Page size of the system; in most cases 4096 bytes. For exotic OS or +# CPU this needs to be changed: +const + PageShift = 12 + PageSize = 1 shl PageShift + PageMask = PageSize-1 + + MemAlign = 8 # also minimal allocatable memory block + + BitsPerPage = PageSize div MemAlign + UnitsPerPage = BitsPerPage div (sizeof(int)*8) + # how many ints do we need to describe a page: + # on 32 bit systems this is only 16 (!) + + TrunkShift = 9 + BitsPerTrunk = 1 shl TrunkShift # needs to be power of 2 and divisible by 64 + TrunkMask = BitsPerTrunk - 1 + IntsPerTrunk = BitsPerTrunk div (sizeof(int)*8) + IntShift = 5 + ord(sizeof(int) == 8) # 5 or 6, depending on int width + IntMask = 1 shl IntShift - 1 + +proc raiseOutOfMem() {.noreturn.} = + quit("out of memory") + +# ------------ platform specific chunk allocation code ----------------------- + +when defined(posix): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2 # Changes are private + + when defined(linux) or defined(aix): + const MAP_ANONYMOUS = 0x20 # don't use a file + elif defined(macosx) or defined(bsd): + const MAP_ANONYMOUS = 0x1000 + elif defined(solaris): + const MAP_ANONYMOUS = 0x100 + else: + {.error: "Port memory manager to your platform".} + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "<sys/mman.h>".} + + proc munmap(adr: pointer, len: int) {.header: "<sys/mman.h>".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + proc osDeallocPages(p: pointer, size: int) {.inline} = + when reallyOsDealloc: munmap(p, size) + +elif defined(windows): + const + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + MEM_TOP_DOWN = 0x100000 + PAGE_READWRITE = 0x04 + + MEM_DECOMMIT = 0x4000 + MEM_RELEASE = 0x8000 + + proc VirtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "<windows.h>", stdcall.} + + proc VirtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32) {.header: "<windows.h>", stdcall.} + + proc osAllocPages(size: int): pointer {.inline.} = + result = VirtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + if result == nil: raiseOutOfMem() + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # according to Microsoft, 0 is the only correct value here: + when reallyOsDealloc: VirtualFree(p, 0, MEM_RELEASE) + +else: + {.error: "Port memory manager to your platform".} + +# --------------------- end of non-portable code ----------------------------- + +# We manage *chunks* of memory. Each chunk is a multiple of the page size. +# Each chunk starts at an address that is divisible by the page size. Chunks +# that are bigger than ``ChunkOsReturn`` are returned back to the operating +# system immediately. + +const + ChunkOsReturn = 256 * PageSize + InitialMemoryRequest = ChunkOsReturn div 2 # < ChunkOsReturn! + SmallChunkSize = PageSize + +type + PTrunk = ptr TTrunk + TTrunk {.final.} = object + next: PTrunk # all nodes are connected with this pointer + key: int # start address at bit 0 + bits: array[0..IntsPerTrunk-1, int] # a bit vector + + TTrunkBuckets = array[0..1023, PTrunk] + TIntSet {.final.} = object + data: TTrunkBuckets + +type + TAlignType = biggestFloat + TFreeCell {.final, pure.} = object + next: ptr TFreeCell # next free cell in chunk (overlaid with refcount) + zeroField: int # 0 means cell is not used (overlaid with typ field) + # 1 means cell is manually managed pointer + + PChunk = ptr TBaseChunk + PBigChunk = ptr TBigChunk + PSmallChunk = ptr TSmallChunk + TBaseChunk {.pure.} = object + prevSize: int # size of previous chunk; for coalescing + size: int # if < PageSize it is a small chunk + used: bool # later will be optimized into prevSize... + + TSmallChunk = object of TBaseChunk + next, prev: PSmallChunk # chunks of the same size + freeList: ptr TFreeCell + free: int # how many bytes remain + acc: int # accumulator for small object allocation + data: TAlignType # start of usable memory + + TBigChunk = object of TBaseChunk # not necessarily > PageSize! + next: PBigChunk # chunks of the same (or bigger) size + prev: PBigChunk + align: int + data: TAlignType # start of usable memory + +template smallChunkOverhead(): expr = sizeof(TSmallChunk)-sizeof(TAlignType) +template bigChunkOverhead(): expr = sizeof(TBigChunk)-sizeof(TAlignType) + +proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + assert(result >= x) + #return ((-x) and (v-1)) +% x + +assert(roundup(14, PageSize) == PageSize) +assert(roundup(15, 8) == 16) +assert(roundup(65, 8) == 72) + +# ------------- chunk table --------------------------------------------------- +# We use a PtrSet of chunk starts and a table[Page, chunksize] for chunk +# endings of big chunks. This is needed by the merging operation. The only +# remaining operation is best-fit for big chunks. Since there is a size-limit +# for big chunks (because greater than the limit means they are returned back +# to the OS), a fixed size array can be used. + +type + PLLChunk = ptr TLLChunk + TLLChunk {.pure.} = object ## *low-level* chunk + size: int # remaining size + acc: int # accumulator + + TAllocator {.final, pure.} = object + llmem: PLLChunk + currMem, maxMem, freeMem: int # memory sizes (allocated from OS) + freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] + freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access + chunkStarts: TIntSet + +proc incCurrMem(a: var TAllocator, bytes: int) {.inline.} = + inc(a.currMem, bytes) + +proc decCurrMem(a: var TAllocator, bytes: int) {.inline.} = + a.maxMem = max(a.maxMem, a.currMem) + dec(a.currMem, bytes) + +proc getMaxMem(a: var TAllocator): int = + # Since we update maxPagesCount only when freeing pages, + # maxPagesCount may not be up to date. Thus we use the + # maximum of these both values here: + return max(a.currMem, a.maxMem) + +var + allocator: TAllocator + +proc llAlloc(a: var TAllocator, size: int): pointer = + # *low-level* alloc for the memory managers data structures. Deallocation + # is never done. + if a.llmem == nil or size > a.llmem.size: + var request = roundup(size+sizeof(TLLChunk), PageSize) + a.llmem = cast[PLLChunk](osAllocPages(request)) + incCurrMem(a, request) + a.llmem.size = request - sizeof(TLLChunk) + a.llmem.acc = sizeof(TLLChunk) + result = cast[pointer](cast[TAddress](a.llmem) + a.llmem.acc) + dec(a.llmem.size, size) + inc(a.llmem.acc, size) + zeroMem(result, size) + +proc IntSetGet(t: TIntSet, key: int): PTrunk = + var it = t.data[key and high(t.data)] + while it != nil: + if it.key == key: return it + it = it.next + result = nil + +proc IntSetPut(t: var TIntSet, key: int): PTrunk = + result = IntSetGet(t, key) + if result == nil: + result = cast[PTrunk](llAlloc(allocator, sizeof(result^))) + result.next = t.data[key and high(t.data)] + t.data[key and high(t.data)] = result + result.key = key + +proc Contains(s: TIntSet, key: int): bool = + var t = IntSetGet(s, key shr TrunkShift) + if t != nil: + var u = key and TrunkMask + result = (t.bits[u shr IntShift] and (1 shl (u and IntMask))) != 0 + else: + result = false + +proc Incl(s: var TIntSet, key: int) = + var t = IntSetPut(s, key shr TrunkShift) + var u = key and TrunkMask + t.bits[u shr IntShift] = t.bits[u shr IntShift] or (1 shl (u and IntMask)) + +proc Excl(s: var TIntSet, key: int) = + var t = IntSetGet(s, key shr TrunkShift) + if t != nil: + var u = key and TrunkMask + t.bits[u shr IntShift] = t.bits[u shr IntShift] and not + (1 shl (u and IntMask)) + +proc ContainsOrIncl(s: var TIntSet, key: int): bool = + var t = IntSetGet(s, key shr TrunkShift) + if t != nil: + var u = key and TrunkMask + result = (t.bits[u shr IntShift] and (1 shl (u and IntMask))) != 0 + if not result: + t.bits[u shr IntShift] = t.bits[u shr IntShift] or + (1 shl (u and IntMask)) + else: + Incl(s, key) + result = false + +# ------------- chunk management ---------------------------------------------- +proc pageIndex(c: PChunk): int {.inline.} = + result = cast[TAddress](c) shr PageShift + +proc pageIndex(p: pointer): int {.inline.} = + result = cast[TAddress](p) shr PageShift + +proc pageAddr(p: pointer): PChunk {.inline.} = + result = cast[PChunk](cast[TAddress](p) and not PageMask) + assert(Contains(allocator.chunkStarts, pageIndex(result))) + +var lastSize = PageSize + +proc requestOsChunks(a: var TAllocator, size: int): PBigChunk = + incCurrMem(a, size) + inc(a.freeMem, size) + result = cast[PBigChunk](osAllocPages(size)) + assert((cast[TAddress](result) and PageMask) == 0) + #zeroMem(result, size) + result.next = nil + result.prev = nil + result.used = false + result.size = size + # update next.prevSize: + var nxt = cast[TAddress](result) +% size + assert((nxt and PageMask) == 0) + var next = cast[PChunk](nxt) + if pageIndex(next) in a.chunkStarts: + #echo("Next already allocated!") + next.prevSize = size + # set result.prevSize: + var prv = cast[TAddress](result) -% lastSize + assert((nxt and PageMask) == 0) + var prev = cast[PChunk](prv) + if pageIndex(prev) in a.chunkStarts and prev.size == lastSize: + #echo("Prev already allocated!") + result.prevSize = lastSize + else: + result.prevSize = 0 # unknown + lastSize = size # for next request + +proc freeOsChunks(a: var TAllocator, p: pointer, size: int) = + # update next.prevSize: + var c = cast[PChunk](p) + var nxt = cast[TAddress](p) +% c.size + assert((nxt and PageMask) == 0) + var next = cast[PChunk](nxt) + if pageIndex(next) in a.chunkStarts: + next.prevSize = 0 # XXX used + excl(a.chunkStarts, pageIndex(p)) + osDeallocPages(p, size) + decCurrMem(a, size) + dec(a.freeMem, size) + #c_fprintf(c_stdout, "[Alloc] back to OS: %ld\n", size) + +proc isAccessible(p: pointer): bool {.inline.} = + result = Contains(allocator.chunkStarts, pageIndex(p)) + +proc contains[T](list, x: T): bool = + var it = list + while it != nil: + if it == x: return true + it = it.next + +when false: + proc writeFreeList(a: TAllocator) = + var it = a.freeChunksList + c_fprintf(c_stdout, "freeChunksList: %p\n", it) + while it != nil: + c_fprintf(c_stdout, "it: %p, next: %p, prev: %p\n", + it, it.next, it.prev) + it = it.next + +proc ListAdd[T](head: var T, c: T) {.inline.} = + assert(c notin head) + assert c.prev == nil + assert c.next == nil + c.next = head + if head != nil: + assert head.prev == nil + head.prev = c + head = c + +proc ListRemove[T](head: var T, c: T) {.inline.} = + assert(c in head) + if c == head: + head = c.next + assert c.prev == nil + if head != nil: head.prev = nil + else: + assert c.prev != nil + c.prev.next = c.next + if c.next != nil: c.next.prev = c.prev + c.next = nil + c.prev = nil + +proc isSmallChunk(c: PChunk): bool {.inline.} = + return c.size <= SmallChunkSize-smallChunkOverhead() + #return c.size < SmallChunkSize + +proc chunkUnused(c: PChunk): bool {.inline.} = + result = not c.used + +proc updatePrevSize(a: var TAllocator, c: PBigChunk, + prevSize: int) {.inline.} = + var ri = cast[PChunk](cast[TAddress](c) +% c.size) + assert((cast[TAddress](ri) and PageMask) == 0) + if isAccessible(ri): + ri.prevSize = prevSize + +proc freeBigChunk(a: var TAllocator, c: PBigChunk) = + var c = c + assert(c.size >= PageSize) + inc(a.freeMem, c.size) + when coalescRight: + var ri = cast[PChunk](cast[TAddress](c) +% c.size) + assert((cast[TAddress](ri) and PageMask) == 0) + if isAccessible(ri) and chunkUnused(ri): + assert(not isSmallChunk(ri)) + if not isSmallChunk(ri): + ListRemove(a.freeChunksList, cast[PBigChunk](ri)) + inc(c.size, ri.size) + excl(a.chunkStarts, pageIndex(ri)) + when coalescLeft: + if c.prevSize != 0: + var le = cast[PChunk](cast[TAddress](c) -% c.prevSize) + assert((cast[TAddress](le) and PageMask) == 0) + if isAccessible(le) and chunkUnused(le): + assert(not isSmallChunk(le)) + if not isSmallChunk(le): + ListRemove(a.freeChunksList, cast[PBigChunk](le)) + inc(le.size, c.size) + excl(a.chunkStarts, pageIndex(c)) + c = cast[PBigChunk](le) + + if c.size < ChunkOsReturn: + incl(a.chunkStarts, pageIndex(c)) + updatePrevSize(a, c, c.size) + ListAdd(a.freeChunksList, c) + c.used = false + else: + freeOsChunks(a, c, c.size) + +proc splitChunk(a: var TAllocator, c: PBigChunk, size: int) = + var rest = cast[PBigChunk](cast[TAddress](c) +% size) + assert(rest notin a.freeChunksList) + # c_fprintf(c_stdout, "to add: %p\n", rest) + # writeFreeList(allocator) + # assert false + rest.size = c.size - size + rest.used = false + rest.next = nil + rest.prev = nil + rest.prevSize = size + updatePrevSize(a, c, rest.size) + c.size = size + incl(a.chunkStarts, pageIndex(rest)) + ListAdd(a.freeChunksList, rest) + +proc getBigChunk(a: var TAllocator, size: int): PBigChunk = + # use first fit for now: + assert((size and PageMask) == 0) + assert(size > 0) + result = a.freeChunksList + block search: + while result != nil: + #if not chunkUnused(result): + # c_fprintf(c_stdout, "%lld\n", int(result.used)) + assert chunkUnused(result) + if result.size == size: + ListRemove(a.freeChunksList, result) + break search + elif result.size > size: + #c_fprintf(c_stdout, "res size: %lld; size: %lld\n", result.size, size) + ListRemove(a.freeChunksList, result) + splitChunk(a, result, size) + break search + result = result.next + assert result != a.freeChunksList + if size < InitialMemoryRequest: + result = requestOsChunks(a, InitialMemoryRequest) + splitChunk(a, result, size) + else: + result = requestOsChunks(a, size) + result.prevSize = 0 # XXX why is this needed? + result.used = true + incl(a.chunkStarts, pageIndex(result)) + dec(a.freeMem, size) + +proc getSmallChunk(a: var TAllocator): PSmallChunk = + var res = getBigChunk(a, PageSize) + assert res.prev == nil + assert res.next == nil + result = cast[PSmallChunk](res) + +# ----------------------------------------------------------------------------- + +proc getCellSize(p: pointer): int {.inline.} = + var c = pageAddr(p) + result = c.size + +proc rawAlloc(a: var TAllocator, requestedSize: int): pointer = + assert(roundup(65, 8) == 72) + assert requestedSize >= sizeof(TFreeCell) + var size = roundup(requestedSize, MemAlign) + #c_fprintf(c_stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): + # allocate a small block: for small chunks, we use only its next pointer + var s = size div MemAlign + var c = a.freeSmallChunks[s] + if c == nil: + c = getSmallChunk(a) + c.freeList = nil + assert c.size == PageSize + c.size = size + c.acc = size + c.free = SmallChunkSize - smallChunkOverhead() - size + c.next = nil + c.prev = nil + ListAdd(a.freeSmallChunks[s], c) + result = addr(c.data) + assert((cast[TAddress](result) and (MemAlign-1)) == 0) + else: + assert c.next != c + #if c.size != size: + # c_fprintf(c_stdout, "csize: %lld; size %lld\n", c.size, size) + assert c.size == size + if c.freeList == nil: + assert(c.acc + smallChunkOverhead() + size <= SmallChunkSize) + result = cast[pointer](cast[TAddress](addr(c.data)) +% c.acc) + inc(c.acc, size) + else: + result = c.freeList + assert(c.freeList.zeroField == 0) + c.freeList = c.freeList.next + dec(c.free, size) + assert((cast[TAddress](result) and (MemAlign-1)) == 0) + if c.free < size: + ListRemove(a.freeSmallChunks[s], c) + else: + size = roundup(requestedSize+bigChunkOverhead(), PageSize) + # allocate a large block + var c = getBigChunk(a, size) + assert c.prev == nil + assert c.next == nil + assert c.size == size + result = addr(c.data) + assert((cast[TAddress](result) and (MemAlign-1)) == 0) + assert(isAccessible(result)) + +proc rawDealloc(a: var TAllocator, p: pointer) = + var c = pageAddr(p) + if isSmallChunk(c): + # `p` is within a small chunk: + var c = cast[PSmallChunk](c) + var s = c.size + var f = cast[ptr TFreeCell](p) + #echo("setting to nil: ", $cast[TAddress](addr(f.zeroField))) + assert(f.zeroField != 0) + f.zeroField = 0 + f.next = c.freeList + c.freeList = f + when overwriteFree: + # set to 0xff to check for usage after free bugs: + c_memset(cast[pointer](cast[int](p) +% sizeof(TFreeCell)), -1'i32, + s -% sizeof(TFreeCell)) + # check if it is not in the freeSmallChunks[s] list: + if c.free < s: + assert c notin a.freeSmallChunks[s div memAlign] + # add it to the freeSmallChunks[s] array: + ListAdd(a.freeSmallChunks[s div memAlign], c) + inc(c.free, s) + else: + inc(c.free, s) + if c.free == SmallChunkSize-smallChunkOverhead(): + ListRemove(a.freeSmallChunks[s div memAlign], c) + c.size = SmallChunkSize + freeBigChunk(a, cast[PBigChunk](c)) + else: + # set to 0xff to check for usage after free bugs: + when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead()) + # free big chunk + freeBigChunk(a, cast[PBigChunk](c)) + +proc isAllocatedPtr(a: TAllocator, p: pointer): bool = + if isAccessible(p): + var c = pageAddr(p) + if not chunkUnused(c): + if isSmallChunk(c): + var c = cast[PSmallChunk](c) + var offset = (cast[TAddress](p) and PageMask) -% smallChunkOverhead() + result = (c.acc >% offset) and (offset %% c.size == 0) and + (cast[ptr TFreeCell](p).zeroField >% 1) + else: + var c = cast[PBigChunk](c) + result = p == addr(c.data) and cast[ptr TFreeCell](p).zeroField >% 1 + +# ---------------------- interface to programs ------------------------------- + +when true: + proc alloc(size: int): pointer = + result = rawAlloc(allocator, size+sizeof(TFreeCell)) + cast[ptr TFreeCell](result).zeroField = 2 # mark it as used + #assert(not isAllocatedPtr(allocator, result)) + result = cast[pointer](cast[TAddress](result) +% sizeof(TFreeCell)) + + proc alloc0(size: int): pointer = + result = talloc.alloc(size) + zeroMem(result, size) + + proc dealloc(p: pointer) = + var x = cast[pointer](cast[TAddress](p) -% sizeof(TFreeCell)) + assert(cast[ptr TFreeCell](x).zeroField == 2) + rawDealloc(allocator, x) + assert(not isAllocatedPtr(allocator, x)) + + proc isAllocatedPtr(p: pointer): bool = + var x = cast[pointer](cast[TAddress](p) -% sizeof(TFreeCell)) + result = isAllocatedPtr(allocator, x) + + proc ptrSize(p: pointer): int = + var x = cast[pointer](cast[TAddress](p) -% sizeof(TFreeCell)) + result = pageAddr(x).size - sizeof(TFreeCell) + + proc realloc(p: pointer, newsize: int): pointer = + if newsize > 0: + result = talloc.alloc(newsize) + if p != nil: + copyMem(result, p, ptrSize(p)) + talloc.dealloc(p) + elif p != nil: + talloc.dealloc(p) + + proc countFreeMem(): int = + # only used for assertions + var it = allocator.freeChunksList + while it != nil: + inc(result, it.size) + it = it.next + + proc getFreeMem(): int = + result = allocator.freeMem + #assert(result == countFreeMem()) + + proc getTotalMem(): int = return allocator.currMem + proc getOccupiedMem(): int = return talloc.getTotalMem() - talloc.getFreeMem() + +when isMainModule: + const iterations = 4000_000 + incl(allocator.chunkStarts, 11) + assert 11 in allocator.chunkStarts + excl(allocator.chunkStarts, 11) + assert 11 notin allocator.chunkStarts + var p: array [1..iterations, pointer] + for i in 4..70: + var x = i * 8 + for j in 1.. iterations: + p[j] = talloc.Alloc(x) + for j in 1..iterations: + assert isAllocatedPtr(p[j]) + echo($i, " used memory: ", $(allocator.currMem)) + for j in countdown(iterations, 1): + #echo("j: ", $j) + talloc.dealloc(p[j]) + assert(not isAllocatedPtr(allocator, p[j])) + echo($i, " after freeing: ", $(allocator.currMem)) + diff --git a/tinyc/config.h b/tinyc/config.h index edaf335bb..161a3e178 100755 --- a/tinyc/config.h +++ b/tinyc/config.h @@ -7,11 +7,13 @@ # define TCC_TARGET_I386 # define CONFIG_TCCDIR "." #elif defined(__i386__) +# define CONFIG_USE_LIBGCC # define TCC_TARGET_I386 # define CONFIG_TCCDIR "/usr/local/lib/tcc" # define GCC_MAJOR 4 # define HOST_I386 1 #else +# define CONFIG_USE_LIBGCC # define TCC_TARGET_X86_64 # define CONFIG_TCCDIR "/usr/local/lib/tcc" # define GCC_MAJOR 4 diff --git a/todo.txt b/todo.txt index 9240286be..2740908a9 100755 --- a/todo.txt +++ b/todo.txt @@ -1,11 +1,13 @@ High priority (version 0.9.0) ============================= + - fix implicit generic routines - fix the streams implementation so that it uses methods - fix overloading resolution - wrong co-/contravariance + Bugs ---- - proc (x: int) is passable to proc (x: var int) !? @@ -103,7 +105,6 @@ Low priority - find a way for easy constructors and destructors; (destructors are much more important than constructors) - code generated for type information is wasteful -- icon installation for the Windows installer Other ideas diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim new file mode 100755 index 000000000..cc1f89a74 --- /dev/null +++ b/tools/nimgrep.nim @@ -0,0 +1,191 @@ +# +# +# Nimrod Grep Utility +# (c) Copyright 2010 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import + os, strutils, parseopt, pegs, re, terminal + +const + Usage = """ +Usage: nimgrep [options] [pattern] [files/directory] +Options: + --find, -f find the pattern (default) + --replace, -r replace the pattern + --peg pattern is a peg (default) + --re pattern is a regular expression + --recursive process directories recursively + --confirm confirm each occurence/replacement; there is a chance + to abort any time without touching the file(s) + --stdin read pattern from stdin (to avoid the shell's confusing + quoting rules) + --word, -w the pattern should have word boundaries + --ignore_case, -i be case insensitive + --ignore_style, -y be style insensitive +""" + +type + TOption = enum + optFind, optReplace, optPeg, optRegex, optRecursive, optConfirm, optStdin, + optWord, optIgnoreCase, optIgnoreStyle + TOptions = set[TOption] + TConfirmEnum = enum + ceAbort, ceYes, ceAll, ceNo, ceNone + +var + filename = "" + pattern = "" + replacement = "" + options: TOptions + +proc ask(msg: string): string = + stdout.write(msg) + result = stdin.readline() + +proc Confirm: TConfirmEnum = + while true: + case normalize(ask("[a]bort; [y]es, a[l]l, [n]o, non[e]: ")) + of "a", "abort": return ceAbort + of "y", "yes": return ceYes + of "l", "all": return ceAll + of "n", "no": return ceNo + of "e", "none": return ceNone + else: nil + +proc highlight(a, b, c: string) = + stdout.write(a) + terminal.WriteStyled(b) + stdout.writeln(c) + +proc countLines(s: string, first = 0, last = s.high): int = + var i = first + while i <= last: + if s[i] == '\13': + inc result + if i < last and s[i+1] == '\10': inc(i) + elif s[i] == '\10': + inc result + inc i + +proc processFile(filename: string) = + var buffer = system.readFile(filename) + if isNil(buffer): quit("cannot open file: " & filename) + var pegp: TPeg + var rep: TRegex + var result: string + + if optRegex in options: + if optIgnoreCase in options: + rep = re(pattern, {reExtended, reIgnoreCase}) + else: + rep = re(pattern) + else: + pegp = peg(pattern) + + if optReplace in options: + result = newString(buffer.len) + setLen(result, 0) + + var line = 1 + var i = 0 + var matches: array[0..re.MaxSubpatterns-1. string] + var reallyReplace = true + while i < buffer.len: + var t: tuple[first, last: int] + if optRegex in options: + quit "to implement" + else: + t = findBounds(buffer, pegp, matches, i) + + if t.first <= 0: break + inc(line, countLines(buffer, i, t.first-1)) + + var wholeMatch = buffer.copy(t.first, t.last) + echo "line ", line, ": ", wholeMatch + + if optReplace in options: + var r = replace(wholeMatch, pegp, replacement) + + if optConfirm in options: + case Confirm() + of ceAbort: + of ceYes: + of ceAll: + reallyReplace = true + of ceNo: + reallyReplace = false + of ceNone: + reallyReplace = false + if reallyReplace: + + + inc(line, countLines(buffer, t.first, t.last)) + + i = t.last+1 + + +proc walker(dir: string) = + for kind, path in walkDir(dir): + case kind + of pcFile: processFile(path) + of pcDirectory: + if optRecursive in options: + walker(path) + else: nil + +proc writeHelp() = quit(Usage) +proc writeVersion() = quit("1.0") + +proc checkOptions(subset: TOptions, a, b: string) = + if subset <= options: + quit("cannot specify both '$#' and '$#'" % [a, b]) + +for kind, key, val in getopt(): + case kind + of cmdArgument: + if options.contains(optStdIn): + filename = key + elif pattern.len == 0: + pattern = key + elif options.contains(optReplace) and replacement.len == 0: + replacement = key + else: + filename = key + of cmdLongOption, cmdShortOption: + case normalize(key) + of "find", "f": incl(options, optFind) + of "replace", "r": incl(options, optReplace) + of "peg": incl(options, optPeg) + of "re": incl(options, optRegex) + of "recursive": incl(options, optRecursive) + of "confirm": incl(options, optConfirm) + of "stdin": incl(options, optStdin) + of "word", "w": incl(options, optWord) + of "ignorecase", "i": incl(options, optIgnoreCase) + of "ignorestyle", "y": incl(options, optIgnoreStyle) + of "help", "h": writeHelp() + of "version", "v": writeVersion() + else: writeHelp() + of cmdEnd: assert(false) # cannot happen + +checkOptions({optFind, optReplace}, "find", "replace") +checkOptions({optPeg, optRegex}, "peg", "re") +checkOptions({optIgnoreCase, optIgnoreStyle}, "ignore_case", "ignore_style") +checkOptions({optIgnoreCase, optPeg}, "ignore_case", "peg") + +if optStdin in options: + pattern = ask("pattern [ENTER to exit]: ") + if IsNil(pattern) or pattern.len == 0: quit(0) + if optReplace in options: + replacement = ask("replacement [supports $1, $# notations]: ") + +if pattern.len == 0: + writeHelp() +else: + if filename.len == 0: filename = os.getCurrentDir() + walker(filename) + diff --git a/tools/niminst.nim b/tools/niminst.nim index 77cdfa16b..e696b6b93 100755 --- a/tools/niminst.nim +++ b/tools/niminst.nim @@ -84,6 +84,7 @@ proc initConfigData(c: var TConfigData) = c.vars = newStringTable(modeStyleInsensitive) proc skipRoot(f: string): string = + # "abc/def/xyz" --> "def/xyz" var i = 0 result = "" for component in split(f, {dirsep, altsep}): diff --git a/tools/nimrepl.nim b/tools/nimrepl.nim index 432ca1356..220307dba 100755 --- a/tools/nimrepl.nim +++ b/tools/nimrepl.nim @@ -39,7 +39,7 @@ proc destroy(widget: PWidget, data: pgpointer){.cdecl.} = proc FileOpenClicked(menuitem: PMenuItem, userdata: pgpointer) {.cdecl.} = var path = ChooseFileToOpen(w) if path != "": - var file: string = readFile(path) + var file = readFile(path) if file != nil: set_text(InputTextBuffer, file, len(file)) else: diff --git a/web/news.txt b/web/news.txt index 4e6c51c56..36fdcc79f 100755 --- a/web/news.txt +++ b/web/news.txt @@ -5,6 +5,8 @@ News 2010-XX-XX Version 0.8.12 released ================================== +Version 0.8.12 has been released! Get it `here <download.html>`_. + Bugfixes -------- @@ -14,6 +16,8 @@ Bugfixes - Bugfix: ``dialogs.ChooseFilesToOpen`` did not work if only one file is selected. - Bugfix: niminst: ``nimrod`` is not default dir for *every* project. +- Bugfix: Multiple yield statements in iterators did not cause local vars to be + copied. Additions @@ -21,7 +25,10 @@ Additions - Added ``re.findAll``, ``pegs.findAll``. - Added ``os.findExe``. -- The Pegs module supports a *captured search loop operator* ``{@}``. +- Added ``strutils.align``, ``strutils.tokenize``, ``strutils.wordWrap``. +- Pegs support a *captured search loop operator* ``{@}``. +- Pegs support new built-ins: ``\letter``, ``\upper``, ``\lower``, + ``\title``, ``\white``. 2010-10-20 Version 0.8.10 released @@ -95,9 +102,6 @@ Additions 2010-03-14 Version 0.8.8 released ================================= -Version 0.8.8 has been released! Get it `here <download.html>`_. - - Bugfixes -------- - The Posix version of ``os.copyFile`` has better error handling. |