diff options
| author | Andreas Rumpf <rumpf_a@web.de> | 2017-12-11 15:12:45 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2017-12-11 15:12:45 +0100 |
| commit | 5e43e0d789a27f9e3635d5ac4644cb20769985b2 (patch) | |
| tree | abbde2857ed443df77a99705ed5e8407f042d8a7 /lib | |
| parent | 94fe5bd1184bd416f54d3fffc0227df6c3d7883a (diff) | |
| parent | 28e0bf9dcd62f387c79f767848cadd8b71d825de (diff) | |
| download | Nim-5e43e0d789a27f9e3635d5ac4644cb20769985b2.tar.gz | |
Merge branch 'devel' into async-improvements
Diffstat (limited to 'lib')
34 files changed, 743 insertions, 396 deletions
diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ebc9f7714..ee6c1a09f 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1226,3 +1226,8 @@ when not defined(booting): macro payload: untyped {.gensym.} = result = parseStmt(e) payload() + +macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped = + result = newCall(callee) + for i in 0 ..< args.len: + result.add args[i] diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 24fc83366..c7f8f336b 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -49,9 +49,6 @@ type RegexError* = object of ValueError ## is raised if the pattern is no valid regular expression. -{.deprecated: [TRegexFlag: RegexFlag, TRegexDesc: RegexDesc, TRegex: Regex, - EInvalidRegEx: RegexError].} - proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = var e: ref RegexError new(e) @@ -470,8 +467,8 @@ proc replacef*(s: string, sub: Regex, by: string): string = prev = match.last + 1 add(result, substr(s, prev)) -proc parallelReplace*(s: string, subs: openArray[ - tuple[pattern: Regex, repl: string]]): string = +proc multiReplace*(s: string, subs: openArray[ + tuple[pattern: Regex, repl: string]]): string = ## Returns a modified copy of ``s`` with the substitutions in ``subs`` ## applied in parallel. result = "" @@ -490,13 +487,20 @@ proc parallelReplace*(s: string, subs: openArray[ # copy the rest: add(result, substr(s, i)) +proc parallelReplace*(s: string, subs: openArray[ + tuple[pattern: Regex, repl: string]]): string {.deprecated.} = + ## Returns a modified copy of ``s`` with the substitutions in ``subs`` + ## applied in parallel. + ## **Deprecated since version 0.18.0**: Use ``multiReplace`` instead. + result = multiReplace(s, subs) + proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = ## reads in the file ``infile``, performs a parallel replacement (calls ## ``parallelReplace``) and writes back to ``outfile``. Raises ``IOError`` if an ## error occurs. This is supposed to be used for quick scripting. var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) + writeFile(outfile, x.multiReplace(subs)) iterator split*(s: string, sep: Regex): string = ## Splits the string ``s`` into substrings. @@ -579,12 +583,12 @@ const ## common regular expressions ## describes an URL when isMainModule: - doAssert match("(a b c)", re"\( .* \)") + doAssert match("(a b c)", rex"\( .* \)") doAssert match("WHiLe", re("while", {reIgnoreCase})) doAssert "0158787".match(re"\d+") doAssert "ABC 0232".match(re"\w+\s+\d+") - doAssert "ABC".match(re"\d+ | \w+") + doAssert "ABC".match(rex"\d+ | \w+") {.push warnings:off.} doAssert matchLen("key", re(reIdentifier)) == 3 diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 13eb1e759..f34efe9a2 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -177,7 +177,7 @@ proc `==`*(x, y: JsRoot): bool {. importcpp: "(# === #)" .} ## and not strings or numbers, this is a *comparison of references*. {. experimental .} -macro `.`*(obj: JsObject, field: static[cstring]): JsObject = +macro `.`*(obj: JsObject, field: untyped): JsObject = ## Experimental dot accessor (get) for type JsObject. ## Returns the value of a property of name `field` from a JsObject `x`. ## @@ -196,14 +196,14 @@ macro `.`*(obj: JsObject, field: static[cstring]): JsObject = helper(`obj`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) let importString = "#." & mangledNames[$field] result = quote do: proc helper(o: JsObject): JsObject {. importcpp: `importString`, gensym .} helper(`obj`) -macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped = +macro `.=`*(obj: JsObject, field, value: untyped): untyped = ## Experimental dot accessor (set) for type JsObject. ## Sets the value of a property of name `field` in a JsObject `x` to `value`. if validJsName($field): @@ -214,7 +214,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped = helper(`obj`, `value`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) let importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: JsObject, v: auto) @@ -222,7 +222,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped = helper(`obj`, `value`) macro `.()`*(obj: JsObject, - field: static[cstring], + field: untyped, args: varargs[JsObject, jsFromAst]): JsObject = ## Experimental "method call" operator for type JsObject. ## Takes the name of a method of the JavaScript object (`field`) and calls @@ -245,7 +245,7 @@ macro `.()`*(obj: JsObject, importString = "#." & $field & "(@)" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] & "(@)" result = quote: proc helper(o: JsObject): JsObject @@ -257,7 +257,7 @@ macro `.()`*(obj: JsObject, result[1].add args[idx].copyNimTree macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V], - field: static[cstring]): V = + field: untyped): V = ## Experimental dot accessor (get) for type JsAssoc. ## Returns the value of a property of name `field` from a JsObject `x`. var importString: string @@ -265,7 +265,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V], importString = "#." & $field else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] result = quote do: proc helper(o: type(`obj`)): `obj`.V @@ -273,7 +273,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V], helper(`obj`) macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V], - field: static[cstring], + field: untyped, value: V): untyped = ## Experimental dot accessor (set) for type JsAssoc. ## Sets the value of a property of name `field` in a JsObject `x` to `value`. @@ -282,7 +282,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V], importString = "#." & $field & " = #" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName(field) + mangledNames[$field] = $mangleJsName($field) importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: type(`obj`), v: `obj`.V) @@ -290,7 +290,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V], helper(`obj`, `value`) macro `.()`*[K: string | cstring, V: proc](obj: JsAssoc[K, V], - field: static[cstring], + field: untyped, args: varargs[untyped]): auto = ## Experimental "method call" operator for type JsAssoc. ## Takes the name of a method of the JavaScript object (`field`) and calls diff --git a/lib/nimbase.h b/lib/nimbase.h index 76192713b..ac2cc097c 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -70,7 +70,7 @@ __clang__ #if defined(_MSC_VER) # pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309) # pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706) -# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090) +# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297) #endif /* ------------------------------------------------------------------------- */ diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 70369b001..2a58854a6 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -31,14 +31,12 @@ type state: TokenClass SourceLanguage* = enum - langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava, + langNone, langNim, langCpp, langCsharp, langC, langJava, langYaml -{.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass, - TGeneralTokenizer: GeneralTokenizer].} const sourceLanguageToStr*: array[SourceLanguage, string] = ["none", - "Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"] + "Nim", "C++", "C#", "C", "Java", "Yaml"] tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace", "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", @@ -398,7 +396,6 @@ type TokenizerFlag = enum hasPreprocessor, hasNestedComments TokenizerFlags = set[TokenizerFlag] -{.deprecated: [TTokenizerFlag: TokenizerFlag, TTokenizerFlags: TokenizerFlags].} proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], flags: TokenizerFlags) = @@ -888,7 +885,7 @@ proc yamlNextToken(g: var GeneralTokenizer) = proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) = case lang of langNone: assert false - of langNim, langNimrod: nimNextToken(g) + of langNim: nimNextToken(g) of langCpp: cppNextToken(g) of langCsharp: csharpNextToken(g) of langC: cNextToken(g) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 53699166f..223fc836a 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -45,8 +45,6 @@ type MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.nimcall.} ## what to do in case of an error FindFileHandler* = proc (filename: string): string {.nimcall.} -{.deprecated: [TRstParseOptions: RstParseOptions, TRstParseOption: RstParseOption, - TMsgKind: MsgKind].} const messages: array[MsgKind, string] = [ @@ -127,8 +125,6 @@ type bufpos*: int line*, col*, baseIndent*: int skipPounds*: bool -{.deprecated: [TTokType: TokType, TToken: Token, TTokenSeq: TokenSeq, - TLexer: Lexer].} proc getThing(L: var Lexer, tok: var Token, s: set[char]) = tok.kind = tkWord @@ -288,10 +284,6 @@ type hasToc*: bool EParseError* = object of ValueError -{.deprecated: [TLevelMap: LevelMap, TSubstitution: Substitution, - TSharedState: SharedState, TRstParser: RstParser, - TMsgHandler: MsgHandler, TFindFileHandler: FindFileHandler, - TMsgClass: MsgClass].} proc whichMsgClass*(k: MsgKind): MsgClass = ## returns which message class `k` belongs to. @@ -341,11 +333,6 @@ proc rstMessage(p: RstParser, msgKind: MsgKind) = p.col + p.tok[p.idx].col, msgKind, p.tok[p.idx].symbol) -when false: - proc corrupt(p: RstParser) = - assert p.indentStack[0] == 0 - for i in 1 .. high(p.indentStack): assert p.indentStack[i] < 1_000 - proc currInd(p: RstParser): int = result = p.indentStack[high(p.indentStack)] diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 1272affdc..e6c95b59e 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -46,7 +46,7 @@ type target*: OutputTarget config*: StringTableRef splitAfter*: int # split too long entries in the TOC - listingCounter: int + listingCounter*: int tocPart*: seq[TocEntry] hasToc*: bool theIndex: string # Contents of the index file to be dumped at the end. @@ -61,6 +61,9 @@ type seenIndexTerms: Table[string, int] ## \ ## Keeps count of same text index terms to generate different identifiers ## for hyperlinks. See renderIndexTerm proc for details. + id*: int ## A counter useful for generating IDs. + onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int; + content: string) PDoc = var RstGenerator ## Alias to type less. @@ -69,8 +72,9 @@ type startLine: int ## The starting line of the code block, by default 1. langStr: string ## Input string used to specify the language. lang: SourceLanguage ## Type of highlighting, by default none. -{.deprecated: [TRstGenerator: RstGenerator, TTocEntry: TocEntry, - TOutputTarget: OutputTarget, TMetaEnum: MetaEnum].} + filename: string + testCmd: string + status: int proc init(p: var CodeBlockParams) = ## Default initialisation of CodeBlockParams to sane values. @@ -133,6 +137,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, g.options = options g.findFile = findFile g.currentSection = "" + g.id = 0 let fileParts = filename.splitFile if fileParts.ext == ".nim": g.currentSection = "Module " & fileParts.name @@ -368,7 +373,6 @@ type ## ## The value indexed by this IndexEntry is a sequence with the real index ## entries found in the ``.idx`` file. -{.deprecated: [TIndexEntry: IndexEntry, TIndexedDocs: IndexedDocs].} proc cmp(a, b: IndexEntry): int = ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. @@ -823,13 +827,20 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = var number: int if parseInt(n.getFieldValue, number) > 0: params.startLine = number - of "file": + of "file", "filename": # The ``file`` option is a Nim extension to the official spec, it acts # like it would for other directives like ``raw`` or ``cvs-table``. This # field is dealt with in ``rst.nim`` which replaces the existing block with # the referenced file, so we only need to ignore it here to avoid incorrect # warning messages. - discard + params.filename = n.getFieldValue.strip + of "test": + params.testCmd = n.getFieldValue.strip + if params.testCmd.len == 0: params.testCmd = "nim c -r $1" + of "status": + var status: int + if parseInt(n.getFieldValue, status) > 0: + params.status = status of "default-language": params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage @@ -901,6 +912,9 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) = var m = n.sons[2].sons[0] assert m.kind == rnLeaf + if params.testCmd.len > 0 and d.onTestSnippet != nil: + d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text) + let (blockStart, blockEnd) = buildLinesHTMLTable(d, params, m.text) dispA(d.target, result, blockStart, "\\begin{rstpre}\n", []) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 65004cbe0..d48274bb9 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -59,9 +59,10 @@ export asyncfutures, asyncstreams ## ## .. code-block::nim ## var future = socket.recv(100) -## future.callback = +## future.addCallback( ## proc () = ## echo(future.read) +## ) ## ## All asynchronous functions returning a ``Future`` will not block. They ## will not however return immediately. An asynchronous function will have diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 8941dca6e..82bc53aaf 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -334,7 +334,7 @@ proc all*[T](futs: varargs[Future[T]]): auto = let totalFutures = len(futs) for fut in futs: - fut.callback = proc(f: Future[T]) = + fut.addCallback proc (f: Future[T]) = inc(completedFutures) if not retFuture.finished: if f.failed: @@ -356,7 +356,7 @@ proc all*[T](futs: varargs[Future[T]]): auto = for i, fut in futs: proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = + fut.addCallback proc (f: Future[T]) = inc(completedFutures) if not retFuture.finished: if f.failed: diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 433931c9d..ba1615651 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -275,10 +275,7 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string lineFut.mget() = newStringOfCap(80) while not client.isClosed: - try: - await processRequest(server, request, client, address, lineFut, callback) - except: - asyncCheck request.mget().respondError(Http500) + await processRequest(server, request, client, address, lineFut, callback) proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure,gcsafe.}, diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index d1207603d..3f213c5ea 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -181,7 +181,7 @@ elif useICC_builtins: proc countSetBits*(x: SomeInteger): int {.inline, nosideeffect.} = - ## Counts the set bits in integer. (also called Hamming weight.) + ## Counts the set bits in integer. (also called `Hamming weight`:idx:.) # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. # like GCC and MSVC when nimvm: diff --git a/lib/pure/cgi.nim b/lib/pure/cgi.nim index fcf2cf99f..5de6aa487 100644 --- a/lib/pure/cgi.nim +++ b/lib/pure/cgi.nim @@ -29,21 +29,8 @@ ## writeLine(stdout, "your password: " & myData["password"]) ## writeLine(stdout, "</body></html>") -import strutils, os, strtabs, cookies - -proc encodeUrl*(s: string): string = - ## Encodes a value to be HTTP safe: This means that characters in the set - ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, - ## a space is converted to ``'+'`` and every other character is encoded as - ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. - result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars - for i in 0..s.len-1: - case s[i] - of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) - of ' ': add(result, '+') - else: - add(result, '%') - add(result, toHex(ord(s[i]), 2)) +import strutils, os, strtabs, cookies, uri +export uri.encodeUrl, uri.decodeUrl proc handleHexChar(c: char, x: var int) {.inline.} = case c @@ -52,28 +39,6 @@ proc handleHexChar(c: char, x: var int) {.inline.} = of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) else: assert(false) -proc decodeUrl*(s: string): string = - ## Decodes a value from its HTTP representation: This means that a ``'+'`` - ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal - ## value) is converted to the character with ordinal number ``xx``, and - ## and every other character is carried over. - result = newString(s.len) - var i = 0 - var j = 0 - while i < s.len: - case s[i] - of '%': - var x = 0 - handleHexChar(s[i+1], x) - handleHexChar(s[i+2], x) - inc(i, 2) - result[j] = chr(x) - of '+': result[j] = ' ' - else: result[j] = s[i] - inc(i) - inc(j) - setLen(result, j) - proc addXmlChar(dest: var string, c: char) {.inline.} = case c of '&': add(dest, "&") @@ -390,8 +355,3 @@ proc existsCookie*(name: string): bool = ## Checks if a cookie of `name` exists. if gcookies == nil: gcookies = parseCookies(getHttpCookie()) result = hasKey(gcookies, name) - -when isMainModule: - const test1 = "abc\L+def xyz" - assert encodeUrl(test1) == "abc%0A%2Bdef+xyz" - assert decodeUrl(encodeUrl(test1)) == test1 diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index d0f5c78e0..06e96ca36 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -66,7 +66,7 @@ proc cycle*[T](s: openArray[T], n: Natural): seq[T] = ## ## Example: ## - ## .. code-block: + ## .. code-block:: ## ## let ## s = @[1, 2, 3] @@ -84,7 +84,7 @@ proc repeat*[T](x: T, n: Natural): seq[T] = ## ## Example: ## - ## .. code-block: + ## .. code-block:: ## ## let ## total = repeat(5, 3) diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index ad199a215..c0934a45b 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -59,8 +59,8 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "", # copy the attributes; when iterating over them these lists # will be modified, so that each attribute is only given one value - var req = split(reqAttr) - var opt = split(optAttr) + var req = splitWhitespace(reqAttr) + var opt = splitWhitespace(optAttr) result = newNimNode(nnkBracket, e) result.add(newStrLitNode("<")) result.add(newStrLitNode(tag)) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 2165c5cb2..7ed250307 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -279,8 +279,9 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, inc(s.count) pkey.events = events -proc unregister*[T](s: Selector[T], fd: int | SocketHandle) = +proc unregister*[T](s: Selector[T], fd: SocketHandle|int) = s.withSelectLock(): + let fd = fd.SocketHandle var pkey = s.getKey(fd) if Event.Read in pkey.events: IOFD_CLR(fd, addr s.rSet) @@ -438,18 +439,19 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2: untyped) = mixin withSelectLock s.withSelectLock(): - var value: ptr T - let fdi = int(fd) - var i = 0 - while i < FD_SETSIZE: - if s.fds[i].ident == fdi: - value = addr(s.fds[i].data) - break - inc(i) - if i != FD_SETSIZE: - body1 - else: - body2 + block: + var value: ptr T + let fdi = int(fd) + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + value = addr(s.fds[i].data) + break + inc(i) + if i != FD_SETSIZE: + body1 + else: + body2 proc getFd*[T](s: Selector[T]): int = diff --git a/lib/pure/json.nim b/lib/pure/json.nim index cea485c43..b5b84863a 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1346,6 +1346,16 @@ proc createJsonIndexer(jsonNode: NimNode, indexNode ) +proc transformJsonIndexer(jsonNode: NimNode): NimNode = + case jsonNode.kind + of nnkBracketExpr: + result = newNimNode(nnkCurlyExpr) + else: + result = jsonNode.copy() + + for child in jsonNode: + result.add(transformJsonIndexer(child)) + template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ast: string) = if node.kind notin kinds: @@ -1524,6 +1534,35 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] = doAssert result.len > 0 +proc processFields(obj: NimNode, + jsonNode: NimNode): seq[NimNode] {.compileTime.} = + ## Process all the fields of an ``ObjectTy`` and any of its + ## parent type's fields (via inheritance). + result = @[] + case obj.kind + of nnkObjectTy: + expectKind(obj[2], nnkRecList) + for field in obj[2]: + let nodes = processObjField(field, jsonNode) + result.add(nodes) + + # process parent type fields + case obj[1].kind + of nnkBracketExpr: + assert $obj[1][0] == "ref" + result.add(processFields(getType(obj[1][1]), jsonNode)) + of nnkSym: + result.add(processFields(getType(obj[1]), jsonNode)) + else: + discard + of nnkTupleTy: + for identDefs in obj: + expectKind(identDefs, nnkIdentDefs) + let nodes = processObjField(identDefs[0], jsonNode) + result.add(nodes) + else: + doAssert false, "Unable to process field type: " & $obj.kind + proc processType(typeName: NimNode, obj: NimNode, jsonNode: NimNode, isRef: bool): NimNode {.compileTime.} = ## Process a type such as ``Sym "float"`` or ``ObjectTy ...``. @@ -1533,20 +1572,21 @@ proc processType(typeName: NimNode, obj: NimNode, ## .. code-block::plain ## ObjectTy ## Empty - ## Empty + ## InheritanceInformation ## RecList ## Sym "events" case obj.kind - of nnkObjectTy: + of nnkObjectTy, nnkTupleTy: # Create object constructor. - result = newNimNode(nnkObjConstr) - result.add(typeName) # Name of the type to construct. + result = + if obj.kind == nnkObjectTy: newNimNode(nnkObjConstr) + else: newNimNode(nnkPar) - # Process each object field and add it as an exprColonExpr - expectKind(obj[2], nnkRecList) - for field in obj[2]: - let nodes = processObjField(field, jsonNode) - result.add(nodes) + if obj.kind == nnkObjectTy: + result.add(typeName) # Name of the type to construct. + + # Process each object/tuple field and add it as an exprColonExpr + result.add(processFields(obj, jsonNode)) # Object might be null. So we need to check for that. if isRef: @@ -1569,25 +1609,14 @@ proc processType(typeName: NimNode, obj: NimNode, `getEnumCall` ) of nnkSym: - case ($typeName).normalize - of "float": - result = quote do: - ( - verifyJsonKind(`jsonNode`, {JFloat, JInt}, astToStr(`jsonNode`)); - if `jsonNode`.kind == JFloat: `jsonNode`.fnum else: `jsonNode`.num.float - ) + let name = ($typeName).normalize + case name of "string": result = quote do: ( verifyJsonKind(`jsonNode`, {JString, JNull}, astToStr(`jsonNode`)); if `jsonNode`.kind == JNull: nil else: `jsonNode`.str ) - of "int": - result = quote do: - ( - verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`)); - `jsonNode`.num.int - ) of "biggestint": result = quote do: ( @@ -1601,12 +1630,36 @@ proc processType(typeName: NimNode, obj: NimNode, `jsonNode`.bval ) else: - doAssert false, "Unable to process nnkSym " & $typeName + if name.startsWith("int") or name.startsWith("uint"): + result = quote do: + ( + verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`)); + `jsonNode`.num.`obj` + ) + elif name.startsWith("float"): + result = quote do: + ( + verifyJsonKind(`jsonNode`, {JInt, JFloat}, astToStr(`jsonNode`)); + if `jsonNode`.kind == JFloat: `jsonNode`.fnum.`obj` else: `jsonNode`.num.`obj` + ) + else: + doAssert false, "Unable to process nnkSym " & $typeName else: doAssert false, "Unable to process type: " & $obj.kind doAssert(not result.isNil(), "processType not initialised.") +import options +proc workaroundMacroNone[T](): Option[T] = + none(T) + +proc depth(n: NimNode, current = 0): int = + result = 1 + for child in n: + let d = 1 + child.depth(current + 1) + if d > result: + result = d + proc createConstructor(typeSym, jsonNode: NimNode): NimNode = ## Accepts a type description, i.e. "ref Type", "seq[Type]", "Type" etc. ## @@ -1616,10 +1669,50 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = # echo("--createConsuctor-- \n", treeRepr(typeSym)) # echo() + if depth(jsonNode) > 150: + error("The `to` macro does not support ref objects with cycles.", jsonNode) + case typeSym.kind of nnkBracketExpr: var bracketName = ($typeSym[0]).normalize case bracketName + of "option": + # TODO: Would be good to verify that this is Option[T] from + # options module I suppose. + let lenientJsonNode = transformJsonIndexer(jsonNode) + + let optionGeneric = typeSym[1] + let value = createConstructor(typeSym[1], jsonNode) + let workaround = bindSym("workaroundMacroNone") # TODO: Nim Bug: This shouldn't be necessary. + + result = quote do: + ( + if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`) + ) + of "table", "orderedtable": + let tableKeyType = typeSym[1] + if ($tableKeyType).cmpIgnoreStyle("string") != 0: + error("JSON doesn't support keys of type " & $tableKeyType) + let tableValueType = typeSym[2] + + let forLoopKey = genSym(nskForVar, "key") + let indexerNode = createJsonIndexer(jsonNode, forLoopKey) + let constructorNode = createConstructor(tableValueType, indexerNode) + + let tableInit = + if bracketName == "table": + bindSym("initTable") + else: + bindSym("initOrderedTable") + + # Create a statement expression containing a for loop. + result = quote do: + ( + var map = `tableInit`[`tableKeyType`, `tableValueType`](); + verifyJsonKind(`jsonNode`, {JObject}, astToStr(`jsonNode`)); + for `forLoopKey` in keys(`jsonNode`.fields): map[`forLoopKey`] = `constructorNode`; + map + ) of "ref": # Ref type. var typeName = $typeSym[1] @@ -1663,12 +1756,23 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode = let obj = getType(typeSym) result = processType(typeSym, obj, jsonNode, false) of nnkSym: + # Handle JsonNode. + if ($typeSym).cmpIgnoreStyle("jsonnode") == 0: + return jsonNode + + # Handle all other types. let obj = getType(typeSym) if obj.kind == nnkBracketExpr: # When `Sym "Foo"` turns out to be a `ref object`. result = createConstructor(obj, jsonNode) else: result = processType(typeSym, obj, jsonNode, false) + of nnkTupleTy: + result = processType(typeSym, typeSym, jsonNode, false) + of nnkPar: + # TODO: The fact that `jsonNode` here works to give a good line number + # is weird. Specifying typeSym should work but doesn't. + error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode) else: doAssert false, "Unable to create constructor for: " & $typeSym.kind @@ -1796,10 +1900,18 @@ macro to*(node: JsonNode, T: typedesc): untyped = expectKind(typeNode, nnkBracketExpr) doAssert(($typeNode[0]).normalize == "typedesc") - result = createConstructor(typeNode[1], node) + # Create `temp` variable to store the result in case the user calls this + # on `parseJson` (see bug #6604). + result = newNimNode(nnkStmtListExpr) + let temp = genSym(nskLet, "temp") + result.add quote do: + let `temp` = `node` + + let constructor = createConstructor(typeNode[1], temp) # TODO: Rename postProcessValue and move it (?) - result = postProcessValue(result) + result.add(postProcessValue(constructor)) + # echo(treeRepr(result)) # echo(toStrLit(result)) when false: diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index e2a5bed96..830820fd1 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -202,13 +202,17 @@ when not defined(js): proc countLogLines(logger: RollingFileLogger): int = result = 0 - for line in logger.file.lines(): + let fp = open(logger.baseName, fmRead) + for line in fp.lines(): result.inc() + fp.close() proc countFiles(filename: string): int = # Example: file.log.1 result = 0 - let (dir, name, ext) = splitFile(filename) + var (dir, name, ext) = splitFile(filename) + if dir == "": + dir = "." for kind, path in walkDir(dir): if kind == pcFile: let llfn = name & ext & ExtSep diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 1e315afb4..b397ef47b 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -491,6 +491,8 @@ const mimes* = { "vrml": "x-world/x-vrml", "wrl": "x-world/x-vrml"} +from strutils import startsWith + proc newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. @@ -498,8 +500,11 @@ proc newMimetypes*(): MimeDB = proc getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext`` - ## could not be found. - result = mimedb.mimes.getOrDefault(ext) + ## could not be found. ``ext`` can start with an optional dot which is ignored. + if ext.startsWith("."): + result = mimedb.mimes.getOrDefault(ext.substr(1)) + else: + result = mimedb.mimes.getOrDefault(ext) if result == "": return default diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 5c162317e..621ef992c 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -145,7 +145,7 @@ type SOBool* = enum ## Boolean socket options. OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, - OptOOBInline, OptReuseAddr, OptReusePort + OptOOBInline, OptReuseAddr, OptReusePort, OptNoDelay ReadLineResult* = enum ## result for readLineAsync ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone @@ -865,6 +865,11 @@ proc close*(socket: Socket) = socket.fd.close() +when defined(posix): + from posix import TCP_NODELAY +else: + from winlean import TCP_NODELAY + proc toCInt*(opt: SOBool): cint = ## Converts a ``SOBool`` into its Socket Option cint representation. case opt @@ -876,6 +881,7 @@ proc toCInt*(opt: SOBool): cint = of OptOOBInline: SO_OOBINLINE of OptReuseAddr: SO_REUSEADDR of OptReusePort: SO_REUSEPORT + of OptNoDelay: TCP_NODELAY proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {. tags: [ReadIOEffect].} = @@ -898,6 +904,12 @@ proc getPeerAddr*(socket: Socket): (string, Port) = proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {. tags: [WriteIOEffect].} = ## Sets option ``opt`` to a boolean value specified by ``value``. + ## + ## .. code-block:: Nim + ## var socket = newSocket() + ## socket.setSockOpt(OptReusePort, true) + ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) + ## var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 8b26552de..a59134007 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -630,7 +630,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## the process has finished. To execute a program without having a ## shell involved, use the `execProcess` proc of the `osproc` ## module. - when defined(linux): + when defined(posix): result = c_system(command) shr 8 else: result = c_system(command) diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index c3bd399db..0d638abb9 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -602,14 +602,13 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} else: return "'" & s.replace("'", "'\"'\"'") & "'" -proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = - ## Quote ``s``, so it can be safely passed to shell. - when defined(Windows): - return quoteShellWindows(s) - elif defined(posix): - return quoteShellPosix(s) - else: - {.error:"quoteShell is not supported on your system".} +when defined(windows) or defined(posix): + proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = + ## Quote ``s``, so it can be safely passed to shell. + when defined(windows): + return quoteShellWindows(s) + else: + return quoteShellPosix(s) when isMainModule: assert quoteShellWindows("aaa") == "aaa" diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index cc4c26161..2a1ce0c58 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -41,6 +41,8 @@ type ## Windows: Named pipes are used so that you can peek ## at the process' output streams. poDemon ## Windows: The program creates no Window. + ## Unix: Start the program as a demon. This is still + ## work in progress! ProcessObj = object of RootObj when defined(windows): @@ -167,8 +169,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. - -proc peekExitCode*(p: Process): int {.tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} ## return -1 if the process is still running. Otherwise the process' exit code ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -231,55 +232,79 @@ proc execProcesses*(cmds: openArray[string], ## executes the commands `cmds` in parallel. Creates `n` processes ## that execute in parallel. The highest return value of all processes ## is returned. Runs `beforeRunEvent` before running each command. - when false: - # poParentStreams causes problems on Posix, so we simply disable it: - var options = options - {poParentStreams} assert n > 0 if n > 1: - var q: seq[Process] - newSeq(q, n) + var i = 0 + var q = newSeq[Process](n) var m = min(n, cmds.len) - for i in 0..m-1: + + when defined(windows): + var w: WOHandleArray + var wcount = m + + while i < m: if beforeRunEvent != nil: beforeRunEvent(i) - q[i] = startProcess(cmds[i], options=options + {poEvalCommand}) - when defined(noBusyWaiting): - var r = 0 - for i in m..high(cmds): - when defined(debugExecProcesses): - var err = "" - var outp = outputStream(q[r]) - while running(q[r]) or not atEnd(outp): - err.add(outp.readLine()) - err.add("\n") - echo(err) - result = max(waitForExit(q[r]), result) - if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - r = (r + 1) mod n - else: - var i = m - while i <= high(cmds): - sleep(50) - for r in 0..n-1: + q[i] = startProcess(cmds[i], options = options + {poEvalCommand}) + when defined(windows): + w[i] = q[i].fProcessHandle + inc(i) + + var ecount = len(cmds) + while ecount > 0: + when defined(windows): + # waiting for all children, get result if any child exits + var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32, + INFINITE) + if ret == WAIT_TIMEOUT: + # must not be happen + discard + elif ret == WAIT_FAILED: + raiseOSError(osLastError()) + else: + var status : cint = 1 + # waiting for all children, get result if any child exits + let res = waitpid(-1, status, 0) + if res > 0: + for r in 0..m-1: + if not isNil(q[r]) and q[r].id == res: + # we updating `exitStatus` manually, so `running()` can work. + if WIFEXITED(status) or WIFSIGNALED(status): + q[r].exitStatus = status + break + else: + let err = osLastError() + if err == OSErrorCode(ECHILD): + # some child exits, we need to check our childs exit codes + discard + elif err == OSErrorCode(EINTR): + # signal interrupted our syscall, lets repeat it + continue + else: + # all other errors are exceptions + raiseOSError(err) + + for r in 0..m-1: + if not isNil(q[r]): if not running(q[r]): - #echo(outputStream(q[r]).readLine()) - result = max(waitForExit(q[r]), result) + result = max(result, q[r].peekExitCode()) if afterRunEvent != nil: afterRunEvent(r, q[r]) - if q[r] != nil: close(q[r]) - if beforeRunEvent != nil: - beforeRunEvent(i) - q[r] = startProcess(cmds[i], options=options + {poEvalCommand}) - inc(i) - if i > high(cmds): break - for j in 0..m-1: - result = max(waitForExit(q[j]), result) - if afterRunEvent != nil: afterRunEvent(j, q[j]) - if q[j] != nil: close(q[j]) + close(q[r]) + if i < len(cmds): + if beforeRunEvent != nil: beforeRunEvent(i) + q[r] = startProcess(cmds[i], + options = options + {poEvalCommand}) + when defined(windows): + w[r] = q[r].fProcessHandle + inc(i) + else: + q[r] = nil + when defined(windows): + for c in r..MAXIMUM_WAIT_OBJECTS - 2: + w[c] = w[c + 1] + dec(wcount) + dec(ecount) else: for i in 0..high(cmds): if beforeRunEvent != nil: @@ -321,6 +346,8 @@ when not defined(useNimRtl): elif not running(p): break close(p) +template streamAccess(p) = + assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams" when defined(Windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: @@ -581,12 +608,15 @@ when defined(Windows) and not defined(useNimRtl): return res proc inputStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.inHandle) proc outputStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.outHandle) proc errorStream(p: Process): Stream = + streamAccess(p) result = newFileHandleStream(p.errHandle) proc execCmd(command: string): int = @@ -682,9 +712,7 @@ elif not defined(useNimRtl): sysEnv: cstringArray workingDir: cstring pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] - optionPoUsePath: bool - optionPoParentStreams: bool - optionPoStdErrToStdOut: bool + options: set[ProcessOption] {.deprecated: [TStartProcessData: StartProcessData].} const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and @@ -749,10 +777,8 @@ elif not defined(useNimRtl): data.pStdin = pStdin data.pStdout = pStdout data.pStderr = pStderr - data.optionPoParentStreams = poParentStreams in options - data.optionPoUsePath = poUsePath in options - data.optionPoStdErrToStdOut = poStdErrToStdOut in options data.workingDir = workingDir + data.options = options when useProcessAuxSpawn: var currentDir = getCurrentDir() @@ -801,19 +827,22 @@ elif not defined(useNimRtl): var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) - chck posix_spawnattr_setpgroup(attr, 0'i32) + if poDemon in data.options: + chck posix_spawnattr_setpgroup(attr, 0'i32) - chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or - POSIX_SPAWN_SETSIGMASK or - POSIX_SPAWN_SETPGROUP) + var flags = POSIX_SPAWN_USEVFORK or + POSIX_SPAWN_SETSIGMASK + if poDemon in data.options: + flags = flags or POSIX_SPAWN_SETPGROUP + chck posix_spawnattr_setflags(attr, flags) - if not data.optionPoParentStreams: + if not (poParentStreams in data.options): chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx) chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if (poStdErrToStdOut in data.options): chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) @@ -823,7 +852,7 @@ elif not defined(useNimRtl): setCurrentDir($data.workingDir) var pid: Pid - if data.optionPoUsePath: + if (poUsePath in data.options): res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) else: res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) @@ -885,7 +914,7 @@ elif not defined(useNimRtl): # Warning: no GC here! # Or anything that touches global structures - all called nim procs # must be marked with stackTrace:off. Inspect C code after making changes. - if not data.optionPoParentStreams: + if not (poParentStreams in data.options): discard close(data.pStdin[writeIdx]) if dup2(data.pStdin[readIdx], readIdx) < 0: startProcessFail(data) @@ -893,7 +922,7 @@ elif not defined(useNimRtl): if dup2(data.pStdout[writeIdx], writeIdx) < 0: startProcessFail(data) discard close(data.pStderr[readIdx]) - if data.optionPoStdErrToStdOut: + if (poStdErrToStdOut in data.options): if dup2(data.pStdout[writeIdx], 2) < 0: startProcessFail(data) else: @@ -907,7 +936,7 @@ elif not defined(useNimRtl): discard close(data.pErrorPipe[readIdx]) discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) - if data.optionPoUsePath: + if (poUsePath in data.options): when defined(uClibc) or defined(linux): # uClibc environment (OpenWrt included) doesn't have the full execvpe let exe = findExe(data.sysCommand) @@ -939,19 +968,22 @@ elif not defined(useNimRtl): if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError()) proc running(p: Process): bool = - var ret : int - var status : cint = 1 - ret = waitpid(p.id, status, WNOHANG) - if ret == int(p.id): - if isExitStatus(status): - p.exitStatus = status - return false - else: - return true - elif ret == 0: - return true # Can't establish status. Assume running. - else: + if p.exitStatus != -3: return false + else: + var ret : int + var status : cint = 1 + ret = waitpid(p.id, status, WNOHANG) + if ret == int(p.id): + if isExitStatus(status): + p.exitStatus = status + return false + else: + return true + elif ret == 0: + return true # Can't establish status. Assume running. + else: + raiseOSError(osLastError()) proc terminate(p: Process) = if kill(p.id, SIGTERM) != 0'i32: @@ -1152,16 +1184,19 @@ elif not defined(useNimRtl): stream = newFileStream(f) proc inputStream(p: Process): Stream = + streamAccess(p) if p.inStream == nil: createStream(p.inStream, p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = + streamAccess(p) if p.outStream == nil: createStream(p.outStream, p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = + streamAccess(p) if p.errStream == nil: createStream(p.errStream, p.errHandle, fmRead) return p.errStream diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 00d007d01..6891e2ff7 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -956,6 +956,7 @@ proc parseInsert(p: var SqlParser): SqlNode = if p.tok.kind == tkParLe: var n = newNode(nkColumnList) parseParIdentList(p, n) + result.add n else: result.add(nil) if isKeyw(p, "default"): @@ -1160,7 +1161,7 @@ proc ra(n: SqlNode, s: var string, indent: int) = else: s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") of nkStringLit: - s.add(escape(n.strVal, "e'", "'")) + s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: s.add("b'" & n.strVal & "'") of nkHexStringLit: @@ -1240,7 +1241,7 @@ proc ra(n: SqlNode, s: var string, indent: int) = if n.sons[2].kind == nkDefault: s.add("default values") else: - s.add("\nvalues ") + s.add("\n") ra(n.sons[2], s, indent) s.add(';') of nkUpdate: diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 6eb3e5311..ea90972fe 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -310,6 +310,8 @@ else: include ioselects/ioselectors_select elif defined(solaris): include ioselects/ioselectors_poll # need to replace it with event ports + elif defined(genode): + include ioselects/ioselectors_select # TODO: use the native VFS layer else: include ioselects/ioselectors_poll diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index d773cc7d8..62ceaa2e8 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -32,10 +32,6 @@ when defined(nimOldSplit): else: {.pragma: deprecatedSplit.} -type - CharSet* {.deprecated.} = set[char] # for compatibility with Nim -{.deprecated: [TCharSet: CharSet].} - const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} ## All the characters that count as whitespace. @@ -78,40 +74,40 @@ proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar, return c in Letters proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericChar".}= + rtl, extern: "nsuIsAlphaNumericChar".} = ## Checks whether or not `c` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. - return c in Letters or c in Digits + return c in Letters+Digits proc isDigit*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitChar".}= + rtl, extern: "nsuIsDigitChar".} = ## Checks whether or not `c` is a number. ## ## This checks 0-9 ASCII characters only. return c in Digits proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiChar".}= + rtl, extern: "nsuIsSpaceAsciiChar".} = ## Checks whether or not `c` is a whitespace character. return c in Whitespace proc isLowerAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiChar".}= + rtl, extern: "nsuIsLowerAsciiChar".} = ## Checks whether or not `c` is a lower case character. ## ## This checks ASCII characters only. return c in {'a'..'z'} proc isUpperAscii*(c: char): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiChar".}= + rtl, extern: "nsuIsUpperAsciiChar".} = ## Checks whether or not `c` is an upper case character. ## ## This checks ASCII characters only. return c in {'A'..'Z'} proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaAsciiStr".}= + rtl, extern: "nsuIsAlphaAsciiStr".} = ## Checks whether or not `s` is alphabetical. ## ## This checks a-z, A-Z ASCII characters only. @@ -123,10 +119,10 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar, result = true for c in s: - result = c.isAlphaAscii() and result + if not c.isAlphaAscii(): return false proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsAlphaNumericStr".}= + rtl, extern: "nsuIsAlphaNumericStr".} = ## Checks whether or not `s` is alphanumeric. ## ## This checks a-z, A-Z, 0-9 ASCII characters only. @@ -142,7 +138,7 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar, return false proc isDigit*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsDigitStr".}= + rtl, extern: "nsuIsDigitStr".} = ## Checks whether or not `s` is a numeric value. ## ## This checks 0-9 ASCII characters only. @@ -158,7 +154,7 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar, return false proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsSpaceAsciiStr".}= + rtl, extern: "nsuIsSpaceAsciiStr".} = ## Checks whether or not `s` is completely whitespace. ## ## Returns true if all characters in `s` are whitespace @@ -172,7 +168,7 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar, return false proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsLowerAsciiStr".}= + rtl, extern: "nsuIsLowerAsciiStr".} = ## Checks whether or not `s` contains all lower case characters. ## ## This checks ASCII characters only. @@ -187,7 +183,7 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, true proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, - rtl, extern: "nsuIsUpperAsciiStr".}= + rtl, extern: "nsuIsUpperAsciiStr".} = ## Checks whether or not `s` contains all upper case characters. ## ## This checks ASCII characters only. @@ -506,16 +502,15 @@ template splitCommon(s, sep, maxsplit, sepLen) = var last = 0 var splits = maxsplit - if len(s) > 0: - while last <= len(s): - var first = last - while last < len(s) and not stringHasSep(s, last, sep): - inc(last) - if splits == 0: last = len(s) - yield substr(s, first, last-1) - if splits == 0: break - dec(splits) - inc(last, sepLen) + while last <= len(s): + var first = last + while last < len(s) and not stringHasSep(s, last, sep): + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) template oldSplit(s, seps, maxsplit) = var last = 0 @@ -673,30 +668,29 @@ template rsplitCommon(s, sep, maxsplit, sepLen) = splits = maxsplit startPos = 0 - if len(s) > 0: - # go to -1 in order to get separators at the beginning - while first >= -1: - while first >= 0 and not stringHasSep(s, first, sep): - dec(first) + # go to -1 in order to get separators at the beginning + while first >= -1: + while first >= 0 and not stringHasSep(s, first, sep): + dec(first) - if splits == 0: - # No more splits means set first to the beginning - first = -1 + if splits == 0: + # No more splits means set first to the beginning + first = -1 - if first == -1: - startPos = 0 - else: - startPos = first + sepLen + if first == -1: + startPos = 0 + else: + startPos = first + sepLen - yield substr(s, startPos, last) + yield substr(s, startPos, last) - if splits == 0: - break + if splits == 0: + break - dec(splits) - dec(first) + dec(splits) + dec(first) - last = first + last = first iterator rsplit*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): string = @@ -824,12 +818,18 @@ proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[st noSideEffect, rtl, extern: "nsuSplitCharSet".} = ## The same as the `split iterator <#split.i,string,set[char],int>`_, but is a ## proc that returns a sequence of substrings. + runnableExamples: + doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"] + doAssert "".split({' '}) == @[""] accumulateResult(split(s, seps, maxsplit)) proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect, rtl, extern: "nsuSplitChar".} = ## The same as the `split iterator <#split.i,string,char,int>`_, but is a proc ## that returns a sequence of substrings. + runnableExamples: + doAssert "a,b,c".split(',') == @["a", "b", "c"] + doAssert "".split(' ') == @[""] accumulateResult(split(s, sep, maxsplit)) proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect, @@ -838,6 +838,13 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff ## ## Substrings are separated by the string `sep`. This is a wrapper around the ## `split iterator <#split.i,string,string,int>`_. + runnableExamples: + doAssert "a,b,c".split(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".split("Elon Musk") == @[""] + doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] + + doAssert "a largely spaced sentence".split(" ", maxsplit=1) == @["a", " largely spaced sentence"] doAssert(sep.len > 0) accumulateResult(split(s, sep, maxsplit)) @@ -906,6 +913,13 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string] ## .. code-block:: nim ## @["Root#Object#Method", "Index"] ## + runnableExamples: + doAssert "a largely spaced sentence".rsplit(" ", maxsplit=1) == @["a largely spaced", "sentence"] + + doAssert "a,b,c".rsplit(",") == @["a", "b", "c"] + doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", "plan ", "canal panama"] + doAssert "".rsplit("Elon Musk") == @[""] + doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"] accumulateResult(rsplit(s, sep, maxsplit)) result.reverse() @@ -1305,14 +1319,13 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0) ## This is often useful for generating some code where the items need to ## be *separated* by `sep`. `sep` is only added if `dest` is longer than ## `startLen`. The following example creates a string describing - ## an array of integers: - ## - ## .. code-block:: nim - ## var arr = "[" - ## for x in items([2, 3, 5, 7, 11]): - ## addSep(arr, startLen=len("[")) - ## add(arr, $x) - ## add(arr, "]") + ## an array of integers. + runnableExamples: + var arr = "[" + for x in items([2, 3, 5, 7, 11]): + addSep(arr, startLen=len("[")) + add(arr, $x) + add(arr, "]") if dest.len > startLen: add(dest, sep) proc allCharsInSet*(s: string, theSet: set[char]): bool = @@ -1730,7 +1743,9 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect, ## ## Even though the algorithm works with any string `s`, it is only useful ## if `s` contains a number. - ## Example: ``insertSep("1000000") == "1_000_000"`` + runnableExamples: + doAssert insertSep("1000000") == "1_000_000" + var L = (s.len-1) div digits + s.len result = newString(L) var j = 0 @@ -1818,6 +1833,8 @@ proc validIdentifier*(s: string): bool {.noSideEffect, ## ## A valid identifier starts with a character of the set `IdentStartChars` ## and is followed by any number of characters of the set `IdentChars`. + runnableExamples: + doAssert "abc_def08".validIdentifier if s[0] in IdentStartChars: for i in 1..s.len-1: if s[i] notin IdentChars: return false @@ -1828,7 +1845,7 @@ proc editDistance*(a, b: string): int {.noSideEffect, ## Returns the edit distance between `a` and `b`. ## ## This uses the `Levenshtein`:idx: distance algorithm with only a linear - ## memory overhead. This implementation is highly optimized! + ## memory overhead. var len1 = a.len var len2 = b.len if len1 > len2: @@ -2007,16 +2024,11 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, ## after the decimal point for Nim's ``float`` type. ## ## If ``precision == -1``, it tries to format it nicely. - ## - ## Examples: - ## - ## .. code-block:: nim - ## - ## let x = 123.456 - ## doAssert x.formatFloat() == "123.4560000000000" - ## doAssert x.formatFloat(ffDecimal, 4) == "123.4560" - ## doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" - ## + runnableExamples: + let x = 123.456 + doAssert x.formatFloat() == "123.4560000000000" + doAssert x.formatFloat(ffDecimal, 4) == "123.4560" + doAssert x.formatFloat(ffScientific, 2) == "1.23e+02" result = formatBiggestFloat(f, format, precision, decimalSep) proc trimZeros*(x: var string) {.noSideEffect.} = @@ -2051,18 +2063,13 @@ proc formatSize*(bytes: int64, ## ## `includeSpace` can be set to true to include the (SI preferred) space ## between the number and the unit (e.g. 1 KiB). - ## - ## Examples: - ## - ## .. code-block:: nim - ## - ## formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" - ## formatSize((2.234*1024*1024).int) == "2.234MiB" - ## formatSize(4096, includeSpace=true) == "4 KiB" - ## formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" - ## formatSize(4096) == "4KiB" - ## formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" - ## + runnableExamples: + doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB" + doAssert formatSize((2.234*1024*1024).int) == "2.234MiB" + doAssert formatSize(4096, includeSpace=true) == "4 KiB" + doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB" + doAssert formatSize(4096) == "4KiB" + doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB" const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"] const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"] var @@ -2156,7 +2163,7 @@ proc formatEng*(f: BiggestFloat, ## formatEng(4100, unit="V") == "4.1e3 V" ## formatEng(4100, unit="") == "4.1e3 " # Space with unit="" ## - ## `decimalSep` is used as the decimal separator + ## `decimalSep` is used as the decimal separator. var absolute: BiggestFloat significand: BiggestFloat @@ -2369,17 +2376,16 @@ proc removeSuffix*(s: var string, chars: set[char] = Newlines) {. rtl, extern: "nsuRemoveSuffixCharSet".} = ## Removes all characters from `chars` from the end of the string `s` ## (in-place). - ## - ## .. code-block:: nim - ## var userInput = "Hello World!*~\r\n" - ## userInput.removeSuffix - ## doAssert userInput == "Hello World!*~" - ## userInput.removeSuffix({'~', '*'}) - ## doAssert userInput == "Hello World!" - ## - ## var otherInput = "Hello!?!" - ## otherInput.removeSuffix({'!', '?'}) - ## doAssert otherInput == "Hello" + runnableExamples: + var userInput = "Hello World!*~\r\n" + userInput.removeSuffix + doAssert userInput == "Hello World!*~" + userInput.removeSuffix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "Hello!?!" + otherInput.removeSuffix({'!', '?'}) + doAssert otherInput == "Hello" if s.len == 0: return var last = s.high while last > -1 and s[last] in chars: last -= 1 @@ -2390,24 +2396,23 @@ proc removeSuffix*(s: var string, c: char) {. ## Removes all occurrences of a single character (in-place) from the end ## of a string. ## - ## .. code-block:: nim - ## var table = "users" - ## table.removeSuffix('s') - ## doAssert table == "user" - ## - ## var dots = "Trailing dots......." - ## dots.removeSuffix('.') - ## doAssert dots == "Trailing dots" + runnableExamples: + var table = "users" + table.removeSuffix('s') + doAssert table == "user" + + var dots = "Trailing dots......." + dots.removeSuffix('.') + doAssert dots == "Trailing dots" removeSuffix(s, chars = {c}) proc removeSuffix*(s: var string, suffix: string) {. rtl, extern: "nsuRemoveSuffixString".} = ## Remove the first matching suffix (in-place) from a string. - ## - ## .. code-block:: nim - ## var answers = "yeses" - ## answers.removeSuffix("es") - ## doAssert answers == "yes" + runnableExamples: + var answers = "yeses" + answers.removeSuffix("es") + doAssert answers == "yes" var newLen = s.len if s.endsWith(suffix): newLen -= len(suffix) @@ -2418,16 +2423,16 @@ proc removePrefix*(s: var string, chars: set[char] = Newlines) {. ## Removes all characters from `chars` from the start of the string `s` ## (in-place). ## - ## .. code-block:: nim - ## var userInput = "\r\n*~Hello World!" - ## userInput.removePrefix - ## doAssert userInput == "*~Hello World!" - ## userInput.removePrefix({'~', '*'}) - ## doAssert userInput == "Hello World!" - ## - ## var otherInput = "?!?Hello!?!" - ## otherInput.removePrefix({'!', '?'}) - ## doAssert otherInput == "Hello!?!" + runnableExamples: + var userInput = "\r\n*~Hello World!" + userInput.removePrefix + doAssert userInput == "*~Hello World!" + userInput.removePrefix({'~', '*'}) + doAssert userInput == "Hello World!" + + var otherInput = "?!?Hello!?!" + otherInput.removePrefix({'!', '?'}) + doAssert otherInput == "Hello!?!" var start = 0 while start < s.len and s[start] in chars: start += 1 if start > 0: s.delete(0, start - 1) @@ -2437,20 +2442,20 @@ proc removePrefix*(s: var string, c: char) {. ## Removes all occurrences of a single character (in-place) from the start ## of a string. ## - ## .. code-block:: nim - ## var ident = "pControl" - ## ident.removePrefix('p') - ## doAssert ident == "Control" + runnableExamples: + var ident = "pControl" + ident.removePrefix('p') + doAssert ident == "Control" removePrefix(s, chars = {c}) proc removePrefix*(s: var string, prefix: string) {. rtl, extern: "nsuRemovePrefixString".} = ## Remove the first matching prefix (in-place) from a string. ## - ## .. code-block:: nim - ## var answers = "yesyes" - ## answers.removePrefix("yes") - ## doAssert answers == "yes" + runnableExamples: + var answers = "yesyes" + answers.removePrefix("yes") + doAssert answers == "yes" if s.startsWith(prefix): s.delete(0, prefix.len - 1) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 7d9c3108b..257c620f7 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -293,33 +293,33 @@ proc runeSubStr*(s: string, pos:int, len:int = int.high): string = if pos < 0: let (o, rl) = runeReverseOffset(s, -pos) if len >= rl: - result = s[o.. s.len-1] + result = s.substr(o, s.len-1) elif len < 0: let e = rl + len if e < 0: result = "" else: - result = s[o.. runeOffset(s, e-(rl+pos) , o)-1] + result = s.substr(o, runeOffset(s, e-(rl+pos) , o)-1) else: - result = s[o.. runeOffset(s, len, o)-1] + result = s.substr(o, runeOffset(s, len, o)-1) else: let o = runeOffset(s, pos) if o < 0: result = "" elif len == int.high: - result = s[o.. s.len-1] + result = s.substr(o, s.len-1) elif len < 0: let (e, rl) = runeReverseOffset(s, -len) discard rl if e <= 0: result = "" else: - result = s[o.. e-1] + result = s.substr(o, e-1) else: var e = runeOffset(s, len, o) if e < 0: e = s.len - result = s[o.. e-1] + result = s.substr(o, e-1) const alphaRanges = [ diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 164a57ecf..a651530c3 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -47,6 +47,49 @@ proc add*(url: var Url, a: Url) {.deprecated.} = url = url / a {.pop.} +proc encodeUrl*(s: string): string = + ## Encodes a value to be HTTP safe: This means that characters in the set + ## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result, + ## a space is converted to ``'+'`` and every other character is encoded as + ## ``'%xx'`` where ``xx`` denotes its hexadecimal value. + result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars + for i in 0..s.len-1: + case s[i] + of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i]) + of ' ': add(result, '+') + else: + add(result, '%') + add(result, toHex(ord(s[i]), 2)) + +proc decodeUrl*(s: string): string = + ## Decodes a value from its HTTP representation: This means that a ``'+'`` + ## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal + ## value) is converted to the character with ordinal number ``xx``, and + ## and every other character is carried over. + proc handleHexChar(c: char, x: var int) {.inline.} = + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: assert(false) + + result = newString(s.len) + var i = 0 + var j = 0 + while i < s.len: + case s[i] + of '%': + var x = 0 + handleHexChar(s[i+1], x) + handleHexChar(s[i+2], x) + inc(i, 2) + result[j] = chr(x) + of '+': result[j] = ' ' + else: result[j] = s[i] + inc(i) + inc(j) + setLen(result, j) + proc parseAuthority(authority: string, result: var Uri) = var i = 0 var inPort = false @@ -328,6 +371,11 @@ proc `$`*(u: Uri): string = when isMainModule: block: + const test1 = "abc\L+def xyz" + doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz" + doAssert decodeUrl(encodeUrl(test1)) == test1 + + block: let str = "http://localhost" let test = parseUri(str) doAssert test.path == "" diff --git a/lib/system.nim b/lib/system.nim index 1b53bf9f5..b9f01c306 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1439,7 +1439,11 @@ const ## is the value that should be passed to `quit <#quit>`_ to indicate ## failure. -var programResult* {.exportc: "nim_program_result".}: int +when defined(nodejs): + var programResult* {.importc: "process.exitCode".}: int + programResult = 0 +else: + var programResult* {.exportc: "nim_program_result".}: int ## modify this variable to specify the exit code of the program ## under normal circumstances. When the program is terminated ## prematurely using ``quit``, this value is ignored. @@ -3525,7 +3529,10 @@ when hasAlloc or defined(nimscript): ## .. code-block:: nim ## var s = "abcdef" ## assert s[1..3] == "bcd" - result = s.substr(s ^^ x.a, s ^^ x.b) + let a = s ^^ x.a + let L = (s ^^ x.b) - a + 1 + result = newString(L) + for i in 0 ..< L: result[i] = s[i + a] proc `[]=`*[T, U](s: var string, x: HSlice[T, U], b: string) = ## slice assignment for strings. If @@ -3752,6 +3759,7 @@ template assert*(cond: bool, msg = "") = ## that ``AssertionError`` is hidden from the effect system, so it doesn't ## produce ``{.raises: [AssertionError].}``. This exception is only supposed ## to be caught by unit testing frameworks. + ## ## The compiler may not generate any code at all for ``assert`` if it is ## advised to do so through the ``-d:release`` or ``--assertions:off`` ## `command line switches <nimc.html#command-line-switches>`_. @@ -3992,3 +4000,38 @@ when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage): proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32", importc: "SetConsoleOutputCP".} discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage + + +when defined(nimHasRunnableExamples): + proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".} + ## A section you should use to mark `runnable example`:idx: code with. + ## + ## - In normal debug and release builds code within + ## a ``runnableExamples`` section is ignored. + ## - The documentation generator is aware of these examples and considers them + ## part of the ``##`` doc comment. As the last step of documentation + ## generation the examples are put into an ``$file_example.nim`` file, + ## compiled and tested. The collected examples are + ## put into their own module to ensure the examples do not refer to + ## non-exported symbols. +else: + template runnableExamples*(body: untyped) = + discard + +template doAssertRaises*(exception, code: untyped): typed = + ## Raises ``AssertionError`` if specified ``code`` does not raise the + ## specified exception. + runnableExamples: + doAssertRaises(ValueError): + raise newException(ValueError, "Hello World") + + try: + block: + code + raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code)) + except exception: + discard + except Exception as exc: + raiseAssert(astToStr(exception) & + " wasn't raised, another error was raised instead by:\n"& + astToStr(code)) \ No newline at end of file diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 19d27e7d2..e274e8e0c 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -8,8 +8,6 @@ # # Low level allocator for Nim. Has been designed to support the GC. -# TODO: -# - make searching for block O(1) {.push profiler:off.} include osalloc @@ -19,14 +17,17 @@ template track(op, address, size) = memTrackerOp(op, address, size) # We manage *chunks* of memory. Each chunk is a multiple of the page size. -# Each chunk starts at an address that is divisible by the page size. Chunks -# that are bigger than ``ChunkOsReturn`` are returned back to the operating -# system immediately. +# Each chunk starts at an address that is divisible by the page size. const - ChunkOsReturn = 256 * PageSize # 1 MB - InitialMemoryRequest = ChunkOsReturn div 2 # < ChunkOsReturn! + InitialMemoryRequest = 128 * PageSize # 0.5 MB SmallChunkSize = PageSize + MaxFli = 30 + MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32' + # everywhere! + MaxSli = 1 shl MaxLog2Sli + FliOffset = 6 + RealFli = MaxFli - FliOffset type PTrunk = ptr Trunk @@ -99,10 +100,12 @@ type MemRegion = object minLargeObj, maxLargeObj: int freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk] + flBitmap: uint32 + slBitmap: array[RealFli, uint32] + matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk currMem, maxMem, freeMem: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly - freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages. @@ -110,7 +113,109 @@ type bottomData: AvlNode heapLinks: HeapLinks -{.deprecated: [TMemRegion: MemRegion].} +const + fsLookupTable: array[byte, int8] = [ + -1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7 + ] + +proc msbit(x: uint32): int {.inline.} = + let a = if x <= 0xff_ff: + (if x <= 0xff: 0 else: 8) + else: + (if x <= 0xff_ff_ff: 16 else: 24) + result = int(fsLookupTable[byte(x shr a)]) + a + +proc lsbit(x: uint32): int {.inline.} = + msbit(x and ((not x) + 1)) + +proc setBit(nr: int; dest: var uint32) {.inline.} = + dest = dest or (1u32 shl (nr and 0x1f)) + +proc clearBit(nr: int; dest: var uint32) {.inline.} = + dest = dest and not (1u32 shl (nr and 0x1f)) + +proc mappingSearch(r, fl, sl: var int) {.inline.} = + #let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1 + # This diverges from the standard TLSF algorithm because we need to ensure + # PageSize alignment: + let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1 + r = r + t + fl = msbit(uint32 r) + sl = (r shr (fl - MaxLog2Sli)) - MaxSli + dec fl, FliOffset + r = r and not t + sysAssert((r and PageMask) == 0, "mappingSearch: still not aligned") + +# See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of +# this algorithm. + +proc mappingInsert(r: int): tuple[fl, sl: int] {.inline.} = + sysAssert((r and PageMask) == 0, "mappingInsert: still not aligned") + result.fl = msbit(uint32 r) + result.sl = (r shr (result.fl - MaxLog2Sli)) - MaxSli + dec result.fl, FliOffset + +template mat(): untyped = a.matrix[fl][sl] + +proc findSuitableBlock(a: MemRegion; fl, sl: var int): PBigChunk {.inline.} = + let tmp = a.slBitmap[fl] and (not 0u32 shl sl) + result = nil + if tmp != 0: + sl = lsbit(tmp) + result = mat() + else: + fl = lsbit(a.flBitmap and (not 0u32 shl (fl + 1))) + if fl > 0: + sl = lsbit(a.slBitmap[fl]) + result = mat() + +template clearBits(sl, fl) = + clearBit(sl, a.slBitmap[fl]) + if a.slBitmap[fl] == 0u32: + # do not forget to cascade: + clearBit(fl, a.flBitmap) + +proc removeChunkFromMatrix(a: var MemRegion; b: PBigChunk) = + let (fl, sl) = mappingInsert(b.size) + if b.next != nil: b.next.prev = b.prev + if b.prev != nil: b.prev.next = b.next + if mat() == b: + mat() = b.next + if mat() == nil: + clearBits(sl, fl) + b.prev = nil + b.next = nil + +proc removeChunkFromMatrix2(a: var MemRegion; b: PBigChunk; fl, sl: int) = + mat() = b.next + if mat() != nil: + mat().prev = nil + else: + clearBits(sl, fl) + b.prev = nil + b.next = nil + +proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) = + let (fl, sl) = mappingInsert(b.size) + b.prev = nil + b.next = mat() + if mat() != nil: + mat().prev = b + mat() = b + setBit(sl, a.slBitmap[fl]) + setBit(fl, a.flBitmap) {.push stack_trace: off.} proc initAllocator() = discard "nothing to do anymore" @@ -203,6 +308,7 @@ proc llDeallocAll(a: var MemRegion) = var next = it.next osDeallocPages(it, PageSize) it = next + a.llmem = nil proc intSetGet(t: IntSet, key: int): PTrunk = var it = t.data[key and high(t.data)] @@ -369,6 +475,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = result.prevSize = 0 or (result.prevSize and 1) # unknown # but do not overwrite 'used' field a.lastSize = size # for next request + sysAssert((cast[int](result) and PageMask) == 0, "requestOschunks: unaligned chunk") proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} = result = contains(a.chunkStarts, pageIndex(p)) @@ -419,7 +526,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = if isAccessible(a, ri) and chunkUnused(ri): sysAssert(not isSmallChunk(ri), "freeBigChunk 3") if not isSmallChunk(ri): - listRemove(a.freeChunksList, cast[PBigChunk](ri)) + removeChunkFromMatrix(a, cast[PBigChunk](ri)) inc(c.size, ri.size) excl(a.chunkStarts, pageIndex(ri)) when coalescLeft: @@ -430,49 +537,44 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") if not isSmallChunk(le): - listRemove(a.freeChunksList, cast[PBigChunk](le)) + removeChunkFromMatrix(a, cast[PBigChunk](le)) inc(le.size, c.size) excl(a.chunkStarts, pageIndex(c)) c = cast[PBigChunk](le) incl(a, a.chunkStarts, pageIndex(c)) updatePrevSize(a, c, c.size) - listAdd(a.freeChunksList, c) + addChunkToMatrix(a, c) # set 'used' to false: c.prevSize = c.prevSize and not 1 proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = var rest = cast[PBigChunk](cast[ByteAddress](c) +% size) - sysAssert(rest notin a.freeChunksList, "splitChunk") rest.size = c.size - size track("rest.origSize", addr rest.origSize, sizeof(int)) + # XXX check if these two nil assignments are dead code given + # addChunkToMatrix's implementation: rest.next = nil rest.prev = nil - # size and not used + # size and not used: rest.prevSize = size sysAssert((size and 1) == 0, "splitChunk 2") + sysAssert((size and PageMask) == 0, + "splitChunk: size is not a multiple of the PageSize") updatePrevSize(a, c, rest.size) c.size = size incl(a, a.chunkStarts, pageIndex(rest)) - listAdd(a.freeChunksList, rest) + addChunkToMatrix(a, rest) proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # use first fit for now: - sysAssert((size and PageMask) == 0, "getBigChunk 1") sysAssert(size > 0, "getBigChunk 2") - result = a.freeChunksList - block search: - while result != nil: - sysAssert chunkUnused(result), "getBigChunk 3" - if result.size == size: - listRemove(a.freeChunksList, result) - break search - elif result.size > size: - listRemove(a.freeChunksList, result) - splitChunk(a, result, size) - break search - result = result.next - sysAssert result != a.freeChunksList, "getBigChunk 4" + var size = size # roundup(size, PageSize) + var fl, sl: int + mappingSearch(size, fl, sl) + sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") + result = findSuitableBlock(a, fl, sl) + if result == nil: if size < InitialMemoryRequest: result = requestOsChunks(a, InitialMemoryRequest) splitChunk(a, result, size) @@ -481,7 +583,10 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # if we over allocated split the chunk: if result.size > size: splitChunk(a, result, size) - + else: + removeChunkFromMatrix2(a, result, fl, sl) + if result.size >= size + PageSize: + splitChunk(a, result, size) # set 'used' to to true: result.prevSize = 1 track("setUsedToFalse", addr result.origSize, sizeof(int)) @@ -572,14 +677,14 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") else: - size = roundup(requestedSize+bigChunkOverhead(), PageSize) + size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block var c = getBigChunk(a, size) sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" - sysAssert c.size == size, "rawAlloc 12" result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") if a.root == nil: a.root = getBottom(a) add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) sysAssert(isAccessible(a, result), "rawAlloc 14") diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 824934966..9ac039e19 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -42,7 +42,8 @@ type # Page size of the system; in most cases 4096 bytes. For exotic OS or # CPU this needs to be changed: const - PageShift = when defined(cpu16): 8 else: 12 + PageShift = when defined(cpu16): 8 else: 12 # \ + # my tests showed no improvments for using larger page sizes. PageSize = 1 shl PageShift PageMask = PageSize-1 @@ -343,7 +344,6 @@ elif defined(gogc): const goFlagNoZero: uint32 = 1 shl 3 proc goRuntimeMallocGC(size: uint, typ: uint, flag: uint32): pointer {.importc: "runtime_mallocgc", dynlib: goLib.} - proc goFree(v: pointer) {.importc: "__go_free", dynlib: goLib.} proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.} @@ -376,7 +376,6 @@ elif defined(gogc): result = goRuntimeMallocGC(roundup(newsize, sizeof(pointer)).uint, 0.uint, goFlagNoZero) copyMem(result, old, oldsize) zeroMem(cast[pointer](cast[ByteAddress](result) +% oldsize), newsize - oldsize) - goFree(old) proc nimGCref(p: pointer) {.compilerproc, inline.} = discard proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard @@ -573,3 +572,11 @@ when not declared(nimNewSeqOfCap): cast[PGenericSeq](result).reserved = cap {.pop.} + +when not declared(ForeignCell): + type ForeignCell* = object + data*: pointer + + proc protect*(x: pointer): ForeignCell = ForeignCell(data: x) + proc dispose*(x: ForeignCell) = discard + proc isNotForeign*(x: ForeignCell): bool = false diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 65a057772..444f11306 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -166,7 +166,7 @@ elif defined(windows): # space heavily, so we now treat Windows as a strange unmap target. when reallyOsDealloc: if virtualFree(p, 0, MEM_RELEASE) == 0: - cprintf "yes, failing!" + cprintf "virtualFree failing!" quit 1 #VirtualFree(p, size, MEM_DECOMMIT) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 3f8e0eff0..56b8ade97 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -259,7 +259,7 @@ proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} = result.reserved = r proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. - compilerRtl.} = + compilerRtl, inl.} = result = seq if result.space < newLen: let r = max(resize(result.space), newLen) @@ -282,10 +282,11 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) gch.tempStack.len = len0 else: - for i in newLen..result.len-1: - forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% - GenericSeqSize +% (i*%elemSize)), - extGetCellType(result).base, waZctDecRef) + if ntfNoRefs notin extGetCellType(result).base.flags: + for i in newLen..result.len-1: + forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +% + GenericSeqSize +% (i*%elemSize)), + extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 016bf5822..f61cc4280 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -255,9 +255,9 @@ when emulatedThreadVars: proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".} # we preallocate a fixed size for thread local storage, so that no heap -# allocations are needed. Currently less than 7K are used on a 64bit machine. +# allocations are needed. Currently less than 16K are used on a 64bit machine. # We use ``float`` for proper alignment: -const nimTlsSize {.intdefine.} = 8000 +const nimTlsSize {.intdefine.} = 16000 type ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float] diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index c3229cc7b..7eb268a9a 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -541,6 +541,7 @@ var SO_DONTLINGER* {.importc, header: "winsock2.h".}: cint SO_EXCLUSIVEADDRUSE* {.importc, header: "winsock2.h".}: cint # disallow local address reuse SO_ERROR* {.importc, header: "winsock2.h".}: cint + TCP_NODELAY* {.importc, header: "winsock2.h".}: cint proc `==`*(x, y: SocketHandle): bool {.borrow.} |