From c6a9a9c30b2b758f3f5da582b2f1a8c4b0d7d503 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 22 May 2018 23:07:02 +0200 Subject: hotfix: fixes SSL initialization for httpclient with --threads:on (#7863) * hotfix: fixes SSL initialization for httpclient with --threads:on --- lib/pure/httpclient.nim | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 3f1a3d067..8530e4c42 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -362,15 +362,13 @@ when not defined(ssl): type SSLContext = ref object var defaultSSLContext {.threadvar.}: SSLContext -when defined(ssl): - defaultSSLContext = newContext(verifyMode = CVerifyNone) - template contextOrDefault(ctx: SSLContext): SSLContext = - var result = ctx - if ctx == nil: - if defaultSSLContext == nil: - defaultSSLContext = newContext(verifyMode = CVerifyNone) +proc getDefaultSSL(): SSLContext = + result = defaultSslContext + when defined(ssl): + if result == nil: + defaultSSLContext = newContext(verifyMode = CVerifyNone) result = defaultSSLContext - result + doAssert result != nil, "failure to initialize the SSL context" proc newProxy*(url: string, auth = ""): Proxy = ## Constructs a new ``TProxy`` object. @@ -480,9 +478,9 @@ proc format(p: MultipartData): tuple[contentType, body: string] = result.body.add("--" & bound & "--\c\L") proc request*(url: string, httpMethod: string, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response - {.deprecated.} = + {.deprecated: "use HttpClient.request instead".} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` @@ -580,7 +578,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", result = parseResponse(s, httpMethod != "HEAD", timeout) proc request*(url: string, httpMethod = HttpGET, extraHeaders = "", - body = "", sslContext = defaultSSLContext, timeout = -1, + body = "", sslContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. @@ -611,7 +609,7 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = result = $parsed proc get*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object @@ -632,7 +630,7 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, lastURL = redirectTo proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. @@ -651,7 +649,7 @@ proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, proc post*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): Response {.deprecated.} = @@ -694,7 +692,7 @@ proc post*(url: string, extraHeaders = "", body = "", proc postContent*(url: string, extraHeaders = "", body = "", maxRedirects = 5, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, multipart: MultipartData = nil): string @@ -717,7 +715,7 @@ proc postContent*(url: string, extraHeaders = "", body = "", return r.body proc downloadFile*(url: string, outputFilename: string, - sslContext: SSLContext = defaultSSLContext, + sslContext: SSLContext = getDefaultSSL(), timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil) {.deprecated.} = ## | Downloads ``url`` and saves it to ``outputFilename`` @@ -817,7 +815,7 @@ type HttpClient* = HttpClientBase[Socket] proc newHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, proxy: Proxy = nil, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil, timeout = -1): HttpClient = ## Creates a new HttpClient instance. ## @@ -844,7 +842,7 @@ proc newHttpClient*(userAgent = defUserAgent, result.bodyStream = newStringStream() result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext type AsyncHttpClient* = HttpClientBase[AsyncSocket] @@ -852,7 +850,7 @@ type {.deprecated: [PAsyncHttpClient: AsyncHttpClient].} proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext, + maxRedirects = 5, sslContext = getDefaultSSL(), proxy: Proxy = nil): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## @@ -876,7 +874,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.bodyStream = newFutureStream[string]("newAsyncHttpClient") result.getBody = true when defined(ssl): - result.sslContext = contextOrDefault(sslContext) + result.sslContext = sslContext proc close*(client: HttpClient | AsyncHttpClient) = ## Closes any connections held by the HTTP client. -- cgit 1.4.1-2-gfad0 From 799674168d969afa73a507d59de7e34b360f1047 Mon Sep 17 00:00:00 2001 From: Tyler Date: Tue, 22 May 2018 19:27:28 -0700 Subject: Patch the PEG library to work with the JS backend (#7866) * Added compiler check on transformFile --- lib/pure/pegs.nim | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 830429842..e13bc8329 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1010,14 +1010,18 @@ proc replace*(s: string, sub: Peg, cb: proc( inc(m) add(result, substr(s, i)) -proc transformFile*(infile, outfile: string, - subs: varargs[tuple[pattern: Peg, repl: string]]) {. - rtl, extern: "npegs$1".} = - ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an - ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string - writeFile(outfile, x.parallelReplace(subs)) +when not defined(js): + proc transformFile*(infile, outfile: string, + subs: varargs[tuple[pattern: Peg, repl: string]]) {. + rtl, extern: "npegs$1".} = + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## error occurs. This is supposed to be used for quick scripting. + ## + ## **Note**: this proc does not exist while using the JS backend. + var x = readFile(infile).string + writeFile(outfile, x.parallelReplace(subs)) + iterator split*(s: string, sep: Peg): string = ## Splits the string `s` into substrings. -- cgit 1.4.1-2-gfad0 From 39a1ad650cb221ece78980fa1b782638fdc84df5 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 08:42:22 +0300 Subject: Faster binarySearch --- lib/pure/algorithm.nim | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 169dcd602..daec452a0 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -86,6 +86,9 @@ proc binarySearch*[T, K](a: openArray[T], key: K, var step = len shr 1 while step > 0: let i = result or step + if cmp(a[i], key) == 0: + return i + if cmp(a[i], key) < 1: result = i step = step shr 1 @@ -94,6 +97,9 @@ proc binarySearch*[T, K](a: openArray[T], key: K, var b = len while result < b: var mid = (result + b) shr 1 + if cmp(a[mid], key) == 0: + return mid + if cmp(a[mid], key) < 0: result = mid + 1 else: -- cgit 1.4.1-2-gfad0 From 6acbe6fb019b149d73bc702335f2768619ac1ac3 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 09:45:51 +0300 Subject: Fixes pegs bugs --- lib/pure/pegs.nim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index e13bc8329..23ed855a4 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1125,7 +1125,7 @@ proc handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 - if L.buf[result] == '\L': inc(result) + if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result proc handleLF(L: var PegLexer, pos: int): int = @@ -1221,7 +1221,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = proc skip(c: var PegLexer) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: case buf[pos] of ' ', '\t': inc(pos) @@ -1242,7 +1242,7 @@ proc getString(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 var buf = c.buf var quote = buf[pos-1] - while true: + while pos < c.buf.len: case buf[pos] of '\\': c.bufpos = pos @@ -1265,7 +1265,7 @@ proc getDollar(c: var PegLexer, tok: var Token) = if buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 - while buf[pos] in {'0'..'9'}: + while pos < c.buf.len and buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') inc(pos) else: @@ -1281,7 +1281,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = if buf[pos] == '^': inc(pos) caret = true - while true: + while pos < c.buf.len: var ch: char case buf[pos] of ']': @@ -1300,7 +1300,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = inc(pos) incl(tok.charset, ch) if buf[pos] == '-': - if buf[pos+1] == ']': + if pos+1 < c.buf.len and buf[pos+1] == ']': incl(tok.charset, '-') inc(pos) else: @@ -1326,10 +1326,10 @@ proc getCharSet(c: var PegLexer, tok: var Token) = proc getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos var buf = c.buf - while true: + while pos < c.buf.len: add(tok.literal, buf[pos]) inc(pos) - if buf[pos] notin strutils.IdentChars: break + if pos < buf.len and buf[pos] notin strutils.IdentChars: break c.bufpos = pos tok.kind = tkIdentifier @@ -1451,8 +1451,8 @@ proc getTok(c: var PegLexer, tok: var Token) = proc arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos - while c.buf[pos] in {'\t', ' '}: inc(pos) - result = c.buf[pos] == '<' and c.buf[pos+1] == '-' + while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) + result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-' # ----------------------------- parser ---------------------------------------- @@ -1475,7 +1475,7 @@ proc pegError(p: PegParser, msg: string, line = -1, col = -1) = proc getTok(p: var PegParser) = getTok(p, p.tok) - if p.tok.kind == tkInvalid: pegError(p, "invalid token") + if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") proc eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) -- cgit 1.4.1-2-gfad0 From df37796d88d024e7511ec134dc5405f9d65cd3e1 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 10:23:19 +0300 Subject: Fixes pegs bugs, continue --- lib/pure/pegs.nim | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 23ed855a4..39c5790ed 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1226,7 +1226,8 @@ proc skip(c: var PegLexer) = of ' ', '\t': inc(pos) of '#': - while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) + while (pos < c.buf.len) and + not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) of '\c': pos = handleCR(c, pos) buf = c.buf @@ -1285,7 +1286,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = var ch: char case buf[pos] of ']': - inc(pos) + if pos < c.buf.len: inc(pos) break of '\\': c.bufpos = pos @@ -1304,7 +1305,10 @@ proc getCharSet(c: var PegLexer, tok: var Token) = incl(tok.charset, '-') inc(pos) else: - inc(pos) + if pos+1 < c.buf.len: + inc(pos) + else: + break var ch2: char case buf[pos] of '\\': @@ -1316,8 +1320,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid break else: - ch2 = buf[pos] - inc(pos) + if pos+1 < c.buf.len: + ch2 = buf[pos] + inc(pos) + else: + break for i in ord(ch)+1 .. ord(ch2): incl(tok.charset, chr(i)) c.bufpos = pos @@ -1334,7 +1341,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = tok.kind = tkIdentifier proc getBuiltin(c: var PegLexer, tok: var Token) = - if c.buf[c.bufpos+1] in strutils.Letters: + if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) tok.kind = tkBuiltin @@ -1347,10 +1354,12 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.modifier = modNone setLen(tok.literal, 0) skip(c) + case c.buf[c.bufpos] of '{': inc(c.bufpos) - if c.buf[c.bufpos] == '@' and c.buf[c.bufpos+1] == '}': + if c.buf[c.bufpos] == '@' and c.bufpos+2 < c.buf.len and + c.buf[c.bufpos+1] == '}': tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") @@ -1383,13 +1392,11 @@ proc getTok(c: var PegLexer, tok: var Token) = getBuiltin(c, tok) of '\'', '"': getString(c, tok) of '$': getDollar(c, tok) - of '\0': - tok.kind = tkEof - tok.literal = "[EOF]" of 'a'..'z', 'A'..'Z', '\128'..'\255': getSymbol(c, tok) if c.buf[c.bufpos] in {'\'', '"'} or - c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and + c.buf[c.bufpos+1] in {'0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1410,7 +1417,7 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '+') of '<': - if c.buf[c.bufpos+1] == '-': + if c.bufpos+2 < c.buf.len and c.buf[c.bufpos+1] == '-': inc(c.bufpos, 2) tok.kind = tkArrow add(tok.literal, "<-") @@ -1445,6 +1452,9 @@ proc getTok(c: var PegLexer, tok: var Token) = inc(c.bufpos) add(tok.literal, '^') else: + if c.bufpos >= c.buf.len: + tok.kind = tkEof + tok.literal = "[EOF]" add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -- cgit 1.4.1-2-gfad0 From a093605ab06ef851bf177fc444711db8435d7c8f Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 12:40:55 +0300 Subject: binarySearch became even better --- lib/pure/algorithm.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index daec452a0..35315947d 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -84,27 +84,31 @@ proc binarySearch*[T, K](a: openArray[T], key: K, if (len and (len - 1)) == 0: # when `len` is a power of 2, a faster shr can be used. var step = len shr 1 + var cmpRes: int while step > 0: let i = result or step - if cmp(a[i], key) == 0: + cmpRes = cmp(a[i], key) + if cmpRes == 0: return i - if cmp(a[i], key) < 1: + if cmpRes < 1: result = i step = step shr 1 - if cmp(a[result], key) != 0: result = -1 + if cmpRes != 0: result = -1 else: var b = len + var cmpRes: int while result < b: var mid = (result + b) shr 1 - if cmp(a[mid], key) == 0: + cmpRes = cmp(a[mid], key) + if cmpRes == 0: return mid - if cmp(a[mid], key) < 0: + if cmpRes < 0: result = mid + 1 else: b = mid - if result >= len or cmp(a[result], key) != 0: result = -1 + if result >= len or cmpRes != 0: result = -1 proc binarySearch*[T](a: openArray[T], key: T): int = ## binary search for `key` in `a`. Returns -1 if not found. -- cgit 1.4.1-2-gfad0 From 17b8bb8b47ef77877c5dac17abaf9998a12eea67 Mon Sep 17 00:00:00 2001 From: data-man Date: Wed, 23 May 2018 12:58:26 +0300 Subject: Fixes binarySearch's bug --- lib/pure/algorithm.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 35315947d..98330b680 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -94,7 +94,7 @@ proc binarySearch*[T, K](a: openArray[T], key: K, if cmpRes < 1: result = i step = step shr 1 - if cmpRes != 0: result = -1 + if cmp(a[result], key) != 0: result = -1 else: var b = len var cmpRes: int @@ -108,7 +108,7 @@ proc binarySearch*[T, K](a: openArray[T], key: K, result = mid + 1 else: b = mid - if result >= len or cmpRes != 0: result = -1 + if result >= len or cmp(a[result], key) != 0: result = -1 proc binarySearch*[T](a: openArray[T], key: T): int = ## binary search for `key` in `a`. Returns -1 if not found. -- cgit 1.4.1-2-gfad0 From 85b7d8fcc4032a0d95af3f25faf1cc850f7d59f5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 23 May 2018 15:28:53 +0100 Subject: Rstgen/xml tree fixes (#7823) * Don't prefix lang names with "lang" in rstgen. * Implements ability to render xmltree w/o \n. Fixes <> for `data-*` attrs. * Various rstgen fixes. * Fixes security vulnerabilities due to not escaping some code. * Adds