diff options
author | Johannes Hofmann <johannes.hofmann@gmx.de> | 2016-09-30 08:37:09 +0200 |
---|---|---|
committer | Johannes Hofmann <johannes.hofmann@gmx.de> | 2016-09-30 08:37:09 +0200 |
commit | 72ee6f78c5ae1d2eff406d3b39fd6820887e22ad (patch) | |
tree | 88a38cc7d0d7bd022006d1edc73820f6d41f755e | |
parent | 1dccbaf9a0cc0256701cfbae5bbb17b4d4e9416a (diff) | |
parent | c035fd93f465ec2bc47645b4b838bec946ee88f0 (diff) | |
download | Nim-72ee6f78c5ae1d2eff406d3b39fd6820887e22ad.tar.gz |
Merge branch 'devel' into unify_waitpid_handling
42 files changed, 989 insertions, 834 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cf30eb89..54b40dcd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,19 +18,6 @@ stages: tags: - windows -build-linux: - stage: build - script: - - sh ci/build.sh - artifacts: - paths: - - bin/nim - - bin/nimd - - compiler/nim - - koch - expire_in: 1 week - tags: - - linux build-windows: stage: build @@ -58,15 +45,7 @@ deploy-windows: - windows - fast -test-linux: - stage: test - <<: *linux_set_path_def - script: - - sh ci/deps.sh - - nim c --taintMode:on tests/testament/tester - - tests/testament/tester --pedantic all - tags: - - linux + test-windows: stage: test @@ -79,18 +58,3 @@ test-windows: - windows - fast -.csources: &csources_definition - stage: test - script: - - apt-get update -qq - - apt-get install -y -qq build-essential git - - nim -v - - ./koch csources - - ./koch xz - artifacts: - paths: - - build/c_code - -csources-linux: - <<: *csources_definition - <<: *linux_set_path_def diff --git a/.travis.yml b/.travis.yml index 0ebeeb995..ebf287502 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ script: - nimble install zip - nimble install opengl - nimble install sdl1 - - nimble install jester + - nimble install jester@#head - nimble install niminst - nim c --taintMode:on tests/testament/tester - tests/testament/tester --pedantic all diff --git a/build_tools.sh b/build_tools.sh new file mode 100644 index 000000000..4d867b0ad --- /dev/null +++ b/build_tools.sh @@ -0,0 +1,3 @@ +./bin/nim c --noNimblePath -p:compiler -o:./bin/nimble dist/nimble/src/nimble.nim +./bin/nim c --noNimblePath -p:compiler -o:./bin/nimsuggest dist/nimsuggest/nimsuggest.nim +./bin/nim c -o:./bin/nimgrep tools/nimgrep.nim diff --git a/ci/deps.bat b/ci/deps.bat index 477c70d1d..bda1fe14f 100644 --- a/ci/deps.bat +++ b/ci/deps.bat @@ -1,4 +1,4 @@ nim e install_nimble.nims nim e tests/test_nimscript.nims nimble update -nimble install -y zip opengl sdl1 jester niminst +nimble install -y zip opengl sdl1 jester@#head niminst diff --git a/ci/deps.sh b/ci/deps.sh index 50680e604..3385a213b 100644 --- a/ci/deps.sh +++ b/ci/deps.sh @@ -14,4 +14,4 @@ export PATH=$(pwd)/bin:$PATH nim e install_nimble.nims nim e tests/test_nimscript.nims nimble update -nimble install zip opengl sdl1 jester niminst +nimble install zip opengl sdl1 jester@#head niminst diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat index 338a975d7..f870e8e88 100644 --- a/ci/nsis_build.bat +++ b/ci/nsis_build.bat @@ -29,7 +29,6 @@ rem move /y build\nim-%NIMVER%.zip web\upload\download rem Grab C sources and nimsuggest git clone --depth 1 https://github.com/nim-lang/csources.git -git clone --depth 1 https://github.com/nim-lang/nimsuggest.git set PATH=%CD%\bin;%PATH% @@ -39,13 +38,11 @@ set PATH=C:\Users\araq\projects\mingw32\bin;%PATH% cd csources call build.bat cd .. -nim c koch || exit /b -koch boot -d:release || exit /b -cd nimsuggest -nim c -d:release --noNimblePath --path:.. nimsuggest || exit /b -copy /y nimsuggest.exe ..\bin || exit /b -cd .. -koch nsis -d:release || exit /b +ReM Rebuilding koch is necessary because it uses its pointer size to determine +ReM which mingw link to put in the NSIS installer. +nim c --out:koch_temp koch || exit /b +koch_temp boot -d:release || exit /b +koch_temp nsis -d:release || exit /b dir build move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b @@ -55,11 +52,7 @@ set PATH=C:\Users\araq\projects\mingw64\bin;%PATH% cd csources call build64.bat cd .. -nim c koch || exit /b -koch boot -d:release || exit /b -cd nimsuggest -nim c -d:release --noNimblePath --path:.. nimsuggest || exit /b -copy /y nimsuggest.exe ..\bin || exit /b -cd .. -koch nsis -d:release || exit /b +nim c --out:koch_temp koch || exit /b +koch_temp boot -d:release || exit /b +koch_temp nsis -d:release || exit /b move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b diff --git a/compiler/ast.nim b/compiler/ast.nim index 131c2a38f..d8939fc60 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1408,6 +1408,7 @@ proc copyNode*(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -1426,6 +1427,7 @@ proc shallowCopy*(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -1445,6 +1447,7 @@ proc copyTree*(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id diff --git a/compiler/installer.ini b/compiler/installer.ini index 4fc03dd1d..cb7b86427 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -50,6 +50,7 @@ Files: "readme.txt;install.txt;contributors.txt;copying.txt" Files: "makefile" Files: "koch.nim" Files: "install_nimble.nims" +Files: "install_tools.nims" Files: "icons/nim.ico" Files: "icons/nim.rc" @@ -77,7 +78,8 @@ Files: "lib" [Other] Files: "examples" -#Files: "dist/nimble" +Files: "dist/nimble" +Files: "dist/nimsuggest" Files: "tests" @@ -98,8 +100,8 @@ BinPath: r"bin;dist\mingw\bin;dist" ; Section | dir | zipFile | size hint (in KB) | url | exe start menu entry Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html" Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip" -Download: r"Support DLL's|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip" -Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" +Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip" +Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" ; for now only NSIS supports optional downloads [UnixBin] diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 97c3b6ddd..ebcff643f 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -155,7 +155,10 @@ proc discardCheck(c: PContext, result: PNode) = else: var n = result while n.kind in skipForDiscardable: n = n.lastSon - localError(n.info, errDiscardValueX, result.typ.typeToString) + if result.typ.kind == tyProc: + localError(n.info, "value of type '" & result.typ.typeToString & "' has to be discarded; for a function call use ()") + else: + localError(n.info, errDiscardValueX, result.typ.typeToString) proc semIf(c: PContext, n: PNode): PNode = result = n diff --git a/compiler/vm.nim b/compiler/vm.nim index 2e2a49db5..efcc55c59 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -171,6 +171,7 @@ proc copyValue(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -1071,7 +1072,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcRepr: decodeB(rkNode) createStr regs[ra] - regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments}) + regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments, renderDocComments}) of opcQuit: if c.mode in {emRepl, emStaticExpr, emStaticStmt}: message(c.debug[pc], hintQuitCalled) diff --git a/config/nim.cfg b/config/nim.cfg index 0373de135..8c8270f3e 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -66,7 +66,7 @@ path="$lib/pure" @end @if unix: - @if not bsd: + @if not bsd or haiku: # -fopenmp gcc.options.linker = "-ldl" gcc.cpp.options.linker = "-ldl" @@ -80,6 +80,14 @@ path="$lib/pure" # at least NetBSD has problems with thread local storage: tlsEmulation:on @end + @if haiku: + # -fopenmp + gcc.options.linker = "-lroot -lnetwork" + gcc.cpp.options.linker = "-lroot -lnetwork" + clang.options.linker = "-lroot -lnetwork" + clang.cpp.options.linker = "-lroot -lnetwork" + tcc.options.linker = "-lroot -lnetwork" + @end @end # Configuration for the Intel C/C++ compiler: diff --git a/install.txt b/install.txt index 67be2221b..833fbf0fb 100644 --- a/install.txt +++ b/install.txt @@ -30,6 +30,15 @@ There are also ``install.sh`` and ``deinstall.sh`` scripts for distributing the files over the UNIX hierarchy. However, updating your Nim installation is more cumbersome then. +To complete the installation you should also build Nim's tools like +``nimsuggest``, ``nimble`` or ``nimgrep`` via:: + + nim e install_tools.nims + +Note that these tools should also end up in your ``PATH`` so adding +``$your_install_dir/bin/nim`` to your ``PATH`` is preferred over the symlink +solution. + Installation on the Macintosh ----------------------------- diff --git a/install_tools.nims b/install_tools.nims new file mode 100644 index 000000000..6ecc545a5 --- /dev/null +++ b/install_tools.nims @@ -0,0 +1,15 @@ + +import ospaths + +mode = ScriptMode.Verbose + +let nimbleExe = "./bin/nimble".toExe +selfExec "c --noNimblePath -p:compiler -o:" & nimbleExe & + " dist/nimble/src/nimble.nim" + +let nimsugExe = "./bin/nimsuggest".toExe +selfExec "c --noNimblePath -p:compiler -o:" & nimsugExe & + " dist/nimsuggest/nimsuggest.nim" + +let nimgrepExe = "./bin/nimgrep".toExe +selfExec "c -o:./bin/nimgrep tools/nimgrep.nim" diff --git a/koch.nim b/koch.nim index fab0e67cb..1e78abbca 100644 --- a/koch.nim +++ b/koch.nim @@ -123,6 +123,8 @@ proc testUnixInstall() = execCleanPath("./koch boot -d:release", destDir / "bin") # check the docs build: execCleanPath("./koch web", destDir / "bin") + # check nimble builds: + execCleanPath("./bin/nim e install_tools.nims") # check the tests work: execCleanPath("./koch tests", destDir / "bin") else: @@ -149,7 +151,7 @@ proc csource(args: string) = exec("$4 cc $1 -r $3 --var:version=$2 --var:mingw=none csource --main:compiler/nim.nim compiler/installer.ini $1" % [args, VersionAsString, compileNimInst, findNim()]) -proc bundleNimble() = +proc bundleNimbleSrc() = if dirExists("dist/nimble/.git"): exec("git --git-dir dist/nimble/.git pull") else: @@ -157,20 +159,33 @@ proc bundleNimble() = let tags = execProcess("git --git-dir dist/nimble/.git tag -l v*").splitLines let tag = tags[^1] exec("git --git-dir dist/nimble/.git checkout " & tag) + +proc bundleNimbleExe() = + bundleNimbleSrc() # now compile Nimble and copy it to $nim/bin for the installer.ini # to pick it up: exec(findNim() & " c dist/nimble/src/nimble.nim") copyExe("dist/nimble/src/nimble".exe, "bin/nimble".exe) +proc bundleNimsuggest(buildExe: bool) = + if dirExists("dist/nimsuggest/.git"): + exec("git --git-dir dist/nimsuggest/.git pull") + else: + exec("git clone https://github.com/nim-lang/nimsuggest.git dist/nimsuggest") + if buildExe: + exec(findNim() & " c --noNimblePath -p:compiler dist/nimsuggest/nimsuggest.nim") + copyExe("dist/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) + proc zip(args: string) = - bundleNimble() + bundleNimbleSrc() exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" % ["tools/niminst/niminst".exe, VersionAsString]) proc xz(args: string) = - bundleNimble() + bundleNimbleSrc() + bundleNimsuggest(false) exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" % @@ -181,7 +196,8 @@ proc buildTool(toolname, args: string) = copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe) proc nsis(args: string) = - bundleNimble() + bundleNimbleExe() + bundleNimsuggest(true) # make sure we have generated the niminst executables: buildTool("tools/niminst/niminst", args) #buildTool("tools/nimgrep", args) diff --git a/lib/arch/arch.nim b/lib/arch/arch.nim index a11bfb21f..c8ae3da1c 100644 --- a/lib/arch/arch.nim +++ b/lib/arch/arch.nim @@ -6,6 +6,9 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. # +# Architecture-specific optimizations and features. +# arch.nim can be imported by only a subset of the +# architectures supported by Nim. when defined(windows): const diff --git a/lib/impure/re.nim b/lib/impure/re.nim index bf397550a..bd86bcdcf 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -359,7 +359,7 @@ proc parallelReplace*(s: string, subs: openArray[ proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## `parallelReplace`) and writes back to `outfile`. Raises ``IOError`` if an ## error occurs. This is supposed to be used for quick scripting. var x = readFile(infile).string writeFile(outfile, x.parallelReplace(subs)) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index f9085de55..8336da1fb 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -156,290 +156,10 @@ export Port, SocketFlag ## * Can't await in a ``except`` body ## * Forward declarations for async procs are broken, ## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. -## * FutureVar[T] needs to be completed manually. # TODO: Check if yielded future is nil and throw a more meaningful exception -# -- Futures - -type - FutureBase* = ref object of RootObj ## Untyped future. - cb: proc () {.closure,gcsafe.} - finished: bool - error*: ref Exception ## Stored exception - errorStackTrace*: string - when not defined(release): - stackTrace: string ## For debugging purposes only. - id: int - fromProc: string - - Future*[T] = ref object of FutureBase ## Typed future. - value: T ## Stored value - - FutureVar*[T] = distinct Future[T] - - FutureError* = object of Exception - cause*: FutureBase - -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - -when not defined(release): - var currentID = 0 - -proc callSoon*(cbproc: proc ()) {.gcsafe.} - -proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = - ## Creates a new future. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - new(result) - result.finished = false - when not defined(release): - result.stackTrace = getStackTrace() - result.id = currentID - result.fromProc = fromProc - currentID.inc() - -proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = - ## Create a new ``FutureVar``. This Future type is ideally suited for - ## situations where you want to avoid unnecessary allocations of Futures. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - result = FutureVar[T](newFuture[T](fromProc)) - -proc clean*[T](future: FutureVar[T]) = - ## Resets the ``finished`` status of ``future``. - Future[T](future).finished = false - Future[T](future).error = nil - -proc checkFinished[T](future: Future[T]) = - ## Checks whether `future` is finished. If it is then raises a - ## ``FutureError``. - when not defined(release): - if future.finished: - var msg = "" - msg.add("An attempt was made to complete a Future more than once. ") - msg.add("Details:") - msg.add("\n Future ID: " & $future.id) - msg.add("\n Created in proc: " & future.fromProc) - msg.add("\n Stack trace to moment of creation:") - msg.add("\n" & indent(future.stackTrace.strip(), 4)) - when T is string: - msg.add("\n Contents (string): ") - msg.add("\n" & indent(future.value.repr, 4)) - msg.add("\n Stack trace to moment of secondary completion:") - msg.add("\n" & indent(getStackTrace().strip(), 4)) - var err = newException(FutureError, msg) - err.cause = future - raise err - -proc complete*[T](future: Future[T], val: T) = - ## Completes ``future`` with value ``val``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.value = val - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*(future: Future[void]) = - ## Completes a void ``future``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*[T](future: FutureVar[T]) = - ## Completes a ``FutureVar``. - template fut: expr = Future[T](future) - checkFinished(fut) - assert(fut.error == nil) - fut.finished = true - if fut.cb != nil: - fut.cb() - -proc fail*[T](future: Future[T], error: ref Exception) = - ## Completes ``future`` with ``error``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - future.finished = true - future.error = error - future.errorStackTrace = - if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) - if future.cb != nil: - future.cb() - else: - # This is to prevent exceptions from being silently ignored when a future - # is discarded. - # TODO: This may turn out to be a bad idea. - # Turns out this is a bad idea. - #raise error - discard - -proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - ## - ## **Note**: You most likely want the other ``callback`` setter which - ## passes ``future`` as a param to the callback. - future.cb = cb - if future.finished: - callSoon(future.cb) - -proc `callback=`*[T](future: Future[T], - cb: proc (future: Future[T]) {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - future.callback = proc () = cb(future) - -proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. - when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") - - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) - else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) - -proc read*[T](future: Future[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished: - if future.error != nil: - injectStacktrace(future) - raise future.error - when T isnot void: - return future.value - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - -proc readError*[T](future: Future[T]): ref Exception = - ## Retrieves the exception stored in ``future``. - ## - ## An ``ValueError`` exception will be thrown if no exception exists - ## in the specified Future. - if future.error != nil: return future.error - else: - raise newException(ValueError, "No error in future.") - -proc mget*[T](future: FutureVar[T]): var T = - ## Returns a mutable value stored in ``future``. - ## - ## Unlike ``read``, this function will not raise an exception if the - ## Future has not been finished. - result = Future[T](future).value - -proc finished*[T](future: Future[T]): bool = - ## Determines whether ``future`` has completed. - ## - ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - future.finished - -proc failed*(future: FutureBase): bool = - ## Determines whether ``future`` completed with an error. - return future.error != nil - -proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future - ## finished with an error. - ## - ## This should be used instead of ``discard`` to discard void futures. - future.callback = - proc () = - if future.failed: - injectStacktrace(future) - raise future.error - -proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once both ``fut1`` and ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`and`") - fut1.callback = - proc () = - if not retFuture.finished: - if fut1.failed: retFuture.fail(fut1.error) - elif fut2.finished: retFuture.complete() - fut2.callback = - proc () = - if not retFuture.finished: - if fut2.failed: retFuture.fail(fut2.error) - elif fut1.finished: retFuture.complete() - return retFuture - -proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once either ``fut1`` or ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`or`") - proc cb[X](fut: Future[X]) = - if fut.failed: retFuture.fail(fut.error) - if not retFuture.finished: retFuture.complete() - fut1.callback = cb[T] - fut2.callback = cb[Y] - return retFuture - -proc all*[T](futs: varargs[Future[T]]): auto = - ## Returns a future which will complete once - ## all futures in ``futs`` complete. - ## - ## If the awaited futures are not ``Future[void]``, the returned future - ## will hold the values of all awaited futures in a sequence. - ## - ## If the awaited futures *are* ``Future[void]``, - ## this proc returns ``Future[void]``. - - when T is void: - var - retFuture = newFuture[void]("asyncdispatch.all") - completedFutures = 0 - - let totalFutures = len(futs) - - for fut in futs: - fut.callback = proc(f: Future[T]) = - if f.failed: - retFuture.fail(f.error) - elif not retFuture.finished: - inc(completedFutures) - - if completedFutures == totalFutures: - retFuture.complete() - - return retFuture - - else: - var - retFuture = newFuture[seq[T]]("asyncdispatch.all") - retValues = newSeq[T](len(futs)) - completedFutures = 0 - - for i, fut in futs: - proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = - if f.failed: - retFuture.fail(f.error) - elif not retFuture.finished: - retValues[i] = f.read() - inc(completedFutures) - - if completedFutures == len(retValues): - retFuture.complete(retValues) - - setCallback(i) - - return retFuture +include includes/asyncfutures type PDispatcherBase = ref object of RootRef @@ -1646,7 +1366,7 @@ proc send*(socket: AsyncFD, data: string, # -- Await Macro include asyncmacro -proc recvLine*(socket: AsyncFD): Future[string] {.async.} = +proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} = ## Reads a line of data from ``socket``. Returned future will complete once ## a full line is read or an error occurs. ## @@ -1664,6 +1384,8 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## ## **Note**: This procedure is mostly used for testing. You likely want to ## use ``asyncnet.recvLine`` instead. + ## + ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead. template addNLIfEmpty(): stmt = if result.len == 0: diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index e417e0b6c..019a18f55 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -360,6 +360,14 @@ proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} = assertReply(await ftp.send("RNFR " & name_from), "350") assertReply(await ftp.send("RNTO " & name_to), "250") +proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} = + ## Delete a file ``filename`` on the remote FTP server + assertReply(await ftp.send("DELE " & filename), "250") + +proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} = + ## Delete a directory ``dir`` on the remote FTP server + assertReply(await ftp.send("RMD " & dir), "250") + proc newAsyncFtpClient*(address: string, port = Port(21), user, pass = ""): AsyncFtpClient = ## Creates a new ``AsyncFtpClient`` object. @@ -380,6 +388,10 @@ when not defined(testing) and isMainModule: await ftp.store("payload.jpg", "payload.jpg") await ftp.retrFile("payload.jpg", "payload2.jpg") await ftp.rename("payload.jpg", "payload_renamed.jpg") + await ftp.store("payload.jpg", "payload_remove.jpg") + await ftp.removeFile("payload_remove.jpg") + await ftp.createDir("deleteme") + await ftp.removeDir("deleteme") echo("Finished") waitFor main(ftp) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 6a7326e83..a658097f9 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -9,6 +9,12 @@ ## This module implements a high performance asynchronous HTTP server. ## +## This HTTP server has not been designed to be used in production, but +## for testing applications locally. Because of this, when deploying your +## application you should use a reverse proxy (for example nginx) instead of +## allowing users to connect directly to this server. +## +## ## Examples ## -------- ## @@ -38,7 +44,7 @@ export httpcore except parseHeader type Request* = object client*: AsyncSocket # TODO: Separate this into a Response object? - reqMethod*: string + reqMethod*: HttpMethod headers*: HttpHeaders protocol*: tuple[orig: string, major, minor: int] url*: Uri @@ -127,7 +133,14 @@ proc processClient(client: AsyncSocket, address: string, var i = 0 for linePart in lineFut.mget.split(' '): case i - of 0: request.reqMethod.shallowCopy(linePart.normalize) + of 0: + try: + # TODO: this is likely slow. + request.reqMethod = parseEnum[HttpMethod]("http" & linePart) + except ValueError: + asyncCheck request.respond(Http400, "Invalid request method. Got: " & + linePart) + continue of 1: parseUri(linePart, request.url) of 2: try: @@ -159,7 +172,7 @@ proc processClient(client: AsyncSocket, address: string, request.client.close() return - if request.reqMethod == "post": + if request.reqMethod == HttpPost: # Check for Expect header if request.headers.hasKey("Expect"): if "100-continue" in request.headers["Expect"]: @@ -178,17 +191,12 @@ proc processClient(client: AsyncSocket, address: string, else: request.body = await client.recv(contentLength) assert request.body.len == contentLength - elif request.reqMethod == "post": + elif request.reqMethod == HttpPost: await request.respond(Http400, "Bad Request. No Content-Length.") continue - case request.reqMethod - of "get", "post", "head", "put", "delete", "trace", "options", - "connect", "patch": - await callback(request) - else: - await request.respond(Http400, "Invalid request method. Got: " & - request.reqMethod) + # Call the user's callback. + await callback(request) if "upgrade" in request.headers.getOrDefault("connection"): return diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index f70afaafa..3d004e84c 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -25,7 +25,7 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} = result = node[0] template createCb(retFutureSym, iteratorNameSym, - name: untyped) = + name, futureVarCompletions: untyped) = var nameIterVar = iteratorNameSym #{.push stackTrace: off.} proc cb {.closure,gcsafe.} = @@ -44,6 +44,8 @@ template createCb(retFutureSym, iteratorNameSym, raise else: retFutureSym.fail(getCurrentException()) + + futureVarCompletions cb() #{.pop.} proc generateExceptionCheck(futSym, @@ -119,8 +121,22 @@ template createVar(result: var NimNode, futSymName: string, result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y useVar(result, futSym, valueReceiver, rootReceiver, fromNode) +proc createFutureVarCompletions(futureVarIdents: seq[NimNode]): NimNode + {.compileTime.} = + result = newStmtList() + # Add calls to complete each FutureVar parameter. + for ident in futureVarIdents: + # Only complete them if they have not been completed already by the user. + result.add newIfStmt( + ( + newCall(newIdentNode("not"), + newDotExpr(ident, newIdentNode("finished"))), + newCall(newIdentNode("complete"), ident) + ) + ) + proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, + subTypeIsVoid: bool, futureVarIdents: seq[NimNode], tryStmt: NimNode): NimNode {.compileTime.} = #echo(node.treeRepr) result = node @@ -134,11 +150,14 @@ proc processBody(node, retFutureSym: NimNode, else: result.add newCall(newIdentNode("complete"), retFutureSym) else: - let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt) + let x = node[0].processBody(retFutureSym, subTypeIsVoid, + futureVarIdents, tryStmt) if x.kind == nnkYieldStmt: result.add x else: result.add newCall(newIdentNode("complete"), retFutureSym, x) + result.add createFutureVarCompletions(futureVarIdents) + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) return # Don't process the children of this return stmt of nnkCommand, nnkCall: @@ -196,7 +215,8 @@ proc processBody(node, retFutureSym: NimNode, # Transform ``except`` body. # TODO: Could we perform some ``await`` transformation here to get it # working in ``except``? - tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, + futureVarIdents, nil) proc processForTry(n: NimNode, i: var int, res: NimNode): bool {.compileTime.} = @@ -207,7 +227,7 @@ proc processBody(node, retFutureSym: NimNode, var skipped = n.skipStmtList() while i < skipped.len: var processed = processBody(skipped[i], retFutureSym, - subTypeIsVoid, n) + subTypeIsVoid, futureVarIdents, n) # Check if we transformed the node into an exception check. # This suggests skipped[i] contains ``await``. @@ -239,7 +259,8 @@ proc processBody(node, retFutureSym: NimNode, else: discard for i in 0 .. <result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) + result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, + futureVarIdents, nil) proc getName(node: NimNode): string {.compileTime.} = case node.kind @@ -252,6 +273,14 @@ proc getName(node: NimNode): string {.compileTime.} = else: error("Unknown name.") +proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = + result = @[] + for i in 1 .. <len(params): + expectKind(params[i], nnkIdentDefs) + if params[i][1].kind == nnkBracketExpr and + ($params[i][1][0].ident).normalize == "futurevar": + result.add(params[i][0]) + proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = ## This macro transforms a single procedure into a closure iterator. ## The ``async`` macro supports a stmtList holding multiple async procedures. @@ -282,6 +311,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = let subtypeIsVoid = returnType.kind == nnkEmpty or (baseType.kind == nnkIdent and returnType[1].ident == !"void") + let futureVarIdents = getFutureVarIdents(prc[3]) + var outerProcBody = newNimNode(nnkStmtList, prc[6]) # -> var retFuture = newFuture[T]() @@ -304,7 +335,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> <proc_body> # -> complete(retFuture, result) var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") - var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, + futureVarIdents, nil) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: if not subtypeIsVoid: @@ -326,6 +358,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> complete(retFuture) procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + procBody.add(createFutureVarCompletions(futureVarIdents)) + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], procBody, nnkIteratorDef) closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) @@ -334,7 +368,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # -> createCb(retFuture) #var cbName = newIdentNode("cb") var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) + newStrLitNode(prc[0].getName), + createFutureVarCompletions(futureVarIdents)) outerProcBody.add procCb # -> return retFuture diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 334f95baa..3b64c278f 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -159,7 +159,9 @@ when defineSsl: await socket.fd.AsyncFd.send(data, flags) proc appeaseSsl(socket: AsyncSocket, flags: set[SocketFlag], - sslError: cint) {.async.} = + sslError: cint): Future[bool] {.async.} = + ## Returns ``true`` if ``socket`` is still connected, otherwise ``false``. + result = true case sslError of SSL_ERROR_WANT_WRITE: await sendPendingSslData(socket, flags) @@ -173,6 +175,7 @@ when defineSsl: elif length == 0: # connection not properly closed by remote side or connection dropped SSL_set_shutdown(socket.sslHandle, SSL_RECEIVED_SHUTDOWN) + result = false else: raiseSSLError("Cannot appease SSL.") @@ -180,13 +183,27 @@ when defineSsl: op: expr) = var opResult {.inject.} = -1.cint while opResult < 0: + # Call the desired operation. opResult = op # Bit hackish here. # TODO: Introduce an async template transformation pragma? + + # Send any remaining pending SSL data. yield sendPendingSslData(socket, flags) + + # If the operation failed, try to see if SSL has some data to read + # or write. if opResult < 0: let err = getSslError(socket.sslHandle, opResult.cint) - yield appeaseSsl(socket, flags, err.cint) + let fut = appeaseSsl(socket, flags, err.cint) + yield fut + if not fut.read(): + # Socket disconnected. + if SocketFlag.SafeDisconn in flags: + break + else: + raiseSSLError("Socket has been disconnected") + proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## Connects ``socket`` to server at ``address:port``. @@ -388,7 +405,7 @@ proc accept*(socket: AsyncSocket, return retFut proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], - flags = {SocketFlag.SafeDisconn}) {.async.} = + flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {.async.} = ## Reads a line of data from ``socket`` into ``resString``. ## ## If a full line is read ``\r\L`` is not @@ -401,13 +418,14 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], ## is read) then line will be set to ``""``. ## The partial line **will be lost**. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: The ``Peek`` flag is not yet implemented. ## ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the ## protocol uses ``\r\L`` to delimit a new line. - ## - ## **Warning**: ``recvLineInto`` currently uses a raw pointer to a string for - ## performance reasons. This will likely change soon to use FutureVars. assert SocketFlag.Peek notin flags ## TODO: assert(not resString.mget.isNil(), "String inside resString future needs to be initialised") @@ -454,6 +472,12 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], else: resString.mget.add socket.buffer[socket.currPos] socket.currPos.inc() + + # Verify that this isn't a DOS attack: #3847. + if resString.mget.len > maxLength: + let msg = "recvLine received more than the specified `maxLength` " & + "allowed." + raise newException(ValueError, msg) else: var c = "" while true: @@ -475,10 +499,17 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], resString.complete() return resString.mget.add c + + # Verify that this isn't a DOS attack: #3847. + if resString.mget.len > maxLength: + let msg = "recvLine received more than the specified `maxLength` " & + "allowed." + raise newException(ValueError, msg) resString.complete() proc recvLine*(socket: AsyncSocket, - flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = + flags = {SocketFlag.SafeDisconn}, + maxLength = MaxLineLength): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once ## a full line is read or an error occurs. ## @@ -492,6 +523,10 @@ proc recvLine*(socket: AsyncSocket, ## is read) then line will be set to ``""``. ## The partial line **will be lost**. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: The ``Peek`` flag is not yet implemented. ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol @@ -501,7 +536,7 @@ proc recvLine*(socket: AsyncSocket, # TODO: Optimise this var resString = newFutureVar[string]("asyncnet.recvLine") resString.mget() = "" - await socket.recvLineInto(resString, flags) + await socket.recvLineInto(resString, flags, maxLength) result = resString.mget() proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = diff --git a/lib/pure/basic2d.nim b/lib/pure/basic2d.nim index 7d74424fa..e4696c6a8 100644 --- a/lib/pure/basic2d.nim +++ b/lib/pure/basic2d.nim @@ -117,13 +117,13 @@ proc safeArccos(v:float):float= template makeBinOpVector(s:expr)= - ## implements binary operators + , - , * and / for vectors + ## implements binary operators ``+``, ``-``, ``*`` and ``/`` for vectors proc s*(a,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a.x,b.x),s(a.y,b.y)) proc s*(a:Vector2d,b:float):Vector2d {.inline,noInit.} = vector2d(s(a.x,b),s(a.y,b)) proc s*(a:float,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a,b.x),s(a,b.y)) template makeBinOpAssignVector(s:expr)= - ## implements inplace binary operators += , -= , /= and *= for vectors + ## implements inplace binary operators ``+=``, ``-=``, ``/=`` and ``*=`` for vectors proc s*(a:var Vector2d,b:Vector2d) {.inline.} = s(a.x,b.x) ; s(a.y,b.y) proc s*(a:var Vector2d,b:float) {.inline.} = s(a.x,b) ; s(a.y,b) diff --git a/lib/pure/basic3d.nim b/lib/pure/basic3d.nim index 424c191f8..f7a9c237c 100644 --- a/lib/pure/basic3d.nim +++ b/lib/pure/basic3d.nim @@ -117,7 +117,6 @@ proc safeArccos(v:float):float= return arccos(clamp(v,-1.0,1.0)) template makeBinOpVector(s:expr)= - ## implements binary operators + , - , * and / for vectors proc s*(a,b:Vector3d):Vector3d {.inline,noInit.} = vector3d(s(a.x,b.x),s(a.y,b.y),s(a.z,b.z)) proc s*(a:Vector3d,b:float):Vector3d {.inline,noInit.} = @@ -126,11 +125,10 @@ template makeBinOpVector(s:expr)= vector3d(s(a,b.x),s(a,b.y),s(a,b.z)) template makeBinOpAssignVector(s:expr)= - ## implements inplace binary operators += , -= , /= and *= for vectors proc s*(a:var Vector3d,b:Vector3d) {.inline.} = - s(a.x,b.x) ; s(a.y,b.y) ; s(a.z,b.z) + s(a.x,b.x); s(a.y,b.y); s(a.z,b.z) proc s*(a:var Vector3d,b:float) {.inline.} = - s(a.x,b) ; s(a.y,b) ; s(a.z,b) + s(a.x,b); s(a.y,b); s(a.z,b) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 27b3b46be..4404a9426 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -8,79 +8,102 @@ # ## This module implements a simple HTTP client that can be used to retrieve -## webpages/other data. -## -## -## **Note**: This module is not ideal, connection is not kept alive so sites with -## many redirects are expensive. As such in the future this module may change, -## and the current procedures will be deprecated. +## webpages and other data. ## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve -## ``http://google.com`` +## ``http://google.com``: ## ## .. code-block:: Nim +## var client = newHttpClient() ## echo(getContent("http://google.com")) ## +## The same action can also be performed asynchronously, simply use the +## ``AsyncHttpClient``: +## +## .. code-block:: Nim +## var client = newAsyncHttpClient() +## echo(await getContent("http://google.com")) +## +## The functionality implemented by ``HttpClient`` and ``AsyncHttpClient`` +## is the same, so you can use whichever one suits you best in the examples +## shown here. +## +## **Note:** You will need to run asynchronous examples in an async proc +## otherwise you will get an ``Undeclared identifier: 'await'`` error. +## ## Using HTTP POST ## =============== ## ## This example demonstrates the usage of the W3 HTML Validator, it -## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to -## the server. +## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to be +## validated to the server. ## ## .. code-block:: Nim +## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") ## -## echo postContent("http://validator.w3.org/check", multipart=data) +## echo client.postContent("http://validator.w3.org/check", multipart=data) ## -## Asynchronous HTTP requests -## ========================== +## Progress reporting +## ================== ## -## You simply have to create a new instance of the ``AsyncHttpClient`` object. -## You may then use ``await`` on the functions defined for that object. -## Keep in mind that the following code needs to be inside an asynchronous -## procedure. -## -## .. code-block::nim +## You may specify a callback procedure to be called during an HTTP request. +## This callback will be executed every second with information about the +## progress of the HTTP request. ## +## .. code-block:: Nim ## var client = newAsyncHttpClient() -## var resp = await client.request("http://google.com") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") +## client.onProgressChanged = onProgressChanged +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## If you would like to remove the callback simply set it to ``nil``. +## +## .. code-block:: Nim +## client.onProgressChanged = nil ## ## SSL/TLS support ## =============== ## This requires the OpenSSL library, fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the ``https`` schema, for example: -## ``https://github.com/``, you also have to compile with ``ssl`` defined like so: +## ``https://github.com/``. +## +## You will also have to compile with ``ssl`` defined like so: ## ``nim c -d:ssl ...``. ## ## Timeouts ## ======== -## Currently all functions support an optional timeout, by default the timeout is set to -## `-1` which means that the function will never time out. The timeout is +## +## Currently only the synchronous functions support a timeout. +## The timeout is ## measured in milliseconds, once it is set any call on a socket which may -## block will be susceptible to this timeout, however please remember that the +## block will be susceptible to this timeout. +## +## It may be surprising but the ## function as a whole can take longer than the specified timeout, only ## individual internal calls on the socket are affected. In practice this means ## that as long as the server is sending data an exception will not be raised, -## if however data does not reach client within the specified timeout an ETimeout -## exception will then be raised. +## if however data does not reach the client within the specified timeout a +## ``TimeoutError`` exception will be raised. ## ## Proxy ## ===== ## -## A proxy can be specified as a param to any of these procedures, the ``newProxy`` -## constructor should be used for this purpose. However, -## currently only basic authentication is supported. +## A proxy can be specified as a param to any of the procedures defined in +## this module. To do this, use the ``newProxy`` constructor. Unfortunately, +## only basic authentication is supported at the moment. import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, - math, random, httpcore + math, random, httpcore, times import asyncnet, asyncdispatch import nativesockets @@ -379,15 +402,18 @@ proc format(p: MultipartData): tuple[header, body: string] = proc request*(url: string, httpMethod: string, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = + userAgent = defUserAgent, proxy: Proxy = nil): Response + {.deprecated.} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. var r = if proxy == nil: parseUri(url) else: proxy.url var hostUrl = if proxy == nil: r else: parseUri(url) - var headers = substr(httpMethod, len("http")).toUpper() + var headers = httpMethod.toUpper() # TODO: Use generateHeaders further down once it supports proxies. var s = newSocket() @@ -471,15 +497,18 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", if body != "": s.send(body) - result = parseResponse(s, httpMethod != "httpHEAD", timeout) + result = parseResponse(s, httpMethod != "HEAD", timeout) proc request*(url: string, httpMethod = httpGET, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = + userAgent = defUserAgent, proxy: Proxy = nil): Response + {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout, userAgent, proxy) @@ -502,12 +531,14 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = proc get*(url: string, extraHeaders = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response = + proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object ## | This proc also handles redirection ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. result = request(url, httpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) var lastURL = url @@ -521,12 +552,14 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): string = + proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.getContent`` instead. var r = get(url, extraHeaders, maxRedirects, sslContext, timeout, userAgent, proxy) if r.status[0] in {'4','5'}: @@ -539,7 +572,7 @@ proc post*(url: string, extraHeaders = "", body = "", sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, - multipart: MultipartData = nil): Response = + multipart: MultipartData = nil): Response {.deprecated.} = ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object. ## | This proc adds the necessary Content-Length header. ## | This proc also handles redirection. @@ -548,6 +581,8 @@ proc post*(url: string, extraHeaders = "", body = "", ## server takes longer than specified an ETimeout exception will be raised. ## | The optional ``multipart`` parameter can be used to create ## ``multipart/form-data`` POSTs comfortably. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.post`` instead. let (mpHeaders, mpBody) = format(multipart) template withNewLine(x): expr = @@ -577,7 +612,8 @@ proc postContent*(url: string, extraHeaders = "", body = "", sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, - multipart: MultipartData = nil): string = + multipart: MultipartData = nil): string + {.deprecated.} = ## | POSTs ``body`` to ``url`` and returns the response's body as a string ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. @@ -585,6 +621,9 @@ proc postContent*(url: string, extraHeaders = "", body = "", ## server takes longer than specified an ETimeout exception will be raised. ## | The optional ``multipart`` parameter can be used to create ## ``multipart/form-data`` POSTs comfortably. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.postContent`` + ## instead. var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout, userAgent, proxy, multipart) if r.status[0] in {'4','5'}: @@ -610,7 +649,7 @@ proc downloadFile*(url: string, outputFilename: string, proc generateHeaders(requestUrl: Uri, httpMethod: string, headers: HttpHeaders, body: string, proxy: Proxy): string = # GET - result = substr(httpMethod, len("http")).toUpper() + result = httpMethod.toUpper() result.add ' ' if proxy.isNil: @@ -653,17 +692,30 @@ proc generateHeaders(requestUrl: Uri, httpMethod: string, add(result, "\c\L") type + ProgressChangedProc*[ReturnType] = + proc (total, progress, speed: BiggestInt): + ReturnType {.closure, gcsafe.} + HttpClientBase*[SocketType] = ref object socket: SocketType connected: bool currentURL: Uri ## Where we are currently connected. - headers*: HttpHeaders + headers*: HttpHeaders ## Headers to send in requests. maxRedirects: int userAgent: string timeout: int ## Only used for blocking HttpClient for now. proxy: Proxy + ## ``nil`` or the callback to call when request progress changes. + when SocketType is Socket: + onProgressChanged*: ProgressChangedProc[void] + else: + onProgressChanged*: ProgressChangedProc[Future[void]] when defined(ssl): sslContext: net.SslContext + contentTotal: BiggestInt + contentProgress: BiggestInt + oneSecondProgress: BiggestInt + lastProgressReport: float type HttpClient* = HttpClientBase[Socket] @@ -692,6 +744,7 @@ proc newHttpClient*(userAgent = defUserAgent, result.maxRedirects = maxRedirects result.proxy = proxy result.timeout = timeout + result.onProgressChanged = nil when defined(ssl): result.sslContext = sslContext @@ -721,6 +774,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.maxRedirects = maxRedirects result.proxy = proxy result.timeout = -1 # TODO + result.onProgressChanged = nil when defined(ssl): result.sslContext = sslContext @@ -730,19 +784,37 @@ proc close*(client: HttpClient | AsyncHttpClient) = client.socket.close() client.connected = false -proc recvFull(socket: Socket | AsyncSocket, +proc reportProgress(client: HttpClient | AsyncHttpClient, + progress: BiggestInt) {.multisync.} = + client.contentProgress += progress + client.oneSecondProgress += progress + if epochTime() - client.lastProgressReport >= 1.0: + if not client.onProgressChanged.isNil: + await client.onProgressChanged(client.contentTotal, + client.contentProgress, + client.oneSecondProgress) + client.oneSecondProgress = 0 + client.lastProgressReport = epochTime() + +proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int): Future[string] {.multisync.} = ## Ensures that all the data requested is read and returned. result = "" while true: if size == result.len: break - when socket is Socket: - let data = socket.recv(size - result.len, timeout) + + let remainingSize = size - result.len + let sizeToRecv = min(remainingSize, net.BufferSize) + + when client.socket is Socket: + let data = client.socket.recv(sizeToRecv, timeout) else: - let data = await socket.recv(size - result.len) + let data = await client.socket.recv(sizeToRecv) if data == "": break # We've been disconnected. result.add data + await reportProgress(client, data.len) + proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string] {.multisync.} = result = "" @@ -770,10 +842,10 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string] httpError("Invalid chunk size: " & chunkSizeStr) inc(i) if chunkSize <= 0: - discard await recvFull(client.socket, 2, client.timeout) # Skip \c\L + discard await recvFull(client, 2, client.timeout) # Skip \c\L break - result.add await recvFull(client.socket, chunkSize, client.timeout) - discard await recvFull(client.socket, 2, client.timeout) # Skip \c\L + result.add await recvFull(client, chunkSize, client.timeout) + discard await recvFull(client, 2, client.timeout) # Skip \c\L # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 @@ -781,6 +853,12 @@ proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, httpVersion: string): Future[string] {.multisync.} = result = "" + # Reset progress from previous requests. + client.contentTotal = 0 + client.contentProgress = 0 + client.oneSecondProgress = 0 + client.lastProgressReport = 0 + if headers.getOrDefault"Transfer-Encoding" == "chunked": result = await parseChunks(client) else: @@ -789,8 +867,9 @@ proc parseBody(client: HttpClient | AsyncHttpClient, var contentLengthHeader = headers.getOrDefault"Content-Length" if contentLengthHeader != "": var length = contentLengthHeader.parseint() + client.contentTotal = length if length > 0: - result = await client.socket.recvFull(length, client.timeout) + result = await client.recvFull(length, client.timeout) if result == "": httpError("Got disconnected while trying to read body.") if result.len != length: @@ -804,7 +883,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient, if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: - buf = await client.socket.recvFull(4000, client.timeout) + buf = await client.recvFull(4000, client.timeout) if buf == "": break result.add(buf) @@ -902,8 +981,6 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## Connection will kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. - ## - ## The returned future will complete once the request is completed. let connectionUrl = if client.proxy.isNil: parseUri(url) else: client.proxy.url let requestUrl = parseUri(url) @@ -933,14 +1010,15 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, if not client.headers.hasKey("user-agent") and client.userAgent != "": client.headers["User-Agent"] = client.userAgent - var headers = generateHeaders(requestUrl, $httpMethod, + var headers = generateHeaders(requestUrl, httpMethod, client.headers, body, client.proxy) await client.socket.send(headers) if body != "": await client.socket.send(body) - result = await parseResponse(client, httpMethod notin {HttpHead, HttpConnect}) + result = await parseResponse(client, + httpMethod.toLower() notin ["head", "connect"]) # Restore the clients proxy in case it was overwritten. client.proxy = savedProxy @@ -950,11 +1028,12 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## Connects to the hostname specified by the URL and performs a request ## using the method specified. ## - ## Connection will kept alive. Further requests on the same ``client`` to + ## Connection will be kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. ## - ## The returned future will complete once the request is completed. + ## When a request is made to a different hostname, the current connection will + ## be closed. result = await request(client, url, $httpMethod, body) proc get*(client: HttpClient | AsyncHttpClient, @@ -964,6 +1043,8 @@ proc get*(client: HttpClient | AsyncHttpClient, ## This procedure will follow redirects up to a maximum number of redirects ## specified in ``client.maxRedirects``. result = await client.request(url, HttpGET) + + # Handle redirects. var lastURL = url for i in 1..client.maxRedirects: if result.status.redirection(): @@ -971,6 +1052,21 @@ proc get*(client: HttpClient | AsyncHttpClient, result = await client.request(redirectTo, HttpGET) lastURL = redirectTo +proc getContent*(client: HttpClient | AsyncHttpClient, + url: string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a GET request. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in ``client.maxRedirects``. + ## + ## A ``HttpRequestError`` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + let resp = await get(client, url) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + else: + return resp.body + proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a POST request. @@ -990,3 +1086,28 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", client.headers["Content-Length"] = $len(xb) result = await client.request(url, HttpPOST, xb) + # Handle redirects. + var lastURL = url + for i in 1..client.maxRedirects: + if result.status.redirection(): + let redirectTo = getNewLocation(lastURL, result.headers) + var meth = if result.status != "307": HttpGet else: HttpPost + result = await client.request(redirectTo, meth, xb) + lastURL = redirectTo + +proc postContent*(client: HttpClient | AsyncHttpClient, url: string, + body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a POST request. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in ``client.maxRedirects``. + ## + ## A ``HttpRequestError`` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + let resp = await post(client, url, body, multipart) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + else: + return resp.body diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 0515eeecd..8147f1c50 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -43,9 +43,8 @@ type ## for specified address. HttpConnect, ## Converts the request connection to a transparent ## TCP/IP tunnel, usually used for proxies. - HttpPatch ## Added in RFC 5789. Can be used to update partial - ## resources. The set of changes is represented in a - ## format called a "patch document". + HttpPatch ## Applies partial modifications to a resource. + {.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost, httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace, httpOptions: HttpOptions, httpConnect: HttpConnect].} @@ -226,7 +225,7 @@ proc `$`*(code: HttpCode): string = ## For example: ## ## .. code-block:: nim - ## doAssert(Http404.status == "404 Not Found") + ## doAssert($Http404 == "404 Not Found") case code.int of 100: "100 Continue" of 101: "101 Switching Protocols" @@ -298,6 +297,9 @@ proc is5xx*(code: HttpCode): bool = ## Determines whether ``code`` is a 5xx HTTP status code. return code.int in {500 .. 599} +proc `$`*(httpMethod: HttpMethod): string = + return (system.`$`(httpMethod))[4 .. ^1].toUpper() + when isMainModule: var test = newHttpHeaders() test["Connection"] = @["Upgrade", "Close"] diff --git a/lib/pure/includes/asyncfutures.nim b/lib/pure/includes/asyncfutures.nim new file mode 100644 index 000000000..fda78c1a1 --- /dev/null +++ b/lib/pure/includes/asyncfutures.nim @@ -0,0 +1,293 @@ + +# TODO: This shouldn't need to be included, but should ideally be exported. +type + FutureBase* = ref object of RootObj ## Untyped future. + cb: proc () {.closure,gcsafe.} + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Exception + cause*: FutureBase + +{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} + +when not defined(release): + var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + +proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = + ## Create a new ``FutureVar``. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + result = FutureVar[T](newFuture[T](fromProc)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the ``finished`` status of ``future``. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## ``FutureError``. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(future.stackTrace.strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent(future.value.repr, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc complete*[T](future: Future[T], val: T) = + ## Completes ``future`` with value ``val``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: Future[void]) = + ## Completes a void ``future``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*[T](future: FutureVar[T]) = + ## Completes a ``FutureVar``. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + if fut.cb != nil: + fut.cb() + +proc complete*[T](future: FutureVar[T], val: T) = + ## Completes a ``FutureVar`` with value ``val``. + ## + ## Any previously stored value will be overwritten. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + fut.value = val + if fut.cb != nil: + fut.cb() + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes ``future`` with ``error``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + if future.cb != nil: + future.cb() + else: + # This is to prevent exceptions from being silently ignored when a future + # is discarded. + # TODO: This may turn out to be a bad idea. + # Turns out this is a bad idea. + #raise error + discard + +proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + callSoon(future.cb) + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc injectStacktrace[T](future: Future[T]) = + # TODO: Come up with something better. + when not defined(release): + var msg = "" + msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + + if not future.errorStackTrace.isNil and future.errorStackTrace != "": + msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) + else: + msg.add("\n Empty or nil stack trace.") + future.error.msg.add(msg) + +proc read*[T](future: Future[T] | FutureVar[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + let fut = Future[T](future) + if fut.finished: + if fut.error != nil: + injectStacktrace(fut) + raise fut.error + when T isnot void: + return fut.value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in ``future``. + ## + ## An ``ValueError`` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in ``future``. + ## + ## Unlike ``read``, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*[T](future: Future[T] | FutureVar[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + (Future[T](future)).finished + +proc failed*(future: FutureBase): bool = + ## Determines whether ``future`` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on ``future`` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of ``discard`` to discard void futures. + future.callback = + proc () = + if future.failed: + injectStacktrace(future) + raise future.error + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if not retFuture.finished: + if fut1.failed: retFuture.fail(fut1.error) + elif fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if not retFuture.finished: + if fut2.failed: retFuture.fail(fut2.error) + elif fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb[X](fut: Future[X]) = + if fut.failed: retFuture.fail(fut.error) + if not retFuture.finished: retFuture.complete() + fut1.callback = cb[T] + fut2.callback = cb[Y] + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + if f.failed: + retFuture.fail(f.error) + elif not retFuture.finished: + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + if f.failed: + retFuture.fail(f.error) + elif not retFuture.finished: + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 7ad7efd23..0b7908c02 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -24,6 +24,8 @@ ## jobj["test"] = newJFloat(0.7) # create or update ## echo($jobj["test"].fnum) ## echo($jobj["key2"].bval) +## echo jobj{"missing key"}.getFNum(0.1) # read a float value using a default +## jobj{"a", "b", "c"} = newJFloat(3.3) # created nested keys ## ## Results in: ## diff --git a/lib/pure/net.nim b/lib/pure/net.nim index a70f60a8e..d4f239c49 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -112,6 +112,7 @@ else: const BufferSize*: int = 4000 ## size of a buffered socket's buffer + MaxLineLength* = 1_000_000 type SocketImpl* = object ## socket type @@ -1006,7 +1007,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = result = recv(socket.fd, addr(c), 1, MSG_PEEK) proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, - flags = {SocketFlag.SafeDisconn}) {. + flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {. tags: [ReadIOEffect, TimeEffect].} = ## Reads a line of data from ``socket``. ## @@ -1021,6 +1022,10 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, ## A timeout can be specified in milliseconds, if data is not received within ## the specified time an ETimeout exception will be raised. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. template addNLIfEmpty() = @@ -1054,8 +1059,15 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, return add(line.string, c) + # Verify that this isn't a DOS attack: #3847. + if line.string.len > maxLength: + let msg = "recvLine received more than the specified `maxLength` " & + "allowed." + raise newException(ValueError, msg) + proc recvLine*(socket: Socket, timeout = -1, - flags = {SocketFlag.SafeDisconn}): TaintedString = + flags = {SocketFlag.SafeDisconn}, + maxLength = MaxLineLength): TaintedString = ## Reads a line of data from ``socket``. ## ## If a full line is read ``\r\L`` is not @@ -1069,9 +1081,13 @@ proc recvLine*(socket: Socket, timeout = -1, ## A timeout can be specified in milliseconds, if data is not received within ## the specified time an ETimeout exception will be raised. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. result = "" - readLine(socket, result, timeout, flags) + readLine(socket, result, timeout, flags, maxLength) proc recvFrom*(socket: Socket, data: var string, length: int, address: var string, port: var Port, flags = 0'i32): int {. diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index cba101fff..506b2cec0 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -377,7 +377,8 @@ proc contains*(s: Selector, key: SelectorKey): bool = proc len*(s: Selector): int = ## Retrieves the number of registered file descriptors in this Selector. - return s.fds.len + when not defined(nimdoc): + return s.fds.len {.deprecated: [TEvent: Event, PSelectorKey: SelectorKey, TReadyInfo: ReadyInfo, PSelector: Selector].} diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 4465e961c..29b955c46 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -130,286 +130,7 @@ export Port, SocketFlag # TODO: Check if yielded future is nil and throw a more meaningful exception -# -- Futures - -type - FutureBase* = ref object of RootObj ## Untyped future. - cb: proc () {.closure,gcsafe.} - finished: bool - error*: ref Exception ## Stored exception - errorStackTrace*: string - when not defined(release): - stackTrace: string ## For debugging purposes only. - id: int - fromProc: string - - Future*[T] = ref object of FutureBase ## Typed future. - value: T ## Stored value - - FutureVar*[T] = distinct Future[T] - - FutureError* = object of Exception - cause*: FutureBase - -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - -when not defined(release): - var currentID = 0 - -proc callSoon*(cbproc: proc ()) {.gcsafe.} - -proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = - ## Creates a new future. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - new(result) - result.finished = false - when not defined(release): - result.stackTrace = getStackTrace() - result.id = currentID - result.fromProc = fromProc - currentID.inc() - -proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = - ## Create a new ``FutureVar``. This Future type is ideally suited for - ## situations where you want to avoid unnecessary allocations of Futures. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - result = FutureVar[T](newFuture[T](fromProc)) - -proc clean*[T](future: FutureVar[T]) = - ## Resets the ``finished`` status of ``future``. - Future[T](future).finished = false - Future[T](future).error = nil - -proc checkFinished[T](future: Future[T]) = - ## Checks whether `future` is finished. If it is then raises a - ## ``FutureError``. - when not defined(release): - if future.finished: - var msg = "" - msg.add("An attempt was made to complete a Future more than once. ") - msg.add("Details:") - msg.add("\n Future ID: " & $future.id) - msg.add("\n Created in proc: " & future.fromProc) - msg.add("\n Stack trace to moment of creation:") - msg.add("\n" & indent(future.stackTrace.strip(), 4)) - when T is string: - msg.add("\n Contents (string): ") - msg.add("\n" & indent(future.value.repr, 4)) - msg.add("\n Stack trace to moment of secondary completion:") - msg.add("\n" & indent(getStackTrace().strip(), 4)) - var err = newException(FutureError, msg) - err.cause = future - raise err - -proc complete*[T](future: Future[T], val: T) = - ## Completes ``future`` with value ``val``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.value = val - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*(future: Future[void]) = - ## Completes a void ``future``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*[T](future: FutureVar[T]) = - ## Completes a ``FutureVar``. - template fut: expr = Future[T](future) - checkFinished(fut) - assert(fut.error == nil) - fut.finished = true - if fut.cb != nil: - fut.cb() - -proc fail*[T](future: Future[T], error: ref Exception) = - ## Completes ``future`` with ``error``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - future.finished = true - future.error = error - future.errorStackTrace = - if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) - if future.cb != nil: - future.cb() - else: - # This is to prevent exceptions from being silently ignored when a future - # is discarded. - # TODO: This may turn out to be a bad idea. - # Turns out this is a bad idea. - #raise error - discard - -proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - ## - ## **Note**: You most likely want the other ``callback`` setter which - ## passes ``future`` as a param to the callback. - future.cb = cb - if future.finished: - callSoon(future.cb) - -proc `callback=`*[T](future: Future[T], - cb: proc (future: Future[T]) {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - future.callback = proc () = cb(future) - -proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. - when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") - - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) - else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) - -proc read*[T](future: Future[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished: - if future.error != nil: - injectStacktrace(future) - raise future.error - when T isnot void: - return future.value - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - -proc readError*[T](future: Future[T]): ref Exception = - ## Retrieves the exception stored in ``future``. - ## - ## An ``ValueError`` exception will be thrown if no exception exists - ## in the specified Future. - if future.error != nil: return future.error - else: - raise newException(ValueError, "No error in future.") - -proc mget*[T](future: FutureVar[T]): var T = - ## Returns a mutable value stored in ``future``. - ## - ## Unlike ``read``, this function will not raise an exception if the - ## Future has not been finished. - result = Future[T](future).value - -proc finished*[T](future: Future[T]): bool = - ## Determines whether ``future`` has completed. - ## - ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - future.finished - -proc failed*(future: FutureBase): bool = - ## Determines whether ``future`` completed with an error. - return future.error != nil - -proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future - ## finished with an error. - ## - ## This should be used instead of ``discard`` to discard void futures. - future.callback = - proc () = - if future.failed: - injectStacktrace(future) - raise future.error - -proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once both ``fut1`` and ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`and`") - fut1.callback = - proc () = - if not retFuture.finished: - if fut1.failed: retFuture.fail(fut1.error) - elif fut2.finished: retFuture.complete() - fut2.callback = - proc () = - if not retFuture.finished: - if fut2.failed: retFuture.fail(fut2.error) - elif fut1.finished: retFuture.complete() - return retFuture - -proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once either ``fut1`` or ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`or`") - proc cb[X](fut: Future[X]) = - if fut.failed: retFuture.fail(fut.error) - if not retFuture.finished: retFuture.complete() - fut1.callback = cb[T] - fut2.callback = cb[Y] - return retFuture - -proc all*[T](futs: varargs[Future[T]]): auto = - ## Returns a future which will complete once - ## all futures in ``futs`` complete. - ## - ## If the awaited futures are not ``Future[void]``, the returned future - ## will hold the values of all awaited futures in a sequence. - ## - ## If the awaited futures *are* ``Future[void]``, - ## this proc returns ``Future[void]``. - - when T is void: - var - retFuture = newFuture[void]("asyncdispatch.all") - completedFutures = 0 - - let totalFutures = len(futs) - - for fut in futs: - fut.callback = proc(f: Future[T]) = - if f.failed: - retFuture.fail(f.error) - elif not retFuture.finished: - inc(completedFutures) - - if completedFutures == totalFutures: - retFuture.complete() - - return retFuture - - else: - var - retFuture = newFuture[seq[T]]("asyncdispatch.all") - retValues = newSeq[T](len(futs)) - completedFutures = 0 - - for i, fut in futs: - proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = - if f.failed: - retFuture.fail(f.error) - elif not retFuture.finished: - retValues[i] = f.read() - inc(completedFutures) - - if completedFutures == len(retValues): - retFuture.complete(retValues) - - setCallback(i) - - return retFuture +include "../includes/asyncfutures" type PDispatcherBase = ref object of RootRef @@ -522,43 +243,44 @@ when defined(windows) or defined(nimdoc): if at == -1: winlean.INFINITE else: at.int32 - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - - GC_unref(customOverlapped) - else: - let errCode = osLastError() - if customOverlapped != nil: + if p.handles.len != 0: + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. if customOverlapped.data.cell.data != nil: system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: raiseOSError(errCode) + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) # Timer processing. processTimers(p) diff --git a/lib/wrappers/linenoise/clinenoise.c b/lib/wrappers/linenoise/clinenoise.c index a03e6f379..c76fc6586 100644 --- a/lib/wrappers/linenoise/clinenoise.c +++ b/lib/wrappers/linenoise/clinenoise.c @@ -116,7 +116,9 @@ #include <sys/types.h> #include <sys/ioctl.h> #include <unistd.h> -#include "clinenoise.h" +#ifndef __LINENOISE_H +# include "clinenoise.h" +#endif #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 #define LINENOISE_MAX_LINE 4096 diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index b7004c6e2..4ac7bf4a8 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -14,7 +14,8 @@ type CompletionCallback* = proc (a2: cstring; a3: ptr Completions) {.cdecl.} -{.compile: "clinenoise.c".} +{.emit: staticRead"clinenoise.h".} +{.emit: staticRead"clinenoise.c".} proc setCompletionCallback*(a2: ptr CompletionCallback) {. importc: "linenoiseSetCompletionCallback".} diff --git a/readme.md b/readme.md index 204b51906..01061bfa2 100644 --- a/readme.md +++ b/readme.md @@ -51,6 +51,14 @@ instead of ``build.sh``. The ``koch`` tool is the Nim build tool, more ``koch`` related options are documented in [doc/koch.rst](doc/koch.rst). +To complete the installation you should also build Nim's tools like +``nimsuggest``, ``nimble`` or ``nimgrep``: + +``` +nim e install_tools.nims +``` + + ## Nimble [Nimble](https://github.com/nim-lang/nimble) is Nim's package manager. For the source based installations where you added Nim's ``bin`` directory to your PATH diff --git a/tests/async/tfuturevar.nim b/tests/async/tfuturevar.nim new file mode 100644 index 000000000..73c0fddf7 --- /dev/null +++ b/tests/async/tfuturevar.nim @@ -0,0 +1,47 @@ +import asyncdispatch + +proc completeOnReturn(fut: FutureVar[string], x: bool) {.async.} = + if x: + fut.mget() = "" + fut.mget.add("foobar") + return + +proc completeOnImplicitReturn(fut: FutureVar[string], x: bool) {.async.} = + if x: + fut.mget() = "" + fut.mget.add("foobar") + +proc failureTest(fut: FutureVar[string], x: bool) {.async.} = + if x: + raise newException(Exception, "Test") + +proc manualComplete(fut: FutureVar[string], x: bool) {.async.} = + if x: + fut.mget() = "Hello World" + fut.complete() + return + +proc main() {.async.} = + var fut: FutureVar[string] + + fut = newFutureVar[string]() + await completeOnReturn(fut, true) + doAssert(fut.read() == "foobar") + + fut = newFutureVar[string]() + await completeOnImplicitReturn(fut, true) + doAssert(fut.read() == "foobar") + + fut = newFutureVar[string]() + let retFut = failureTest(fut, true) + yield retFut + doAssert(fut.read().isNil) + doAssert(fut.finished) + + fut = newFutureVar[string]() + await manualComplete(fut, true) + doAssert(fut.read() == "Hello World") + + +waitFor main() + diff --git a/tests/async/tupcoming_async.nim b/tests/async/tupcoming_async.nim index e53482dae..137794afd 100644 --- a/tests/async/tupcoming_async.nim +++ b/tests/async/tupcoming_async.nim @@ -44,19 +44,8 @@ when defined(upcoming): ev.setEvent() proc timerTest() = - var timeout = 200 - var errorRate = 40.0 - var start = epochTime() waitFor(waitTimer(200)) - var finish = epochTime() - var lowlimit = float(timeout) - float(timeout) * errorRate / 100.0 - var highlimit = float(timeout) + float(timeout) * errorRate / 100.0 - var elapsed = (finish - start) * 1_000 # convert to milliseconds - if elapsed >= lowlimit and elapsed < highlimit: - echo "OK" - else: - echo "timerTest: Timeout = " & $(elapsed) & ", but must be inside of [" & - $lowlimit & ", " & $highlimit & ")" + echo "OK" proc eventTest() = var event = newAsyncEvent() diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index 9412a5afa..dd9a6139a 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -7,6 +7,8 @@ from net import TimeoutError import httpclient, asyncdispatch +const manualTests = false + proc asyncTest() {.async.} = var client = newAsyncHttpClient() var resp = await client.request("http://example.com/") @@ -20,12 +22,40 @@ proc asyncTest() {.async.} = resp = await client.request("https://google.com/") doAssert(resp.code.is2xx or resp.code.is3xx) + + # getContent + try: + discard await client.getContent("https://google.com/404") + doAssert(false, "HttpRequestError should have been raised") + except HttpRequestError: + discard + except: + doAssert(false, "HttpRequestError should have been raised") + + + # Multipart test. + var data = newMultipartData() + data["output"] = "soap12" + data["uploaded_file"] = ("test.html", "text/html", + "<html><head></head><body><p>test</p></body></html>") + resp = await client.post("http://validator.w3.org/check", multipart=data) + doAssert(resp.code.is2xx) + + # onProgressChanged + when manualTests: + proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = + echo("Downloaded ", progress, " of ", total) + echo("Current rate: ", speed div 1000, "kb/s") + client.onProgressChanged = onProgressChanged + discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") + client.close() # Proxy test - #client = newAsyncHttpClient(proxy = newProxy("http://51.254.106.76:80/")) - #var resp = await client.request("https://github.com") - #echo resp + #when manualTests: + # client = newAsyncHttpClient(proxy = newProxy("http://51.254.106.76:80/")) + # var resp = await client.request("https://github.com") + # echo resp proc syncTest() = var client = newHttpClient() @@ -41,6 +71,31 @@ proc syncTest() = resp = client.request("https://google.com/") doAssert(resp.code.is2xx or resp.code.is3xx) + # getContent + try: + discard client.getContent("https://google.com/404") + doAssert(false, "HttpRequestError should have been raised") + except HttpRequestError: + discard + except: + doAssert(false, "HttpRequestError should have been raised") + + # Multipart test. + var data = newMultipartData() + data["output"] = "soap12" + data["uploaded_file"] = ("test.html", "text/html", + "<html><head></head><body><p>test</p></body></html>") + resp = client.post("http://validator.w3.org/check", multipart=data) + doAssert(resp.code.is2xx) + + # onProgressChanged + when manualTests: + proc onProgressChanged(total, progress, speed: BiggestInt) = + echo("Downloaded ", progress, " of ", total) + echo("Current rate: ", speed div 1000, "kb/s") + client.onProgressChanged = onProgressChanged + discard client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") + client.close() # Timeout test. @@ -56,21 +111,3 @@ proc syncTest() = syncTest() waitFor(asyncTest()) - -#[ - - else: - #downloadFile("http://force7.de/nim/index.html", "nimindex.html") - #downloadFile("http://www.httpwatch.com/", "ChunkTest.html") - #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com", - # "validator.html") - - #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& - # charset=%28detect+automatically%29&doctype=Inline&group=0") - - var data = newMultipartData() - data["output"] = "soap12" - data["uploaded_file"] = ("test.html", "text/html", - "<html><head></head><body><p>test</p></body></html>") - - echo postContent("http://validator.w3.org/check", multipart=data)]# diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index a1f564fbc..8dff722ec 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -135,7 +135,7 @@ proc processFile(filename: string) = if optVerbose in options: stdout.writeLine(filename) stdout.flushFile() - var pegp: TPeg + var pegp: Peg var rep: Regex var result: string @@ -213,7 +213,7 @@ proc hasRightExt(filename: string, exts: seq[string]): bool = if os.cmpPaths(x, y) == 0: return true proc styleInsensitive(s: string): string = - template addx: stmt = + template addx = result.add(s[i]) inc(i) result = "" diff --git a/tools/niminst/buildsh.tmpl b/tools/niminst/buildsh.tmpl index 13dfe5226..220ecdb7f 100644 --- a/tools/niminst/buildsh.tmpl +++ b/tools/niminst/buildsh.tmpl @@ -92,6 +92,7 @@ case $uos in ;; *haiku* ) myos="haiku" + LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" ;; *) echo 2>&1 "Error: unknown operating system: $uos" diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 4c8dfcddf..01efa88d4 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -447,7 +447,8 @@ proc readCFiles(c: var ConfigData, osA, cpuA: int) = # HACK: we conditionally add ``-lm -ldl``, so remove them from the # linker flags: c.linker.flags = c.linker.flags.replaceWord("-lm").replaceWord( - "-ldl").strip + "-ldl").replaceWord("-lroot").replaceWord( + "-lnetwork").strip else: if cmpIgnoreStyle(k.key, "libpath") == 0: c.libpath = k.value diff --git a/tools/niminst/nsis.tmpl b/tools/niminst/nsis.tmpl index 0897d36a8..64dcc8f8b 100644 --- a/tools/niminst/nsis.tmpl +++ b/tools/niminst/nsis.tmpl @@ -35,16 +35,16 @@ ; Default installation folder ; This is changed later (in .onInit) to the root directory, if possible. - InstallDir "$PROGRAMFILES?{when sizeof(int) == 8: "64" else: ""}\?{c.name}-?{c.version}" + InstallDir "$PROGRAMFILES?{when sizeof(int) == 8: "64" else: ""}\?{c.name}-?{c.version}\" ; Get installation folder from registry if available InstallDirRegKey HKCU "Software\c.name\c.version" "" - ; Request user level application privileges. - RequestExecutionLevel user + ; Request admin level application privileges. + RequestExecutionLevel admin ; Allow installation to the root drive directory. - AllowRootDirInstall true + AllowRootDirInstall false ; Maximum compression! SetCompressor /SOLID /FINAL lzma @@ -159,7 +159,7 @@ ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\dist\mingw" ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\dist\mingw\bin" ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\bin" - ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\dist\babel" + ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$PROFILE\.nimble\bin" SectionEnd ; The downloadable sections. These sections are automatically generated by @@ -246,7 +246,7 @@ ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\dist\mingw" ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\dist\mingw\bin" ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\bin" - ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\dist\babel" + ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$PROFILE\.nimble\bin" SectionEnd ;-------------------------------- @@ -254,5 +254,5 @@ Function .onInit ${GetRoot} "$EXEDIR" $R0 - strCpy $INSTDIR "$R0\?{c.name}" + ;strCpy $INSTDIR "$R0\?{c.name}" FunctionEnd diff --git a/web/download.rst b/web/download.rst index 583398c20..11c130c23 100644 --- a/web/download.rst +++ b/web/download.rst @@ -45,6 +45,7 @@ Now open a terminal and follow these instructions: ``cd ~/programs/nim``. * run ``sh build.sh``. * Add ``$your_install_dir/bin`` to your PATH. +* To build associated tools like ``nimble`` and ``nimsuggest`` run ``nim e install_tools.nims``. After restarting your terminal, you should be able to run ``nim -v`` which should show you the version of Nim you just installed. diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst index dea893ed1..ac0f06176 100644 --- a/web/news/version_0_15_released.rst +++ b/web/news/version_0_15_released.rst @@ -3,24 +3,23 @@ Version 0.15.0 released .. container:: metadata - Posted by Dominik Picheta on 17/09/2016 + Posted by Dominik Picheta and Andreas Rumpf on 17/09/2016 Some text here. Changes affecting backwards compatibility ----------------------------------------- -- The ``json`` module uses an ``OrderedTable`` rather than a ``Table`` +- The ``json`` module now uses an ``OrderedTable`` rather than a ``Table`` for JSON objects. -- De-deprecated ``re.nim`` because we have too much code using it - and it got the basic API right. - -- ``split`` with ``set[char]`` as a delimiter in ``strutils.nim`` - no longer strips and splits characters out of the target string +- The ``split`` `(doc) <http://nim-lang.org/docs/strutils.html#split,string,set[char],int>`_ + procedure in the ``strutils`` module (with a delimiter of type + ``set[char]``) no longer strips and splits characters out of the target string by the entire set of characters. Instead, it now behaves in a similar fashion to ``split`` with ``string`` and ``char`` delimiters. Use ``splitWhitespace`` to get the old behaviour. + - The command invocation syntax will soon apply to open brackets and curlies too. This means that code like ``a [i]`` will be interpreted as ``a([i])`` and not as ``a[i]`` anymore. Likewise @@ -29,92 +28,142 @@ Changes affecting backwards compatibility Warning: a [b] will be parsed as command syntax; spacing is deprecated - See `<https://github.com/nim-lang/Nim/issues/3898>`_ for the relevant - discussion. + See `Issue #3898 <https://github.com/nim-lang/Nim/issues/3898>`_ for the + relevant discussion. + - Overloading the special operators ``.``, ``.()``, ``.=``, ``()`` now - should be enabled via ``{.experimental.}``. + needs to be enabled via the ``{.experimental.}`` pragma. + - ``immediate`` templates and macros are now deprecated. - Instead use ``untyped`` parameters. -- The metatype ``expr`` is deprecated. Use ``untyped`` instead. -- The metatype ``stmt`` is deprecated. Use ``typed`` instead. + Use ``untyped`` `(doc) <http://nim-lang.org/docs/manual.html#templates-typed-vs-untyped-parameters>`_ + parameters instead. + +- The metatype ``expr`` is deprecated. Use ``untyped`` + `(doc) <http://nim-lang.org/docs/manual.html#templates-typed-vs-untyped-parameters>`_ instead. + +- The metatype ``stmt`` is deprecated. Use ``typed`` + `(doc) <http://nim-lang.org/docs/manual.html#templates-typed-vs-untyped-parameters>`_ instead. + - The compiler is now more picky when it comes to ``tuple`` types. The following code used to compile, now it's rejected: .. code-block:: nim import tables - var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64] ]() + var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64]]() rocketaims["hi"] = {(-1.int8, 0.int8): 0.int64}.toTable() -Instead be consistent in your tuple usage and use tuple names for tuples -that have tuple name: +Instead be consistent in your tuple usage and use tuple names for named tuples: .. code-block:: nim import tables - var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64] ]() + var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64]]() rocketaims["hi"] = {(k: -1.int8, v: 0.int8): 0.int64}.toTable() -- Now when you compile console application for Windows, console output +- Now when you compile console applications for Windows, console output encoding is automatically set to UTF-8. -- Unhandled exceptions in JavaScript are now thrown regardless ``noUnhandledHandler`` - is defined. But now they do their best to provide a readable stack trace. +- Unhandled exceptions in JavaScript are now thrown regardless of whether + ``noUnhandledHandler`` is defined. But the stack traces should be much more + readable now. + +- In JavaScript, the ``system.alert`` procedure has been deprecated. + Use ``dom.alert`` instead. + +- De-deprecated ``re.nim`` because there is too much code using it + and it got the basic API right. + +- The type of ``headers`` field in the ``AsyncHttpClient`` type + `(doc) <http://nim-lang.org/docs/httpclient.html#AsyncHttpClient>`_ + has been changed + from a string table to the specialised ``HttpHeaders`` type. + +- The ``httpclient.request`` + `(doc) <http://nim-lang.org/docs/httpclient.html#request,AsyncHttpClient,string,string,string>`_ + procedure which takes the ``httpMethod`` as a string + value no longer requires it to be prefixed with ``"http"`` + (or similar). -- In JavaScript ``system.alert`` is deprecated. Use ``dom.alert`` instead. +- Converting a ``HttpMethod`` + `(doc) <nim-lang.org/docs/httpcore.html#HttpMethod>`_ + value to a string using the ``$`` operator will + give string values without the ``"Http"`` prefix now. -- ``AsyncHttpClient.headers`` type is now ``HttpHeaders``. +- The ``Request`` + `(doc) <http://nim-lang.org/docs/asynchttpserver.html#Request>`_ + object defined in the ``asynchttpserver`` module now uses + the ``HttpMethod`` type for the request method. Library Additions ----------------- -- Added ``readHeaderRow`` and ``rowEntry`` to ``parsecsv.nim`` to provide +- Added ``readHeaderRow`` and ``rowEntry`` to the ``parsecsv`` + `(doc) <http://nim-lang.org/docs/parsecsv.html>`_ module + to provide a lightweight alternative to python's ``csv.DictReader``. -- Added ``setStdIoUnbuffered`` proc to ``system.nim`` to enable unbuffered I/O. -- Added ``center`` and ``rsplit`` to ``strutils.nim`` to - provide similar Python functionality for Nim's strings. +- Added ``setStdIoUnbuffered`` proc to the ``system`` module to enable + unbuffered I/O. + +- Added ``center`` and ``rsplit`` to the ``strutils`` + `(doc) <http://nim-lang.org/docs/strutils.html>`_ module + to provide similar Python functionality for Nim's strings. - Added ``isTitle``, ``title``, ``swapCase``, ``isUpper``, ``toUpper``, ``isLower``, ``toLower``, ``isAlpha``, ``isSpace``, and ``capitalize`` - to ``unicode.nim`` to provide unicode aware case manipulation and case + to the ``unicode.nim`` + `(doc) <http://nim-lang.org/docs/unicode.html>`_ module + to provide unicode aware case manipulation and case testing. -- Added a new module ``lib/pure/strmisc.nim`` to hold uncommon string +- Added a new module ``strmisc`` + `(doc) <http://nim-lang.org/docs/strmisc.html>`_ + to hold uncommon string operations. Currently contains ``partition``, ``rpartition`` and ``expandTabs``. -- Split out ``walkFiles`` in ``os.nim`` to three separate procs in order - to make a clear distinction of functionality. ``walkPattern`` iterates +- Split out ``walkFiles`` in the ``os`` + `(doc) <http://nim-lang.org/docs/os.html>`_ module to three separate procs + in order to make a clear distinction of functionality. ``walkPattern`` iterates over both files and directories, while ``walkFiles`` now only iterates over files and ``walkDirs`` only iterates over directories. -- Added synchronous ``HttpClient`` in the ``httpclient`` module. +- Added a synchronous ``HttpClient`` in the ``httpclient`` + `(doc) <http://nim-lang.org/docs/httpclient.html>`_ + module. The old + ``get``, ``post`` and similar procedures are now deprecated in favour of it. - Added a new macro called ``multisync`` allowing you to write procedures for -synchronous and asynchronous sockets with no duplication. + synchronous and asynchronous sockets with no duplication. + +- The ``async`` macro will now complete ``FutureVar[T]`` parameters + automatically unless they have been completed already. Compiler Additions ------------------ - The ``-d/--define`` flag can now optionally take a value to be used by code at compile time. + `(doc) <http://nim-lang.org/docs/manual.html#implementation-specific-pragmas-compile-time-define-pragmas>`_ Nimscript Additions ------------------- -- Finally it's possible to dis/enable specific hints and warnings in - Nimscript via the procs ``warning`` and ``hint``. +- It's possible to enable and disable specific hints and warnings in + Nimscript via the ``warning`` and ``hint`` procedures. + - Nimscript exports a proc named ``patchFile`` which can be used to patch modules or include files for different Nimble packages, including the ``stdlib`` package. - Language Additions ------------------ - Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of (optional) compile time defines. + `(doc) <http://nim-lang.org/docs/manual.html#implementation-specific-pragmas-compile-time-define-pragmas>`_ + - If the first statement is an ``import system`` statement then ``system`` is not imported implicitly anymore. This allows for code like ``import system except echo`` or ``from system import nil``. @@ -122,6 +171,11 @@ Language Additions Bugfixes -------- +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-06-22+..+2016-09-28%22+>`_. + - Fixed "RFC: should startsWith and endsWith work with characters?" (`#4252 <https://github.com/nim-lang/Nim/issues/4252>`_) @@ -347,3 +401,30 @@ Bugfixes (`#4626 <https://github.com/nim-lang/Nim/issues/4626>`_) - Fixed "module securehash not gcsafe" (`#4760 <https://github.com/nim-lang/Nim/issues/4760>`_) + +- Fixed "Nimble installation failed on Windows x86." + (`#4764 <https://github.com/nim-lang/Nim/issues/4764>`_) +- Fixed "Recent changes to marshal module break old marshalled data" + (`#4779 <https://github.com/nim-lang/Nim/issues/4779>`_) +- Fixed "tnewasyncudp.nim test loops forever" + (`#4777 <https://github.com/nim-lang/Nim/issues/4777>`_) +- Fixed "Wrong poll timeout behavior in asyncdispatch" + (`#4262 <https://github.com/nim-lang/Nim/issues/4262>`_) +- Fixed "Standalone await shouldn't read future" + (`#4170 <https://github.com/nim-lang/Nim/issues/4170>`_) +- Fixed "Regression: httpclient fails to compile without -d:ssl" + (`#4797 <https://github.com/nim-lang/Nim/issues/4797>`_) +- Fixed "C Error on declaring array of heritable objects with bitfields" + (`#3567 <https://github.com/nim-lang/Nim/issues/3567>`_) +- Fixed "Corruption when using Channels and Threads" + (`#4776 <https://github.com/nim-lang/Nim/issues/4776>`_) +- Fixed "Sometimes Channel tryRecv() erroneously reports no messages available on the first call on Windows" + (`#4746 <https://github.com/nim-lang/Nim/issues/4746>`_) +- Fixed "Improve error message of functions called without parenthesis" + (`#4813 <https://github.com/nim-lang/Nim/issues/4813>`_) +- Fixed "Docgen doesn't find doc comments in macro generated procs" + (`#4803 <https://github.com/nim-lang/Nim/issues/4803>`_) +- Fixed "asynchttpserver may consume unbounded memory reading headers" + (`#3847 <https://github.com/nim-lang/Nim/issues/3847>`_) +- Fixed "TLS connection to api.clashofclans.com hangs forever." + (`#4587 <https://github.com/nim-lang/Nim/issues/4587>`_) |