From a20326e2682486be95c33cf5cb23fe1cafe70979 Mon Sep 17 00:00:00 2001 From: pgkos Date: Wed, 25 Oct 2017 18:54:34 +0200 Subject: Allow parsing URIs without authority --- lib/pure/uri.nim | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 4b2e4e052..2af5412d1 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -135,9 +135,8 @@ proc parseUri*(uri: string, result: var Uri) = i.inc(2) # Skip // var authority = "" i.inc parseUntil(uri, authority, {'/', '?', '#'}, i) - if authority == "": - raise newException(ValueError, "Expected authority got nothing.") - parseAuthority(authority, result) + if authority.len > 0: + parseAuthority(authority, result) else: result.opaque = true @@ -412,6 +411,15 @@ when isMainModule: doAssert test.hostname == "github.com" doAssert test.port == "dom96" doAssert test.path == "/packages" + + block: + let str = "file:///foo/bar/baz.txt" + let test = parseUri(str) + doAssert test.scheme == "file" + doAssert test.username == "" + doAssert test.hostname == "" + doAssert test.port == "" + doAssert test.path == "/foo/bar/baz.txt" # Remove dot segments tests block: @@ -524,4 +532,4 @@ when isMainModule: doAssert "https://example.com/about/staff.html?".parseUri().isAbsolute == true doAssert "https://example.com/about/staff.html?parameters".parseUri().isAbsolute == true - echo("All good!") \ No newline at end of file + echo("All good!") -- cgit 1.4.1-2-gfad0 From 3db460f5045e790b54ea3825713c87e92679e8ab Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 16:17:48 +0000 Subject: The AsyncFD type now implies that the underlying FD is registered. * `asyncdispatch.register` won't attempt to register an ``AsyncFD``, but instead assume that it is already registered. --- changelog.md | 6 ++++++ lib/pure/asyncdispatch.nim | 25 ++++++++++++++++++++++--- lib/pure/asyncfile.nim | 15 +++++++-------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index 49cd4123a..1bef6794c 100644 --- a/changelog.md +++ b/changelog.md @@ -12,6 +12,12 @@ `getBool`, `getFloat`, `getBiggestInt`. Also `getInt` procedure was added. - `reExtended` is no longer default for the `re` constructor in the `re` module. + + +### Library changes + +- The `AsyncFD` type now reflects the fact that the underlying FD is registered + in the async dispatcher. - The overloading rules changed slightly so that constrained generics are preferred over unconstrained generics. (Bug #6526) - It is now possible to forward declare object types so that mutually diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 4c96aa614..7211fabc7 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -219,7 +219,7 @@ when defined(windows) or defined(nimdoc): PCustomOverlapped* = ref CustomOverlapped - AsyncFD* = distinct int + AsyncFD* = distinct int ## An FD that is registered in the dispatcher. PostCallbackData = object ioPort: Handle @@ -262,13 +262,22 @@ when defined(windows) or defined(nimdoc): setGlobalDispatcher(newDispatcher()) result = gDisp - proc register*(fd: AsyncFD) = + proc register*(fd: cint | SocketHandle | AsyncFD): AsyncFD {.discardable.} = ## Registers ``fd`` with the dispatcher. + ## + ## By convention, an ``AsyncFD`` is said to be already registered in the + ## dispatcher. This procedure will raise an exception if ``fd`` has already + ## been registered, but only if the type of the ``fd`` isn't ``AsyncFD``. let p = getGlobalDispatcher() + when fd is AsyncFD: + if fd in p.handles: + return + if createIoCompletionPort(fd.Handle, p.ioPort, cast[CompletionKey](fd), 1) == 0: raiseOSError(osLastError()) p.handles.incl(fd) + return fd.AsyncFD proc verifyPresence(fd: AsyncFD) = ## Ensures that file descriptor has been registered with the dispatcher. @@ -753,6 +762,9 @@ when defined(windows) or defined(nimdoc): ## Unregisters ``fd``. getGlobalDispatcher().handles.excl(fd) + proc contains*(disp: PDispatcher, fd: AsyncFd | SocketHandle): bool = + return fd.SocketHandle in disp.handles + {.push stackTrace:off.} proc waitableCallback(param: pointer, timerOrWaitFired: WINBOOL): void {.stdcall.} = @@ -1091,10 +1103,14 @@ else: setGlobalDispatcher(newDispatcher()) result = gDisp - proc register*(fd: AsyncFD) = + proc register*(fd: cint | SocketHandle | AsyncFD): AsyncFD {.discardable.} = let p = getGlobalDispatcher() + when fd is AsyncFD: + if fd.SocketHandle in p.selector: + return var data = newAsyncData() p.selector.registerHandle(fd.SocketHandle, {}, data) + return fd.AsyncFD proc closeSocket*(sock: AsyncFD) = let disp = getGlobalDispatcher() @@ -1106,6 +1122,9 @@ else: proc unregister*(ev: AsyncEvent) = getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + + proc contains*(disp: PDispatcher, fd: AsyncFd | SocketHandle): bool = + return fd.SocketHandle in disp.selector proc addRead*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 9f4da16a3..6cd62efa4 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -81,11 +81,10 @@ proc getFileSize*(f: AsyncFile): int64 = else: result = lseek(f.fd.cint, 0, SEEK_END) -proc newAsyncFile*(fd: AsyncFd): AsyncFile = +proc newAsyncFile*(fd: cint | AsyncFd): AsyncFile = ## Creates `AsyncFile` with a previously opened file descriptor `fd`. new result - result.fd = fd - register(result.fd) + result.fd = register(result.fd) proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using @@ -97,16 +96,16 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = when useWinUnicode: let fd = createFileW(newWideCString(filename), desiredAccess, FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + nil, creationDisposition, flags, 0) else: let fd = createFileA(filename, desiredAccess, FILE_SHARE_READ, - nil, creationDisposition, flags, 0).AsyncFd + nil, creationDisposition, flags, 0) - if fd.Handle == INVALID_HANDLE_VALUE: + if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) - result = newAsyncFile(fd) + result = newAsyncFile(fd.cint) if mode == fmAppend: result.offset = getFileSize(result) @@ -115,7 +114,7 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = getPosixFlags(mode) # RW (Owner), RW (Group), R (Other) let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH - let fd = open(filename, flags, perm).AsyncFD + let fd = open(filename, flags, perm) if fd.cint == -1: raiseOSError(osLastError()) -- cgit 1.4.1-2-gfad0 From d6870f2e8957b88e0ebc6b4e3308361862b48524 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 16:25:03 +0000 Subject: Multiple improvements to selectors. * Added ``getFd`` procedure for retrieving the underlying selector's FD. * Selectors module's procedures now accept an ``int`` as well as a ``SocketHandle``. * ReadyKey now contains the error code for Event.Error events. --- changelog.md | 2 ++ lib/pure/ioselects/ioselectors_epoll.nim | 21 +++++++++++++++++---- lib/pure/ioselects/ioselectors_kqueue.nim | 18 +++++++++++++++--- lib/pure/ioselects/ioselectors_poll.nim | 10 +++++++--- lib/pure/ioselects/ioselectors_select.nim | 9 ++++++--- lib/pure/selectors.nim | 23 +++++++++++++++-------- 6 files changed, 62 insertions(+), 21 deletions(-) diff --git a/changelog.md b/changelog.md index 1bef6794c..c43eb8b2c 100644 --- a/changelog.md +++ b/changelog.md @@ -16,6 +16,8 @@ ### Library changes +- The `ReadyKey` type in the selectors module now contains an ``errorCode`` + field to help distinguish between ``Event.Error`` events. - The `AsyncFD` type now reflects the fact that the underlying FD is registered in the async dispatcher. - The overloading rules changed slightly so that constrained generics are diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 35cdace09..144b4ce8d 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -141,7 +141,7 @@ template checkFd(s, f) = if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) @@ -156,7 +156,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, raiseIOSelectorsError(osLastError()) inc(s.count) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} let fdi = int(fd) @@ -391,9 +391,19 @@ proc selectInto*[T](s: Selector[T], timeout: int, let pevents = resTable[i].events var pkey = addr(s.fds[fdi]) doAssert(pkey.ident != 0) - var rkey = ReadyKey(fd: int(fdi), events: {}) + var rkey = ReadyKey(fd: fdi, events: {}) if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + if (pevents and EPOLLHUP) != 0: + rkey.errorCode = ECONNRESET.OSErrorCode + else: + # Try reading SO_ERROR from fd. + var error: cint + var size = sizeof(error).SockLen + if getsockopt(fdi.SocketHandle, SOL_SOCKET, SO_ERROR, addr(error), + addr(size)) == 0'i32: + rkey.errorCode = error.OSErrorCode + rkey.events.incl(Event.Error) if (pevents and EPOLLOUT) != 0: rkey.events.incl(Event.Write) @@ -481,7 +491,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -515,3 +525,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + +proc getFd*[T](s: Selector[T]): int = + return s.epollFd.int \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 3e2ec64a8..c0b50ab15 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -217,7 +217,7 @@ else: raiseIOSelectorsError(osLastError()) s.changes.setLen(0) -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = let fdi = int(fd) s.checkFd(fdi) @@ -235,7 +235,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, when not declared(CACHE_EVENTS): flushKQueue(s) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -503,6 +503,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, if (kevent.flags and EV_ERROR) != 0: rkey.events = {Event.Error} + rkey.errorCode = kevent.data.OSErrorCode case kevent.filter: of EVFILT_READ: @@ -569,6 +570,13 @@ proc selectInto*[T](s: Selector[T], timeout: int, doAssert(true, "Unsupported kqueue filter in the queue!") if (kevent.flags and EV_EOF) != 0: + if kevent.fflags != 0: + rkey.errorCode = kevent.fflags.OSErrorCode + else: + # This assumes we are dealing with sockets. + # TODO: For future-proofing it might be a good idea to give the + # user access to the raw `kevent`. + rkey.errorCode = ECONNRESET.OSErrorCode rkey.events.incl(Event.Error) results[k] = rkey @@ -585,7 +593,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -619,3 +627,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + + +proc getFd*[T](s: Selector[T]): int = + return s.kqFD.int \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index cc06aa592..66d52b352 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -141,7 +141,7 @@ template checkFd(s, f) = if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = var fdi = int(fd) s.checkFd(fdi) @@ -149,7 +149,7 @@ proc registerHandle*[T](s: Selector[T], fd: SocketHandle, setKey(s, fdi, events, 0, data) if events != {}: s.pollAdd(fdi.cint, events) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -280,7 +280,7 @@ template isEmpty*[T](s: Selector[T]): bool = (s.count == 0) proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = - return s.fds[fd].ident != 0 + return s.fds[fd.int].ident != 0 proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = let fdi = int(fd) @@ -314,3 +314,7 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body1 else: body2 + + +proc getFd*[T](s: Selector[T]): int = + return -1 \ No newline at end of file diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 017d08117..2165c5cb2 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -229,7 +229,7 @@ proc delKey[T](s: Selector[T], fd: SocketHandle) = doAssert(i < FD_SETSIZE, "Descriptor [" & $int(fd) & "] is not registered in the queue!") -proc registerHandle*[T](s: Selector[T], fd: SocketHandle, +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = when not defined(windows): let fdi = int(fd) @@ -255,7 +255,7 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = IOFD_SET(ev.rsock, addr s.rSet) inc(s.count) -proc updateHandle*[T](s: Selector[T], fd: SocketHandle, +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) = let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User, Event.Oneshot, Event.Error} @@ -279,7 +279,7 @@ proc updateHandle*[T](s: Selector[T], fd: SocketHandle, inc(s.count) pkey.events = events -proc unregister*[T](s: Selector[T], fd: SocketHandle) = +proc unregister*[T](s: Selector[T], fd: int | SocketHandle) = s.withSelectLock(): var pkey = s.getKey(fd) if Event.Read in pkey.events: @@ -451,3 +451,6 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, else: body2 + +proc getFd*[T](s: Selector[T]): int = + return -1 \ No newline at end of file diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 3a7d1523f..6eb3e5311 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -54,9 +54,9 @@ when defined(nimdoc): Timer, ## Timer descriptor is completed Signal, ## Signal is raised Process, ## Process is finished - Vnode, ## BSD specific file change happens + Vnode, ## BSD specific file change User, ## User event is raised - Error, ## Error happens while waiting, for descriptor + Error, ## Error occurred while waiting for descriptor VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) @@ -69,6 +69,8 @@ when defined(nimdoc): ## An object which holds result for descriptor fd* : int ## file/socket descriptor events*: set[Event] ## set of events + errorCode*: OSErrorCode ## additional error code information for + ## Error events SelectEvent* = object ## An object which holds user defined event @@ -79,13 +81,14 @@ when defined(nimdoc): proc close*[T](s: Selector[T]) = ## Closes the selector. - proc registerHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event], - data: T) = + proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = ## Registers file/socket descriptor ``fd`` to selector ``s`` ## with events set in ``events``. The ``data`` is application-defined ## data, which will be passed when an event is triggered. - proc updateHandle*[T](s: Selector[T], fd: SocketHandle, events: set[Event]) = + proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = ## Update file/socket descriptor ``fd``, registered in selector ## ``s`` with new events set ``event``. @@ -221,11 +224,15 @@ when defined(nimdoc): proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = ## Determines whether selector contains a file descriptor. + proc getFd*[T](s: Selector[T]): int = + ## Retrieves the underlying selector's file descriptor. + ## + ## For *poll* and *select* selectors ``-1`` is returned. + else: when hasThreadSupport: import locks - type SharedArray[T] = UncheckedArray[T] @@ -234,7 +241,6 @@ else: proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) - type Event* {.pure.} = enum Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, @@ -247,6 +253,7 @@ else: ReadyKey* = object fd* : int events*: set[Event] + errorCode*: OSErrorCode SelectorKey[T] = object ident: int @@ -264,7 +271,7 @@ else: msg.add("Internal Error\n") var err = newException(IOSelectorsException, msg) raise err - + proc setNonBlocking(fd: cint) {.inline.} = setBlocking(fd.SocketHandle, false) -- cgit 1.4.1-2-gfad0 From 66d7091b7994b3e5a1d4ede1b50cbf59780c9c16 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 16:27:43 +0000 Subject: setBlocking(false) is called on AsyncFD in newAsyncSocket proc. --- changelog.md | 3 ++- lib/pure/asyncnet.nim | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index c43eb8b2c..cc6732fc1 100644 --- a/changelog.md +++ b/changelog.md @@ -12,7 +12,8 @@ `getBool`, `getFloat`, `getBiggestInt`. Also `getInt` procedure was added. - `reExtended` is no longer default for the `re` constructor in the `re` module. - +- `newAsyncSocket` taking an `AsyncFD` now runs `setBlocking(false)` on the + fd. ### Library changes diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 5be457d2a..650bb3340 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -140,9 +140,18 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket = ## Creates a new ``AsyncSocket`` based on the supplied params. + ## + ## The supplied ``fd``'s non-blocking state will be enabled implicitly. + ## + ## **Note**: This procedure will **NOT** register ``fd`` with the global + ## async dispatcher. You need to do this manually. If you have used + ## ``newAsyncNativeSocket`` to create ``fd`` then it's already registered. + ## The reason for this is that the ``AsyncFD`` type is a special type for an + ## FD that signifies that its been registered. assert fd != osInvalidSocket.AsyncFD new(result) result.fd = fd.SocketHandle + fd.SocketHandle.setBlocking(false) result.isBuffered = buffered result.domain = domain result.sockType = sockType -- cgit 1.4.1-2-gfad0 From e5a27c96fbd8a3bdd48b1974595381c1a335aaa5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 16:29:34 +0000 Subject: Implements nativesockets.accept. --- changelog.md | 2 ++ lib/pure/nativesockets.nim | 17 +++++++++++++++-- lib/pure/net.nim | 10 +++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index cc6732fc1..a9212efbd 100644 --- a/changelog.md +++ b/changelog.md @@ -21,6 +21,8 @@ field to help distinguish between ``Event.Error`` events. - The `AsyncFD` type now reflects the fact that the underlying FD is registered in the async dispatcher. +- Implemented an `accept` proc that works on a `SocketHandle` in + ``nativesockets``. - The overloading rules changed slightly so that constrained generics are preferred over unconstrained generics. (Bug #6526) - It is now possible to forward declare object types so that mutually diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 6c8701843..790ad627d 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -187,12 +187,12 @@ proc toSockType*(protocol: Protocol): SockType = proc newNativeSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, protocol: Protocol = IPPROTO_TCP): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. socket(toInt(domain), toInt(sockType), toInt(protocol)) proc newNativeSocket*(domain: cint, sockType: cint, protocol: cint): SocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. + ## Creates a new socket; returns `osInvalidSocket` if an error occurs. ## ## Use this overload if one of the enums specified above does ## not contain what you need. @@ -666,6 +666,19 @@ proc selectWrite*(writefds: var seq[SocketHandle], pruneSocketSet(writefds, (wr)) +proc accept*(fd: SocketHandle): (SocketHandle, string) = + ## Accepts a new client connection. + ## + ## Returns (osInvalidSocket, "") if an error occurred. + var sockAddress: Sockaddr_in + var addrLen = sizeof(sockAddress).SockLen + var sock = accept(fd, cast[ptr SockAddr](addr(sockAddress)), + addr(addrLen)) + if sock == osInvalidSocket: + return (osInvalidSocket, "") + else: + return (sock, $inet_ntoa(sockAddress.sin_addr)) + when defined(Windows): var wsa: WSAData if wsaStartup(0x0101'i16, addr wsa) != 0: raiseOSError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b8d05642b..5c162317e 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -753,10 +753,8 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, ## flag is specified then this error will not be raised and instead ## accept will be called again. assert(client != nil) - var sockAddress: Sockaddr_in - var addrLen = sizeof(sockAddress).SockLen - var sock = accept(server.fd, cast[ptr SockAddr](addr(sockAddress)), - addr(addrLen)) + let ret = accept(server.fd) + let sock = ret[0] if sock == osInvalidSocket: let err = osLastError() @@ -764,6 +762,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, acceptAddr(server, client, address, flags) raiseOSError(err) else: + address = ret[1] client.fd = sock client.isBuffered = server.isBuffered @@ -776,9 +775,6 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, let ret = SSLAccept(client.sslHandle) socketError(client, ret, false) - # Client socket is set above. - address = $inet_ntoa(sockAddress.sin_addr) - when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. -- cgit 1.4.1-2-gfad0 From 325e4520ec74cd49a21c389bb2d137a74536f062 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 16:30:56 +0000 Subject: Implements ``asyncdispatch.getIoHandler`` and assert on nil futures. --- changelog.md | 3 +++ lib/pure/asyncdispatch.nim | 8 ++++++++ lib/pure/asyncfutures.nim | 1 + 3 files changed, 12 insertions(+) diff --git a/changelog.md b/changelog.md index a9212efbd..847a36d84 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,9 @@ in the async dispatcher. - Implemented an `accept` proc that works on a `SocketHandle` in ``nativesockets``. +- Implemented ``getIoHandler`` proc in the ``asyncdispatch`` module that allows + you to retrieve the underlying IO Completion Port or ``Selector[AsyncData]`` + object in the specified dispatcher. - The overloading rules changed slightly so that constrained generics are preferred over unconstrained generics. (Bug #6526) - It is now possible to forward declare object types so that mutually diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 7211fabc7..cee692980 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -262,6 +262,11 @@ when defined(windows) or defined(nimdoc): setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Handle = + ## Returns the underlying IO Completion Port handle (Windows) or selector + ## (Unix) for the specified dispatcher. + return disp.ioPort + proc register*(fd: cint | SocketHandle | AsyncFD): AsyncFD {.discardable.} = ## Registers ``fd`` with the dispatcher. ## @@ -1103,6 +1108,9 @@ else: setGlobalDispatcher(newDispatcher()) result = gDisp + proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = + return disp.selector + proc register*(fd: cint | SocketHandle | AsyncFD): AsyncFD {.discardable.} = let p = getGlobalDispatcher() when fd is AsyncFD: diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index bebd19611..8941dca6e 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -281,6 +281,7 @@ proc asyncCheck*[T](future: Future[T]) = ## finished with an error. ## ## This should be used instead of ``discard`` to discard void futures. + assert(not future.isNil, "Future is nil") future.callback = proc () = if future.failed: -- cgit 1.4.1-2-gfad0 From a1e2ec887282d38cae21be0ca3b56718fb36b9cd Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 17:51:34 +0000 Subject: Fix typo in asyncfile. --- lib/pure/asyncfile.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 6cd62efa4..3a9d26378 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -84,7 +84,7 @@ proc getFileSize*(f: AsyncFile): int64 = proc newAsyncFile*(fd: cint | AsyncFd): AsyncFile = ## Creates `AsyncFile` with a previously opened file descriptor `fd`. new result - result.fd = register(result.fd) + result.fd = register(fd) proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using -- cgit 1.4.1-2-gfad0 From 4cc8c15064d421987f2b617b06285ecac94b1a4a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 17:52:02 +0000 Subject: Remove upcoming folder. --- lib/upcoming/asyncdispatch.nim | 1630 ---------------------------------------- 1 file changed, 1630 deletions(-) delete mode 100644 lib/upcoming/asyncdispatch.nim diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim deleted file mode 100644 index 4e3b06173..000000000 --- a/lib/upcoming/asyncdispatch.nim +++ /dev/null @@ -1,1630 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -include "system/inclrtl" - -import os, tables, strutils, times, heapqueue, lists, options, asyncstreams -import asyncfutures except callSoon - -import nativesockets, net, deques - -export Port, SocketFlag -export asyncfutures, asyncstreams - -#{.injectStmt: newGcInvariant().} - -## AsyncDispatch -## ************* -## -## This module implements asynchronous IO. This includes a dispatcher, -## a ``Future`` type implementation, and an ``async`` macro which allows -## asynchronous code to be written in a synchronous style with the ``await`` -## keyword. -## -## The dispatcher acts as a kind of event loop. You must call ``poll`` on it -## (or a function which does so for you such as ``waitFor`` or ``runForever``) -## in order to poll for any outstanding events. The underlying implementation -## is based on epoll on Linux, IO Completion Ports on Windows and select on -## other operating systems. -## -## The ``poll`` function will not, on its own, return any events. Instead -## an appropriate ``Future`` object will be completed. A ``Future`` is a -## type which holds a value which is not yet available, but which *may* be -## available in the future. You can check whether a future is finished -## by using the ``finished`` function. When a future is finished it means that -## either the value that it holds is now available or it holds an error instead. -## The latter situation occurs when the operation to complete a future fails -## with an exception. You can distinguish between the two situations with the -## ``failed`` function. -## -## Future objects can also store a callback procedure which will be called -## automatically once the future completes. -## -## Futures therefore can be thought of as an implementation of the proactor -## pattern. In this -## pattern you make a request for an action, and once that action is fulfilled -## a future is completed with the result of that action. Requests can be -## made by calling the appropriate functions. For example: calling the ``recv`` -## function will create a request for some data to be read from a socket. The -## future which the ``recv`` function returns will then complete once the -## requested amount of data is read **or** an exception occurs. -## -## Code to read some data from a socket may look something like this: -## -## .. code-block::nim -## var future = socket.recv(100) -## future.callback = -## proc () = -## echo(future.read) -## -## All asynchronous functions returning a ``Future`` will not block. They -## will not however return immediately. An asynchronous function will have -## code which will be executed before an asynchronous request is made, in most -## cases this code sets up the request. -## -## In the above example, the ``recv`` function will return a brand new -## ``Future`` instance once the request for data to be read from the socket -## is made. This ``Future`` instance will complete once the requested amount -## of data is read, in this case it is 100 bytes. The second line sets a -## callback on this future which will be called once the future completes. -## All the callback does is write the data stored in the future to ``stdout``. -## The ``read`` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. -## -## Asynchronous procedures -## ----------------------- -## -## Asynchronous procedures remove the pain of working with callbacks. They do -## this by allowing you to write asynchronous code the same way as you would -## write synchronous code. -## -## An asynchronous procedure is marked using the ``{.async.}`` pragma. -## When marking a procedure with the ``{.async.}`` pragma it must have a -## ``Future[T]`` return type or no return type at all. If you do not specify -## a return type then ``Future[void]`` is assumed. -## -## Inside asynchronous procedures ``await`` can be used to call any -## procedures which return a -## ``Future``; this includes asynchronous procedures. When a procedure is -## "awaited", the asynchronous procedure it is awaited in will -## suspend its execution -## until the awaited procedure's Future completes. At which point the -## asynchronous procedure will resume its execution. During the period -## when an asynchronous procedure is suspended other asynchronous procedures -## will be run by the dispatcher. -## -## The ``await`` call may be used in many contexts. It can be used on the right -## hand side of a variable declaration: ``var data = await socket.recv(100)``, -## in which case the variable will be set to the value of the future -## automatically. It can be used to await a ``Future`` object, and it can -## be used to await a procedure returning a ``Future[void]``: -## ``await socket.send("foobar")``. -## -## Discarding futures -## ------------------ -## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the ``asyncCheck`` procedure instead of the ``discard`` keyword. -## -## Examples -## -------- -## -## For examples take a look at the documentation for the modules implementing -## asynchronous IO. A good place to start is the -## `asyncnet module `_. -## -## Limitations/Bugs -## ---------------- -## -## * The effect system (``raises: []``) does not work with async procedures. -## * Can't await in a ``except`` body -## * Forward declarations for async procs are broken, -## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. -## * FutureVar[T] needs to be completed manually. - -# TODO: Check if yielded future is nil and throw a more meaningful exception - -type - PDispatcherBase = ref object of RootRef - timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] - callbacks: Deque[proc ()] - -proc processTimers(p: PDispatcherBase) {.inline.} = - #Process just part if timers at a step - var count = p.timers.len - let t = epochTime() - while count > 0 and t >= p.timers[0].finishAt: - p.timers.pop().fut.complete() - dec count - -proc processPendingCallbacks(p: PDispatcherBase) = - while p.callbacks.len > 0: - var cb = p.callbacks.popFirst() - cb() - -proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = - # If dispatcher has active timers this proc returns the timeout - # of the nearest timer. Returns `timeout` otherwise. - result = timeout - if p.timers.len > 0: - let timerTimeout = p.timers[0].finishAt - let curTime = epochTime() - if timeout == -1 or (curTime + (timeout / 1000)) > timerTimeout: - result = int((timerTimeout - curTime) * 1000) - if result < 0: result = 0 - -proc callSoon(cbproc: proc ()) {.gcsafe.} - -proc initCallSoonProc = - if asyncfutures.getCallSoonProc().isNil: - asyncfutures.setCallSoonProc(callSoon) - -when defined(windows) or defined(nimdoc): - import winlean, sets, hashes - type - CompletionKey = ULONG_PTR - - CompletionData* = object - fd*: AsyncFD # TODO: Rename this. - cb*: proc (fd: AsyncFD, bytesTransferred: Dword, - errcode: OSErrorCode) {.closure,gcsafe.} - cell*: ForeignCell # we need this `cell` to protect our `cb` environment, - # when using RegisterWaitForSingleObject, because - # waiting is done in different thread. - - PDispatcher* = ref object of PDispatcherBase - ioPort: Handle - handles: HashSet[AsyncFD] - - CustomOverlapped = object of OVERLAPPED - data*: CompletionData - - PCustomOverlapped* = ref CustomOverlapped - - AsyncFD* = distinct int - - PostCallbackData = object - ioPort: Handle - handleFd: AsyncFD - waitFd: Handle - ovl: PCustomOverlapped - PostCallbackDataPtr = ptr PostCallbackData - - AsyncEventImpl = object - hEvent: Handle - hWaiter: Handle - pcd: PostCallbackDataPtr - AsyncEvent* = ptr AsyncEventImpl - - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - {.deprecated: [TCompletionKey: CompletionKey, TAsyncFD: AsyncFD, - TCustomOverlapped: CustomOverlapped, TCompletionData: CompletionData].} - - proc hash(x: AsyncFD): Hash {.borrow.} - proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.} - - proc newDispatcher*(): PDispatcher = - ## Creates a new Dispatcher instance. - new result - result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) - result.handles = initSet[AsyncFD]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](64) - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setGlobalDispatcher*(disp: PDispatcher) = - if not gDisp.isNil: - assert gDisp.callbacks.len == 0 - gDisp = disp - initCallSoonProc() - - proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: - setGlobalDispatcher(newDispatcher()) - result = gDisp - - proc register*(fd: AsyncFD) = - ## Registers ``fd`` with the dispatcher. - let p = getGlobalDispatcher() - if createIoCompletionPort(fd.Handle, p.ioPort, - cast[CompletionKey](fd), 1) == 0: - raiseOSError(osLastError()) - p.handles.incl(fd) - - proc verifyPresence(fd: AsyncFD) = - ## Ensures that file descriptor has been registered with the dispatcher. - let p = getGlobalDispatcher() - if fd notin p.handles: - raise newException(ValueError, - "Operation performed on a socket which has not been registered with" & - " the dispatcher yet.") - - proc hasPendingOperations*(): bool = - ## Returns `true` if the global dispatcher has pending operations. - let p = getGlobalDispatcher() - p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0 - - proc poll*(timeout = 500) = - ## Waits for completion events and processes them. Raises ``ValueError`` - ## if there are no pending operations. - let p = getGlobalDispatcher() - if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0: - raise newException(ValueError, - "No handles or timers registered in dispatcher.") - - let at = p.adjustedTimeout(timeout) - var llTimeout = - if at == -1: winlean.INFINITE - else: at.int32 - - if p.handles.len != 0: - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - - GC_unref(customOverlapped) - else: - let errCode = osLastError() - if customOverlapped != nil: - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - GC_unref(customOverlapped) - else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: raiseOSError(errCode) - - # Timer processing. - processTimers(p) - # Callback queue processing - processPendingCallbacks(p) - - var acceptEx*: WSAPROC_ACCEPTEX - var connectEx*: WSAPROC_CONNECTEX - var getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS - - proc initPointer(s: SocketHandle, fun: var pointer, guid: var GUID): bool = - # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c - var bytesRet: Dword - fun = nil - result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, - sizeof(GUID).Dword, addr fun, sizeof(pointer).Dword, - addr bytesRet, nil, nil) == 0 - - proc initAll() = - let dummySock = newNativeSocket() - if dummySock == INVALID_SOCKET: - raiseOSError(osLastError()) - var fun: pointer = nil - if not initPointer(dummySock, fun, WSAID_CONNECTEX): - raiseOSError(osLastError()) - connectEx = cast[WSAPROC_CONNECTEX](fun) - if not initPointer(dummySock, fun, WSAID_ACCEPTEX): - raiseOSError(osLastError()) - acceptEx = cast[WSAPROC_ACCEPTEX](fun) - if not initPointer(dummySock, fun, WSAID_GETACCEPTEXSOCKADDRS): - raiseOSError(osLastError()) - getAcceptExSockAddrs = cast[WSAPROC_GETACCEPTEXSOCKADDRS](fun) - close(dummySock) - - proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - ## Reads **up to** ``size`` bytes from ``socket``. Returned future will - ## complete once all the data requested is read, a part of the data has been - ## read, or the socket has disconnected in which case the future will - ## complete with a value of ``""``. - ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. - - - # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very - # unreliable. - # * Still need to implement message-oriented socket disconnection, - # '\0' in the message currently signifies a socket disconnect. Who - # knows what will happen when someone sends that to our socket. - verifyPresence(socket) - assert SocketFlag.Peek notin flags, "Peek not supported on Windows." - - var retFuture = newFuture[string]("recv") - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](alloc0(size)) - dataBuf.len = size.ULONG - - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - if bytesCount == 0 and dataBuf.buf[0] == '\0': - retFuture.complete("") - else: - var data = newString(bytesCount) - assert bytesCount <= size - copyMem(addr data[0], addr dataBuf.buf[0], bytesCount) - retFuture.complete($data) - else: - if flags.isDisconnectionError(errcode): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - if dataBuf.buf != nil: - dealloc dataBuf.buf - dataBuf.buf = nil - ) - - let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - addr flagsio, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - if dataBuf.buf != nil: - dealloc dataBuf.buf - dataBuf.buf = nil - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0: - # Request completed immediately. - if bytesReceived != 0: - var data = newString(bytesReceived) - assert bytesReceived <= size - copyMem(addr data[0], addr dataBuf.buf[0], bytesReceived) - retFuture.complete($data) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete("") - return retFuture - - proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must - ## at least be of that size. Returned future will complete once all the - ## data requested is read, a part of the data has been read, or the socket - ## has disconnected in which case the future will complete with a value of - ## ``0``. - ## - ## **Warning**: The ``Peek`` socket flag is not supported on Windows. - - - # Things to note: - # * When WSARecv completes immediately then ``bytesReceived`` is very - # unreliable. - # * Still need to implement message-oriented socket disconnection, - # '\0' in the message currently signifies a socket disconnect. Who - # knows what will happen when someone sends that to our socket. - verifyPresence(socket) - assert SocketFlag.Peek notin flags, "Peek not supported on Windows." - - var retFuture = newFuture[int]("recvInto") - - #buf[] = '\0' - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](buf) - dataBuf.len = size.ULONG - - var bytesReceived: Dword - var flagsio = flags.toOSFlags().Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete(bytesCount) - else: - if flags.isDisconnectionError(errcode): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - if dataBuf.buf != nil: - dataBuf.buf = nil - ) - - let ret = WSARecv(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - addr flagsio, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - if dataBuf.buf != nil: - dataBuf.buf = nil - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - elif ret == 0: - # Request completed immediately. - if bytesReceived != 0: - assert bytesReceived <= size - retFuture.complete(bytesReceived) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete(bytesReceived) - return retFuture - - proc send*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future - ## will complete once all data has been sent. - ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, - ## you must use GC_ref/GC_unref calls to avoid early freeing of the buffer. - verifyPresence(socket) - var retFuture = newFuture[void]("send") - - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](buf) - dataBuf.len = size.ULONG - - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - if flags.isDisconnectionError(errcode): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - lowFlags, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - if flags.isDisconnectionError(err): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to ``socket``. The returned future will complete once all - ## data has been sent. - verifyPresence(socket) - var retFuture = newFuture[void]("send") - - var dataBuf: TWSABuf - dataBuf.buf = data - GC_ref(data) # we need to protect data until send operation is completed - # or failed. - dataBuf.len = data.len.ULONG - - var bytesReceived, lowFlags: Dword - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - GC_unref(data) # if operation completed `data` must be released. - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - if flags.isDisconnectionError(errcode): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, - lowFlags, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - GC_unref(data) # if operation failed `data` must be released, because - # completion routine will not be called. - if flags.isDisconnectionError(err): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, - saddrLen: Socklen, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to specified destination ``saddr``, using - ## socket ``socket``. The returned future will complete once all data - ## has been sent. - verifyPresence(socket) - var retFuture = newFuture[void]("sendTo") - var dataBuf: TWSABuf - dataBuf.buf = cast[cstring](data) - dataBuf.len = size.ULONG - var bytesSent = 0.Dword - var lowFlags = 0.Dword - - # we will preserve address in our stack - var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes - var stalen: cint = cint(saddrLen) - zeroMem(addr(staddr[0]), 128) - copyMem(addr(staddr[0]), saddr, saddrLen) - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let ret = WSASendTo(socket.SocketHandle, addr dataBuf, 1, addr bytesSent, - lowFlags, cast[ptr SockAddr](addr(staddr[0])), - stalen, cast[POVERLAPPED](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - retFuture.complete() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - return retFuture - - proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, - saddr: ptr SockAddr, saddrLen: ptr SockLen, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Receives a datagram data from ``socket`` into ``buf``, which must - ## be at least of size ``size``, address of datagram's sender will be - ## stored into ``saddr`` and ``saddrLen``. Returned future will complete - ## once one datagram has been received, and will return size of packet - ## received. - verifyPresence(socket) - var retFuture = newFuture[int]("recvFromInto") - - var dataBuf = TWSABuf(buf: cast[cstring](data), len: size.ULONG) - - var bytesReceived = 0.Dword - var lowFlags = 0.Dword - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - assert bytesCount <= size - retFuture.complete(bytesCount) - else: - # datagram sockets don't have disconnection, - # so we can just raise an exception - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - ) - - let res = WSARecvFrom(socket.SocketHandle, addr dataBuf, 1, - addr bytesReceived, addr lowFlags, - saddr, cast[ptr cint](saddrLen), - cast[POVERLAPPED](ol), nil) - if res == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) - else: - # Request completed immediately. - if bytesReceived != 0: - assert bytesReceived <= size - retFuture.complete(bytesReceived) - else: - if hasOverlappedIoCompleted(cast[POVERLAPPED](ol)): - retFuture.complete(bytesReceived) - return retFuture - - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection and the remote address of the client. - ## The future will complete when the connection is successfully accepted. - ## - ## The resulting client socket is automatically registered to the - ## dispatcher. - ## - ## The ``accept`` call may result in an error if the connecting socket - ## disconnects during the duration of the ``accept``. If the ``SafeDisconn`` - ## flag is specified then this error will not be raised and instead - ## accept will be called again. - verifyPresence(socket) - var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - - var clientSock = newNativeSocket() - if clientSock == osInvalidSocket: raiseOSError(osLastError()) - - const lpOutputLen = 1024 - var lpOutputBuf = newString(lpOutputLen) - var dwBytesReceived: Dword - let dwReceiveDataLength = 0.Dword # We don't want any data to be read. - let dwLocalAddressLength = Dword(sizeof(Sockaddr_in6) + 16) - let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in6) + 16) - - template failAccept(errcode) = - if flags.isDisconnectionError(errcode): - var newAcceptFut = acceptAddr(socket, flags) - newAcceptFut.callback = - proc () = - if newAcceptFut.failed: - retFuture.fail(newAcceptFut.readError) - else: - retFuture.complete(newAcceptFut.read) - else: - retFuture.fail(newException(OSError, osErrorMsg(errcode))) - - template completeAccept() {.dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).SockLen) - if setoptRet != 0: - let errcode = osLastError() - discard clientSock.closeSocket() - failAccept(errcode) - else: - var localSockaddr, remoteSockaddr: ptr SockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr localSockaddr, addr localLen, - addr remoteSockaddr, addr remoteLen) - try: - let address = getAddrString(remoteSockAddr) - register(clientSock.AsyncFD) - retFuture.complete((address: address, client: clientSock.AsyncFD)) - except: - # getAddrString may raise - clientSock.close() - retFuture.fail(getCurrentException()) - - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if not retFuture.finished: - if errcode == OSErrorCode(-1): - completeAccept() - else: - failAccept(errcode) - ) - - # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx - let ret = acceptEx(socket.SocketHandle, clientSock, addr lpOutputBuf[0], - dwReceiveDataLength, - dwLocalAddressLength, - dwRemoteAddressLength, - addr dwBytesReceived, cast[POVERLAPPED](ol)) - - if not ret: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - failAccept(err) - GC_unref(ol) - else: - completeAccept() - # We don't deallocate ``ol`` here because even though this completed - # immediately poll will still be notified about its completion and it will - # free ``ol``. - - return retFuture - - proc closeSocket*(socket: AsyncFD) = - ## Closes a socket and ensures that it is unregistered. - socket.SocketHandle.close() - getGlobalDispatcher().handles.excl(socket) - - proc unregister*(fd: AsyncFD) = - ## Unregisters ``fd``. - getGlobalDispatcher().handles.excl(fd) - - {.push stackTrace:off.} - proc waitableCallback(param: pointer, - timerOrWaitFired: WINBOOL): void {.stdcall.} = - var p = cast[PostCallbackDataPtr](param) - discard postQueuedCompletionStatus(p.ioPort, timerOrWaitFired.Dword, - ULONG_PTR(p.handleFd), - cast[pointer](p.ovl)) - {.pop.} - - proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: Dword) = - let p = getGlobalDispatcher() - var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword - var hEvent = wsaCreateEvent() - if hEvent == 0: - raiseOSError(osLastError()) - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - pcd.ioPort = p.ioPort - pcd.handleFd = fd - var ol = PCustomOverlapped() - GC_ref(ol) - - ol.data = CompletionData(fd: fd, cb: - proc(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - # we excluding our `fd` because cb(fd) can register own handler - # for this `fd` - p.handles.excl(fd) - # unregisterWait() is called before callback, because appropriate - # winsockets function can re-enable event. - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx - if unregisterWait(pcd.waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - if cb(fd): - # callback returned `true`, so we free all allocated resources - deallocShared(cast[pointer](pcd)) - if not wsaCloseEvent(hEvent): - raiseOSError(osLastError()) - # pcd.ovl will be unrefed in poll(). - else: - # callback returned `false` we need to continue - if p.handles.contains(fd): - # new callback was already registered with `fd`, so we free all - # allocated resources. This happens because in callback `cb` - # addRead/addWrite was called with same `fd`. - deallocShared(cast[pointer](pcd)) - if not wsaCloseEvent(hEvent): - raiseOSError(osLastError()) - else: - # we need to include `fd` again - p.handles.incl(fd) - # and register WaitForSingleObject again - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): - # pcd.ovl will be unrefed in poll() - let err = osLastError() - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - else: - # we incref `pcd.ovl` and `protect` callback one more time, - # because it will be unrefed and disposed in `poll()` after - # callback finishes. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - ) - # We need to protect our callback environment value, so GC will not free it - # accidentally. - ol.data.cell = system.protect(rawEnv(ol.data.cb)) - - # This is main part of `hacky way` is using WSAEventSelect, so `hEvent` - # will be signaled when appropriate `mask` events will be triggered. - if wsaEventSelect(fd.SocketHandle, hEvent, mask) != 0: - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - - pcd.ovl = ol - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) - raiseOSError(err) - p.handles.incl(fd) - - proc addRead*(fd: AsyncFD, cb: Callback) = - ## Start watching the file descriptor for read availability and then call - ## the callback ``cb``. - ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), - ## so if you can avoid it, please do it. Use `addRead` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be - ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.recv() - ## or asyncdispatch.accept(), because they are using IOCP, please use - ## nativesockets.recv() and nativesockets.accept() instead. - ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `read` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE) - - proc addWrite*(fd: AsyncFD, cb: Callback) = - ## Start watching the file descriptor for write availability and then call - ## the callback ``cb``. - ## - ## This is not ``pure`` mechanism for Windows Completion Ports (IOCP), - ## so if you can avoid it, please do it. Use `addWrite` only if really - ## need it (main usecase is adaptation of `unix like` libraries to be - ## asynchronous on Windows). - ## If you use this function, you dont need to use asyncdispatch.send() - ## or asyncdispatch.connect(), because they are using IOCP, please use - ## nativesockets.send() and nativesockets.connect() instead. - ## - ## Be sure your callback ``cb`` returns ``true``, if you want to remove - ## watch of `write` notifications, and ``false``, if you want to continue - ## receiving notifies. - registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE) - - template registerWaitableHandle(p, hEvent, flags, pcd, timeout, - handleCallback) = - let handleFD = AsyncFD(hEvent) - pcd.ioPort = p.ioPort - pcd.handleFd = handleFD - var ol = PCustomOverlapped() - GC_ref(ol) - ol.data.fd = handleFD - ol.data.cb = handleCallback - # We need to protect our callback environment value, so GC will not free it - # accidentally. - ol.data.cell = system.protect(rawEnv(ol.data.cb)) - - pcd.ovl = ol - if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), timeout.Dword, flags): - let err = osLastError() - GC_unref(ol) - deallocShared(cast[pointer](pcd)) - discard closeHandle(hEvent) - raiseOSError(err) - p.handles.incl(handleFD) - - template closeWaitable(handle: untyped) = - let waitFd = pcd.waitFd - deallocShared(cast[pointer](pcd)) - p.handles.excl(fd) - if unregisterWait(waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - discard closeHandle(handle) - raiseOSError(err) - if closeHandle(handle) == 0: - raiseOSError(osLastError()) - - proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Registers callback ``cb`` to be called when timer expired. - ## ``timeout`` - timeout value in milliseconds. - ## ``oneshot`` - `true`, to generate only one timeout event, `false`, to - ## generate timeout events periodically. - - doAssert(timeout > 0) - let p = getGlobalDispatcher() - - var hEvent = createEvent(nil, 1, 0, nil) - if hEvent == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - if oneshot: flags = flags or WT_EXECUTEONLYONCE - - proc timercb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - let res = cb(fd) - if res or oneshot: - closeWaitable(hEvent) - else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - - registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) - - proc addProcess*(pid: int, cb: Callback) = - ## Registers callback ``cb`` to be called when process with pid ``pid`` - ## exited. - let p = getGlobalDispatcher() - let procFlags = SYNCHRONIZE - var hProcess = openProcess(procFlags, 0, pid.Dword) - if hProcess == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - - proc proccb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - closeWaitable(hProcess) - discard cb(fd) - - registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) - - proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent`` object. - ## New ``AsyncEvent`` object is not automatically registered with - ## dispatcher like ``AsyncSocket``. - var sa = SECURITY_ATTRIBUTES( - nLength: sizeof(SECURITY_ATTRIBUTES).cint, - bInheritHandle: 1 - ) - var event = createEvent(addr(sa), 0'i32, 0'i32, nil) - if event == INVALID_HANDLE_VALUE: - raiseOSError(osLastError()) - result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) - result.hEvent = event - - proc setEvent*(ev: AsyncEvent) = - ## Set event ``ev`` to signaled state. - if setEvent(ev.hEvent) == 0: - raiseOSError(osLastError()) - - proc unregister*(ev: AsyncEvent) = - ## Unregisters event ``ev``. - doAssert(ev.hWaiter != 0, "Event is not registered in the queue!") - let p = getGlobalDispatcher() - p.handles.excl(AsyncFD(ev.hEvent)) - if unregisterWait(ev.hWaiter) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - raiseOSError(err) - ev.hWaiter = 0 - - proc close*(ev: AsyncEvent) = - ## Closes event ``ev``. - let res = closeHandle(ev.hEvent) - deallocShared(cast[pointer](ev)) - if res == 0: - raiseOSError(osLastError()) - - proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Registers callback ``cb`` to be called when ``ev`` will be signaled - doAssert(ev.hWaiter == 0, "Event is already registered in the queue!") - - let p = getGlobalDispatcher() - let hEvent = ev.hEvent - - var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData))) - var flags = WT_EXECUTEINWAITTHREAD.Dword - - proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = - if ev.hWaiter != 0: - if cb(fd): - # we need this check to avoid exception, if `unregister(event)` was - # called in callback. - deallocShared(cast[pointer](pcd)) - if ev.hWaiter != 0: - unregister(ev) - else: - # if callback returned `false`, then it wants to be called again, so - # we need to ref and protect `pcd.ovl` again, because it will be - # unrefed and disposed in `poll()`. - GC_ref(pcd.ovl) - pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb)) - else: - # if ev.hWaiter == 0, then event was unregistered before `poll()` call. - deallocShared(cast[pointer](pcd)) - - registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) - ev.hWaiter = pcd.waitFd - - initAll() -else: - import ioselectors - from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, - MSG_NOSIGNAL - const - InitCallbackListSize = 4 # initial size of callbacks sequence, - # associated with file/socket descriptor. - InitDelayedCallbackListSize = 64 # initial size of delayed callbacks - # queue. - type - AsyncFD* = distinct cint - Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} - - AsyncData = object - readList: seq[Callback] - writeList: seq[Callback] - - AsyncEvent* = distinct SelectEvent - - PDispatcher* = ref object of PDispatcherBase - selector: Selector[AsyncData] - {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} - - proc `==`*(x, y: AsyncFD): bool {.borrow.} - proc `==`*(x, y: AsyncEvent): bool {.borrow.} - - template newAsyncData(): AsyncData = - AsyncData( - readList: newSeqOfCap[Callback](InitCallbackListSize), - writeList: newSeqOfCap[Callback](InitCallbackListSize) - ) - - proc newDispatcher*(): PDispatcher = - new result - result.selector = newSelector[AsyncData]() - result.timers.newHeapQueue() - result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize) - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setGlobalDispatcher*(disp: PDispatcher) = - if not gDisp.isNil: - assert gDisp.callbacks.len == 0 - gDisp = disp - initCallSoonProc() - - proc getGlobalDispatcher*(): PDispatcher = - if gDisp.isNil: - setGlobalDispatcher(newDispatcher()) - result = gDisp - - proc register*(fd: AsyncFD) = - let p = getGlobalDispatcher() - var data = newAsyncData() - p.selector.registerHandle(fd.SocketHandle, {}, data) - - proc closeSocket*(sock: AsyncFD) = - let disp = getGlobalDispatcher() - disp.selector.unregister(sock.SocketHandle) - sock.SocketHandle.close() - - proc unregister*(fd: AsyncFD) = - getGlobalDispatcher().selector.unregister(fd.SocketHandle) - - proc unregister*(ev: AsyncEvent) = - getGlobalDispatcher().selector.unregister(SelectEvent(ev)) - - proc addRead*(fd: AsyncFD, cb: Callback) = - let p = getGlobalDispatcher() - var newEvents = {Event.Read} - withData(p.selector, fd.SocketHandle, adata) do: - adata.readList.add(cb) - newEvents.incl(Event.Read) - if len(adata.writeList) != 0: newEvents.incl(Event.Write) - do: - raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, newEvents) - - proc addWrite*(fd: AsyncFD, cb: Callback) = - let p = getGlobalDispatcher() - var newEvents = {Event.Write} - withData(p.selector, fd.SocketHandle, adata) do: - adata.writeList.add(cb) - newEvents.incl(Event.Write) - if len(adata.readList) != 0: newEvents.incl(Event.Read) - do: - raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, newEvents) - - proc hasPendingOperations*(): bool = - let p = getGlobalDispatcher() - not p.selector.isEmpty() or p.timers.len != 0 or p.callbacks.len != 0 - - template processBasicCallbacks(ident, rwlist: untyped) = - # Process pending descriptor's and AsyncEvent callbacks. - # Invoke every callback stored in `rwlist`, until first one - # returned `false`, which means callback wants to stay - # alive. In such case all remaining callbacks will be added - # to `rwlist` again, in the order they have been inserted. - # - # `rwlist` associated with file descriptor MUST BE emptied before - # dispatching callback (See https://github.com/nim-lang/Nim/issues/5128), - # or it can be possible to fall into endless cycle. - var curList: seq[Callback] - - withData(p.selector, ident, adata) do: - shallowCopy(curList, adata.rwlist) - adata.rwlist = newSeqOfCap[Callback](InitCallbackListSize) - - let newLength = max(len(curList), InitCallbackListSize) - var newList = newSeqOfCap[Callback](newLength) - - for cb in curList: - if len(newList) > 0: - newList.add(cb) - else: - if not cb(fd.AsyncFD): - newList.add(cb) - - withData(p.selector, ident, adata) do: - # descriptor still present in queue. - adata.rwlist = newList & adata.rwlist - rLength = len(adata.readList) - wLength = len(adata.writeList) - do: - # descriptor was unregistered in callback via `unregister()`. - rLength = -1 - wLength = -1 - - template processCustomCallbacks(ident: untyped) = - # Process pending custom event callbacks. Custom events are - # {Event.Timer, Event.Signal, Event.Process, Event.Vnode}. - # There can be only one callback registered with one descriptor, - # so there no need to iterate over list. - var curList: seq[Callback] - - withData(p.selector, ident, adata) do: - shallowCopy(curList, adata.readList) - adata.readList = newSeqOfCap[Callback](InitCallbackListSize) - - let newLength = len(curList) - var newList = newSeqOfCap[Callback](newLength) - - var cb = curList[0] - if not cb(fd.AsyncFD): - newList.add(cb) - - withData(p.selector, ident, adata) do: - # descriptor still present in queue. - adata.readList = newList & adata.readList - if len(adata.readList) == 0: - # if no callbacks registered with descriptor, unregister it. - p.selector.unregister(fd) - do: - # descriptor was unregistered in callback via `unregister()`. - discard - - proc poll*(timeout = 500) = - var keys: array[64, ReadyKey] - - let p = getGlobalDispatcher() - when ioselSupportedPlatform: - let customSet = {Event.Timer, Event.Signal, Event.Process, - Event.Vnode} - - if p.selector.isEmpty() and p.timers.len == 0 and p.callbacks.len == 0: - raise newException(ValueError, - "No handles or timers registered in dispatcher.") - - if not p.selector.isEmpty(): - var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) - var i = 0 - while i < count: - var custom = false - let fd = keys[i].fd - let events = keys[i].events - var rLength = 0 # len(data.readList) after callback - var wLength = 0 # len(data.writeList) after callback - - if Event.Read in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - - if Event.Write in events or events == {Event.Error}: - processBasicCallbacks(fd, writeList) - - if Event.User in events or events == {Event.Error}: - processBasicCallbacks(fd, readList) - custom = true - if rLength == 0: - p.selector.unregister(fd) - - when ioselSupportedPlatform: - if (customSet * events) != {}: - custom = true - processCustomCallbacks(fd) - - # because state `data` can be modified in callback we need to update - # descriptor events with currently registered callbacks. - if not custom: - var newEvents: set[Event] = {} - if rLength != -1 and wLength != -1: - if rLength > 0: incl(newEvents, Event.Read) - if wLength > 0: incl(newEvents, Event.Write) - p.selector.updateHandle(SocketHandle(fd), newEvents) - inc(i) - - # Timer processing. - processTimers(p) - # Callback queue processing - processPendingCallbacks(p) - - proc recv*(socket: AsyncFD, size: int, - flags = {SocketFlag.SafeDisconn}): Future[string] = - var retFuture = newFuture[string]("recv") - - var readBuffer = newString(size) - - proc cb(sock: AsyncFD): bool = - result = true - let res = recv(sock.SocketHandle, addr readBuffer[0], size.cint, - flags.toOSFlags()) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete("") - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - elif res == 0: - # Disconnected - retFuture.complete("") - else: - readBuffer.setLen(res) - retFuture.complete(readBuffer) - # TODO: The following causes a massive slowdown. - #if not cb(socket): - addRead(socket, cb) - return retFuture - - proc recvInto*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = - var retFuture = newFuture[int]("recvInto") - - proc cb(sock: AsyncFD): bool = - result = true - let res = recv(sock.SocketHandle, buf, size.cint, - flags.toOSFlags()) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete(0) - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - retFuture.complete(res) - # TODO: The following causes a massive slowdown. - #if not cb(socket): - addRead(socket, cb) - return retFuture - - proc send*(socket: AsyncFD, buf: pointer, size: int, - flags = {SocketFlag.SafeDisconn}): Future[void] = - var retFuture = newFuture[void]("send") - - var written = 0 - - proc cb(sock: AsyncFD): bool = - result = true - let netSize = size-written - var d = cast[cstring](buf) - let res = send(sock.SocketHandle, addr d[written], netSize.cint, - MSG_NOSIGNAL) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - written.inc(res) - if res != netSize: - result = false # We still have data to send. - else: - retFuture.complete() - # TODO: The following causes crashes. - #if not cb(socket): - addWrite(socket, cb) - return retFuture - - proc send*(socket: AsyncFD, data: string, - flags = {SocketFlag.SafeDisconn}): Future[void] = - var retFuture = newFuture[void]("send") - - var written = 0 - - proc cb(sock: AsyncFD): bool = - result = true - let netSize = data.len-written - var d = data.cstring - let res = send(sock.SocketHandle, addr d[written], netSize.cint, - MSG_NOSIGNAL) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - if flags.isDisconnectionError(lastError): - retFuture.complete() - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - written.inc(res) - if res != netSize: - result = false # We still have data to send. - else: - retFuture.complete() - # TODO: The following causes crashes. - #if not cb(socket): - addWrite(socket, cb) - return retFuture - - proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr, - saddrLen: SockLen, - flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` of size ``size`` in bytes to specified destination - ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``. - ## The returned future will complete once all data has been sent. - var retFuture = newFuture[void]("sendTo") - - # we will preserve address in our stack - var staddr: array[128, char] # SOCKADDR_STORAGE size is 128 bytes - var stalen = saddrLen - zeroMem(addr(staddr[0]), 128) - copyMem(addr(staddr[0]), saddr, saddrLen) - - proc cb(sock: AsyncFD): bool = - result = true - let res = sendto(sock.SocketHandle, data, size, MSG_NOSIGNAL, - cast[ptr SockAddr](addr(staddr[0])), stalen) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false # We still want this callback to be called. - else: - retFuture.complete() - - addWrite(socket, cb) - return retFuture - - proc recvFromInto*(socket: AsyncFD, data: pointer, size: int, - saddr: ptr SockAddr, saddrLen: ptr SockLen, - flags = {SocketFlag.SafeDisconn}): Future[int] = - ## Receives a datagram data from ``socket`` into ``data``, which must - ## be at least of size ``size`` in bytes, address of datagram's sender - ## will be stored into ``saddr`` and ``saddrLen``. Returned future will - ## complete once one datagram has been received, and will return size - ## of packet received. - var retFuture = newFuture[int]("recvFromInto") - proc cb(sock: AsyncFD): bool = - result = true - let res = recvfrom(sock.SocketHandle, data, size.cint, flags.toOSFlags(), - saddr, saddrLen) - if res < 0: - let lastError = osLastError() - if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - result = false - else: - retFuture.complete(res) - addRead(socket, cb) - return retFuture - - proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}): - Future[tuple[address: string, client: AsyncFD]] = - var retFuture = newFuture[tuple[address: string, - client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = - result = true - var sockAddress: Sockaddr_storage - var addrLen = sizeof(sockAddress).Socklen - var client = accept(sock.SocketHandle, - cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) - if client == osInvalidSocket: - let lastError = osLastError() - assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} - if lastError.int32 == EINTR: - return false - else: - if flags.isDisconnectionError(lastError): - return false - else: - retFuture.fail(newException(OSError, osErrorMsg(lastError))) - else: - try: - let address = getAddrString(cast[ptr SockAddr](addr sockAddress)) - register(client.AsyncFD) - retFuture.complete((address, client.AsyncFD)) - except: - # getAddrString may raise - client.close() - retFuture.fail(getCurrentException()) - addRead(socket, cb) - return retFuture - - when ioselSupportedPlatform: - - proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = - ## Start watching for timeout expiration, and then call the - ## callback ``cb``. - ## ``timeout`` - time in milliseconds, - ## ``oneshot`` - if ``true`` only one event will be dispatched, - ## if ``false`` continuous events every ``timeout`` milliseconds. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerTimer(timeout, oneshot, data) - - proc addSignal*(signal: int, cb: Callback) = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb``. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerSignal(signal, data) - - proc addProcess*(pid: int, cb: Callback) = - ## Start watching for process exit with pid ``pid``, and then call - ## the callback ``cb``. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerProcess(pid, data) - - proc newAsyncEvent*(): AsyncEvent = - ## Creates new ``AsyncEvent``. - result = AsyncEvent(newSelectEvent()) - - proc setEvent*(ev: AsyncEvent) = - ## Sets new ``AsyncEvent`` to signaled state. - setEvent(SelectEvent(ev)) - - proc close*(ev: AsyncEvent) = - ## Closes ``AsyncEvent`` - close(SelectEvent(ev)) - - proc addEvent*(ev: AsyncEvent, cb: Callback) = - ## Start watching for event ``ev``, and call callback ``cb``, when - ## ev will be set to signaled state. - let p = getGlobalDispatcher() - var data = newAsyncData() - data.readList.add(cb) - p.selector.registerEvent(SelectEvent(ev), data) - -# Common procedures between current and upcoming asyncdispatch -include includes.asynccommon - -proc sleepAsync*(ms: int): Future[void] = - ## Suspends the execution of the current async procedure for the next - ## ``ms`` milliseconds. - var retFuture = newFuture[void]("sleepAsync") - let p = getGlobalDispatcher() - p.timers.push((epochTime() + (ms / 1000), retFuture)) - return retFuture - -proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] = - ## Returns a future which will complete once ``fut`` completes or after - ## ``timeout`` milliseconds has elapsed. - ## - ## If ``fut`` completes first the returned future will hold true, - ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned - ## future will hold false. - - var retFuture = newFuture[bool]("asyncdispatch.`withTimeout`") - var timeoutFuture = sleepAsync(timeout) - fut.callback = - proc () = - if not retFuture.finished: retFuture.complete(true) - timeoutFuture.callback = - proc () = - if not retFuture.finished: retFuture.complete(false) - return retFuture - -proc accept*(socket: AsyncFD, - flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection. - ## The future will complete when the connection is successfully accepted. - var retFut = newFuture[AsyncFD]("accept") - var fut = acceptAddr(socket, flags) - fut.callback = - proc (future: Future[tuple[address: string, client: AsyncFD]]) = - assert future.finished - if future.failed: - retFut.fail(future.error) - else: - retFut.complete(future.read.client) - return retFut - -# -- Await Macro -include asyncmacro - -proc readAll*(future: FutureStream[string]): Future[string] {.async.} = - ## Returns a future that will complete when all the string data from the - ## specified future stream is retrieved. - result = "" - while true: - let (hasValue, value) = await future.read() - if hasValue: - result.add(value) - else: - break - -proc recvLine*(socket: AsyncFD): Future[string] {.async.} = - ## Reads a line of data from ``socket``. Returned future will complete once - ## a full line is read or an error occurs. - ## - ## If a full line is read ``\r\L`` is not - ## added to ``line``, however if solely ``\r\L`` is read then ``line`` - ## will be set to it. - ## - ## If the socket is disconnected, ``line`` will be set to ``""``. - ## - ## If the socket is disconnected in the middle of a line (before ``\r\L`` - ## is read) then line will be set to ``""``. - ## The partial line **will be lost**. - ## - ## **Warning**: This assumes that lines are delimited by ``\r\L``. - ## - ## **Note**: This procedure is mostly used for testing. You likely want to - ## use ``asyncnet.recvLine`` instead. - - template addNLIfEmpty(): typed = - if result.len == 0: - result.add("\c\L") - - result = "" - var c = "" - while true: - c = await recv(socket, 1) - if c.len == 0: - return "" - if c == "\r": - c = await recv(socket, 1) - assert c == "\l" - addNLIfEmpty() - return - elif c == "\L": - addNLIfEmpty() - return - add(result, c) - -proc callSoon(cbproc: proc ()) = - ## Schedule `cbproc` to be called as soon as possible. - ## The callback is called when control returns to the event loop. - getGlobalDispatcher().callbacks.addLast(cbproc) - -proc runForever*() = - ## Begins a never ending global dispatcher poll loop. - while true: - poll() - -proc waitFor*[T](fut: Future[T]): T = - ## **Blocks** the current thread until the specified future completes. - while not fut.finished: - poll() - - fut.read -- cgit 1.4.1-2-gfad0 From 94fe5bd1184bd416f54d3fffc0227df6c3d7883a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 24 Nov 2017 22:47:38 +0000 Subject: Fix asyncdispatch docgen --- lib/pure/asyncdispatch.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cee692980..65004cbe0 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -281,7 +281,7 @@ when defined(windows) or defined(nimdoc): if createIoCompletionPort(fd.Handle, p.ioPort, cast[CompletionKey](fd), 1) == 0: raiseOSError(osLastError()) - p.handles.incl(fd) + p.handles.incl(fd.AsyncFD) return fd.AsyncFD proc verifyPresence(fd: AsyncFD) = @@ -1638,4 +1638,4 @@ proc waitFor*[T](fut: Future[T]): T = fut.read -{.deprecated: [setEvent: trigger].} \ No newline at end of file +{.deprecated: [setEvent: trigger].} -- cgit 1.4.1-2-gfad0 From 86fb8bf723194fb1c5a21baa18b7624bfd9ca9d6 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 17 Jan 2018 16:48:02 +0000 Subject: Revert 3db460f5045e790b54ea382 as requested by @Araq. --- changelog.md | 5 ----- lib/pure/asyncdispatch.nim | 28 ++++++++-------------------- lib/pure/asyncfile.nim | 11 ++++++----- lib/pure/asyncnet.nim | 2 -- 4 files changed, 14 insertions(+), 32 deletions(-) diff --git a/changelog.md b/changelog.md index 4dc2310c3..ad3773b0e 100644 --- a/changelog.md +++ b/changelog.md @@ -14,13 +14,8 @@ module. - `newAsyncSocket` taking an `AsyncFD` now runs `setBlocking(false)` on the fd. - -### Library changes - - The `ReadyKey` type in the selectors module now contains an ``errorCode`` field to help distinguish between ``Event.Error`` events. -- The `AsyncFD` type now reflects the fact that the underlying FD is registered - in the async dispatcher. - Implemented an `accept` proc that works on a `SocketHandle` in ``nativesockets``. - Implemented ``getIoHandler`` proc in the ``asyncdispatch`` module that allows diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 0539706a9..598b0195b 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -222,7 +222,7 @@ when defined(windows) or defined(nimdoc): PCustomOverlapped* = ref CustomOverlapped - AsyncFD* = distinct int ## An FD that is registered in the dispatcher. + AsyncFD* = distinct int PostCallbackData = object ioPort: Handle @@ -270,22 +270,14 @@ when defined(windows) or defined(nimdoc): ## (Unix) for the specified dispatcher. return disp.ioPort - proc register*(fd: cint | SocketHandle | AsyncFD): AsyncFD {.discardable.} = + proc register*(fd: AsyncFD) = ## Registers ``fd`` with the dispatcher. - ## - ## By convention, an ``AsyncFD`` is said to be already registered in the - ## dispatcher. This procedure will raise an exception if ``fd`` has already - ## been registered, but only if the type of the ``fd`` isn't ``AsyncFD``. let p = getGlobalDispatcher() - when fd is AsyncFD: - if fd in p.handles: - return if createIoCompletionPort(fd.Handle, p.ioPort, cast[CompletionKey](fd), 1) == 0: raiseOSError(osLastError()) - p.handles.incl(fd.AsyncFD) - return fd.AsyncFD + p.handles.incl(fd) proc verifyPresence(fd: AsyncFD) = ## Ensures that file descriptor has been registered with the dispatcher. @@ -771,8 +763,8 @@ when defined(windows) or defined(nimdoc): ## Unregisters ``fd``. getGlobalDispatcher().handles.excl(fd) - proc contains*(disp: PDispatcher, fd: AsyncFd | SocketHandle): bool = - return fd.SocketHandle in disp.handles + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = + return fd in disp.handles {.push stackTrace:off.} proc waitableCallback(param: pointer, @@ -994,7 +986,7 @@ when defined(windows) or defined(nimdoc): proc newAsyncEvent*(): AsyncEvent = ## Creates a new thread-safe ``AsyncEvent`` object. ## - ## New ``AsyncEvent`` object is not automatically registered with # TODO: Why? -- DP + ## New ``AsyncEvent`` object is not automatically registered with ## dispatcher like ``AsyncSocket``. var sa = SECURITY_ATTRIBUTES( nLength: sizeof(SECURITY_ATTRIBUTES).cint, @@ -1115,14 +1107,10 @@ else: proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = return disp.selector - proc register*(fd: cint | SocketHandle | AsyncFD): AsyncFD {.discardable.} = + proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() - when fd is AsyncFD: - if fd.SocketHandle in p.selector: - return var data = newAsyncData() p.selector.registerHandle(fd.SocketHandle, {}, data) - return fd.AsyncFD proc closeSocket*(sock: AsyncFD) = let disp = getGlobalDispatcher() @@ -1135,7 +1123,7 @@ else: proc unregister*(ev: AsyncEvent) = getGlobalDispatcher().selector.unregister(SelectEvent(ev)) - proc contains*(disp: PDispatcher, fd: AsyncFd | SocketHandle): bool = + proc contains*(disp: PDispatcher, fd: AsyncFd): bool = return fd.SocketHandle in disp.selector proc addRead*(fd: AsyncFD, cb: Callback) = diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 3a9d26378..6ce9e8f75 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -81,10 +81,11 @@ proc getFileSize*(f: AsyncFile): int64 = else: result = lseek(f.fd.cint, 0, SEEK_END) -proc newAsyncFile*(fd: cint | AsyncFd): AsyncFile = +proc newAsyncFile*(fd: AsyncFd): AsyncFile = ## Creates `AsyncFile` with a previously opened file descriptor `fd`. new result - result.fd = register(fd) + result.fd = fd + register(fd) proc openAsync*(filename: string, mode = fmRead): AsyncFile = ## Opens a file specified by the path in ``filename`` using @@ -105,7 +106,7 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) - result = newAsyncFile(fd.cint) + result = newAsyncFile(fd.AsyncFd) if mode == fmAppend: result.offset = getFileSize(result) @@ -115,10 +116,10 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = # RW (Owner), RW (Group), R (Other) let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH let fd = open(filename, flags, perm) - if fd.cint == -1: + if fd == -1: raiseOSError(osLastError()) - result = newAsyncFile(fd) + result = newAsyncFile(fd.AsyncFd) proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = ## Read ``size`` bytes from the specified file asynchronously starting at diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index adee34b9e..bdbf47004 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -146,8 +146,6 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET, ## **Note**: This procedure will **NOT** register ``fd`` with the global ## async dispatcher. You need to do this manually. If you have used ## ``newAsyncNativeSocket`` to create ``fd`` then it's already registered. - ## The reason for this is that the ``AsyncFD`` type is a special type for an - ## FD that signifies that its been registered. assert fd != osInvalidSocket.AsyncFD new(result) result.fd = fd.SocketHandle -- cgit 1.4.1-2-gfad0 From 12b11fd848f68ce0543c6e0dc705b6d445e00146 Mon Sep 17 00:00:00 2001 From: cooldome Date: Mon, 29 Jan 2018 04:59:49 +0000 Subject: Fix compiler crash on try expression with infix as (Fixes #7116) (#7112) * Fix compiler crash * make sure type is not lost --- compiler/transf.nim | 2 +- tests/exception/texcas.nim | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 14ff58c90..94899c5d4 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -716,7 +716,7 @@ proc transformExceptBranch(c: PTransf, n: PNode): PTransNode = result = transformSons(c, n) if n[0].isInfixAs(): let excTypeNode = n[0][1] - let actions = newTransNode(nkStmtList, n[1].info, 2) + let actions = newTransNode(nkStmtListExpr, n[1], 2) # Generating `let exc = (excType)(getCurrentException())` # -> getCurrentException() let excCall = PTransNode(callCodegenProc("getCurrentException", ast.emptyNode)) diff --git a/tests/exception/texcas.nim b/tests/exception/texcas.nim index 4b4ebe448..fee45af3f 100644 --- a/tests/exception/texcas.nim +++ b/tests/exception/texcas.nim @@ -21,5 +21,13 @@ proc test2() = testTemplate(Exception) doAssert(not declared(foobar)) + +proc testTryAsExpr(i: int) = + let x = try: i + except ValueError as ex: + echo(ex.msg) + -1 + test[Exception]() -test2() \ No newline at end of file +test2() +testTryAsExpr(5) -- cgit 1.4.1-2-gfad0 From 02c1f120eb3aa853012aaaa3bf368107347d5217 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 30 Jan 2018 01:41:32 +0300 Subject: Fixes nimsuggest#78 (#7151) --- compiler/pragmas.nim | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 810c4c416..cf289a506 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -658,7 +658,7 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym = proc semCustomPragma(c: PContext, n: PNode): PNode = assert(n.kind in nkPragmaCallKinds + {nkIdent}) - + if n.kind == nkIdent: result = newTree(nkCall, n) elif n.kind == nkExprColonExpr: @@ -667,14 +667,15 @@ proc semCustomPragma(c: PContext, n: PNode): PNode = else: result = n - result = c.semOverloadedCall(c, result, n, {skTemplate}, {}) - if sfCustomPragma notin result[0].sym.flags: + let r = c.semOverloadedCall(c, result, n, {skTemplate}, {}) + if r.isNil or sfCustomPragma notin r[0].sym.flags: invalidPragma(n) - - if n.kind == nkIdent: - result = result[0] - elif n.kind == nkExprColonExpr: - result.kind = n.kind # pragma(arg) -> pragma: arg + else: + result = r + if n.kind == nkIdent: + result = result[0] + elif n.kind == nkExprColonExpr: + result.kind = n.kind # pragma(arg) -> pragma: arg proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, validPragmas: TSpecialWords): bool = @@ -1016,7 +1017,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, else: sym.flags.incl sfUsed of wLiftLocals: discard else: invalidPragma(it) - else: + else: n.sons[i] = semCustomPragma(c, it) -- cgit 1.4.1-2-gfad0 From 8fdc2919360653f5444752b3fd80f12bfe4816ba Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 30 Jan 2018 10:30:18 +0100 Subject: fixes the tracking of 'occupied memory' --- lib/system/alloc.nim | 102 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 6 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index f1f3cdb6c..33e7b3898 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -104,7 +104,7 @@ type slBitmap: array[RealFli, uint32] matrix: array[RealFli, array[MaxSli, PBigChunk]] llmem: PLLChunk - currMem, maxMem, freeMem: int # memory sizes (allocated from OS) + currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly chunkStarts: IntSet root, deleted, last, freeAvlNodes: PAvlNode @@ -421,7 +421,7 @@ const nimMaxHeap {.intdefine.} = 0 proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when not defined(emscripten): if not a.blockChunkSizeIncrease: - let usedMem = a.currMem # - a.freeMem + let usedMem = a.occ #a.currMem # - a.freeMem when nimMaxHeap != 0: if usedMem > nimMaxHeap * 1024 * 1024: raiseOutOfMem() @@ -567,7 +567,6 @@ proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) = addChunkToMatrix(a, rest) proc getBigChunk(a: var MemRegion, size: int): PBigChunk = - # use first fit for now: sysAssert(size > 0, "getBigChunk 2") var size = size # roundup(size, PageSize) var fl, sl: int @@ -627,6 +626,85 @@ else: c = c.next result = true +when false: + var + rsizes: array[50_000, int] + rsizesLen: int + + proc trackSize(size: int) = + rsizes[rsizesLen] = size + inc rsizesLen + + proc untrackSize(size: int) = + for i in 0 .. rsizesLen-1: + if rsizes[i] == size: + rsizes[i] = rsizes[rsizesLen-1] + dec rsizesLen + return + c_fprintf(stdout, "%ld\n", size) + sysAssert(false, "untracked size!") +else: + template trackSize(x) = discard + template untrackSize(x) = discard + +when false: + # not yet used by the GCs + proc rawTryAlloc(a: var MemRegion; requestedSize: int): pointer = + sysAssert(allocInv(a), "rawAlloc: begin") + sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") + sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small") + var size = roundup(requestedSize, MemAlign) + inc a.occ, size + trackSize(size) + sysAssert(size >= requestedSize, "insufficient allocated size!") + #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size) + if size <= SmallChunkSize-smallChunkOverhead(): + # allocate a small block: for small chunks, we use only its next pointer + var s = size div MemAlign + var c = a.freeSmallChunks[s] + if c == nil: + result = nil + else: + sysAssert c.size == size, "rawAlloc 6" + if c.freeList == nil: + sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, + "rawAlloc 7") + result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + inc(c.acc, size) + else: + result = c.freeList + sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") + c.freeList = c.freeList.next + dec(c.free, size) + sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + if c.free < size: + listRemove(a.freeSmallChunks[s], c) + sysAssert(allocInv(a), "rawAlloc: end listRemove test") + sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + size == 0, "rawAlloc 21") + sysAssert(allocInv(a), "rawAlloc: end small size") + else: + inc size, bigChunkOverhead() + var fl, sl: int + mappingSearch(size, fl, sl) + sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk") + let c = findSuitableBlock(a, fl, sl) + if c != nil: + removeChunkFromMatrix2(a, c, fl, sl) + if c.size >= size + PageSize: + splitChunk(a, c, size) + # set 'used' to to true: + c.prevSize = 1 + incl(a, a.chunkStarts, pageIndex(c)) + dec(a.freeMem, size) + result = addr(c.data) + sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + if a.root == nil: a.root = getBottom(a) + add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + else: + result = nil + proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(allocInv(a), "rawAlloc: begin") sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken") @@ -676,6 +754,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") + inc a.occ, size + trackSize(c.size) else: size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize) # allocate a large block @@ -687,6 +767,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") if a.root == nil: a.root = getBottom(a) add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + inc a.occ, c.size + trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") sysAssert(allocInv(a), "rawAlloc: end") when logAlloc: cprintf("var pointer_%p = alloc(%ld)\n", result, requestedSize) @@ -703,6 +785,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) = # `p` is within a small chunk: var c = cast[PSmallChunk](c) var s = c.size + dec a.occ, s + untrackSize(s) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 3") var f = cast[ptr FreeCell](p) @@ -733,6 +818,9 @@ proc rawDealloc(a: var MemRegion, p: pointer) = when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead()) # free big chunk var c = cast[PBigChunk](c) + dec a.occ, c.size + untrackSize(c.size) + sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case B)" a.deleted = getBottom(a) del(a, a.root, cast[int](addr(c.data))) freeBigChunk(a, c) @@ -851,7 +939,8 @@ proc deallocOsPages(a: var MemRegion) = proc getFreeMem(a: MemRegion): int {.inline.} = result = a.freeMem proc getTotalMem(a: MemRegion): int {.inline.} = result = a.currMem proc getOccupiedMem(a: MemRegion): int {.inline.} = - result = a.currMem - a.freeMem + result = a.occ + # a.currMem - a.freeMem # ---------------------- thread memory region ------------------------------- @@ -893,7 +982,7 @@ template instantiateForRegion(allocator: untyped) = #sysAssert(result == countFreeMem()) proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return getTotalMem() - getFreeMem() + proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() proc getMaxMem*(): int = return getMaxMem(allocator) # -------------------- shared heap region ---------------------------------- @@ -944,7 +1033,8 @@ template instantiateForRegion(allocator: untyped) = sharedMemStatsShared(sharedHeap.currMem) proc getOccupiedSharedMem(): int = - sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) + sharedMemStatsShared(sharedHeap.occ) + #sharedMemStatsShared(sharedHeap.currMem - sharedHeap.freeMem) {.pop.} {.pop.} -- cgit 1.4.1-2-gfad0 From 4f08ed63cfc81f8da6658b2823c0d7fbc7f959ab Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 30 Jan 2018 10:30:43 +0100 Subject: M&S GC: collect earlier under memory pressure --- lib/system/gc_ms.nim | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 6f28601d0..75f9c6749 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -125,7 +125,7 @@ when BitsPerPage mod (sizeof(int)*8) != 0: {.error: "(BitsPerPage mod BitsPerUnit) should be zero!".} # forward declarations: -proc collectCT(gch: var GcHeap) {.benign.} +proc collectCT(gch: var GcHeap; size: int) {.benign.} proc forAllChildren(cell: PCell, op: WalkOp) {.benign.} proc doOperation(p: pointer, op: WalkOp) {.benign.} proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.} @@ -277,7 +277,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = incTypeSize typ, size acquire(gch) gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1") - collectCT(gch) + collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT @@ -332,7 +332,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = acquire(gch) - collectCT(gch) + collectCT(gch, newsize + sizeof(Cell)) var ol = usrToCell(old) sysAssert(ol.typ != nil, "growObj: 1") gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") @@ -494,8 +494,9 @@ proc collectCTBody(gch: var GcHeap) = gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold) sysAssert(allocInv(gch.region), "collectCT: end") -proc collectCT(gch: var GcHeap) = - if getOccupiedMem(gch.region) >= gch.cycleThreshold and gch.recGcLock == 0: +proc collectCT(gch: var GcHeap; size: int) = + if (getOccupiedMem(gch.region) >= gch.cycleThreshold or + size > getFreeMem(gch.region)) and gch.recGcLock == 0: collectCTBody(gch) when not defined(useNimRtl): @@ -530,7 +531,7 @@ when not defined(useNimRtl): acquire(gch) var oldThreshold = gch.cycleThreshold gch.cycleThreshold = 0 # forces cycle collection - collectCT(gch) + collectCT(gch, 0) gch.cycleThreshold = oldThreshold release(gch) -- cgit 1.4.1-2-gfad0 From d84ace8a5b7d1a875b5beaf1330e1d80931f3ef9 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 30 Jan 2018 17:25:41 +0300 Subject: Fixes #7140 (#7154) --- compiler/sempass2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index d427750e4..e560c18c8 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -747,7 +747,7 @@ proc track(tracked: PEffects, n: PNode) = # may not look like an assignment, but it is: let arg = n.sons[1] initVarViaNew(tracked, arg) - if {tfNeedsInit} * arg.typ.lastSon.flags != {}: + if arg.typ.len != 0 and {tfNeedsInit} * arg.typ.lastSon.flags != {}: if a.sym.magic == mNewSeq and n[2].kind in {nkCharLit..nkUInt64Lit} and n[2].intVal == 0: # var s: seq[notnil]; newSeq(s, 0) is a special case! -- cgit 1.4.1-2-gfad0 From a37e47d069a6403d7e24ba8691969df3a85d066d Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 30 Jan 2018 17:55:11 +0300 Subject: Undeprecate readChar. Closes #7072 (#7156) --- lib/system.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 4e071e802..079b06b48 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3025,9 +3025,9 @@ when not defined(JS): #and not defined(nimscript): proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. - proc readChar*(f: File): char {.tags: [ReadIOEffect], deprecated.} - ## Reads a single character from the stream `f`. **Deprecated** since - ## version 0.16.2. Use some variant of ``readBuffer`` instead. + proc readChar*(f: File): char {.tags: [ReadIOEffect].} + ## Reads a single character from the stream `f`. Should not be used in + ## performance sensitive code. proc flushFile*(f: File) {.tags: [WriteIOEffect].} ## Flushes `f`'s buffer. -- cgit 1.4.1-2-gfad0 From a605fa8ba6e0b6205d3f3d8b141242874451957b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 30 Jan 2018 19:51:38 +0100 Subject: fixes #7129 --- compiler/cgen.nim | 1 - compiler/commands.nim | 2 +- compiler/jsgen.nim | 6 +++--- compiler/options.nim | 1 + 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 630426cfd..12e640d96 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -908,7 +908,6 @@ proc addIntTypes(result: var Rope) {.inline.} = platform.CPU[targetCPU].intSize.rope]) proc getCopyright(cfile: Cfile): Rope = - const copyrightYear = "2017" if optCompileOnly in gGlobalOptions: result = ("/* Generated by Nim Compiler v$1 */$N" & "/* (c) " & copyrightYear & " Andreas Rumpf */$N" & diff --git a/compiler/commands.nim b/compiler/commands.nim index 45c0f586c..aa7d13703 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -54,7 +54,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; const HelpMessage = "Nim Compiler Version $1 [$2: $3]\n" & - "Copyright (c) 2006-2017 by Andreas Rumpf\n" + "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n" const Usage = slurp"../doc/basicopt.txt".replace("//", "") diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d58008516..6f40e7031 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2390,7 +2390,7 @@ proc genHeader(target: TTarget): Rope = if target == targetJS: result = ( "/* Generated by the Nim Compiler v$1 */$n" & - "/* (c) 2017 Andreas Rumpf */$n$n" & + "/* (c) " & copyrightYear & " Andreas Rumpf */$n$n" & "var framePtr = null;$n" & "var excHandler = 0;$n" & "var lastJSError = null;$n" & @@ -2406,7 +2406,7 @@ proc genHeader(target: TTarget): Rope = else: result = (" Date: Wed, 31 Jan 2018 14:29:08 +0100 Subject: nimTypeNames feature: show generics properly --- compiler/ccgtypes.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 8a60143fb..3a001ebf7 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -970,7 +970,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) discard cgsym(m, "TNimType") if isDefined("nimTypeNames"): - var typename = typeToString(origType, preferName) + var typename = typeToString(if origType.typeInst != nil: origType.typeInst + else: origType, preferName) if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil: typename = "anon ref object from " & $origType.skipTypes(skipPtrs).sym.info addf(m.s[cfsTypeInit3], "$1.name = $2;$n", -- cgit 1.4.1-2-gfad0 From 94038545bec1bffa9bf045be8a5e52b79e9f217c Mon Sep 17 00:00:00 2001 From: GULPF Date: Wed, 31 Jan 2018 16:29:42 +0100 Subject: Fixes codegen bug with literal negative zero, fixes #7079 (#7158) * Fixes #7079 * Fix handling of neg zero during constant folding --- compiler/jsgen.nim | 16 +++++++++++----- compiler/rodutils.nim | 16 ++++++++++------ compiler/semfold.nim | 1 + tests/ccgbugs/t7079.nim | 9 +++++++++ 4 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 tests/ccgbugs/t7079.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 6f40e7031..75880a15a 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2284,11 +2284,17 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = r.kind = resExpr of nkFloatLit..nkFloat64Lit: let f = n.floatVal - if f != f: r.res = rope"NaN" - elif f == 0.0: r.res = rope"0.0" - elif f == 0.5 * f: - if f > 0.0: r.res = rope"Infinity" - else: r.res = rope"-Infinity" + case classify(f) + of fcNaN: + r.res = rope"NaN" + of fcNegZero: + r.res = rope"-0.0" + of fcZero: + r.res = rope"0.0" + of fcInf: + r.res = rope"Infinity" + of fcNegInf: + r.res = rope"-Infinity" else: r.res = rope(f.toStrMaxPrecision) r.kind = resExpr of nkCallKinds: diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index 0456e9349..0d9c7112f 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -8,18 +8,22 @@ # ## Serialization utilities for the compiler. -import strutils +import strutils, math proc c_snprintf(s: cstring; n:uint; frmt: cstring): cint {.importc: "snprintf", header: "", nodecl, varargs.} proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string = - if f != f: + case classify(f) + of fcNaN: result = "NAN" - elif f == 0.0: + of fcNegZero: + result = "-0.0" & literalPostfix + of fcZero: result = "0.0" & literalPostfix - elif f == 0.5 * f: - if f > 0.0: result = "INF" - else: result = "-INF" + of fcInf: + result = "INF" + of fcNegInf: + result = "-INF" else: when defined(nimNoArrayToCstringConversion): result = newString(81) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 55cdc334c..59c55010f 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -225,6 +225,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode = of mDivF64: if getFloat(b) == 0.0: if getFloat(a) == 0.0: result = newFloatNodeT(NaN, n) + elif getFloat(b).classify == fcNegZero: result = newFloatNodeT(-Inf, n) else: result = newFloatNodeT(Inf, n) else: result = newFloatNodeT(getFloat(a) / getFloat(b), n) diff --git a/tests/ccgbugs/t7079.nim b/tests/ccgbugs/t7079.nim new file mode 100644 index 000000000..bfa1b77a6 --- /dev/null +++ b/tests/ccgbugs/t7079.nim @@ -0,0 +1,9 @@ +discard """ + action: run + targets: '''c js''' +""" + +import math +let x = -0.0 +doAssert classify(x) == fcNegZero +doAssert classify(1 / -0.0) == fcNegInf \ No newline at end of file -- cgit 1.4.1-2-gfad0 From 8d8df5807b4641410fb04a0fc2199f965f450909 Mon Sep 17 00:00:00 2001 From: pqflx3 Date: Wed, 31 Jan 2018 10:38:37 -0500 Subject: Fixes #7121 (#7148) * Replace ftell and fseek with (windows) _ftelli64, _fseeki64 and (posix) ftello, fseeko * disable large file test --- lib/system/sysio.nim | 22 ++++++++++++++-------- tests/system/io.nim | 25 ++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index f638b299c..71cbb1c21 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -47,10 +47,16 @@ when not declared(c_fwrite): # C routine that is used here: proc c_fread(buf: pointer, size, n: csize, f: File): csize {. importc: "fread", header: "", tags: [ReadIOEffect].} -proc c_fseek(f: File, offset: clong, whence: cint): cint {. - importc: "fseek", header: "", tags: [].} -proc c_ftell(f: File): clong {. - importc: "ftell", header: "", tags: [].} +when defined(windows): + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "_fseeki64", header: "", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "", tags: [].} +else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseeko", header: "", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftello", header: "", tags: [].} proc c_ferror(f: File): cint {. importc: "ferror", header: "", tags: [].} proc c_setvbuf(f: File, buf: pointer, mode: cint, size: csize): cint {. @@ -210,12 +216,12 @@ proc readAllBuffer(file: File): string = result.add(buffer) break -proc rawFileSize(file: File): int = +proc rawFileSize(file: File): int64 = # this does not raise an error opposed to `getFileSize` var oldPos = c_ftell(file) discard c_fseek(file, 0, 2) # seek the end of the file result = c_ftell(file) - discard c_fseek(file, clong(oldPos), 0) + discard c_fseek(file, oldPos, 0) proc endOfFile(f: File): bool = var c = c_fgetc(f) @@ -223,7 +229,7 @@ proc endOfFile(f: File): bool = return c < 0'i32 #result = c_feof(f) != 0 -proc readAllFile(file: File, len: int): string = +proc readAllFile(file: File, len: int64): string = # We acquire the filesize beforehand and hope it doesn't change. # Speeds things up. result = newString(len) @@ -363,7 +369,7 @@ proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = result = f != nil proc setFilePos(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) = - if c_fseek(f, clong(pos), cint(relativeTo)) != 0: + if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos(f: File): int64 = diff --git a/tests/system/io.nim b/tests/system/io.nim index b0ccfda9f..3d4df806b 100644 --- a/tests/system/io.nim +++ b/tests/system/io.nim @@ -1,5 +1,5 @@ import - unittest, osproc, streams, os + unittest, osproc, streams, os, strformat const STRING_DATA = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." const TEST_FILE = "tests/testdata/string.txt" @@ -23,3 +23,26 @@ suite "io": test "file": check: readFile(TEST_FILE) == STRING_DATA + + +proc verifyFileSize(sz: int64) = + # issue 7121, large file size (2-4GB and >4Gb) + const fn = "tmpfile112358" + let size_in_mb = sz div 1_000_000 + + when defined(windows): + discard execProcess(&"fsutil file createnew {fn} {sz}" ) + else: + discard execProcess(&"dd if=/dev/zero of={fn} bs=1000000 count={size_in_mb}") + + doAssert os.getFileSize(fn) == sz # Verify OS filesize by string + + var f = open(fn) + doAssert f.getFileSize() == sz # Verify file handle filesize + f.close() + + os.removeFile(fn) + +#disable tests for automatic testers +#for s in [50_000_000'i64, 3_000_000_000, 5_000_000_000]: +# verifyFileSize(s) -- cgit 1.4.1-2-gfad0 From 60c7bbc8b7e75951b952a34051a8445592a68fc4 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 31 Jan 2018 18:39:01 +0300 Subject: Jump to definition on import will open the imported module (#7155) --- compiler/importer.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/importer.nim b/compiler/importer.nim index 46d675b27..3d7f62464 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -11,7 +11,7 @@ import intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups, - semdata, passes, renderer, modulepaths + semdata, passes, renderer, modulepaths, sigmatch proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode @@ -149,7 +149,7 @@ proc myImportModule(c: PContext, n: PNode): PSym = localError(n.info, errGenerated, "A module cannot import itself") if sfDeprecated in result.flags: message(n.info, warnDeprecated, result.name.s) - #suggestSym(n.info, result, false) + suggestSym(n.info, result, c.graph.usageSym, false) proc impMod(c: PContext; it: PNode) = let m = myImportModule(c, it) -- cgit 1.4.1-2-gfad0 From 90c1edff8bad05dc21c0c5a79fc026cac2b5fe4c Mon Sep 17 00:00:00 2001 From: konqoro Date: Thu, 1 Feb 2018 15:02:32 +0200 Subject: Fix the names of the float checks pragmas. (#7170) --- doc/manual/types.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/types.txt b/doc/manual/types.txt index 1477995dd..8a90dee05 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -181,11 +181,11 @@ Nim exceptions: `FloatInvalidOpError`:idx:, `FloatDivByZeroError`:idx:, and `FloatInexactError`:idx:. These exceptions inherit from the `FloatingPointError`:idx: base class. -Nim provides the pragmas `NaNChecks`:idx: and `InfChecks`:idx: to control +Nim provides the pragmas `nanChecks`:idx: and `infChecks`:idx: to control whether the IEEE exceptions are ignored or trap a Nim exception: .. code-block:: nim - {.NanChecks: on, InfChecks: on.} + {.nanChecks: on, infChecks: on.} var a = 1.0 var b = 0.0 echo b / b # raises FloatInvalidOpError @@ -195,7 +195,7 @@ In the current implementation ``FloatDivByZeroError`` and ``FloatInexactError`` are never raised. ``FloatOverflowError`` is raised instead of ``FloatDivByZeroError``. There is also a `floatChecks`:idx: pragma that is a short-cut for the -combination of ``NaNChecks`` and ``InfChecks`` pragmas. ``floatChecks`` are +combination of ``nanChecks`` and ``infChecks`` pragmas. ``floatChecks`` are turned off as default. The only operations that are affected by the ``floatChecks`` pragma are -- cgit 1.4.1-2-gfad0 From 5460bd2764db46048b9a3ecdd1d17a20a4b48b79 Mon Sep 17 00:00:00 2001 From: cooldome Date: Thu, 1 Feb 2018 17:34:37 +0000 Subject: Small performance improvement in sempass2 (#7168) --- compiler/sempass2.nim | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index e560c18c8..25525d412 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -593,17 +593,14 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) = notNilCheck(tracked, n, paramType) proc breaksBlock(n: PNode): bool = - case n.kind - of nkStmtList, nkStmtListExpr: - for c in n: - if breaksBlock(c): return true - of nkBreakStmt, nkReturnStmt, nkRaiseStmt: - return true - of nkCallKinds: - if n.sons[0].kind == nkSym and sfNoReturn in n.sons[0].sym.flags: - return true - else: - discard + # sematic check doesn't allow statements after raise, break, return or + # call to noreturn proc, so it is safe to check just the last statements + var it = n + while it.kind in {nkStmtList, nkStmtListExpr} and it.len > 0: + it = it.lastSon + + result = it.kind in {nkBreakStmt, nkReturnStmt, nkRaiseStmt} or + it.kind in nkCallKinds and it[0].kind == nkSym and sfNoReturn in it[0].sym.flags proc trackCase(tracked: PEffects, n: PNode) = track(tracked, n.sons[0]) -- cgit 1.4.1-2-gfad0 From 7fc80f8f8685dcc6e6c8b02479af3f8c308af5ae Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 08:07:31 +0100 Subject: manual: do not mention the VTable types which are not implemented yet --- doc/manual/generics.txt | 89 +++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 1ba62bb5c..09fecbd95 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -314,7 +314,7 @@ The concept types can be parametric just like the regular generic types: .. code-block:: nim ### matrixalgo.nim - + import typetraits type @@ -360,7 +360,7 @@ The concept types can be parametric just like the regular generic types: template Rows*(M: type Matrix): expr = M.M template Cols*(M: type Matrix): expr = M.N template ValueType*(M: type Matrix): typedesc = M.T - + ------------- ### usage.nim @@ -582,60 +582,61 @@ the concept body: proc log(format: static[string], varargs[distinct StringRef]) -VTable types ------------- +.. + VTable types + ------------ -Concepts allow Nim to define a great number of algorithms, using only -static polymorphism and without erasing any type information or sacrificing -any execution speed. But when polymorphic collections of objects are required, -the user must use one of the provided type erasure techniques - either common -base types or VTable types. + Concepts allow Nim to define a great number of algorithms, using only + static polymorphism and without erasing any type information or sacrificing + any execution speed. But when polymorphic collections of objects are required, + the user must use one of the provided type erasure techniques - either common + base types or VTable types. -VTable types are represented as "fat pointers" storing a reference to an -object together with a reference to a table of procs implementing a set of -required operations (the so called vtable). + VTable types are represented as "fat pointers" storing a reference to an + object together with a reference to a table of procs implementing a set of + required operations (the so called vtable). -In contrast to other programming languages, the vtable in Nim is stored -externally to the object, allowing you to create multiple different vtable -views for the same object. Thus, the polymorphism in Nim is unbounded - -any type can implement an unlimited number of protocols or interfaces not -originally envisioned by the type's author. + In contrast to other programming languages, the vtable in Nim is stored + externally to the object, allowing you to create multiple different vtable + views for the same object. Thus, the polymorphism in Nim is unbounded - + any type can implement an unlimited number of protocols or interfaces not + originally envisioned by the type's author. -Any concept type can be turned into a VTable type by using the ``vtref`` -or the ``vtptr`` compiler magics. Under the hood, these magics generate -a converter type class, which converts the regular instances of the matching -types to the corresponding VTable type. + Any concept type can be turned into a VTable type by using the ``vtref`` + or the ``vtptr`` compiler magics. Under the hood, these magics generate + a converter type class, which converts the regular instances of the matching + types to the corresponding VTable type. -.. code-block:: nim - type - IntEnumerable = vtref Enumerable[int] + .. code-block:: nim + type + IntEnumerable = vtref Enumerable[int] - MyObject = object - enumerables: seq[IntEnumerable] - streams: seq[OutputStream.vtref] + MyObject = object + enumerables: seq[IntEnumerable] + streams: seq[OutputStream.vtref] - proc addEnumerable(o: var MyObject, e: IntEnumerable) = - o.enumerables.add e + proc addEnumerable(o: var MyObject, e: IntEnumerable) = + o.enumerables.add e - proc addStream(o: var MyObject, e: OutputStream.vtref) = - o.streams.add e + proc addStream(o: var MyObject, e: OutputStream.vtref) = + o.streams.add e -The procs that will be included in the vtable are derived from the concept -body and include all proc calls for which all param types were specified as -concrete types. All such calls should include exactly one param of the type -matched against the concept (not necessarily in the first position), which -will be considered the value bound to the vtable. + The procs that will be included in the vtable are derived from the concept + body and include all proc calls for which all param types were specified as + concrete types. All such calls should include exactly one param of the type + matched against the concept (not necessarily in the first position), which + will be considered the value bound to the vtable. -Overloads will be created for all captured procs, accepting the vtable type -in the position of the captured underlying object. + Overloads will be created for all captured procs, accepting the vtable type + in the position of the captured underlying object. -Under these rules, it's possible to obtain a vtable type for a concept with -unbound type parameters or one instantiated with metatypes (type classes), -but it will include a smaller number of captured procs. A completely empty -vtable will be reported as an error. + Under these rules, it's possible to obtain a vtable type for a concept with + unbound type parameters or one instantiated with metatypes (type classes), + but it will include a smaller number of captured procs. A completely empty + vtable will be reported as an error. -The ``vtref`` magic produces types which can be bound to ``ref`` types and -the ``vtptr`` magic produced types bound to ``ptr`` types. + The ``vtref`` magic produces types which can be bound to ``ref`` types and + the ``vtptr`` magic produced types bound to ``ptr`` types. Symbol lookup in generics -- cgit 1.4.1-2-gfad0 From bd1dfa4b3809d79ea01cd8ef87dde8d02d1e8447 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 09:29:05 +0100 Subject: better type inference for numerical types; prerequisitive for version 1 --- changelog.md | 3 +++ compiler/sigmatch.nim | 32 ++++++++++++++++++++++++--- doc/manual/generics.txt | 29 ++++++++++++++++++++++++ tests/generics/tspecial_numeric_inference.nim | 12 ++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 tests/generics/tspecial_numeric_inference.nim diff --git a/changelog.md b/changelog.md index ef51eee7a..c2211b510 100644 --- a/changelog.md +++ b/changelog.md @@ -235,3 +235,6 @@ styledEcho "Red on Green.", resetStyle - ``\n`` is now only the single line feed character like in most other programming languages. The new platform specific newline escape sequence is written as ``\p``. This change only affects the Windows platform. +- Type inference for generic type parameters involving numeric types is now symetric. See + [Generic type inference for numeric types](https://nim-lang.org/docs/manual.html#generics-generic-type-inference-fornumeric-types) + for more information. diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index fffe92b2f..5d5367460 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -913,6 +913,25 @@ proc isCovariantPtr(c: var TCandidate, f, a: PType): bool = else: return false +proc maxNumericType(prev, candidate: PType): PType = + let c = candidate.skipTypes({tyRange}) + template greater(s) = + if c.kind in s: result = c + case prev.kind + of tyInt: greater({tyInt64}) + of tyInt8: greater({tyInt, tyInt16, tyInt32, tyInt64}) + of tyInt16: greater({tyInt, tyInt32, tyInt64}) + of tyInt32: greater({tyInt64}) + + of tyUInt: greater({tyUInt64}) + of tyUInt8: greater({tyUInt, tyUInt16, tyUInt32, tyUInt64}) + of tyUInt16: greater({tyUInt, tyUInt32, tyUInt64}) + of tyUInt32: greater({tyUInt64}) + + of tyFloat32: greater({tyFloat64, tyFloat128}) + of tyFloat64: greater({tyFloat128}) + else: discard + proc typeRelImpl(c: var TCandidate, f, aOrig: PType, flags: TTypeRelFlags = {}): TTypeRelation = # typeRel can be used to establish various relationships between types: @@ -1602,9 +1621,16 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType, elif x.kind == tyGenericParam: result = isGeneric else: - result = typeRel(c, x, a) # check if it fits - if result > isGeneric: result = isGeneric - + # Special type binding rule for numeric types. + # See section "Generic type inference for numeric types" of the + # manual for further details: + let rebinding = maxNumericType(x.skipTypes({tyRange}), a) + if rebinding != nil: + put(c, f, rebinding) + result = isGeneric + else: + result = typeRel(c, x, a) # check if it fits + if result > isGeneric: result = isGeneric of tyStatic: let prev = PType(idTableGet(c.bindings, f)) if prev == nil: diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 09fecbd95..80124bc94 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -63,6 +63,8 @@ The following example shows a generic binary tree can be modelled: for str in preorder(root): stdout.writeLine(str) +The ``T`` is called a `generic type parameter `:idx:. + Is operator ----------- @@ -710,3 +712,30 @@ definition): But a ``bind`` is rarely useful because symbol binding from the definition scope is the default. + + +Generic type inference for numeric types +---------------------------------------- + +A `numeric`:idx: type is any signed, unsigned integer type, floating point +type or a subrange thereof. Let ``maxNumericType(T1, T2)`` be the "greater" +type of ``T1`` and ``T2``, that is the type that uses more bits. For +example ``maxNumericType(int32, int64) == int64``. ``maxNumericType`` is only +defined for numeric types of the same class (signed, unsigned, floating point). +``maxNumericType`` strips away subranges, +``maxNumericType(subrangeof(int16), int8)`` produces ``int16`` not its +subrange. The definition ``maxNumericType`` is extended to take a variable +number of arguments in the obvious way; +``maxNumericType(x, y, z) == maxNumericType(maxNumericType(x, y), z)``. + +A generic type parameter ``T`` that is bound to multiple numeric types ``N1``, +``N2``, ``N3``, ... during type checking is inferred to +be ``maxNumericType(N1, N2, N3, ...)``. This special type inference rule ensures +that the builtin arithmetic operators can be written in an intuitive way: + +.. code-block:: nim + proc `@`[T: int|int16|int32](x, y: T): T + + 4'i32 @ 6'i64 # inferred to be of type ``int64`` + + 4'i64 @ 6'i32 # inferred to be of type ``int64`` diff --git a/tests/generics/tspecial_numeric_inference.nim b/tests/generics/tspecial_numeric_inference.nim new file mode 100644 index 000000000..d93544ba4 --- /dev/null +++ b/tests/generics/tspecial_numeric_inference.nim @@ -0,0 +1,12 @@ +discard """ + output: '''int64 +int64''' +""" + +import typetraits + +proc `@`[T: SomeInteger](x, y: T): T = x + +echo(type(5'i64 @ 6'i32)) + +echo(type(5'i32 @ 6'i64)) -- cgit 1.4.1-2-gfad0 From 1b22a3b346cc1e4260cc92b0d55e667543e4acc4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 11:00:42 +0100 Subject: disabled non-documented overloading rule for templates and macros --- compiler/sigmatch.nim | 29 ++++++++++++++++------------- lib/pure/securehash.nim | 2 +- tests/macros/tgettypeinst.nim | 4 ++++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 5d5367460..1e32cfcf2 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1859,19 +1859,22 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, var r = typeRel(m, f, a) - if r != isNone and m.calleeSym != nil and - m.calleeSym.kind in {skMacro, skTemplate}: - # XXX: duplicating this is ugly, but we cannot (!) move this - # directly into typeRel using return-like templates - incMatches(m, r) - if f.kind == tyStmt: - return arg - elif f.kind == tyTypeDesc: - return arg - elif f.kind == tyStatic: - return arg.typ.n - else: - return argSemantized # argOrig + when false: + # This special typing rule for macros and templates is not documented + # anywhere and breaks symmetry. + if r != isNone and m.calleeSym != nil and + m.calleeSym.kind in {skMacro, skTemplate}: + # XXX: duplicating this is ugly, but we cannot (!) move this + # directly into typeRel using return-like templates + incMatches(m, r) + if f.kind == tyStmt: + return arg + elif f.kind == tyTypeDesc: + return arg + elif f.kind == tyStatic: + return arg.typ.n + else: + return argSemantized # argOrig # If r == isBothMetaConvertible then we rerun typeRel. # bothMetaCounter is for safety to avoid any infinite loop, diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index 57c1f3631..b18095ff6 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -67,7 +67,7 @@ proc innerHash(state: var Sha1State, w: var Sha1Buffer) = var round = 0 template rot(value, bits: uint32): uint32 = - (value shl bits) or (value shr (32 - bits)) + (value shl bits) or (value shr (32u32 - bits)) template sha1(fun, val: uint32) = let t = rot(a, 5) + fun + e + val + w[round] diff --git a/tests/macros/tgettypeinst.nim b/tests/macros/tgettypeinst.nim index 2f1abe193..d54b99d77 100644 --- a/tests/macros/tgettypeinst.nim +++ b/tests/macros/tgettypeinst.nim @@ -1,6 +1,10 @@ discard """ + disabled: "true" """ +# disabled: relied on undocumented overloading rules. Too much work +# to make this sane. + import macros, strUtils proc symToIdent(x: NimNode): NimNode = -- cgit 1.4.1-2-gfad0 From 212457f5e01d1c52db590da3b9d7f8dd538d9269 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 12:53:38 +0100 Subject: the .deprecated pragma for procs now supports a user-definable deprecation message --- changelog.md | 9 +++++++++ compiler/pragmas.nim | 5 ++++- compiler/suggest.nim | 14 +++++++++++++- compiler/trees.nim | 2 +- lib/system.nim | 1 - 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index c2211b510..a3f16738f 100644 --- a/changelog.md +++ b/changelog.md @@ -238,3 +238,12 @@ styledEcho "Red on Green.", resetStyle - Type inference for generic type parameters involving numeric types is now symetric. See [Generic type inference for numeric types](https://nim-lang.org/docs/manual.html#generics-generic-type-inference-fornumeric-types) for more information. +- The ``deprecated`` pragma now supports a user-definable warning message for procs. + +```nim + +proc bar {.deprecated: "use foo instead".} = + return + +bar() +``` diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index cf289a506..d52d8b614 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -810,7 +810,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, of wExplain: sym.flags.incl sfExplain of wDeprecated: - if it.kind in nkPragmaCallKinds: deprecatedStmt(c, it) + if sym != nil and sym.kind in routineKinds: + if it.kind in nkPragmaCallKinds: discard getStrLitNode(c, it) + incl(sym.flags, sfDeprecated) + elif it.kind in nkPragmaCallKinds: deprecatedStmt(c, it) elif sym != nil: incl(sym.flags, sfDeprecated) else: incl(c.module.flags, sfDeprecated) of wVarargs: diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 28ce3c591..af31495aa 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -33,6 +33,7 @@ # included from sigmatch.nim import algorithm, prefixmatches +from wordrecg import wDeprecated when defined(nimsuggest): import passes, tables # importer @@ -479,12 +480,23 @@ proc suggestSym*(info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.in isDecl: suggestResult(symToSuggest(s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0)) +proc warnAboutDeprecated(info: TLineInfo; s: PSym) = + if s.kind in routineKinds: + let n = s.ast[pragmasPos] + if n.kind != nkEmpty: + for it in n: + if whichPragma(it) == wDeprecated and it.safeLen == 2 and + it[1].kind in {nkStrLit..nkTripleStrLit}: + message(info, warnDeprecated, it[1].strVal & "; " & s.name.s) + return + message(info, warnDeprecated, s.name.s) + proc markUsed(info: TLineInfo; s: PSym; usageSym: var PSym) = incl(s.flags, sfUsed) if s.kind == skEnumField and s.owner != nil: incl(s.owner.flags, sfUsed) if {sfDeprecated, sfError} * s.flags != {}: - if sfDeprecated in s.flags: message(info, warnDeprecated, s.name.s) + if sfDeprecated in s.flags: warnAboutDeprecated(info, s) if sfError in s.flags: localError(info, errWrongSymbolX, s.name.s) when defined(nimsuggest): suggestSym(info, s, usageSym, false) diff --git a/compiler/trees.nim b/compiler/trees.nim index 577ea75ee..f69108942 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -118,7 +118,7 @@ proc isRange*(n: PNode): bool {.inline.} = result = true proc whichPragma*(n: PNode): TSpecialWord = - let key = if n.kind == nkExprColonExpr: n.sons[0] else: n + let key = if n.kind in nkPragmaCallKinds and n.len > 0: n.sons[0] else: n if key.kind == nkIdent: result = whichKeyword(key.ident) proc unnestStmts(n, result: PNode) = diff --git a/lib/system.nim b/lib/system.nim index 079b06b48..63511f71c 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3769,7 +3769,6 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = # by ``assert``. type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - {.deprecated: [THide: Hide].} Hide(raiseAssert)(msg) template assert*(cond: bool, msg = "") = -- cgit 1.4.1-2-gfad0 From c671356d51488c96b4749a4d109b00d924e1f739 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 08:07:31 +0100 Subject: manual: do not mention the VTable types which are not implemented yet --- doc/manual/generics.txt | 89 +++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 1ba62bb5c..09fecbd95 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -314,7 +314,7 @@ The concept types can be parametric just like the regular generic types: .. code-block:: nim ### matrixalgo.nim - + import typetraits type @@ -360,7 +360,7 @@ The concept types can be parametric just like the regular generic types: template Rows*(M: type Matrix): expr = M.M template Cols*(M: type Matrix): expr = M.N template ValueType*(M: type Matrix): typedesc = M.T - + ------------- ### usage.nim @@ -582,60 +582,61 @@ the concept body: proc log(format: static[string], varargs[distinct StringRef]) -VTable types ------------- +.. + VTable types + ------------ -Concepts allow Nim to define a great number of algorithms, using only -static polymorphism and without erasing any type information or sacrificing -any execution speed. But when polymorphic collections of objects are required, -the user must use one of the provided type erasure techniques - either common -base types or VTable types. + Concepts allow Nim to define a great number of algorithms, using only + static polymorphism and without erasing any type information or sacrificing + any execution speed. But when polymorphic collections of objects are required, + the user must use one of the provided type erasure techniques - either common + base types or VTable types. -VTable types are represented as "fat pointers" storing a reference to an -object together with a reference to a table of procs implementing a set of -required operations (the so called vtable). + VTable types are represented as "fat pointers" storing a reference to an + object together with a reference to a table of procs implementing a set of + required operations (the so called vtable). -In contrast to other programming languages, the vtable in Nim is stored -externally to the object, allowing you to create multiple different vtable -views for the same object. Thus, the polymorphism in Nim is unbounded - -any type can implement an unlimited number of protocols or interfaces not -originally envisioned by the type's author. + In contrast to other programming languages, the vtable in Nim is stored + externally to the object, allowing you to create multiple different vtable + views for the same object. Thus, the polymorphism in Nim is unbounded - + any type can implement an unlimited number of protocols or interfaces not + originally envisioned by the type's author. -Any concept type can be turned into a VTable type by using the ``vtref`` -or the ``vtptr`` compiler magics. Under the hood, these magics generate -a converter type class, which converts the regular instances of the matching -types to the corresponding VTable type. + Any concept type can be turned into a VTable type by using the ``vtref`` + or the ``vtptr`` compiler magics. Under the hood, these magics generate + a converter type class, which converts the regular instances of the matching + types to the corresponding VTable type. -.. code-block:: nim - type - IntEnumerable = vtref Enumerable[int] + .. code-block:: nim + type + IntEnumerable = vtref Enumerable[int] - MyObject = object - enumerables: seq[IntEnumerable] - streams: seq[OutputStream.vtref] + MyObject = object + enumerables: seq[IntEnumerable] + streams: seq[OutputStream.vtref] - proc addEnumerable(o: var MyObject, e: IntEnumerable) = - o.enumerables.add e + proc addEnumerable(o: var MyObject, e: IntEnumerable) = + o.enumerables.add e - proc addStream(o: var MyObject, e: OutputStream.vtref) = - o.streams.add e + proc addStream(o: var MyObject, e: OutputStream.vtref) = + o.streams.add e -The procs that will be included in the vtable are derived from the concept -body and include all proc calls for which all param types were specified as -concrete types. All such calls should include exactly one param of the type -matched against the concept (not necessarily in the first position), which -will be considered the value bound to the vtable. + The procs that will be included in the vtable are derived from the concept + body and include all proc calls for which all param types were specified as + concrete types. All such calls should include exactly one param of the type + matched against the concept (not necessarily in the first position), which + will be considered the value bound to the vtable. -Overloads will be created for all captured procs, accepting the vtable type -in the position of the captured underlying object. + Overloads will be created for all captured procs, accepting the vtable type + in the position of the captured underlying object. -Under these rules, it's possible to obtain a vtable type for a concept with -unbound type parameters or one instantiated with metatypes (type classes), -but it will include a smaller number of captured procs. A completely empty -vtable will be reported as an error. + Under these rules, it's possible to obtain a vtable type for a concept with + unbound type parameters or one instantiated with metatypes (type classes), + but it will include a smaller number of captured procs. A completely empty + vtable will be reported as an error. -The ``vtref`` magic produces types which can be bound to ``ref`` types and -the ``vtptr`` magic produced types bound to ``ptr`` types. + The ``vtref`` magic produces types which can be bound to ``ref`` types and + the ``vtptr`` magic produced types bound to ``ptr`` types. Symbol lookup in generics -- cgit 1.4.1-2-gfad0 From 274fafb2df9d433f59e5729ebc05200eeeca4e45 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 13:12:30 +0100 Subject: fixes #6961 --- compiler/ast.nim | 2 +- compiler/cgen.nim | 2 +- compiler/extccomp.nim | 2 +- compiler/gorgeimpl.nim | 2 +- compiler/jsgen.nim | 2 +- compiler/modules.nim | 2 +- compiler/rodread.nim | 2 +- compiler/rodwrite.nim | 2 +- lib/pure/securehash.nim | 195 +--------------------------------------------- lib/pure/sha1.nim | 195 ++++++++++++++++++++++++++++++++++++++++++++++ lib/system.nim | 1 - tools/niminst/niminst.nim | 2 +- web/website.ini | 2 +- 13 files changed, 208 insertions(+), 203 deletions(-) create mode 100644 lib/pure/sha1.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 5c70bda18..a19dbf7de 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - msgs, hashes, nversion, options, strutils, securehash, ropes, idents, + msgs, hashes, nversion, options, strutils, sha1, ropes, idents, intsets, idgen type diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 12e640d96..6051f2804 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -11,7 +11,7 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, - nversion, nimsets, msgs, securehash, bitsets, idents, types, + nversion, nimsets, msgs, sha1, bitsets, idents, types, ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel, tables, sets, ndi diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 62990593d..3a6fcde5a 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, os, strutils, osproc, platform, condsyms, options, msgs, - securehash, streams + sha1, streams #from debuginfo import writeDebugInfo diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim index 2c51752cd..9cc6eb2ba 100644 --- a/compiler/gorgeimpl.nim +++ b/compiler/gorgeimpl.nim @@ -9,7 +9,7 @@ ## Module that implements ``gorge`` for the compiler. -import msgs, securehash, os, osproc, streams, strutils, options +import msgs, sha1, os, osproc, streams, strutils, options proc readOutput(p: Process): (string, int) = result[0] = "" diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 75880a15a..3288241d9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,7 +31,7 @@ implements the required case distinction. import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, - nversion, nimsets, msgs, securehash, bitsets, idents, types, os, + nversion, nimsets, msgs, sha1, bitsets, idents, types, os, times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings diff --git a/compiler/modules.nim b/compiler/modules.nim index 4763ac79b..e94c5c162 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -10,7 +10,7 @@ ## Implements the module handling, including the caching of modules. import - ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options, + ast, astalgo, magicsys, sha1, rodread, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs when false: diff --git a/compiler/rodread.nim b/compiler/rodread.nim index dfa8fc52b..5abac1d79 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -90,7 +90,7 @@ import os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, - ropes, idents, securehash, idgen, types, rodutils, memfiles, tables + ropes, idents, sha1, idgen, types, rodutils, memfiles, tables type TReasonForRecompile* = enum ## all the reasons that can trigger recompilation diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 9aed33ec9..24d897fb1 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -13,7 +13,7 @@ import intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform, - condsyms, ropes, idents, securehash, rodread, passes, idgen, + condsyms, ropes, idents, sha1, rodread, passes, idgen, rodutils, modulepaths from modulegraphs import ModuleGraph diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index 57c1f3631..06da80e4c 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -1,195 +1,6 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -import strutils -const Sha1DigestSize = 20 +## This module is a deprecated alias for the ``sha1`` module. +{.deprecated.} -type - Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] - SecureHash* = distinct Sha1Digest - -# Copyright (c) 2011, Micael Hildenborg -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Micael Hildenborg nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Ported to Nim by Erik O'Leary - -type - Sha1State* = array[0 .. 5-1, uint32] - Sha1Buffer = array[0 .. 80-1, uint32] - -template clearBuffer(w: Sha1Buffer, len = 16) = - zeroMem(addr(w), len * sizeof(uint32)) - -proc init*(result: var Sha1State) = - result[0] = 0x67452301'u32 - result[1] = 0xefcdab89'u32 - result[2] = 0x98badcfe'u32 - result[3] = 0x10325476'u32 - result[4] = 0xc3d2e1f0'u32 - -proc innerHash(state: var Sha1State, w: var Sha1Buffer) = - var - a = state[0] - b = state[1] - c = state[2] - d = state[3] - e = state[4] - - var round = 0 - - template rot(value, bits: uint32): uint32 = - (value shl bits) or (value shr (32 - bits)) - - template sha1(fun, val: uint32) = - let t = rot(a, 5) + fun + e + val + w[round] - e = d - d = c - c = rot(b, 30) - b = a - a = t - - template process(body: untyped) = - w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) - body - inc(round) - - template wrap(dest, value: untyped) = - let v = dest + value - dest = v - - while round < 16: - sha1((b and c) or (not b and d), 0x5a827999'u32) - inc(round) - - while round < 20: - process: - sha1((b and c) or (not b and d), 0x5a827999'u32) - - while round < 40: - process: - sha1(b xor c xor d, 0x6ed9eba1'u32) - - while round < 60: - process: - sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) - - while round < 80: - process: - sha1(b xor c xor d, 0xca62c1d6'u32) - - wrap state[0], a - wrap state[1], b - wrap state[2], c - wrap state[3], d - wrap state[4], e - -proc sha1(src: cstring; len: int): Sha1Digest = - #Initialize state - var state: Sha1State - init(state) - - #Create w buffer - var w: Sha1Buffer - - #Loop through all complete 64byte blocks. - let byteLen = len - let endOfFullBlocks = byteLen - 64 - var endCurrentBlock = 0 - var currentBlock = 0 - - while currentBlock <= endOfFullBlocks: - endCurrentBlock = currentBlock + 64 - - var i = 0 - while currentBlock < endCurrentBlock: - w[i] = uint32(src[currentBlock+3]) or - uint32(src[currentBlock+2]) shl 8'u32 or - uint32(src[currentBlock+1]) shl 16'u32 or - uint32(src[currentBlock]) shl 24'u32 - currentBlock += 4 - inc(i) - - innerHash(state, w) - - #Handle last and not full 64 byte block if existing - endCurrentBlock = byteLen - currentBlock - clearBuffer(w) - var lastBlockBytes = 0 - - while lastBlockBytes < endCurrentBlock: - - var value = uint32(src[lastBlockBytes + currentBlock]) shl - ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value - inc(lastBlockBytes) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( - 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - ) - - if endCurrentBlock >= 56: - innerHash(state, w) - clearBuffer(w) - - w[15] = uint32(byteLen) shl 3 - innerHash(state, w) - - # Store hash in result pointer, and make sure we get in in the correct order - # on both endian models. - for i in 0 .. Sha1DigestSize-1: - result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) - -proc sha1(src: string): Sha1Digest = - ## Calculate SHA1 from input string - sha1(src, src.len) - -proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) -proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) -proc `$`*(self: SecureHash): string = - result = "" - for v in Sha1Digest(self): - result.add(toHex(int(v), 2)) - -proc parseSecureHash*(hash: string): SecureHash = - for i in 0 ..< Sha1DigestSize: - Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) - -proc `==`*(a, b: SecureHash): bool = - # Not a constant-time comparison, but that's acceptable in this context - Sha1Digest(a) == Sha1Digest(b) - - -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 +include sha1 diff --git a/lib/pure/sha1.nim b/lib/pure/sha1.nim new file mode 100644 index 000000000..b18095ff6 --- /dev/null +++ b/lib/pure/sha1.nim @@ -0,0 +1,195 @@ +# +# +# The Nim Compiler +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import strutils + +const Sha1DigestSize = 20 + +type + Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] + SecureHash* = distinct Sha1Digest + +# Copyright (c) 2011, Micael Hildenborg +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Micael Hildenborg nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Ported to Nim by Erik O'Leary + +type + Sha1State* = array[0 .. 5-1, uint32] + Sha1Buffer = array[0 .. 80-1, uint32] + +template clearBuffer(w: Sha1Buffer, len = 16) = + zeroMem(addr(w), len * sizeof(uint32)) + +proc init*(result: var Sha1State) = + result[0] = 0x67452301'u32 + result[1] = 0xefcdab89'u32 + result[2] = 0x98badcfe'u32 + result[3] = 0x10325476'u32 + result[4] = 0xc3d2e1f0'u32 + +proc innerHash(state: var Sha1State, w: var Sha1Buffer) = + var + a = state[0] + b = state[1] + c = state[2] + d = state[3] + e = state[4] + + var round = 0 + + template rot(value, bits: uint32): uint32 = + (value shl bits) or (value shr (32u32 - bits)) + + template sha1(fun, val: uint32) = + let t = rot(a, 5) + fun + e + val + w[round] + e = d + d = c + c = rot(b, 30) + b = a + a = t + + template process(body: untyped) = + w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) + body + inc(round) + + template wrap(dest, value: untyped) = + let v = dest + value + dest = v + + while round < 16: + sha1((b and c) or (not b and d), 0x5a827999'u32) + inc(round) + + while round < 20: + process: + sha1((b and c) or (not b and d), 0x5a827999'u32) + + while round < 40: + process: + sha1(b xor c xor d, 0x6ed9eba1'u32) + + while round < 60: + process: + sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) + + while round < 80: + process: + sha1(b xor c xor d, 0xca62c1d6'u32) + + wrap state[0], a + wrap state[1], b + wrap state[2], c + wrap state[3], d + wrap state[4], e + +proc sha1(src: cstring; len: int): Sha1Digest = + #Initialize state + var state: Sha1State + init(state) + + #Create w buffer + var w: Sha1Buffer + + #Loop through all complete 64byte blocks. + let byteLen = len + let endOfFullBlocks = byteLen - 64 + var endCurrentBlock = 0 + var currentBlock = 0 + + while currentBlock <= endOfFullBlocks: + endCurrentBlock = currentBlock + 64 + + var i = 0 + while currentBlock < endCurrentBlock: + w[i] = uint32(src[currentBlock+3]) or + uint32(src[currentBlock+2]) shl 8'u32 or + uint32(src[currentBlock+1]) shl 16'u32 or + uint32(src[currentBlock]) shl 24'u32 + currentBlock += 4 + inc(i) + + innerHash(state, w) + + #Handle last and not full 64 byte block if existing + endCurrentBlock = byteLen - currentBlock + clearBuffer(w) + var lastBlockBytes = 0 + + while lastBlockBytes < endCurrentBlock: + + var value = uint32(src[lastBlockBytes + currentBlock]) shl + ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value + inc(lastBlockBytes) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( + 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + ) + + if endCurrentBlock >= 56: + innerHash(state, w) + clearBuffer(w) + + w[15] = uint32(byteLen) shl 3 + innerHash(state, w) + + # Store hash in result pointer, and make sure we get in in the correct order + # on both endian models. + for i in 0 .. Sha1DigestSize-1: + result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) + +proc sha1(src: string): Sha1Digest = + ## Calculate SHA1 from input string + sha1(src, src.len) + +proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) +proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) +proc `$`*(self: SecureHash): string = + result = "" + for v in Sha1Digest(self): + result.add(toHex(int(v), 2)) + +proc parseSecureHash*(hash: string): SecureHash = + for i in 0 ..< Sha1DigestSize: + Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) + +proc `==`*(a, b: SecureHash): bool = + # Not a constant-time comparison, but that's acceptable in this context + Sha1Digest(a) == Sha1Digest(b) + + +when isMainModule: + let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") + doAssert hash1 == hash1 + doAssert parseSecureHash($hash1) == hash1 diff --git a/lib/system.nim b/lib/system.nim index 079b06b48..63511f71c 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3769,7 +3769,6 @@ proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = # by ``assert``. type Hide = proc (msg: string) {.noinline, raises: [], noSideEffect, tags: [].} - {.deprecated: [THide: Hide].} Hide(raiseAssert)(msg) template assert*(cond: bool, msg = "") = diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 9c15326b0..c2fe79087 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -15,7 +15,7 @@ when haveZipLib: import os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - securehash + sha1 const maxOS = 20 # max number of OSes diff --git a/web/website.ini b/web/website.ini index 273c3223d..4420915ab 100644 --- a/web/website.ini +++ b/web/website.ini @@ -65,7 +65,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient;pure/lenientops" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix;pure/distros;pure/oswalkdir" srcdoc2: "pure/collections/heapqueue" -srcdoc2: "pure/fenv;pure/securehash;impure/rdstdin;pure/strformat" +srcdoc2: "pure/fenv;pure/sha1;impure/rdstdin;pure/strformat" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile;js/asyncjs" -- cgit 1.4.1-2-gfad0 From 4f9ae61695bc7a03b15a8d26ee1e702582db76af Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 2 Feb 2018 17:34:54 +0100 Subject: fixes #6939 --- lib/pure/os.nim | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a5db4ed22..f8936f549 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -139,10 +139,15 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired. - for ext in extensions: - result = addFileExt(exe, ext) - if existsFile(result): return - var path = string(getEnv("PATH")) + template checkCurrentDir() = + for ext in extensions: + result = addFileExt(exe, ext) + if existsFile(result): return + when defined(posix): + if '/' in exe: checkCurrentDir() + else: + checkCurrentDir() + let path = string(getEnv("PATH")) for candidate in split(path, PathSep): when defined(windows): var x = (if candidate[0] == '"' and candidate[^1] == '"': @@ -824,7 +829,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}): string {.tags: [ReadDirEffect].} = - ## Recursively walks over the directory `dir` and yields for each file + ## Recursively walks over the directory `dir` and yields for each file ## or directory in `dir`. ## The full path for each file or directory is returned. ## **Warning**: -- cgit 1.4.1-2-gfad0 From 510392bc6086179f37b4d3680da155d40a526673 Mon Sep 17 00:00:00 2001 From: codetriage-readme-bot Date: Fri, 2 Feb 2018 11:23:39 -0600 Subject: Add CodeTriage badge to nim-lang/nim Adds a badge showing the number of people helping this repo on CodeTriage. [![Open Source Helpers](https://www.codetriage.com/nim-lang/nim/badges/users.svg)](https://www.codetriage.com/nim-lang/nim) ## What is CodeTriage? CodeTriage is an Open Source app that is designed to make contributing to Open Source projects easier. It works by sending subscribers a few open issues in their inbox. If subscribers get busy, there is an algorithm that backs off issue load so they do not get overwhelmed [Read more about the CodeTriage project](https://www.codetriage.com/what). ## Why am I getting this PR? Your project was picked by the human, @schneems. They selected it from the projects submitted to https://www.codetriage.com and hand edited the PR. How did your project get added to [CodeTriage](https://www.codetriage.com/what)? Roughly 4 months ago, [@milan15](https://github.com/milan15) added this project to CodeTriage in order to start contributing. Since then, 4 people have subscribed to help this repo. ## What does adding a badge accomplish? Adding a badge invites people to help contribute to your project. It also lets developers know that others are invested in the longterm success and maintainability of the project. You can see an example of a CodeTriage badge on these popular OSS READMEs: - [![](https://www.codetriage.com/rails/rails/badges/users.svg)](https://www.codetriage.com/rails/rails) https://github.com/rails/rails - [![](https://www.codetriage.com/crystal-lang/crystal/badges/users.svg)](https://www.codetriage.com/crystal-lang/crystal) https://github.com/crystal-lang/crystal ## Have a question or comment? While I am a bot, this PR was manually reviewed and monitored by a human - @schneems. My job is writing commit messages and handling PR logistics. If you have any questions, you can reply back to this PR and they will be answered by @schneems. If you do not want a badge right now, no worries, close the PR, you will not hear from me again. Thanks for making your project Open Source! Any feedback is greatly appreciated. --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 1d4ef4f5b..ebcf5f802 100644 --- a/readme.md +++ b/readme.md @@ -95,6 +95,7 @@ This project exists thanks to all the people who contribute. [Read on to find ou [![Backers on Open Collective](https://opencollective.com/nim/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/nim/sponsors/badge.svg)](#sponsors) [![Setup a bounty via Bountysource][badge-nim-bountysource]][nim-bountysource] [![Donate Bitcoins][badge-nim-bitcoin]][nim-bitcoin] +[![Open Source Helpers](https://www.codetriage.com/nim-lang/nim/badges/users.svg)](https://www.codetriage.com/nim-lang/nim) We welcome all contributions to Nim regardless of how small or large they are. Everything from spelling fixes to new modules to be included in the -- cgit 1.4.1-2-gfad0 From 992fd38487afc706d3271beda8189dadfca53592 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 3 Feb 2018 07:57:47 +0100 Subject: make tests green again --- compiler/sigmatch.nim | 32 ++++++++++++++++---------------- tests/macros/tgettypeinst.nim | 4 ---- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 1e32cfcf2..123e11c68 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1859,22 +1859,22 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, var r = typeRel(m, f, a) - when false: - # This special typing rule for macros and templates is not documented - # anywhere and breaks symmetry. - if r != isNone and m.calleeSym != nil and - m.calleeSym.kind in {skMacro, skTemplate}: - # XXX: duplicating this is ugly, but we cannot (!) move this - # directly into typeRel using return-like templates - incMatches(m, r) - if f.kind == tyStmt: - return arg - elif f.kind == tyTypeDesc: - return arg - elif f.kind == tyStatic: - return arg.typ.n - else: - return argSemantized # argOrig + # This special typing rule for macros and templates is not documented + # anywhere and breaks symmetry. It's hard to get rid of though, my + # custom seqs example fails to compile without this: + if r != isNone and m.calleeSym != nil and + m.calleeSym.kind in {skMacro, skTemplate}: + # XXX: duplicating this is ugly, but we cannot (!) move this + # directly into typeRel using return-like templates + incMatches(m, r) + if f.kind == tyStmt: + return arg + elif f.kind == tyTypeDesc: + return arg + elif f.kind == tyStatic: + return arg.typ.n + else: + return argSemantized # argOrig # If r == isBothMetaConvertible then we rerun typeRel. # bothMetaCounter is for safety to avoid any infinite loop, diff --git a/tests/macros/tgettypeinst.nim b/tests/macros/tgettypeinst.nim index d54b99d77..2f1abe193 100644 --- a/tests/macros/tgettypeinst.nim +++ b/tests/macros/tgettypeinst.nim @@ -1,10 +1,6 @@ discard """ - disabled: "true" """ -# disabled: relied on undocumented overloading rules. Too much work -# to make this sane. - import macros, strUtils proc symToIdent(x: NimNode): NimNode = -- cgit 1.4.1-2-gfad0 From cfa96c9b13414c4274a6f5e28bbdeb35d21891df Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 4 Feb 2018 00:49:47 +0100 Subject: fixes the RST in the manual --- doc/manual/generics.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generics.txt b/doc/manual/generics.txt index 80124bc94..30055f035 100644 --- a/doc/manual/generics.txt +++ b/doc/manual/generics.txt @@ -63,7 +63,7 @@ The following example shows a generic binary tree can be modelled: for str in preorder(root): stdout.writeLine(str) -The ``T`` is called a `generic type parameter `:idx:. +The ``T`` is called a `generic type parameter`:idx:. Is operator -- cgit 1.4.1-2-gfad0 From 326b7dc5562d3f3d2b5848723ceaeba5b8182be1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 4 Feb 2018 07:04:50 +0100 Subject: improve the error messages regarding type mismatches in overloading resolution --- compiler/semcall.nim | 18 ++++++++++++++++-- compiler/sigmatch.nim | 8 ++++++-- tests/concepts/t3330.nim | 37 +++++++++++++++++++++++++++++------- tests/errmsgs/tdetailed_position.nim | 22 +++++++++++++++++++++ 4 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 tests/errmsgs/tdetailed_position.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index c580f8fd5..a8ab2f742 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -106,6 +106,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, errors.safeAdd(CandidateError( sym: sym, unmatchedVarParam: int z.mutabilityProblem, + firstMismatch: z.firstMismatch, diagnostics: z.diagnostics)) else: # Symbol table has been modified. Restart and pre-calculate all syms @@ -154,7 +155,20 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): else: add(candidates, err.sym.getProcHeader(prefer)) add(candidates, "\n") - if err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len: + if err.firstMismatch != 0 and n.len > 2: + add(candidates, "first type mismatch at position: " & $err.firstMismatch & + "\nrequired type: ") + if err.firstMismatch < err.sym.typ.len: + candidates.add typeToString(err.sym.typ.sons[err.firstMismatch]) + else: + candidates.add "none" + if err.firstMismatch < n.len: + candidates.add "\nbut expression '" + candidates.add renderTree(n[err.firstMismatch]) + candidates.add "' is of type: " + candidates.add typeToString(n[err.firstMismatch].typ) + candidates.add "\n" + elif err.unmatchedVarParam != 0 and err.unmatchedVarParam < n.len: add(candidates, "for a 'var' type a variable needs to be passed, but '" & renderTree(n[err.unmatchedVarParam]) & "' is immutable\n") for diag in err.diagnostics: @@ -189,7 +203,7 @@ proc bracketNotFoundError(c: PContext; n: PNode) = while symx != nil: if symx.kind in routineKinds: errors.add(CandidateError(sym: symx, - unmatchedVarParam: 0, + unmatchedVarParam: 0, firstMismatch: 0, diagnostics: nil)) symx = nextOverloadIter(o, c, headSymbol) if errors.len == 0: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 123e11c68..090f30f16 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -24,7 +24,7 @@ type CandidateError* = object sym*: PSym - unmatchedVarParam*: int + unmatchedVarParam*, firstMismatch*: int diagnostics*: seq[string] CandidateErrors* = seq[CandidateError] @@ -67,7 +67,9 @@ type # or when the explain pragma is used. may be # triggered with an idetools command in the # future. - inheritancePenalty: int # to prefer closest father object type + inheritancePenalty: int # to prefer closest father object type + firstMismatch*: int # position of the first type mismatch for + # better error messages TTypeRelFlag* = enum trDontBind @@ -2249,6 +2251,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, n.sons[a], nOrig.sons[a]) if arg == nil: m.state = csNoMatch + m.firstMismatch = f return if m.baseTypeMatch: #assert(container == nil) @@ -2303,6 +2306,7 @@ proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) = else: # no default value m.state = csNoMatch + m.firstMismatch = f break else: # use default value: diff --git a/tests/concepts/t3330.nim b/tests/concepts/t3330.nim index bf8bffc7d..6c9883618 100644 --- a/tests/concepts/t3330.nim +++ b/tests/concepts/t3330.nim @@ -1,25 +1,48 @@ discard """ errormsg: "type mismatch: got (Bar[system.int])" nimout: ''' -t3330.nim(40, 4) Error: type mismatch: got (Bar[system.int]) +t3330.nim(63, 4) Error: type mismatch: got (Bar[system.int]) but expected one of: proc test(foo: Foo[int]) -t3330.nim(25, 8) Hint: Non-matching candidates for add(k, string, T) +t3330.nim(48, 8) Hint: Non-matching candidates for add(k, string, T) proc add(x: var string; y: string) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(x: var string; y: char) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(result: var string; x: int64) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(result: var string; x: float) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add(x: var string; y: cstring) +first type mismatch at position: 1 +required type: var string +but expression 'k' is of type: Alias proc add[T](x: var seq[T]; y: openArray[T]) +first type mismatch at position: 1 +required type: var seq[T] +but expression 'k' is of type: Alias proc add[T](x: var seq[T]; y: T) +first type mismatch at position: 1 +required type: var seq[T] +but expression 'k' is of type: Alias -t3330.nim(25, 8) template/generic instantiation from here -t3330.nim(32, 6) Foo: 'bar.value' cannot be assigned to -t3330.nim(25, 8) template/generic instantiation from here -t3330.nim(33, 6) Foo: 'bar.x' cannot be assigned to -''' +t3330.nim(48, 8) template/generic instantiation from here +t3330.nim(55, 6) Foo: 'bar.value' cannot be assigned to +t3330.nim(48, 8) template/generic instantiation from here +t3330.nim(56, 6) Foo: 'bar.x' cannot be assigned to + +expression: test(bar)''' """ + type Foo[T] = concept k add(k, string, T) diff --git a/tests/errmsgs/tdetailed_position.nim b/tests/errmsgs/tdetailed_position.nim new file mode 100644 index 000000000..d6db94c3e --- /dev/null +++ b/tests/errmsgs/tdetailed_position.nim @@ -0,0 +1,22 @@ + +discard """ +cmd: "nim check $file" +errormsg: "type mismatch: got (int literal(1), int literal(2), int literal(3))" +nimout: ''' +but expected one of: +proc main(a, b, c: string) +first type mismatch at position: 1 +required type: string +but expression '1' is of type: int literal(1) + +expression: main(1, 2, 3) +''' +""" + +const + myconst = "abcdefghijklmnopqrstuvwxyz" + +proc main(a, b, c: string) {.deprecated: "use foo " & "instead " & myconst.} = + return + +main(1, 2, 3) -- cgit 1.4.1-2-gfad0 From c1679edad20741a2229c83066a24820f29ad2f9b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 4 Feb 2018 07:23:47 +0100 Subject: fixes #7080 --- compiler/msgs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 4e6226122..936b0172a 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -26,7 +26,7 @@ type errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, errExceptionExpected, errExceptionAlreadyHandled, errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, - errInvalidNumberOfYieldExpr, errCannotReturnExpr, + errInvalidNumberOfYieldExpr, errCannotReturnExpr, errNoReturnWithReturnTypeNotAllowed, errAttemptToRedefine, errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, @@ -1025,6 +1025,9 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string, handleError(msg, eh, s) proc fatal*(info: TLineInfo, msg: TMsgKind, arg = "") = + # this fixes bug #7080 so that it is at least obvious 'fatal' + # was executed. + errorOutputs = {eStdOut, eStdErr} liMessage(info, msg, arg, doAbort) proc globalError*(info: TLineInfo, msg: TMsgKind, arg = "") = -- cgit 1.4.1-2-gfad0 From 31263ef8f3ed7dfea7b4e635aede5a1d627db031 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 4 Feb 2018 07:35:32 +0100 Subject: fixes #5450 --- compiler/semobjconstr.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 46e6ef25d..63a4eb99a 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -260,6 +260,10 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = result = newNodeIT(nkObjConstr, n.info, t) for child in n: result.add child + if t == nil: + localError(n.info, errGenerated, "object constructor needs an object type") + return + t = skipTypes(t, {tyGenericInst, tyAlias, tySink}) if t.kind == tyRef: t = skipTypes(t.sons[0], {tyGenericInst, tyAlias, tySink}) if t.kind != tyObject: -- cgit 1.4.1-2-gfad0 From e2cf3b94f22f9d028500a3b8628c3cd0b381d65f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Feb 2018 08:47:35 +0100 Subject: improve the error message for thread local variables --- compiler/msgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 936b0172a..954883b01 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -370,7 +370,7 @@ const errXhasSideEffects: "\'$1\' can have side effects", errIteratorExpected: "iterator within for loop context expected", errLetNeedsInit: "'let' symbol requires an initialization", - errThreadvarCannotInit: "a thread var cannot be initialized explicitly", + errThreadvarCannotInit: "a thread var cannot be initialized explicitly; this would only run for the main thread", errWrongSymbolX: "usage of \'$1\' is a user-defined error", errIllegalCaptureX: "illegal capture '$1'", errXCannotBeClosure: "'$1' cannot have 'closure' calling convention", -- cgit 1.4.1-2-gfad0 From d72578b3ddfe3ce955975bebd05ea3e3313d0e10 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Feb 2018 16:49:06 +0100 Subject: more chatty error message for the 'discard' checking --- compiler/semstmts.nim | 5 ++++- tests/discard/tneedsdiscard.nim | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 287f81b76..74cca6d0c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -150,9 +150,12 @@ proc discardCheck(c: PContext, result: PNode) = while n.kind in skipForDiscardable: n = n.lastSon var s = "expression '" & $n & "' is of type '" & result.typ.typeToString & "' and has to be discarded" + if result.info.line != n.info.line or + result.info.fileIndex != n.info.fileIndex: + s.add "; start of expression here: " & $result.info if result.typ.kind == tyProc: s.add "; for a function call use ()" - localError(n.info, s) + localError(n.info, s) proc semIf(c: PContext, n: PNode): PNode = result = n diff --git a/tests/discard/tneedsdiscard.nim b/tests/discard/tneedsdiscard.nim index 509e8233b..8d59e7fec 100644 --- a/tests/discard/tneedsdiscard.nim +++ b/tests/discard/tneedsdiscard.nim @@ -1,6 +1,6 @@ discard """ line: 10 - errormsg: '''expression 'open(f, "arg.txt", fmRead, -1)' is of type 'bool' and has to be discarded''' + errormsg: '''expression 'open(f, "arg.txt", fmRead, -1)' is of type 'bool' and has to be discarded; start of expression here: tneedsdiscard.nim(7, 2)''' """ proc p = -- cgit 1.4.1-2-gfad0 From 49ad131fe23a69bcbc3da68fba0b9c4a1df864b8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Feb 2018 21:12:56 +0100 Subject: mark Nim version with .intdefine --- lib/system.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 63511f71c..a0330be76 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1955,13 +1955,13 @@ const ## that you cannot compare a floating point value to this value ## and expect a reasonable result - use the `classify` procedure ## in the module ``math`` for checking for NaN. - NimMajor*: int = 0 + NimMajor* {.intdefine.}: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 17 + NimMinor* {.intdefine.}: int = 17 ## is the minor number of Nim's version. - NimPatch*: int = 3 + NimPatch* {.intdefine.}: int = 3 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch -- cgit 1.4.1-2-gfad0 From fc52dd646385b5b1ca5dc7c451c124a863047b14 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Feb 2018 21:13:18 +0100 Subject: Tut 1: tiny improvements --- doc/sets_fragment.txt | 2 ++ doc/tut1.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/sets_fragment.txt b/doc/sets_fragment.txt index 435807e1a..5c341b7c8 100644 --- a/doc/sets_fragment.txt +++ b/doc/sets_fragment.txt @@ -1,9 +1,11 @@ The set type models the mathematical notion of a set. The set's basetype can only be an ordinal type of a certain size, namely: + * ``int8``-``int16`` * ``uint8``/``byte``-``uint16`` * ``char`` * ``enum`` + or equivalent. The reason is that sets are implemented as high performance bit vectors. Attempting to declare a set with a larger type will result in an error: diff --git a/doc/tut1.rst b/doc/tut1.rst index f935e7935..cbfef183e 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1032,7 +1032,7 @@ procs for these conversions. Type Conversion --------------- -Conversion between basic types is performed by using the +Conversion between numerical types is performed by using the type as a function: .. code-block:: nim -- cgit 1.4.1-2-gfad0 From fa0f5d0238d5656ace070b86ecc9cdec60568fdb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Feb 2018 21:21:22 +0100 Subject: fixes #6946 --- compiler/sem.nim | 15 ++++++++++----- compiler/semstmts.nim | 2 +- compiler/types.nim | 16 ++++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index 0e97a66b2..39b40b2e6 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -215,12 +215,17 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym -proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) = - let t = typeAllowed(typ, kind) +proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind; + flags: TTypeAllowedFlags = {}) = + let t = typeAllowed(typ, kind, flags) if t != nil: - if t == typ: localError(info, "invalid type: '" & typeToString(typ) & "'") - else: localError(info, "invalid type: '" & typeToString(t) & - "' in this context: '" & typeToString(typ) & "'") + if t == typ: + localError(info, "invalid type: '" & typeToString(typ) & + "' for " & substr($kind, 2).toLowerAscii) + else: + localError(info, "invalid type: '" & typeToString(t) & + "' in this context: '" & typeToString(typ) & + "' for " & substr($kind, 2).toLowerAscii) proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} = typeAllowedCheck(typ.n.info, typ, skProc) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 74cca6d0c..f4614272c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -534,7 +534,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = # this can only happen for errornous var statements: if typ == nil: continue - typeAllowedCheck(a.info, typ, symkind) + typeAllowedCheck(a.info, typ, symkind, if c.matchedConcept != nil: {taConcept} else: {}) liftTypeBoundOps(c, typ, a.info) var tup = skipTypes(typ, {tyGenericInst, tyAlias, tySink}) if a.kind == nkVarTuple: diff --git a/compiler/types.nim b/compiler/types.nim index cbbfa8631..452e95dfd 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1058,11 +1058,12 @@ proc commonSuperclass*(a, b: PType): PType = y = y.sons[0] type - TTypeAllowedFlag = enum + TTypeAllowedFlag* = enum taField, - taHeap + taHeap, + taConcept - TTypeAllowedFlags = set[TTypeAllowedFlag] + TTypeAllowedFlags* = set[TTypeAllowedFlag] proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, flags: TTypeAllowedFlags = {}): PType @@ -1130,7 +1131,10 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, of tyVoid: if taField notin flags: result = t of tyTypeClasses: - if not (tfGenericTypeParam in t.flags or taField notin flags): result = t + if tfGenericTypeParam in t.flags or taConcept in flags: #or taField notin flags: + discard + elif kind notin {skParam, skResult}: + result = t of tyGenericBody, tyGenericParam, tyGenericInvocation, tyNone, tyForward, tyFromExpr: result = t @@ -1178,11 +1182,11 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = nil of tyUnused, tyOptAsRef: internalError("typeAllowedAux") -proc typeAllowed*(t: PType, kind: TSymKind): PType = +proc typeAllowed*(t: PType, kind: TSymKind; flags: TTypeAllowedFlags = {}): PType = # returns 'nil' on success and otherwise the part of the type that is # wrong! var marker = initIntSet() - result = typeAllowedAux(marker, t, kind, {}) + result = typeAllowedAux(marker, t, kind, flags) proc align(address, alignment: BiggestInt): BiggestInt = result = (address + (alignment - 1)) and not (alignment - 1) -- cgit 1.4.1-2-gfad0 From 2199f8328059ffa62400acc7342b36dd50aed7e2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 6 Feb 2018 09:15:54 +0100 Subject: improve error message for twrongcolon --- compiler/parser.nim | 4 ++-- compiler/semcall.nim | 4 ++-- tests/errmsgs/twrongcolon.nim | 11 +++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 tests/errmsgs/twrongcolon.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index b1cfd7609..4974abcc3 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -914,7 +914,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = optInd(p, result) addSon(result, parseTypeDesc(p)) else: - addSon(result, ast.emptyNode) + addSon(result, newNodeP(nkEmpty, p)) if p.tok.tokType != tkEquals and withBothOptional notin flags: parMessage(p, errColonOrEqualsExpected, p.tok) if p.tok.tokType == tkEquals: @@ -922,7 +922,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = optInd(p, result) addSon(result, parseExpr(p)) else: - addSon(result, ast.emptyNode) + addSon(result, newNodeP(nkEmpty, p)) proc parseTuple(p: var TParser, indentAllowed = false): PNode = #| inlTupleDecl = 'tuple' diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a8ab2f742..01f291017 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -277,9 +277,9 @@ proc resolveOverloads(c: PContext, n, orig: PNode, if overloadsState == csEmpty and result.state == csEmpty: if nfDotField in n.flags and nfExplicitCall notin n.flags: - localError(n.info, errUndeclaredField, considerQuotedIdent(f).s) + localError(n.info, errUndeclaredField, considerQuotedIdent(f, n).s) else: - localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f).s) + localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f, n).s) return elif result.state != csMatch: if nfExprCall in n.flags: diff --git a/tests/errmsgs/twrongcolon.nim b/tests/errmsgs/twrongcolon.nim new file mode 100644 index 000000000..6f5cc3e5d --- /dev/null +++ b/tests/errmsgs/twrongcolon.nim @@ -0,0 +1,11 @@ +discard """ +errormsg: "in expression '(" +nimout: ''' +Error: in expression '( + 890)': identifier expected, but found '' +''' + +line: 11 +""" + +var n: int : 890 -- cgit 1.4.1-2-gfad0 From 70e8640244e48e8d62c5af42101c9df2b57224d9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 6 Feb 2018 17:40:31 +0100 Subject: move new sha1 module into the new 'std' namespace --- changelog.md | 2 + compiler/ast.nim | 2 +- compiler/cgen.nim | 2 +- compiler/extccomp.nim | 2 +- compiler/gorgeimpl.nim | 2 +- compiler/jsgen.nim | 2 +- compiler/modules.nim | 2 +- compiler/rodread.nim | 2 +- compiler/rodwrite.nim | 2 +- doc/lib.rst | 2 +- lib/pure/securehash.nim | 2 +- lib/pure/sha1.nim | 195 ---------------------------------------------- lib/std/sha1.nim | 195 ++++++++++++++++++++++++++++++++++++++++++++++ tools/niminst/niminst.nim | 2 +- web/website.ini | 2 +- 15 files changed, 209 insertions(+), 207 deletions(-) delete mode 100644 lib/pure/sha1.nim create mode 100644 lib/std/sha1.nim diff --git a/changelog.md b/changelog.md index a3f16738f..15ee5fcfe 100644 --- a/changelog.md +++ b/changelog.md @@ -247,3 +247,5 @@ proc bar {.deprecated: "use foo instead".} = bar() ``` + +- The ``securehash`` module is now deprecated. Instead import ``std / sha1``. diff --git a/compiler/ast.nim b/compiler/ast.nim index a19dbf7de..0639ebf8e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - msgs, hashes, nversion, options, strutils, sha1, ropes, idents, + msgs, hashes, nversion, options, strutils, std / sha1, ropes, idents, intsets, idgen type diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 6051f2804..83c8501a4 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -11,7 +11,7 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, - nversion, nimsets, msgs, sha1, bitsets, idents, types, + nversion, nimsets, msgs, std / sha1, bitsets, idents, types, ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel, tables, sets, ndi diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 3a6fcde5a..8b5a3bf3d 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, os, strutils, osproc, platform, condsyms, options, msgs, - sha1, streams + std / sha1, streams #from debuginfo import writeDebugInfo diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim index 9cc6eb2ba..80302b4b5 100644 --- a/compiler/gorgeimpl.nim +++ b/compiler/gorgeimpl.nim @@ -9,7 +9,7 @@ ## Module that implements ``gorge`` for the compiler. -import msgs, sha1, os, osproc, streams, strutils, options +import msgs, std / sha1, os, osproc, streams, strutils, options proc readOutput(p: Process): (string, int) = result[0] = "" diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 3288241d9..dc74fa933 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,7 +31,7 @@ implements the required case distinction. import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, - nversion, nimsets, msgs, sha1, bitsets, idents, types, os, + nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings diff --git a/compiler/modules.nim b/compiler/modules.nim index e94c5c162..cba152e21 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -10,7 +10,7 @@ ## Implements the module handling, including the caching of modules. import - ast, astalgo, magicsys, sha1, rodread, msgs, cgendata, sigmatch, options, + ast, astalgo, magicsys, std / sha1, rodread, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs when false: diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 5abac1d79..6e952606e 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -90,7 +90,7 @@ import os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, - ropes, idents, sha1, idgen, types, rodutils, memfiles, tables + ropes, idents, std / sha1, idgen, types, rodutils, memfiles, tables type TReasonForRecompile* = enum ## all the reasons that can trigger recompilation diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 24d897fb1..b9f33236d 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -13,7 +13,7 @@ import intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform, - condsyms, ropes, idents, sha1, rodread, passes, idgen, + condsyms, ropes, idents, std / sha1, rodread, passes, idgen, rodutils, modulepaths from modulegraphs import ModuleGraph diff --git a/doc/lib.rst b/doc/lib.rst index 755c11899..0932b25e4 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -384,7 +384,7 @@ Cryptography and Hashing * `base64 `_ This module implements a base64 encoder and decoder. -* `securehash `_ +* `sha1 `_ This module implements a sha1 encoder and decoder. diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index 06da80e4c..c6cde599a 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -3,4 +3,4 @@ ## This module is a deprecated alias for the ``sha1`` module. {.deprecated.} -include sha1 +include "../std/sha1" diff --git a/lib/pure/sha1.nim b/lib/pure/sha1.nim deleted file mode 100644 index b18095ff6..000000000 --- a/lib/pure/sha1.nim +++ /dev/null @@ -1,195 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import strutils - -const Sha1DigestSize = 20 - -type - Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] - SecureHash* = distinct Sha1Digest - -# Copyright (c) 2011, Micael Hildenborg -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Micael Hildenborg nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Ported to Nim by Erik O'Leary - -type - Sha1State* = array[0 .. 5-1, uint32] - Sha1Buffer = array[0 .. 80-1, uint32] - -template clearBuffer(w: Sha1Buffer, len = 16) = - zeroMem(addr(w), len * sizeof(uint32)) - -proc init*(result: var Sha1State) = - result[0] = 0x67452301'u32 - result[1] = 0xefcdab89'u32 - result[2] = 0x98badcfe'u32 - result[3] = 0x10325476'u32 - result[4] = 0xc3d2e1f0'u32 - -proc innerHash(state: var Sha1State, w: var Sha1Buffer) = - var - a = state[0] - b = state[1] - c = state[2] - d = state[3] - e = state[4] - - var round = 0 - - template rot(value, bits: uint32): uint32 = - (value shl bits) or (value shr (32u32 - bits)) - - template sha1(fun, val: uint32) = - let t = rot(a, 5) + fun + e + val + w[round] - e = d - d = c - c = rot(b, 30) - b = a - a = t - - template process(body: untyped) = - w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) - body - inc(round) - - template wrap(dest, value: untyped) = - let v = dest + value - dest = v - - while round < 16: - sha1((b and c) or (not b and d), 0x5a827999'u32) - inc(round) - - while round < 20: - process: - sha1((b and c) or (not b and d), 0x5a827999'u32) - - while round < 40: - process: - sha1(b xor c xor d, 0x6ed9eba1'u32) - - while round < 60: - process: - sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) - - while round < 80: - process: - sha1(b xor c xor d, 0xca62c1d6'u32) - - wrap state[0], a - wrap state[1], b - wrap state[2], c - wrap state[3], d - wrap state[4], e - -proc sha1(src: cstring; len: int): Sha1Digest = - #Initialize state - var state: Sha1State - init(state) - - #Create w buffer - var w: Sha1Buffer - - #Loop through all complete 64byte blocks. - let byteLen = len - let endOfFullBlocks = byteLen - 64 - var endCurrentBlock = 0 - var currentBlock = 0 - - while currentBlock <= endOfFullBlocks: - endCurrentBlock = currentBlock + 64 - - var i = 0 - while currentBlock < endCurrentBlock: - w[i] = uint32(src[currentBlock+3]) or - uint32(src[currentBlock+2]) shl 8'u32 or - uint32(src[currentBlock+1]) shl 16'u32 or - uint32(src[currentBlock]) shl 24'u32 - currentBlock += 4 - inc(i) - - innerHash(state, w) - - #Handle last and not full 64 byte block if existing - endCurrentBlock = byteLen - currentBlock - clearBuffer(w) - var lastBlockBytes = 0 - - while lastBlockBytes < endCurrentBlock: - - var value = uint32(src[lastBlockBytes + currentBlock]) shl - ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value - inc(lastBlockBytes) - - w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( - 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) - ) - - if endCurrentBlock >= 56: - innerHash(state, w) - clearBuffer(w) - - w[15] = uint32(byteLen) shl 3 - innerHash(state, w) - - # Store hash in result pointer, and make sure we get in in the correct order - # on both endian models. - for i in 0 .. Sha1DigestSize-1: - result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) - -proc sha1(src: string): Sha1Digest = - ## Calculate SHA1 from input string - sha1(src, src.len) - -proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) -proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) -proc `$`*(self: SecureHash): string = - result = "" - for v in Sha1Digest(self): - result.add(toHex(int(v), 2)) - -proc parseSecureHash*(hash: string): SecureHash = - for i in 0 ..< Sha1DigestSize: - Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) - -proc `==`*(a, b: SecureHash): bool = - # Not a constant-time comparison, but that's acceptable in this context - Sha1Digest(a) == Sha1Digest(b) - - -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim new file mode 100644 index 000000000..b18095ff6 --- /dev/null +++ b/lib/std/sha1.nim @@ -0,0 +1,195 @@ +# +# +# The Nim Compiler +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import strutils + +const Sha1DigestSize = 20 + +type + Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] + SecureHash* = distinct Sha1Digest + +# Copyright (c) 2011, Micael Hildenborg +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Micael Hildenborg nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Ported to Nim by Erik O'Leary + +type + Sha1State* = array[0 .. 5-1, uint32] + Sha1Buffer = array[0 .. 80-1, uint32] + +template clearBuffer(w: Sha1Buffer, len = 16) = + zeroMem(addr(w), len * sizeof(uint32)) + +proc init*(result: var Sha1State) = + result[0] = 0x67452301'u32 + result[1] = 0xefcdab89'u32 + result[2] = 0x98badcfe'u32 + result[3] = 0x10325476'u32 + result[4] = 0xc3d2e1f0'u32 + +proc innerHash(state: var Sha1State, w: var Sha1Buffer) = + var + a = state[0] + b = state[1] + c = state[2] + d = state[3] + e = state[4] + + var round = 0 + + template rot(value, bits: uint32): uint32 = + (value shl bits) or (value shr (32u32 - bits)) + + template sha1(fun, val: uint32) = + let t = rot(a, 5) + fun + e + val + w[round] + e = d + d = c + c = rot(b, 30) + b = a + a = t + + template process(body: untyped) = + w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) + body + inc(round) + + template wrap(dest, value: untyped) = + let v = dest + value + dest = v + + while round < 16: + sha1((b and c) or (not b and d), 0x5a827999'u32) + inc(round) + + while round < 20: + process: + sha1((b and c) or (not b and d), 0x5a827999'u32) + + while round < 40: + process: + sha1(b xor c xor d, 0x6ed9eba1'u32) + + while round < 60: + process: + sha1((b and c) or (b and d) or (c and d), 0x8f1bbcdc'u32) + + while round < 80: + process: + sha1(b xor c xor d, 0xca62c1d6'u32) + + wrap state[0], a + wrap state[1], b + wrap state[2], c + wrap state[3], d + wrap state[4], e + +proc sha1(src: cstring; len: int): Sha1Digest = + #Initialize state + var state: Sha1State + init(state) + + #Create w buffer + var w: Sha1Buffer + + #Loop through all complete 64byte blocks. + let byteLen = len + let endOfFullBlocks = byteLen - 64 + var endCurrentBlock = 0 + var currentBlock = 0 + + while currentBlock <= endOfFullBlocks: + endCurrentBlock = currentBlock + 64 + + var i = 0 + while currentBlock < endCurrentBlock: + w[i] = uint32(src[currentBlock+3]) or + uint32(src[currentBlock+2]) shl 8'u32 or + uint32(src[currentBlock+1]) shl 16'u32 or + uint32(src[currentBlock]) shl 24'u32 + currentBlock += 4 + inc(i) + + innerHash(state, w) + + #Handle last and not full 64 byte block if existing + endCurrentBlock = byteLen - currentBlock + clearBuffer(w) + var lastBlockBytes = 0 + + while lastBlockBytes < endCurrentBlock: + + var value = uint32(src[lastBlockBytes + currentBlock]) shl + ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or value + inc(lastBlockBytes) + + w[lastBlockBytes shr 2] = w[lastBlockBytes shr 2] or ( + 0x80'u32 shl ((3'u32 - uint32(lastBlockBytes and 3)) shl 3) + ) + + if endCurrentBlock >= 56: + innerHash(state, w) + clearBuffer(w) + + w[15] = uint32(byteLen) shl 3 + innerHash(state, w) + + # Store hash in result pointer, and make sure we get in in the correct order + # on both endian models. + for i in 0 .. Sha1DigestSize-1: + result[i] = uint8((int(state[i shr 2]) shr ((3-(i and 3)) * 8)) and 255) + +proc sha1(src: string): Sha1Digest = + ## Calculate SHA1 from input string + sha1(src, src.len) + +proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) +proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) +proc `$`*(self: SecureHash): string = + result = "" + for v in Sha1Digest(self): + result.add(toHex(int(v), 2)) + +proc parseSecureHash*(hash: string): SecureHash = + for i in 0 ..< Sha1DigestSize: + Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) + +proc `==`*(a, b: SecureHash): bool = + # Not a constant-time comparison, but that's acceptable in this context + Sha1Digest(a) == Sha1Digest(b) + + +when isMainModule: + let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") + doAssert hash1 == hash1 + doAssert parseSecureHash($hash1) == hash1 diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index c2fe79087..c2816a0ef 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -15,7 +15,7 @@ when haveZipLib: import os, osproc, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - sha1 + std / sha1 const maxOS = 20 # max number of OSes diff --git a/web/website.ini b/web/website.ini index 4420915ab..7edaaa712 100644 --- a/web/website.ini +++ b/web/website.ini @@ -65,7 +65,7 @@ srcdoc2: "pure/asyncfile;pure/asyncftpclient;pure/lenientops" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix;pure/distros;pure/oswalkdir" srcdoc2: "pure/collections/heapqueue" -srcdoc2: "pure/fenv;pure/sha1;impure/rdstdin;pure/strformat" +srcdoc2: "pure/fenv;std/sha1;impure/rdstdin;pure/strformat" srcdoc2: "pure/segfaults" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" srcdoc2: "pure/bitops;pure/nimtracker;pure/punycode;pure/volatile;js/asyncjs" -- cgit 1.4.1-2-gfad0 From 40e3b5798a66095ded8f5ba0f45ba211f07d531d Mon Sep 17 00:00:00 2001 From: Volodymyr Melnychuk Date: Fri, 9 Feb 2018 10:34:26 +0200 Subject: Fix undefined reference with MinGw (#7175) * fix undefined reference with mingw * use fseek, ftell for x86 and _fseeki64, _ftelli64 for amd64 --- lib/system/sysio.nim | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 71cbb1c21..285bf1adc 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -48,10 +48,16 @@ when not declared(c_fwrite): proc c_fread(buf: pointer, size, n: csize, f: File): csize {. importc: "fread", header: "", tags: [ReadIOEffect].} when defined(windows): - proc c_fseek(f: File, offset: int64, whence: cint): cint {. - importc: "_fseeki64", header: "", tags: [].} - proc c_ftell(f: File): int64 {. - importc: "_ftelli64", header: "", tags: [].} + when not defined(amd64): + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "fseek", header: "", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "ftell", header: "", tags: [].} + else: + proc c_fseek(f: File, offset: int64, whence: cint): cint {. + importc: "_fseeki64", header: "", tags: [].} + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "", tags: [].} else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "fseeko", header: "", tags: [].} -- cgit 1.4.1-2-gfad0