From 0229dfd1997ef9a2229c144a875fe51db206fdcd Mon Sep 17 00:00:00 2001 From: Ico Doornekamp Date: Wed, 9 Jan 2019 10:46:23 +0100 Subject: epoll selector starts with reasonable fd set size (1024) and increases in powers of two when needed. This prevents the selector to allocate large amounts of memory at startup on systems with a high RLIMIT_NOFILE setting (#10194) --- lib/pure/ioselects/ioselectors_epoll.nim | 22 +++++++++++++++++++--- lib/pure/selectors.nim | 3 +++ 2 files changed, 22 insertions(+), 3 deletions(-) (limited to 'lib/pure') diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 16d901ff0..ffd60120e 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -53,6 +53,7 @@ when hasThreadSupport: SelectorImpl[T] = object epollFD: cint maxFD: int + numFD: int fds: ptr SharedArray[SelectorKey[T]] count: int Selector*[T] = ptr SelectorImpl[T] @@ -61,6 +62,7 @@ else: SelectorImpl[T] = object epollFD: cint maxFD: int + numFD: int fds: seq[SelectorKey[T]] count: int Selector*[T] = ref SelectorImpl[T] @@ -76,6 +78,8 @@ proc newSelector*[T](): Selector[T] = raiseOsError(osLastError()) var maxFD = int(a.rlim_max) doAssert(maxFD > 0) + # Start with a reasonable size, checkFd() will grow this on demand + const numFD = 1024 var epollFD = epoll_create(MAX_EPOLL_EVENTS) if epollFD < 0: @@ -85,14 +89,16 @@ proc newSelector*[T](): Selector[T] = result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) result.epollFD = epollFD result.maxFD = maxFD - result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.numFD = numFD + result.fds = allocSharedArray[SelectorKey[T]](numFD) else: result = Selector[T]() result.epollFD = epollFD result.maxFD = maxFD - result.fds = newSeq[SelectorKey[T]](maxFD) + result.numFD = numFD + result.fds = newSeq[SelectorKey[T]](numFD) - for i in 0 ..< maxFD: + for i in 0 ..< numFD: result.fds[i].ident = InvalidIdent proc close*[T](s: Selector[T]) = @@ -127,6 +133,16 @@ template checkFd(s, f) = # FD if there is too many. -- DP if f >= s.maxFD: raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + if f >= s.numFD: + var numFD = s.numFD + while numFD <= f: numFD *= 2 + when hasThreadSupport: + s.fds = reallocSharedArray(s.fds, numFD) + else: + s.fds.setLen(numFD) + for i in s.numFD ..< numFD: + s.fds[i].ident = InvalidIdent + s.numFD = numFD proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event], data: T) = diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index e4c2b2124..b9c834127 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -239,6 +239,9 @@ else: proc allocSharedArray[T](nsize: int): ptr SharedArray[T] = result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize)) + proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] = + result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize)) + proc deallocSharedArray[T](sa: ptr SharedArray[T]) = deallocShared(cast[pointer](sa)) type -- cgit 1.4.1-2-gfad0