epoll selector starts with reasonable fd set size (1024) and increases in powers of two when needed. This prevents the selector to allocate large amounts of memory at startup on systems with a high RLIMIT_NOFILE setting (#10194)

2 files changed, 22 insertions, 3 deletions

diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim
index 16d901ff0..ffd60120e 100644
--- a/lib/pure/ioselects/ioselectors_epoll.nim
+++ b/lib/pure/ioselects/ioselectors_epoll.nim
@@ -53,6 +53,7 @@ when hasThreadSupport:
     SelectorImpl[T] = object
       epollFD: cint
       maxFD: int
+      numFD: int
       fds: ptr SharedArray[SelectorKey[T]]
       count: int
     Selector*[T] = ptr SelectorImpl[T]
@@ -61,6 +62,7 @@ else:
     SelectorImpl[T] = object
       epollFD: cint
       maxFD: int
+      numFD: int
       fds: seq[SelectorKey[T]]
       count: int
     Selector*[T] = ref SelectorImpl[T]
@@ -76,6 +78,8 @@ proc newSelector*[T](): Selector[T] =
     raiseOsError(osLastError())
   var maxFD = int(a.rlim_max)
   doAssert(maxFD > 0)
+  # Start with a reasonable size, checkFd() will grow this on demand
+  const numFD = 1024
 
   var epollFD = epoll_create(MAX_EPOLL_EVENTS)
   if epollFD < 0:
@@ -85,14 +89,16 @@ proc newSelector*[T](): Selector[T] =
     result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T])))
     result.epollFD = epollFD
     result.maxFD = maxFD
-    result.fds = allocSharedArray[SelectorKey[T]](maxFD)
+    result.numFD = numFD
+    result.fds = allocSharedArray[SelectorKey[T]](numFD)
   else:
     result = Selector[T]()
     result.epollFD = epollFD
     result.maxFD = maxFD
-    result.fds = newSeq[SelectorKey[T]](maxFD)
+    result.numFD = numFD
+    result.fds = newSeq[SelectorKey[T]](numFD)
 
-  for i in 0 ..< maxFD:
+  for i in 0 ..< numFD:
     result.fds[i].ident = InvalidIdent
 
 proc close*[T](s: Selector[T]) =
@@ -127,6 +133,16 @@ template checkFd(s, f) =
   # FD if there is too many. -- DP
   if f >= s.maxFD:
     raiseIOSelectorsError("Maximum number of descriptors is exhausted!")
+  if f >= s.numFD:
+    var numFD = s.numFD
+    while numFD <= f: numFD *= 2
+    when hasThreadSupport:
+      s.fds = reallocSharedArray(s.fds, numFD)
+    else:
+      s.fds.setLen(numFD)
+    for i in s.numFD ..< numFD:
+      s.fds[i].ident = InvalidIdent
+    s.numFD = numFD
 
 proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle,
                         events: set[Event], data: T) =
diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim
index e4c2b2124..b9c834127 100644
--- a/lib/pure/selectors.nim
+++ b/lib/pure/selectors.nim
@@ -239,6 +239,9 @@ else:
     proc allocSharedArray[T](nsize: int): ptr SharedArray[T] =
       result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize))
 
+    proc reallocSharedArray[T](sa: ptr SharedArray[T], nsize: int): ptr SharedArray[T] =
+      result = cast[ptr SharedArray[T]](reallocShared(sa, sizeof(T) * nsize))
+
     proc deallocSharedArray[T](sa: ptr SharedArray[T]) =
       deallocShared(cast[pointer](sa))
   type