fix #14320 (tasyncawait.nim is recently very flaky) + avoid hardcoding service ports everywhere + flakyAssert (#14327)

* hotfix #14320 tasyncawait.nim is recently very flaky
* fix #14327
* add flakyAssert

2 files changed, 37 insertions, 0 deletions

diff --git a/testament/lib/stdtest/netutils.nim b/testament/lib/stdtest/netutils.nim
new file mode 100644
index 000000000..eb913a56a
--- /dev/null
+++ b/testament/lib/stdtest/netutils.nim
@@ -0,0 +1,12 @@
+import std/[nativesockets, asyncdispatch, os]
+
+proc bindAvailablePort*(handle: SocketHandle, port = Port(0)): Port =
+  block:
+    var name: Sockaddr_in
+    name.sin_family = typeof(name.sin_family)(toInt(AF_INET))
+    name.sin_port = htons(uint16(port))
+    name.sin_addr.s_addr = htonl(INADDR_ANY)
+    if bindAddr(handle, cast[ptr SockAddr](addr(name)),
+                sizeof(name).Socklen) < 0'i32:
+      raiseOSError(osLastError())
+  result = getLocalAddr(handle, AF_INET)[1]
diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim
new file mode 100644
index 000000000..8083a63e8
--- /dev/null
+++ b/testament/lib/stdtest/testutils.nim
@@ -0,0 +1,25 @@
+import std/private/miscdollars
+
+template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
+  ## API to deal with flaky or failing tests. This avoids disabling entire tests
+  ## altogether so that at least the parts that are working are kept being
+  ## tested. This also avoids making CI fail periodically for tests known to
+  ## be flaky. Finally, for known failures, passing `notifySuccess = true` will
+  ## log that the test succeeded, which may indicate that a bug was fixed
+  ## "by accident" and should be looked into.
+  const info = instantiationInfo(-1, true)
+  const expr = astToStr(cond)
+  if cond and not notifySuccess:
+    discard # silent success
+  else:
+    var msg2 = ""
+    toLocation(msg2, info.filename, info.line, info.column)
+    if cond:
+      # a flaky test is failing, we still report it but we don't fail CI
+      msg2.add " FLAKY_SUCCESS "
+    else:
+      # a previously failing test is now passing, a pre-existing bug might've been
+      # fixed by accidend
+      msg2.add " FLAKY_FAILURE "
+    msg2.add $expr & " " & msg
+    echo msg2