Merge branch 'devel' into sighashes

5 files changed, 133 insertions, 62 deletions

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index 01088c2e7..1367bc411 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -893,9 +893,11 @@ when defined(windows) or defined(nimdoc):
               deallocShared(cast[pointer](pcd))
               raiseOSError(osLastError())
             else:
-              # we ref pcd.ovl one more time, because it will be unrefed in
-              # poll()
+              # we incref `pcd.ovl` and `protect` callback one more time,
+              # because it will be unrefed and disposed in `poll()` after
+              # callback finishes.
               GC_ref(pcd.ovl)
+              pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
     )
     # We need to protect our callback environment value, so GC will not free it
     # accidentally.
@@ -956,17 +958,8 @@ when defined(windows) or defined(nimdoc):
   initAll()
 else:
   import selectors
-  when defined(windows):
-    import winlean
-    const
-      EINTR = WSAEINPROGRESS
-      EINPROGRESS = WSAEINPROGRESS
-      EWOULDBLOCK = WSAEWOULDBLOCK
-      EAGAIN = EINPROGRESS
-      MSG_NOSIGNAL = 0
-  else:
-    from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK,
-                      MSG_NOSIGNAL
+  from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK,
+                    MSG_NOSIGNAL
 
   type
     AsyncFD* = distinct cint
diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim
index 906a9d23e..fda41dadb 100644
--- a/lib/pure/dynlib.nim
+++ b/lib/pure/dynlib.nim
@@ -11,20 +11,22 @@
 ## libraries. On POSIX this uses the ``dlsym`` mechanism, on
 ## Windows ``LoadLibrary``.
 
+import strutils
+
 type
   LibHandle* = pointer ## a handle to a dynamically loaded library
 
 {.deprecated: [TLibHandle: LibHandle].}
 
-proc loadLib*(path: string, global_symbols=false): LibHandle
+proc loadLib*(path: string, global_symbols=false): LibHandle {.gcsafe.}
   ## loads a library from `path`. Returns nil if the library could not
   ## be loaded.
 
-proc loadLib*(): LibHandle
+proc loadLib*(): LibHandle {.gcsafe.}
   ## gets the handle from the current executable. Returns nil if the
   ## library could not be loaded.
 
-proc unloadLib*(lib: LibHandle)
+proc unloadLib*(lib: LibHandle) {.gcsafe.}
   ## unloads the library `lib`
 
 proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} =
@@ -34,7 +36,7 @@ proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} =
   e.msg = "could not find symbol: " & $name
   raise e
 
-proc symAddr*(lib: LibHandle, name: cstring): pointer
+proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.}
   ## retrieves the address of a procedure/variable from `lib`. Returns nil
   ## if the symbol could not be found.
 
@@ -44,6 +46,28 @@ proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer =
   result = symAddr(lib, name)
   if result == nil: raiseInvalidLibrary(name)
 
+proc libCandidates*(s: string, dest: var seq[string]) =
+  ## given a library name pattern `s` write possible library names to `dest`.
+  var le = strutils.find(s, '(')
+  var ri = strutils.find(s, ')', le+1)
+  if le >= 0 and ri > le:
+    var prefix = substr(s, 0, le - 1)
+    var suffix = substr(s, ri + 1)
+    for middle in split(substr(s, le + 1, ri - 1), '|'):
+      libCandidates(prefix & middle & suffix, dest)
+  else:
+    add(dest, s)
+
+proc loadLibPattern*(pattern: string, global_symbols=false): LibHandle =
+  ## loads a library with name matching `pattern`, similar to what `dlimport`
+  ## pragma does. Returns nil if the library could not be loaded.
+  ## Warning: this proc uses the GC and so cannot be used to load the GC.
+  var candidates = newSeq[string]()
+  libCandidates(pattern, candidates)
+  for c in candidates:
+    result = loadLib(c, global_symbols)
+    if not result.isNil: break
+
 when defined(posix):
   #
   # =========================================================================
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 863a8a6f4..5e10f2291 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -90,8 +90,8 @@ when defineSsl:
 
     SslContext* = ref object
       context*: SslCtx
-      extraInternalIndex: int
       referencedData: HashSet[int]
+      extraInternal: SslContextExtraInternal
 
     SslAcceptResult* = enum
       AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess
@@ -103,6 +103,10 @@ when defineSsl:
 
     SslServerGetPskFunc* = proc(identity: string): string
 
+    SslContextExtraInternal = ref object of RootRef
+      serverGetPskFunc: SslServerGetPskFunc
+      clientGetPskFunc: SslClientGetPskFunc
+
   {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode,
     TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext,
     TSSLAcceptResult: SSLAcceptResult].}
@@ -240,11 +244,6 @@ when defineSsl:
   ErrLoadBioStrings()
   OpenSSL_add_all_algorithms()
 
-  type
-    SslContextExtraInternal = ref object of RootRef
-      serverGetPskFunc: SslServerGetPskFunc
-      clientGetPskFunc: SslClientGetPskFunc
-
   proc raiseSSLError*(s = "") =
     ## Raises a new SSL error.
     if s != "":
@@ -257,12 +256,6 @@ when defineSsl:
     var errStr = ErrErrorString(err, nil)
     raise newException(SSLError, $errStr)
 
-  proc getExtraDataIndex*(ctx: SSLContext): int =
-    ## Retrieves unique index for storing extra data in SSLContext.
-    result = SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil).int
-    if result < 0:
-      raiseSSLError()
-
   proc getExtraData*(ctx: SSLContext, index: int): RootRef =
     ## Retrieves arbitrary data stored inside SSLContext.
     if index notin ctx.referencedData:
@@ -347,15 +340,11 @@ when defineSsl:
     discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY)
     newCTX.loadCertificates(certFile, keyFile)
 
-    result = SSLContext(context: newCTX, extraInternalIndex: 0,
-        referencedData: initSet[int]())
-    result.extraInternalIndex = getExtraDataIndex(result)
-
-    let extraInternal = new(SslContextExtraInternal)
-    result.setExtraData(result.extraInternalIndex, extraInternal)
+    result = SSLContext(context: newCTX, referencedData: initSet[int](),
+      extraInternal: new(SslContextExtraInternal))
 
   proc getExtraInternal(ctx: SSLContext): SslContextExtraInternal =
-    return SslContextExtraInternal(ctx.getExtraData(ctx.extraInternalIndex))
+    return ctx.extraInternal
 
   proc destroyContext*(ctx: SSLContext) =
     ## Free memory referenced by SSLContext.
@@ -379,7 +368,7 @@ when defineSsl:
 
   proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar;
     max_psk_len: cuint): cuint {.cdecl.} =
-    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0)
+    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX)
     let hintString = if hint == nil: nil else: $hint
     let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString)
     if psk.len.cuint > max_psk_len:
@@ -398,8 +387,6 @@ when defineSsl:
     ##
     ## Only used in PSK ciphersuites.
     ctx.getExtraInternal().clientGetPskFunc = fun
-    assert ctx.extraInternalIndex == 0,
-          "The pskClientCallback assumes the extraInternalIndex is 0"
     ctx.context.SSL_CTX_set_psk_client_callback(
         if fun == nil: nil else: pskClientCallback)
 
@@ -407,7 +394,7 @@ when defineSsl:
     return ctx.getExtraInternal().serverGetPskFunc
 
   proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} =
-    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0)
+    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX)
     let pskString = (ctx.serverGetPskFunc)($identity)
     if psk.len.cint > max_psk_len:
       return 0
diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim
index 050712902..87865c005 100644
--- a/lib/pure/smtp.nim
+++ b/lib/pure/smtp.nim
@@ -256,24 +256,48 @@ proc close*(smtp: AsyncSmtp) {.async.} =
   smtp.sock.close()
 
 when not defined(testing) and isMainModule:
-  #var msg = createMessage("Test subject!",
-  #     "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"])
-  #echo(msg)
-
-  #var smtpConn = connect("localhost", Port 25, false, true)
-  #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg)
-
-  #echo(decode("a17sm3701420wbe.12"))
-  proc main() {.async.} =
-    var client = newAsyncSmtp("smtp.gmail.com", Port(465), true)
+  # To test with a real SMTP service, create a smtp.ini file, e.g.:
+  # username = ""
+  # password = ""
+  # smtphost = "smtp.gmail.com"
+  # port = 465
+  # use_tls = true
+  # sender = ""
+  # recipient = ""
+
+  import parsecfg
+
+  proc `[]`(c: Config, key: string): string = c.getSectionValue("", key)
+
+  let
+    conf = loadConfig("smtp.ini")
+    msg = createMessage("Hello from Nim's SMTP!",
+      "Hello!\n Is this awesome or what?", @[conf["recipient"]])
+
+  assert conf["smtphost"] != ""
+
+  proc async_test() {.async.} =
+    let client = newAsyncSmtp(
+      conf["smtphost"],
+      conf["port"].parseInt.Port,
+      conf["use_tls"].parseBool
+    )
     await client.connect()
-    await client.auth("johndoe", "foo")
-    var msg = createMessage("Hello from Nim's SMTP!",
-                            "Hello!!!!.\n Is this awesome or what?",
-                            @["blah@gmail.com"])
-    echo(msg)
-    await client.sendMail("blah@gmail.com", @["blah@gmail.com"], $msg)
-
+    await client.auth(conf["username"], conf["password"])
+    await client.sendMail(conf["sender"], @[conf["recipient"]], $msg)
     await client.close()
-
-  waitFor main()
+    echo "async email sent"
+
+  proc sync_test() =
+    var smtpConn = connect(
+      conf["smtphost"],
+      conf["port"].parseInt.Port,
+      conf["use_tls"].parseBool,
+      true, # debug
+    )
+    smtpConn.auth(conf["username"], conf["password"])
+    smtpConn.sendmail(conf["sender"], @[conf["recipient"]], $msg)
+    echo "sync email sent"
+
+  waitFor async_test()
+  sync_test()
diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim
index d4734c3e3..16cf91d40 100644
--- a/lib/pure/terminal.nim
+++ b/lib/pure/terminal.nim
@@ -13,6 +13,8 @@
 ## Windows API.
 ## Changing the style is permanent even after program termination! Use the
 ## code ``system.addQuitProc(resetAttributes)`` to restore the defaults.
+## Similarly, if you hide the cursor, make sure to unhide it with
+## ``showCursor`` before quitting.
 
 import macros
 
@@ -29,6 +31,8 @@ when defined(windows):
     BACKGROUND_GREEN = 32
     BACKGROUND_RED = 64
     BACKGROUND_INTENSITY = 128
+    FOREGROUND_RGB = FOREGROUND_RED or FOREGROUND_GREEN or FOREGROUND_BLUE
+    BACKGROUND_RGB = BACKGROUND_RED or BACKGROUND_GREEN or BACKGROUND_BLUE
 
   type
     SHORT = int16
@@ -49,6 +53,10 @@ when defined(windows):
       srWindow: SMALL_RECT
       dwMaximumWindowSize: COORD
 
+    CONSOLE_CURSOR_INFO = object
+      dwSize: DWORD
+      bVisible: WINBOOL
+
   proc duplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE,
                        hTargetProcessHandle: HANDLE, lpTargetHandle: ptr HANDLE,
                        dwDesiredAccess: DWORD, bInheritHandle: WINBOOL,
@@ -60,6 +68,14 @@ when defined(windows):
     lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall,
     dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".}
 
+  proc getConsoleCursorInfo(hConsoleOutput: HANDLE,
+      lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{.
+      stdcall, dynlib: "kernel32", importc: "GetConsoleCursorInfo".}
+
+  proc setConsoleCursorInfo(hConsoleOutput: HANDLE,
+      lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{.
+      stdcall, dynlib: "kernel32", importc: "SetConsoleCursorInfo".}
+
   proc terminalWidthIoctl*(handles: openArray[Handle]): int =
     var csbi: CONSOLE_SCREEN_BUFFER_INFO
     for h in handles:
@@ -179,6 +195,30 @@ else:
       return w
     return 80                               #Finally default to venerable value
 
+when defined(windows):
+  proc setCursorVisibility(f: File, visible: bool) =
+    var ccsi: CONSOLE_CURSOR_INFO
+    let h = conHandle(f)
+    if getConsoleCursorInfo(h, addr(ccsi)) == 0:
+      raiseOSError(osLastError())
+    ccsi.bVisible = if visible: 1 else: 0
+    if setConsoleCursorInfo(h, addr(ccsi)) == 0:
+      raiseOSError(osLastError())
+
+proc hideCursor*(f: File) =
+  ## Hides the cursor.
+  when defined(windows):
+    setCursorVisibility(f, false)
+  else:
+    f.write("\e[?25l")
+
+proc showCursor*(f: File) =
+  ## Shows the cursor.
+  when defined(windows):
+    setCursorVisibility(f, true)
+  else:
+    f.write("\e[?25h")
+
 proc setCursorPos*(f: File, x, y: int) =
   ## Sets the terminal's cursor to the (x,y) position.
   ## (0,0) is the upper left of the screen.
@@ -369,12 +409,13 @@ proc setStyle*(f: File, style: set[Style]) =
   ## Sets the terminal style.
   when defined(windows):
     let h = conHandle(f)
+    var old = getAttributes(h) and (FOREGROUND_RGB or BACKGROUND_RGB)
     var a = 0'i16
     if styleBright in style: a = a or int16(FOREGROUND_INTENSITY)
     if styleBlink in style: a = a or int16(BACKGROUND_INTENSITY)
     if styleReverse in style: a = a or 0x4000'i16 # COMMON_LVB_REVERSE_VIDEO
     if styleUnderscore in style: a = a or 0x8000'i16 # COMMON_LVB_UNDERSCORE
-    discard setConsoleTextAttribute(h, a)
+    discard setConsoleTextAttribute(h, old or a)
   else:
     for s in items(style):
       f.write("\e[" & $ord(s) & 'm')
@@ -423,7 +464,7 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) =
   ## Sets the terminal's foreground color.
   when defined(windows):
     let h = conHandle(f)
-    var old = getAttributes(h) and not 0x0007
+    var old = getAttributes(h) and not FOREGROUND_RGB
     if bright:
       old = old or FOREGROUND_INTENSITY
     const lookup: array[ForegroundColor, int] = [
@@ -445,7 +486,7 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) =
   ## Sets the terminal's background color.
   when defined(windows):
     let h = conHandle(f)
-    var old = getAttributes(h) and not 0x0070
+    var old = getAttributes(h) and not BACKGROUND_RGB
     if bright:
       old = old or BACKGROUND_INTENSITY
     const lookup: array[BackgroundColor, int] = [
@@ -558,6 +599,8 @@ proc getch*(): char =
     discard fd.tcsetattr(TCSADRAIN, addr oldMode)
 
 # Wrappers assuming output to stdout:
+template hideCursor*() = hideCursor(stdout)
+template showCursor*() = showCursor(stdout)
 template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y)
 template setCursorXPos*(x: int)   = setCursorXPos(stdout, x)
 when defined(windows):