diff options
| -rw-r--r-- | tools/nimgrab.nim | 32 | ||||
| -rw-r--r-- | tools/urldownloader.nim | 431 |
2 files changed, 454 insertions, 9 deletions
diff --git a/tools/nimgrab.nim b/tools/nimgrab.nim index 2fa50e89a..69824369e 100644 --- a/tools/nimgrab.nim +++ b/tools/nimgrab.nim @@ -1,15 +1,29 @@ -import os, asyncdispatch, httpclient -proc syncDownload(url, file: string) = - var client = newHttpClient() - proc onProgressChanged(total, progress, speed: BiggestInt) = - echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" - echo clamp(int(progress*100 div total), 0, 100), "%" +when defined(windows): + import os, urldownloader - client.onProgressChanged = onProgressChanged - client.downloadFile(url, file) - echo "100%" + proc syncDownload(url, file: string) = + proc progress(status: DownloadStatus, progress: uint, total: uint, + message: string) {.procvar, gcsafe.} = + echo "Downloading " & url + echo clamp(int(progress.BiggestInt*100 div total.BiggestInt), 0, 100), "%" + + downloadToFile(url, file, {optUseCache}, progress) + echo "100%" + +else: + import os, asyncdispatch, httpclient + + proc syncDownload(url, file: string) = + var client = newHttpClient() + proc onProgressChanged(total, progress, speed: BiggestInt) = + echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" + echo clamp(int(progress*100 div total), 0, 100), "%" + + client.onProgressChanged = onProgressChanged + client.downloadFile(url, file) + echo "100%" if os.paramCount() != 2: quit "Usage: nimgrab <url> <file>" diff --git a/tools/urldownloader.nim b/tools/urldownloader.nim new file mode 100644 index 000000000..46d67c821 --- /dev/null +++ b/tools/urldownloader.nim @@ -0,0 +1,431 @@ + +# +# +# Windows native FTP/HTTP/HTTPS file downloader +# (c) Copyright 2017 Eugene Kabanov +# +# See the file "LICENSE", included in this +# distribution, for details about the copyright. +# + +## This module implements native Windows FTP/HTTP/HTTPS downloading feature, +## using ``urlmon.UrlDownloadToFile()``. +## +## + +when not (defined(windows) or defined(nimdoc)): + {.error: "Platform is not supported.".} + +import os + +type + DownloadOptions* = enum + ## Available download options + optUseCache, ## Use Windows cache. + optUseProgressCallback, ## Report progress via callback. + optIgnoreSecurity ## Ignore HTTPS security problems. + + DownloadStatus* = enum + ## Available download status sent to ``progress`` callback. + statusProxyDetecting, ## Automatic Proxy detection. + statusCookieSent ## Cookie will be sent with request. + statusResolving, ## Resolving URL with DNS. + statusConnecting, ## Establish connection to server. + statusRedirecting ## HTTP redirection pending. + statusRequesting, ## Sending request to server. + statusMimetypeAvailable, ## Mimetype received from server. + statusBeginDownloading, ## Download process starting. + statusDownloading, ## Download process pending. + statusEndDownloading, ## Download process finished. + statusCacheAvailable ## File found in Windows cache. + statusUnsupported ## Unsupported status. + statusError ## Error happens. + + DownloadProgressCallback* = proc(status: DownloadStatus, progress: uint, + progressMax: uint, + message: string) + ## Progress callback. + ## + ## status + ## Indicate current stage of downloading process. + ## + ## progress + ## Number of bytes currently downloaded. Available only, if ``status`` is + ## ``statusBeginDownloading``, ``statusDownloading`` or + ## ``statusEndDownloading``. + ## + ## progressMax + ## Number of bytes expected to download. Available only, if ``status`` is + ## ``statusBeginDownloading``, ``statusDownloading`` or + ## ``statusEndDownloading``. + ## + ## message + ## Status message, which depends on ``status`` code. + ## + ## Available messages' values: + ## + ## statusResolving + ## URL hostname to be resolved. + ## statusConnecting + ## IP address + ## statusMimetypeAvailable + ## Downloading resource MIME type. + ## statusCacheAvailable + ## Path to filename stored in Windows cache. + +type + UUID = array[4, uint32] + + LONG = clong + ULONG = culong + HRESULT = clong + DWORD = uint32 + OLECHAR = uint16 + OLESTR = ptr OLECHAR + LPWSTR = OLESTR + UINT = cuint + REFIID = ptr UUID + +const + E_NOINTERFACE = 0x80004002'i32 + E_NOTIMPL = 0x80004001'i32 + S_OK = 0x00000000'i32 + + CP_UTF8 = 65001'u32 + + IID_IUnknown = UUID([0'u32, 0'u32, 192'u32, 1174405120'u32]) + IID_IBindStatusCallback = UUID([2045430209'u32, 298760953'u32, + 2852160140'u32, 195644160'u32]) + + BINDF_GETNEWESTVERSION = 0x00000010'u32 + BINDF_IGNORESECURITYPROBLEM = 0x00000100'u32 + BINDF_RESYNCHRONIZE = 0x00000200'u32 + BINDF_NO_UI = 0x00000800'u32 + BINDF_SILENTOPERATION = 0x00001000'u32 + BINDF_PRAGMA_NO_CACHE = 0x00002000'u32 + + ERROR_FILE_NOT_FOUND = 2 + ERROR_ACCESS_DENIED = 5 + + BINDSTATUS_FINDINGRESOURCE = 1 + BINDSTATUS_CONNECTING = 2 + BINDSTATUS_REDIRECTING = 3 + BINDSTATUS_BEGINDOWNLOADDATA = 4 + BINDSTATUS_DOWNLOADINGDATA = 5 + BINDSTATUS_ENDDOWNLOADDATA = 6 + BINDSTATUS_SENDINGREQUEST = 11 + BINDSTATUS_MIMETYPEAVAILABLE = 13 + BINDSTATUS_CACHEFILENAMEAVAILABLE = 14 + BINDSTATUS_PROXYDETECTING = 32 + BINDSTATUS_COOKIE_SENT = 34 + +type + STGMEDIUM = object + tymed: DWORD + pstg: pointer + pUnkForRelease: pointer + + SECURITY_ATTRIBUTES = object + nLength*: uint32 + lpSecurityDescriptor*: pointer + bInheritHandle*: int32 + + BINDINFO = object + cbSize: ULONG + stgmedData: STGMEDIUM + szExtraInfo: LPWSTR + grfBindInfoF: DWORD + dwBindVerb: DWORD + szCustomVerb: LPWSTR + cbstgmedData: DWORD + dwOptions: DWORD + dwOptionsFlags: DWORD + dwCodePage: DWORD + securityAttributes: SECURITY_ATTRIBUTES + iid: UUID + pUnk: pointer + dwReserved: DWORD + + IBindStatusCallback = object + vtable: ptr IBindStatusCallbackVTable + options: set[DownloadOptions] + objectRefCount: ULONG + binfoFlags: DWORD + progressCallback: DownloadProgressCallback + + PIBindStatusCallback = ptr IBindStatusCallback + LPBINDSTATUSCALLBACK = PIBindStatusCallback + + IBindStatusCallbackVTable = object + QueryInterface: proc (self: PIBindStatusCallback, + riid: ptr UUID, + pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} + AddRef: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} + Release: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} + OnStartBinding: proc(self: PIBindStatusCallback, + dwReserved: DWORD, pib: pointer): HRESULT + {.gcsafe, stdcall.} + GetPriority: proc(self: PIBindStatusCallback, pnPriority: ptr LONG): HRESULT + {.gcsafe, stdcall.} + OnLowResource: proc(self: PIBindStatusCallback, dwReserved: DWORD): HRESULT + {.gcsafe, stdcall.} + OnProgress: proc(self: PIBindStatusCallback, ulProgress: ULONG, + ulProgressMax: ULONG, ulStatusCode: ULONG, + szStatusText: LPWSTR): HRESULT + {.gcsafe, stdcall.} + OnStopBinding: proc(self: PIBindStatusCallback, hresult: HRESULT, + szError: LPWSTR): HRESULT + {.gcsafe, stdcall.} + GetBindInfo: proc(self: PIBindStatusCallback, grfBINDF: ptr DWORD, + pbindinfo: ptr BINDINFO): HRESULT + {.gcsafe, stdcall.} + OnDataAvailable: proc(self: PIBindStatusCallback, grfBSCF: DWORD, + dwSize: DWORD, pformatetc: pointer, + pstgmed: pointer): HRESULT + {.gcsafe, stdcall.} + OnObjectAvailable: proc(self: PIBindStatusCallback, riid: REFIID, + punk: pointer): HRESULT + {.gcsafe, stdcall.} + +template FAILED(hr: HRESULT): bool = + (hr < 0) + +proc URLDownloadToFile(pCaller: pointer, szUrl: LPWSTR, szFileName: LPWSTR, + dwReserved: DWORD, + lpfnCb: LPBINDSTATUSCALLBACK): HRESULT + {.stdcall, dynlib: "urlmon.dll", importc: "URLDownloadToFileW".} + +proc WideCharToMultiByte(CodePage: UINT, dwFlags: DWORD, + lpWideCharStr: ptr OLECHAR, cchWideChar: cint, + lpMultiByteStr: ptr char, cbMultiByte: cint, + lpDefaultChar: ptr char, + lpUsedDefaultChar: ptr uint32): cint + {.stdcall, dynlib: "kernel32.dll", importc: "WideCharToMultiByte".} + +proc MultiByteToWideChar(CodePage: UINT, dwFlags: DWORD, + lpMultiByteStr: ptr char, cbMultiByte: cint, + lpWideCharStr: ptr OLECHAR, cchWideChar: cint): cint + {.stdcall, dynlib: "kernel32.dll", importc: "MultiByteToWideChar".} +proc DeleteUrlCacheEntry(lpszUrlName: LPWSTR): int32 + {.stdcall, dynlib: "wininet.dll", importc: "DeleteUrlCacheEntryW".} + +proc `==`(a, b: UUID): bool = + result = false + if a[0] == b[0] and a[1] == b[1] and + a[2] == b[2] and a[3] == b[3]: + result = true + +proc `$`(bstr: LPWSTR): string = + var buffer: char + var count = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(buffer), 0, + nil, nil) + if count == 0: + raiseOsError(osLastError()) + else: + result = newStringOfCap(count + 8) + let res = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(result[0]), count, + nil, nil) + if res == 0: + raiseOsError(osLastError()) + result.setLen(res - 1) + +proc toBstring(str: string): LPWSTR = + var buffer: OLECHAR + var count = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, + addr(buffer), 0) + if count == 0: + raiseOsError(osLastError()) + else: + result = cast[LPWSTR](alloc0((count + 1) * sizeof(OLECHAR))) + let res = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, + result, count) + if res == 0: + raiseOsError(osLastError()) + +proc freeBstring(bstr: LPWSTR) = + dealloc(bstr) + +proc getStatus(scode: ULONG): DownloadStatus = + case scode + of 0: result = statusError + of BINDSTATUS_PROXYDETECTING: result = statusProxyDetecting + of BINDSTATUS_REDIRECTING: result = statusRedirecting + of BINDSTATUS_COOKIE_SENT: result = statusCookieSent + of BINDSTATUS_FINDINGRESOURCE: result = statusResolving + of BINDSTATUS_CONNECTING: result = statusConnecting + of BINDSTATUS_SENDINGREQUEST: result = statusRequesting + of BINDSTATUS_MIMETYPEAVAILABLE: result = statusMimetypeAvailable + of BINDSTATUS_BEGINDOWNLOADDATA: result = statusBeginDownloading + of BINDSTATUS_DOWNLOADINGDATA: result = statusDownloading + of BINDSTATUS_ENDDOWNLOADDATA: result = statusEndDownloading + of BINDSTATUS_CACHEFILENAMEAVAILABLE: result = statusCacheAvailable + else: result = statusUnsupported + +proc addRef(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = + inc(self.objectRefCount) + result = self.objectRefCount + +proc release(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = + dec(self.objectRefCount) + result = self.objectRefCount + +proc queryInterface(self: PIBindStatusCallback, riid: ptr UUID, + pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} = + pvObject[] = nil + + if riid[] == IID_IUnknown: + pvObject[] = cast[pointer](self) + elif riid[] == IID_IBindStatusCallback: + pvObject[] = cast[pointer](self) + + if not isNil(pvObject[]): + discard addRef(self) + result = S_OK + else: + result = E_NOINTERFACE + +proc onStartBinding(self: PIBindStatusCallback, dwReserved: DWORD, + pib: pointer): HRESULT {.gcsafe, stdcall.} = + result = S_OK + +proc getPriority(self: PIBindStatusCallback, + pnPriority: ptr LONG): HRESULT {.gcsafe, stdcall.} = + result = E_NOTIMPL + +proc onLowResource(self: PIBindStatusCallback, + dwReserved: DWORD): HRESULT {.gcsafe, stdcall.} = + result = S_OK + +proc onStopBinding(self: PIBindStatusCallback, + hresult: HRESULT, szError: LPWSTR): HRESULT + {.gcsafe, stdcall.} = + result = S_OK + +proc getBindInfo(self: PIBindStatusCallback, + grfBINDF: ptr DWORD, pbindinfo: ptr BINDINFO): HRESULT + {.gcsafe, stdcall.} = + var cbSize = pbindinfo.cbSize + zeroMem(cast[pointer](pbindinfo), cbSize) + pbindinfo.cbSize = cbSize + grfBINDF[] = self.binfoFlags + result = S_OK + +proc onDataAvailable(self: PIBindStatusCallback, + grfBSCF: DWORD, dwSize: DWORD, pformatetc: pointer, + pstgmed: pointer): HRESULT {.gcsafe, stdcall.} = + result = S_OK + +proc onObjectAvailable(self: PIBindStatusCallback, + riid: REFIID, punk: pointer): HRESULT + {.gcsafe, stdcall.} = + result = S_OK + +proc onProgress(self: PIBindStatusCallback, + ulProgress: ULONG, ulProgressMax: ULONG, ulStatusCode: ULONG, + szStatusText: LPWSTR): HRESULT {.gcsafe, stdcall.} = + var message: string + if optUseProgressCallback in self.options: + if not isNil(szStatusText): + message = $szStatusText + else: + message = "" + self.progressCallback(getStatus(ulStatusCode), uint(ulProgress), + uint(ulProgressMax), message) + result = S_OK + +proc newBindStatusCallback(): IBindStatusCallback = + result = IBindStatusCallback() + result.vtable = cast[ptr IBindStatusCallbackVTable]( + alloc0(sizeof(IBindStatusCallbackVTable)) + ) + result.vtable.QueryInterface = queryInterface + result.vtable.AddRef = addRef + result.vtable.Release = release + result.vtable.OnStartBinding = onStartBinding + result.vtable.GetPriority = getPriority + result.vtable.OnLowResource = onLowResource + result.vtable.OnStopBinding = onStopBinding + result.vtable.GetBindInfo = getBindInfo + result.vtable.OnDataAvailable = onDataAvailable + result.vtable.OnObjectAvailable = onObjectAvailable + result.vtable.OnProgress = onProgress + result.objectRefCount = 1 + +proc freeBindStatusCallback(v: var IBindStatusCallback) = + dealloc(v.vtable) + +proc downloadToFile*(szUrl: string, szFileName: string, + options: set[DownloadOptions] = {}, + progresscb: DownloadProgressCallback = nil) = + ## Downloads from URL specified in ``szUrl`` to local filesystem path + ## specified in ``szFileName``. + ## + ## szUrl + ## URL to download, international names are supported. + ## szFileName + ## Destination path for downloading resource. + ## options + ## Downloading options. Currently only 2 options supported. + ## progresscb + ## Callback procedure, which will be called throughout the download + ## process, indicating status and progress. + ## + ## Available downloading options: + ## + ## optUseCache + ## Try to use Windows cache when downloading. + ## optIgnoreSecurity + ## Ignore HTTPS security problems, e.g. self-signed HTTPS certificate. + ## + var bszUrl = szUrl.toBstring() + var bszFile = szFileName.toBstring() + var bstatus = newBindStatusCallback() + + bstatus.options = {} + + if optUseCache notin options: + bstatus.options.incl(optUseCache) + let res = DeleteUrlCacheEntry(bszUrl) + if res == 0: + let err = osLastError() + if err.int notin {ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND}: + freeBindStatusCallback(bstatus) + freeBstring(bszUrl) + freeBstring(bszFile) + raiseOsError(err) + + bstatus.binfoFlags = BINDF_GETNEWESTVERSION or BINDF_RESYNCHRONIZE or + BINDF_PRAGMA_NO_CACHE or BINDF_NO_UI or + BINDF_SILENTOPERATION + + if optIgnoreSecurity in options: + bstatus.binfoFlags = bstatus.binfoFlags or BINDF_IGNORESECURITYPROBLEM + + if not isNil(progresscb): + bstatus.options.incl(optUseProgressCallback) + bstatus.progressCallback = progresscb + + let res = URLDownloadToFile(nil, bszUrl, bszFile, 0, addr bstatus) + if FAILED(res): + freeBindStatusCallback(bstatus) + freeBstring(bszUrl) + freeBstring(bszFile) + raiseOsError(OSErrorCode(res)) + + freeBindStatusCallback(bstatus) + freeBstring(bszUrl) + freeBstring(bszFile) + +when isMainModule: + proc progress(status: DownloadStatus, progress: uint, progressMax: uint, + message: string) {.procvar,gcsafe.} = + const downset: set[DownloadStatus] = {statusBeginDownloading, + statusDownloading, statusEndDownloading} + if status in downset: + var message = "Downloaded " & $progress & " of " & $progressMax & "\c" + stdout.write(message) + else: + echo "Status [" & $status & "] message = [" & $message & "]" + + downloadToFile("https://nim-lang.org/download/mingw64-6.3.0.7z", + "test.zip", {optUseCache}, progress) |