# # # Nim's Runtime Library # (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements an advanced facility for executing OS processes ## and process communication. ## ## **See also:** ## * `os module `_ ## * `streams module `_ ## * `memfiles module `_ include "system/inclrtl" import strutils, os, strtabs, streams, cpuinfo export quoteShell, quoteShellWindows, quoteShellPosix when defined(windows): import winlean else: import posix when defined(linux): import linux type ProcessOption* = enum ## Options that can be passed to `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. poEchoCmd, ## Echo the command before execution. poUsePath, ## Asks system to search for executable using PATH environment ## variable. ## On Windows, this is the default. poEvalCommand, ## Pass `command` directly to the shell, without quoting. ## Use it only if `command` comes from trusted source. poStdErrToStdOut, ## Merge stdout and stderr to the stdout stream. poParentStreams, ## Use the parent's streams. poInteractive, ## Optimize the buffer handling for responsiveness for ## UI applications. Currently this only affects ## Windows: Named pipes are used so that you can peek ## at the process' output streams. poDaemon ## Windows: The program creates no Window. ## Unix: Start the program as a daemon. This is still ## work in progress! ProcessObj = object of RootObj when defined(windows): fProcessHandle: Handle fThreadHandle: Handle inHandle, outHandle, errHandle: FileHandle id: Handle else: inHandle, outHandle, errHandle: FileHandle id: Pid inStream, outStream, errStream: owned(Stream) exitStatus: cint exitFlag: bool options: set[ProcessOption] Process* = ref ProcessObj ## Represents an operating system process. const poDemon* {.deprecated.} = poDaemon ## Nim versions before 0.20 ## used the wrong spelling ("demon"). ## Now `ProcessOption` uses the correct spelling ("daemon"), ## and this is needed just for backward compatibility. proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): TaintedString {. rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. ## ## **WARNING:** This function uses `poEvalCommand` by default for backward ## compatibility. ## Make sure to pass options explicitly. ## ## See also: ## * `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process)>`_ ## * `execCmd proc <#execCmd,string>`_ ## ## Example: ## ## .. code-block:: Nim ## let outp = execProcess("nim", args=["c", "-r", "mytestfile.nim"], options={poUsePath}) ## let outp_shell = execProcess("nim c -r mytestfile.nim") ## # Note: outp may have an interleave of text from the nim compile ## # and any output from mytestfile when it runs proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## Executes ``command`` and returns its error code. ## ## Standard input, output, error streams are inherited from the calling process. ## This operation is also often called `system`:idx:. ## ## See also: ## * `execCmdEx proc <#execCmdEx,string,set[ProcessOption]>`_ ## * `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execProcess proc ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: ## ## .. code-block:: Nim ## let errC = execCmd("nim c -r mytestfile.nim") proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): owned(Process) {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory ## is used (default). `args` are the command line arguments that are passed to the ## process. On many operating systems, the first command line argument is the ## name of the executable. `args` should *not* contain this argument! ## `env` is the environment that will be passed to the process. ## If ``env == nil`` (default) the environment is inherited of ## the parent process. `options` are additional flags that may be passed ## to `startProcess`. See the documentation of `ProcessOption<#ProcessOption>`_ ## for the meaning of these flags. ## ## You need to `close <#close,Process>`_ the process when done. ## ## Note that you can't pass any `args` if you use the option ## ``poEvalCommand``, which invokes the system shell to run the specified ## `command`. In this situation you have to concatenate manually the contents ## of `args` to `command` carefully escaping/quoting any special characters, ## since it will be passed *as is* to the system shell. Each system/shell may ## feature different escaping rules, so try to avoid this kind of shell ## invocation if possible as it leads to non portable software. ## ## Return value: The newly created process object. Nil is never returned, ## but ``OSError`` is raised in case of an error. ## ## See also: ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process)>`_ ## * `execProcess proc ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## ## **WARNING:** If the process has not finished executing, this will forcibly ## terminate the process. Doing so may result in zombie processes and ## `pty leaks `_. proc suspend*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Suspends the process `p`. ## ## See also: ## * `resume proc <#resume,Process>`_ ## * `terminate proc <#terminate,Process>`_ ## * `kill proc <#kill,Process>`_ proc resume*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Resumes the process `p`. ## ## See also: ## * `suspend proc <#suspend,Process>`_ ## * `terminate proc <#terminate,Process>`_ ## * `kill proc <#kill,Process>`_ proc terminate*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Stop the process `p`. ## ## On Posix OSes the procedure sends ``SIGTERM`` to the process. ## On Windows the Win32 API function ``TerminateProcess()`` ## is called to stop the process. ## ## See also: ## * `suspend proc <#suspend,Process>`_ ## * `resume proc <#resume,Process>`_ ## * `kill proc <#kill,Process>`_ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## Kill the process `p`. ## ## On Posix OSes the procedure sends ``SIGKILL`` to the process. ## On Windows ``kill`` is simply an alias for `terminate() <#terminate,Process>`_. ## ## See also: ## * `suspend proc <#suspend,Process>`_ ## * `resume proc <#resume,Process>`_ ## * `terminate proc <#terminate,Process>`_ proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} ## Returns true iff the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = ## Returns `p`'s process ID. ## ## See also: ## * `os.getCurrentProcessId proc `_ return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, extern: "nosp$1", tags: [].} ## Waits for the process to finish and returns `p`'s error code. ## ## **WARNING**: Be careful when using `waitForExit` for processes created without ## `poParentStreams` because they may fill output buffers, causing deadlock. ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s input stream for writing to. ## ## **WARNING**: The returned `Stream` should not be closed manually as it ## is closed when closing the Process ``p``. ## ## See also: ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s output stream for reading from. ## ## **WARNING**: The returned `Stream` should not be closed manually as it ## is closed when closing the Process ``p``. ## ## See also: ## * `inputStream proc <#inputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ proc errorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## Returns ``p``'s error stream for reading from. ## ## **WARNING**: The returned `Stream` should not be closed manually as it ## is closed when closing the Process ``p``. ## ## See also: ## * `inputStream proc <#inputStream,Process>`_ ## * `outputStream proc <#outputStream,Process>`_ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## ## **WARNING**: The returned `FileHandle` should not be closed manually as ## it is closed when closing the Process ``p``. ## ## See also: ## * `outputHandle proc <#outputHandle,Process>`_ ## * `errorHandle proc <#errorHandle,Process>`_ result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", tags: [].} = ## Returns ``p``'s output file handle for reading from. ## ## **WARNING**: The returned `FileHandle` should not be closed manually as ## it is closed when closing the Process ``p``. ## ## See also: ## * `inputHandle proc <#inputHandle,Process>`_ ## * `errorHandle proc <#errorHandle,Process>`_ result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", tags: [].} = ## Returns ``p``'s error file handle for reading from. ## ## **WARNING**: The returned `FileHandle` should not be closed manually as ## it is closed when closing the Process ``p``. ## ## See also: ## * `inputHandle proc <#inputHandle,Process>`_ ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle proc countProcessors*(): int {.rtl, extern: "nosp$1".} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. result = cpuinfo.countProcessors() proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## ## The highest (absolute) return value of all processes is returned. ## Runs `beforeRunEvent` before running each command. assert n > 0 if n > 1: var i = 0 var q = newSeq[Process](n) var idxs = newSeq[int](n) # map process index to cmds index when defined(windows): var w: WOHandleArray var m = min(min(n, MAXIMUM_WAIT_OBJECTS), cmds.len) var wcount = m else: var m = min(n, cmds.len) while i < m: if beforeRunEvent != nil: beforeRunEvent(i) q[i] = startProcess(cmds[i], options = options + {poEvalCommand}) idxs[i] = i when defined(windows): w[i] = q[i].fProcessHandle inc(i) var ecount = len(cmds) while ecount > 0: var rexit = -1 when defined(windows): # waiting for all children, get result if any child exits var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32, INFINITE) if ret == WAIT_TIMEOUT: # must not be happen discard elif ret == WAIT_FAILED: raiseOSError(osLastError()) else: var status: int32 for r in 0..m-1: if not isNil(q[r]) and q[r].fProcessHandle == w[ret]: discard getExitCodeProcess(q[r].fProcessHandle, status) q[r].exitFlag = true q[r].exitStatus = status rexit = r break else: var status: cint = 1 # waiting for all children, get result if any child exits let res = waitpid(-1, status, 0) if res > 0: for r in 0..m-1: if not isNil(q[r]) and q[r].id == res: if WIFEXITED(status) or WIFSIGNALED(status): q[r].exitFlag = true q[r].exitStatus = status rexit = r break else: let err = osLastError() if err == OSErrorCode(ECHILD): # some child exits, we need to check our childs exit codes for r in 0..m-1: if (not isNil(q[r])) and (not running(q[r])): q[r].exitFlag = true q[r].exitStatus = status rexit = r break elif err == OSErrorCode(EINTR): # signal interrupted our syscall, lets repeat it continue else: # all other errors are exceptions raiseOSError(err) if rexit >= 0: result = max(result, abs(q[rexit].peekExitCode())) if afterRunEvent != nil: afterRunEvent(idxs[rexit], q[rexit]) close(q[rexit]) if i < len(cmds): if beforeRunEvent != nil: beforeRunEvent(i) q[rexit] = startProcess(cmds[i], options = options + {poEvalCommand}) idxs[rexit] = i when defined(windows): w[rexit] = q[rexit].fProcessHandle inc(i) else: when defined(windows): for k in 0..wcount - 1: if w[k] == q[rexit].fProcessHandle: w[k] = w[wcount - 1] w[wcount - 1] = 0 dec(wcount) break q[rexit] = nil dec(ecount) else: for i in 0..high(cmds): if beforeRunEvent != nil: beforeRunEvent(i) var p = startProcess(cmds[i], options=options + {poEvalCommand}) result = max(abs(waitForExit(p)), result) if afterRunEvent != nil: afterRunEvent(i, p) close(p) when not defined(useNimRtl): proc execProcess(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): TaintedString = var p = startProcess(command, workingDir=workingDir, args=args, env=env, options=options) var outp = outputStream(p) result = TaintedString"" var line = newStringOfCap(120).TaintedString while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): result.string.add(line.string) result.string.add("\n") elif not running(p): break close(p) template streamAccess(p) = assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams" when defined(Windows) and not defined(useNimRtl): # We need to implement a handle stream for Windows: type FileHandleStream = ref object of StreamObj handle: Handle atTheEnd: bool proc hsClose(s: Stream) = discard # nothing to do here proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd proc hsReadData(s: Stream, buffer: pointer, bufLen: int): int = var s = FileHandleStream(s) if s.atTheEnd: return 0 var br: int32 var a = winlean.readFile(s.handle, buffer, bufLen.cint, addr br, nil) # TRUE and zero bytes returned (EOF). # TRUE and n (>0) bytes returned (good data). # FALSE and bytes returned undefined (system error). if a == 0 and br != 0: raiseOSError(osLastError()) s.atTheEnd = br == 0 #< bufLen result = br proc hsWriteData(s: Stream, buffer: pointer, bufLen: int) = var s = FileHandleStream(s) var bytesWritten: int32 var a = winlean.writeFile(s.handle, buffer, bufLen.cint, addr bytesWritten, nil) if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = new(result) result.handle = handle result.closeImpl = hsClose result.atEndImpl = hsAtEnd result.readDataImpl = hsReadData result.writeDataImpl = hsWriteData proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) for i in 0..high(args): result.add(' ') result.add(quoteShell(args[i])) proc buildEnv(env: StringTableRef): tuple[str: cstring, len: int] = var L = 0 for key, val in pairs(env): inc(L, key.len + val.len + 2) var str = cast[cstring](alloc0(L+2)) L = 0 for key, val in pairs(env): var x = key & "=" & val copyMem(addr(str[L]), cstring(x), x.len+1) # copy \0 inc(L, x.len+1) (str, L) #proc open_osfhandle(osh: Handle, mode: int): int {. # importc: "_open_osfhandle", header: "".} #var # O_WRONLY {.importc: "_O_WRONLY", header: "".}: int # O_RDONLY {.importc: "_O_RDONLY", header: "".}: int proc myDup(h: Handle; inherit: WINBOOL=1): Handle = let thisProc = getCurrentProcess() if duplicateHandle(thisProc, h, thisProc, addr result,0,inherit, DUPLICATE_SAME_ACCESS) == 0: raiseOSError(osLastError()) proc createAllPipeHandles(si: var STARTUPINFO; stdin, stdout, stderr: var Handle; hash: int) = var sa: SECURITY_ATTRIBUTES sa.nLength = sizeof(SECURITY_ATTRIBUTES).cint sa.lpSecurityDescriptor = nil sa.bInheritHandle = 1 let pipeOutName = newWideCString(r"\\.\pipe\stdout" & $hash) let pipeInName = newWideCString(r"\\.\pipe\stdin" & $hash) let pipeOut = createNamedPipe(pipeOutName, dwOpenMode=PIPE_ACCESS_INBOUND or FILE_FLAG_WRITE_THROUGH, dwPipeMode=PIPE_NOWAIT, nMaxInstances=1, nOutBufferSize=1024, nInBufferSize=1024, nDefaultTimeOut=0,addr sa) if pipeOut == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) let pipeIn = createNamedPipe(pipeInName, dwOpenMode=PIPE_ACCESS_OUTBOUND or FILE_FLAG_WRITE_THROUGH, dwPipeMode=PIPE_NOWAIT, nMaxInstances=1, nOutBufferSize=1024, nInBufferSize=1024, nDefaultTimeOut=0,addr sa) if pipeIn == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) si.hStdOutput = createFileW(pipeOutName, FILE_WRITE_DATA or SYNCHRONIZE, 0, addr sa, OPEN_EXISTING, # very important flag! FILE_ATTRIBUTE_NORMAL, 0 # no template file for OPEN_EXISTING ) if si.hStdOutput == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) si.hStdError = myDup(si.hStdOutput) si.hStdInput = createFileW(pipeInName, FILE_READ_DATA or SYNCHRONIZE, 0, addr sa, OPEN_EXISTING, # very important flag! FILE_ATTRIBUTE_NORMAL, 0 # no template file for OPEN_EXISTING ) if si.hStdOutput == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) stdin = myDup(pipeIn, 0) stdout = myDup(pipeOut, 0) discard closeHandle(pipeIn) discard closeHandle(pipeOut) stderr = stdout proc createPipeHandles(rdHandle, wrHandle: var Handle) = var sa: SECURITY_ATTRIBUTES sa.nLength = sizeof(SECURITY_ATTRIBUTES).cint sa.lpSecurityDescriptor = nil sa.bInheritHandle = 1 if createPipe(rdHandle, wrHandle, sa, 0) == 0'i32: raiseOSError(osLastError()) proc fileClose(h: Handle) {.inline.} = if h > 4: discard closeHandle(h) proc startProcess(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): owned Process = var si: STARTUPINFO procInfo: PROCESS_INFORMATION success: int hi, ho, he: Handle new(result) result.options = options result.exitFlag = true si.cb = sizeof(si).cint if poParentStreams notin options: si.dwFlags = STARTF_USESTDHANDLES # STARTF_USESHOWWINDOW or if poInteractive notin options: createPipeHandles(si.hStdInput, hi) createPipeHandles(ho, si.hStdOutput) if poStdErrToStdOut in options: si.hStdError = si.hStdOutput he = ho else: createPipeHandles(he, si.hStdError) if setHandleInformation(he, DWORD(1), DWORD(0)) == 0'i32: raiseOSError(osLastError()) if setHandleInformation(hi, DWORD(1), DWORD(0)) == 0'i32: raiseOSError(osLastError()) if setHandleInformation(ho, DWORD(1), DWORD(0)) == 0'i32: raiseOSError(osLastError()) else: createAllPipeHandles(si, hi, ho, he, cast[int](result)) result.inHandle = FileHandle(hi) result.outHandle = FileHandle(ho) result.errHandle = FileHandle(he) else: si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) result.inHandle = FileHandle(si.hStdInput) result.outHandle = FileHandle(si.hStdOutput) result.errHandle = FileHandle(si.hStdError) var cmdl: cstring var cmdRoot: string if poEvalCommand in options: cmdl = command assert args.len == 0 else: cmdRoot = buildCommandLine(command, args) cmdl = cstring(cmdRoot) var wd: cstring = nil var e = (str: nil.cstring, len: -1) if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) when useWinUnicode: var tmp = newWideCString(cmdl) var ee = if e.str.isNil: nil else: newWideCString(e.str, e.len) var wwd = newWideCString(wd) var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT if poDaemon in options: flags = flags or CREATE_NO_WINDOW success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, ee, wwd, si, procInfo) else: success = winlean.createProcessA(nil, cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, e, wd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: fileClose(si.hStdInput) fileClose(si.hStdOutput) if poStdErrToStdOut notin options: fileClose(si.hStdError) if e.str != nil: dealloc(e.str) if success == 0: if poInteractive in result.options: close(result) const errInvalidParameter = 87.int const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, "Requested command not found: '$1'. OS error:" % command) else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess result.fThreadHandle = procInfo.hThread result.id = procInfo.dwProcessId result.exitFlag = false proc close(p: Process) = if poParentStreams notin p.options: discard closeHandle(p.inHandle) discard closeHandle(p.outHandle) discard closeHandle(p.errHandle) discard closeHandle(p.fThreadHandle) discard closeHandle(p.fProcessHandle) proc suspend(p: Process) = discard suspendThread(p.fThreadHandle) proc resume(p: Process) = discard resumeThread(p.fThreadHandle) proc running(p: Process): bool = if p.exitFlag: return false else: var x = waitForSingleObject(p.fProcessHandle, 0) return x == WAIT_TIMEOUT proc terminate(p: Process) = if running(p): discard terminateProcess(p.fProcessHandle, 0) proc kill(p: Process) = terminate(p) proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: return p.exitStatus let res = waitForSingleObject(p.fProcessHandle, timeout.int32) if res == WAIT_TIMEOUT: terminate(p) var status: int32 discard getExitCodeProcess(p.fProcessHandle, status) if status != STILL_ACTIVE: p.exitFlag = true p.exitStatus = status discard closeHandle(p.fThreadHandle) discard closeHandle(p.fProcessHandle) result = status else: result = -1 proc peekExitCode(p: Process): int = if p.exitFlag: return p.exitStatus result = -1 var b = waitForSingleObject(p.fProcessHandle, 0) == WAIT_TIMEOUT if not b: var status: int32 discard getExitCodeProcess(p.fProcessHandle, status) p.exitFlag = true p.exitStatus = status discard closeHandle(p.fThreadHandle) discard closeHandle(p.fProcessHandle) result = status proc inputStream(p: Process): Stream = streamAccess(p) if p.inStream == nil: p.inStream = newFileHandleStream(p.inHandle) result = p.inStream proc outputStream(p: Process): Stream = streamAccess(p) if p.outStream == nil: p.outStream = newFileHandleStream(p.outHandle) result = p.outStream proc errorStream(p: Process): Stream = streamAccess(p) if p.errStream == nil: p.errStream = newFileHandleStream(p.errHandle) result = p.errStream proc execCmd(command: string): int = var si: STARTUPINFO procInfo: PROCESS_INFORMATION process: Handle L: int32 si.cb = sizeof(si).cint si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) when useWinUnicode: var c = newWideCString(command) var res = winlean.createProcessW(nil, c, nil, nil, 0, NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) else: var res = winlean.createProcessA(nil, command, nil, nil, 0, NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: process = procInfo.hProcess discard closeHandle(procInfo.hThread) if waitForSingleObject(process, INFINITE) != -1: discard getExitCodeProcess(process, L) result = int(L) else: result = -1 discard closeHandle(process) proc select(readfds: var seq[Process], timeout = 500): int = assert readfds.len <= MAXIMUM_WAIT_OBJECTS var rfds: WOHandleArray for i in 0..readfds.len()-1: rfds[i] = readfds[i].outHandle #fProcessHandle var ret = waitForMultipleObjects(readfds.len.int32, addr(rfds), 0'i32, timeout.int32) case ret of WAIT_TIMEOUT: return 0 of WAIT_FAILED: raiseOSError(osLastError()) else: var i = ret - WAIT_OBJECT_0 readfds.del(i) return 1 proc hasData*(p: Process): bool = var x: int32 if peekNamedPipe(p.outHandle, lpTotalBytesAvail=addr x): result = x > 0 elif not defined(useNimRtl): const readIdx = 0 writeIdx = 1 proc isExitStatus(status: cint): bool = WIFEXITED(status) or WIFSIGNALED(status) proc envToCStringArray(t: StringTableRef): cstringArray = result = cast[cstringArray](alloc0((t.len + 1) * sizeof(cstring))) var i = 0 for key, val in pairs(t): var x = key & "=" & val result[i] = cast[cstring](alloc(x.len+1)) copyMem(result[i], addr(x[0]), x.len+1) inc(i) proc envToCStringArray(): cstringArray = var counter = 0 for key, val in envPairs(): inc counter result = cast[cstringArray](alloc0((counter + 1) * sizeof(cstring))) var i = 0 for key, val in envPairs(): var x = key.string & "=" & val.string result[i] = cast[cstring](alloc(x.len+1)) copyMem(result[i], addr(x[0]), x.len+1) inc(i) type StartProcessData = object sysCommand: string sysArgs: cstringArray sysEnv: cstringArray workingDir: cstring pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] options: set[ProcessOption] const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr StartProcessData) {. tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): owned Process = var pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options result.exitFlag = true if poParentStreams notin options: if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or pipe(pStderr) != 0'i32: raiseOSError(osLastError()) var sysCommand: string var sysArgsRaw: seq[string] if poEvalCommand in options: const useShPath {.strdefine.} = when not defined(android): "/bin/sh" else: "/system/bin/sh" sysCommand = useShPath sysArgsRaw = @[sysCommand, "-c", command] assert args.len == 0, "`args` has to be empty when using poEvalCommand." else: sysCommand = command sysArgsRaw = @[command] for arg in args.items: sysArgsRaw.add arg var pid: Pid var sysArgs = allocCStringArray(sysArgsRaw) defer: deallocCStringArray(sysArgs) var sysEnv = if env == nil: envToCStringArray() else: envToCStringArray(env) defer: deallocCStringArray(sysEnv) var data: StartProcessData shallowCopy(data.sysCommand, sysCommand) data.sysArgs = sysArgs data.sysEnv = sysEnv data.pStdin = pStdin data.pStdout = pStdout data.pStderr = pStderr data.workingDir = workingDir data.options = options when useProcessAuxSpawn: var currentDir = getCurrentDir() pid = startProcessAuxSpawn(data) if workingDir.len > 0: setCurrentDir(currentDir) else: pid = startProcessAuxFork(data) # Parent process. Copy process information. if poEchoCmd in options: when declared(echo): echo(command, " ", join(args, " ")) result.id = pid result.exitFlag = false if poParentStreams in options: # does not make much sense, but better than nothing: result.inHandle = 0 result.outHandle = 1 if poStdErrToStdOut in options: result.errHandle = result.outHandle else: result.errHandle = 2 else: result.inHandle = pStdin[writeIdx] result.outHandle = pStdout[readIdx] if poStdErrToStdOut in options: result.errHandle = result.outHandle discard close(pStderr[readIdx]) else: result.errHandle = pStderr[readIdx] discard close(pStderr[writeIdx]) discard close(pStdin[readIdx]) discard close(pStdout[writeIdx]) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid = var attr: Tposix_spawnattr var fops: Tposix_spawn_file_actions template chck(e: untyped) = if e != 0'i32: raiseOSError(osLastError()) chck posix_spawn_file_actions_init(fops) chck posix_spawnattr_init(attr) var mask: Sigset chck sigemptyset(mask) chck posix_spawnattr_setsigmask(attr, mask) if poDaemon in data.options: chck posix_spawnattr_setpgroup(attr, 0'i32) var flags = POSIX_SPAWN_USEVFORK or POSIX_SPAWN_SETSIGMASK if poDaemon in data.options: flags = flags or POSIX_SPAWN_SETPGROUP chck posix_spawnattr_setflags(attr, flags) if not (poParentStreams in data.options): chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx) chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) if poStdErrToStdOut in data.options: chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) var res: cint if data.workingDir.len > 0: setCurrentDir($data.workingDir) var pid: Pid if (poUsePath in data.options): res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) else: res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) if res != 0'i32: raiseOSError(OSErrorCode(res), data.sysCommand) return pid else: proc startProcessAuxFork(data: StartProcessData): Pid = if pipe(data.pErrorPipe) != 0: raiseOSError(osLastError()) defer: discard close(data.pErrorPipe[readIdx]) var pid: Pid var dataCopy = data when defined(useClone): const stackSize = 65536 let stackEnd = cast[clong](alloc(stackSize)) let stack = cast[pointer](stackEnd + stackSize) let fn: pointer = startProcessAfterFork pid = clone(fn, stack, cint(CLONE_VM or CLONE_VFORK or SIGCHLD), pointer(addr dataCopy), nil, nil, nil) discard close(data.pErrorPipe[writeIdx]) dealloc(stack) else: pid = fork() if pid == 0: startProcessAfterFork(addr(dataCopy)) exitnow(1) discard close(data.pErrorPipe[writeIdx]) if pid < 0: raiseOSError(osLastError()) var error: cint let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): raiseOSError(osLastError(), "Could not find command: '$1'. OS error: $2" % [$data.sysCommand, $strerror(error)]) return pid {.push stacktrace: off, profiler: off.} proc startProcessFail(data: ptr StartProcessData) = var error: cint = errno discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) exitnow(1) when not defined(uClibc) and (not defined(linux) or defined(android)): var environ {.importc.}: cstringArray proc startProcessAfterFork(data: ptr StartProcessData) = # Warning: no GC here! # Or anything that touches global structures - all called nim procs # must be marked with stackTrace:off. Inspect C code after making changes. if not (poParentStreams in data.options): discard close(data.pStdin[writeIdx]) if dup2(data.pStdin[readIdx], readIdx) < 0: startProcessFail(data) discard close(data.pStdout[readIdx]) if dup2(data.pStdout[writeIdx], writeIdx) < 0: startProcessFail(data) discard close(data.pStderr[readIdx]) if (poStdErrToStdOut in data.options): if dup2(data.pStdout[writeIdx], 2) < 0: startProcessFail(data) else: if dup2(data.pStderr[writeIdx], 2) < 0: startProcessFail(data) if data.workingDir.len > 0: if chdir(data.workingDir) < 0: startProcessFail(data) discard close(data.pErrorPipe[readIdx]) discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) if (poUsePath in data.options): when defined(uClibc) or defined(linux): # uClibc environment (OpenWrt included) doesn't have the full execvpe let exe = findExe(data.sysCommand) discard execve(exe, data.sysArgs, data.sysEnv) else: # MacOSX doesn't have execvpe, so we need workaround. # On MacOSX we can arrive here only from fork, so this is safe: environ = data.sysEnv discard execvp(data.sysCommand, data.sysArgs) else: discard execve(data.sysCommand, data.sysArgs, data.sysEnv) startProcessFail(data) {.pop} proc close(p: Process) = if poParentStreams notin p.options: if p.inStream != nil: close(p.inStream) else: discard close(p.inHandle) if p.outStream != nil: close(p.outStream) else: discard close(p.outHandle) if p.errStream != nil: close(p.errStream) else: discard close(p.errHandle) proc suspend(p: Process) = if kill(p.id, SIGSTOP) != 0'i32: raiseOSError(osLastError()) proc resume(p: Process) = if kill(p.id, SIGCONT) != 0'i32: raiseOSError(osLastError()) proc running(p: Process): bool = if p.exitFlag: return false else: var status: cint = 1 let ret = waitpid(p.id, status, WNOHANG) if ret == int(p.id): if isExitStatus(status): p.exitFlag = true p.exitStatus = status return false else: return true elif ret == 0: return true # Can't establish status. Assume running. else: raiseOSError(osLastError()) proc terminate(p: Process) = if kill(p.id, SIGTERM) != 0'i32: raiseOSError(osLastError()) proc kill(p: Process) = if kill(p.id, SIGKILL) != 0'i32: raiseOSError(osLastError()) when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or defined(dragonfly): import kqueue, times proc waitForExit(p: Process, timeout: int = -1): int = if p.exitFlag: return exitStatusLikeShell(p.exitStatus) if timeout == -1: var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status else: var kqFD = kqueue() if kqFD == -1: raiseOSError(osLastError()) var kevIn = KEvent(ident: p.id.uint, filter: EVFILT_PROC, flags: EV_ADD, fflags: NOTE_EXIT) var kevOut: KEvent var tmspec: Timespec if timeout >= 1000: tmspec.tv_sec = posix.Time(timeout div 1_000) tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 else: tmspec.tv_sec = posix.Time(0) tmspec.tv_nsec = (timeout * 1_000_000) try: while true: var status: cint = 1 var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, addr(tmspec)) if count < 0: let err = osLastError() if err.cint != EINTR: raiseOSError(osLastError()) elif count == 0: # timeout expired, so we trying to kill process if posix.kill(p.id, SIGKILL) == -1: raiseOSError(osLastError()) if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status break else: if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status break else: raiseOSError(osLastError()) finally: discard posix.close(kqFD) result = exitStatusLikeShell(p.exitStatus) else: import times const hasThreadSupport = compileOption("threads") and not defined(nimscript) proc waitForExit(p: Process, timeout: int = -1): int = template adjustTimeout(t, s, e: Timespec) = var diff: int var b: Timespec b.tv_sec = e.tv_sec b.tv_nsec = e.tv_nsec e.tv_sec = e.tv_sec - s.tv_sec if e.tv_nsec >= s.tv_nsec: e.tv_nsec -= s.tv_nsec else: if e.tv_sec == posix.Time(0): raise newException(ValueError, "System time was modified") else: diff = s.tv_nsec - e.tv_nsec e.tv_nsec = 1_000_000_000 - diff t.tv_sec = t.tv_sec - e.tv_sec if t.tv_nsec >= e.tv_nsec: t.tv_nsec -= e.tv_nsec else: t.tv_sec = t.tv_sec - posix.Time(1) diff = e.tv_nsec - t.tv_nsec t.tv_nsec = 1_000_000_000 - diff s.tv_sec = b.tv_sec s.tv_nsec = b.tv_nsec if p.exitFlag: return exitStatusLikeShell(p.exitStatus) if timeout == -1: var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status else: var nmask, omask: Sigset var sinfo: SigInfo var stspec, enspec, tmspec: Timespec discard sigemptyset(nmask) discard sigemptyset(omask) discard sigaddset(nmask, SIGCHLD) when hasThreadSupport: if pthread_sigmask(SIG_BLOCK, nmask, omask) == -1: raiseOSError(osLastError()) else: if sigprocmask(SIG_BLOCK, nmask, omask) == -1: raiseOSError(osLastError()) if timeout >= 1000: tmspec.tv_sec = posix.Time(timeout div 1_000) tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000 else: tmspec.tv_sec = posix.Time(0) tmspec.tv_nsec = (timeout * 1_000_000) try: if clock_gettime(CLOCK_REALTIME, stspec) == -1: raiseOSError(osLastError()) while true: let res = sigtimedwait(nmask, sinfo, tmspec) if res == SIGCHLD: if sinfo.si_pid == p.id: var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status break else: # we have SIGCHLD, but not for process we are waiting, # so we need to adjust timeout value and continue if clock_gettime(CLOCK_REALTIME, enspec) == -1: raiseOSError(osLastError()) adjustTimeout(tmspec, stspec, enspec) elif res < 0: let err = osLastError() if err.cint == EINTR: # we have received another signal, so we need to # adjust timeout and continue if clock_gettime(CLOCK_REALTIME, enspec) == -1: raiseOSError(osLastError()) adjustTimeout(tmspec, stspec, enspec) elif err.cint == EAGAIN: # timeout expired, so we trying to kill process if posix.kill(p.id, SIGKILL) == -1: raiseOSError(osLastError()) var status: cint = 1 if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) p.exitFlag = true p.exitStatus = status break else: raiseOSError(err) finally: when hasThreadSupport: if pthread_sigmask(SIG_UNBLOCK, nmask, omask) == -1: raiseOSError(osLastError()) else: if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: raiseOSError(osLastError()) result = exitStatusLikeShell(p.exitStatus) proc peekExitCode(p: Process): int = var status = cint(0) result = -1 if p.exitFlag: return exitStatusLikeShell(p.exitStatus) var ret = waitpid(p.id, status, WNOHANG) if ret > 0: if isExitStatus(status): p.exitFlag = true p.exitStatus = status result = exitStatusLikeShell(status) proc createStream(stream: var owned(Stream), handle: var FileHandle, fileMode: FileMode) = var f: File if not open(f, handle, fileMode): raiseOSError(osLastError()) stream = newFileStream(f) proc inputStream(p: Process): Stream = streamAccess(p) if p.inStream == nil: createStream(p.inStream, p.inHandle, fmWrite) return p.inStream proc outputStream(p: Process): Stream = streamAccess(p) if p.outStream == nil: createStream(p.outStream, p.outHandle, fmRead) return p.outStream proc errorStream(p: Process): Stream = streamAccess(p) if p.errStream == nil: createStream(p.errStream, p.errHandle, fmRead) return p.errStream proc csystem(cmd: cstring): cint {.nodecl, importc: "system", header: "".} proc execCmd(command: string): int = when defined(linux): let tmp = csystem(command) result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: result = csystem(command) proc createFdSet(fd: var TFdSet, s: seq[Process], m: var int) = FD_ZERO(fd) for i in items(s): m = max(m, int(i.outHandle)) FD_SET(cint(i.outHandle), fd) proc pruneProcessSet(s: var seq[Process], fd: var TFdSet) = var i = 0 var L = s.len while i < L: if FD_ISSET(cint(s[i].outHandle), fd) == 0'i32: s[i] = s[L-1] dec(L) else: inc(i) setLen(s, L) proc select(readfds: var seq[Process], timeout = 500): int = var tv: Timeval tv.tv_sec = posix.Time(0) tv.tv_usec = Suseconds(timeout * 1000) var rd: TFdSet var m = 0 createFdSet((rd), readfds, m) if timeout != -1: result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) else: result = int(select(cint(m+1), addr(rd), nil, nil, nil)) pruneProcessSet(readfds, (rd)) proc hasData*(p: Process): bool = var rd: TFdSet FD_ZERO(rd) let m = max(0, int(p.outHandle)) FD_SET(cint(p.outHandle), rd) result = int(select(cint(m+1), addr(rd), nil, nil, nil)) == 1 proc execCmdEx*(command: string, options: set[ProcessOption] = { poStdErrToStdOut, poUsePath}): tuple[ output: TaintedString, exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = ## A convenience proc that runs the `command`, grabs all its output and ## exit code and returns both. ## ## See also: ## * `execCmd proc <#execCmd,string>`_ ## * `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execProcess proc ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## ## Example: ## ## .. code-block:: Nim ## let (outp, errC) = execCmdEx("nim c -r mytestfile.nim") var p = startProcess(command, options=options + {poEvalCommand}) var outp = outputStream(p) # There is no way to provide input for the child process # anymore. Closing it will create EOF on stdin instead of eternal # blocking. close inputStream(p) result = (TaintedString"", -1) var line = newStringOfCap(120).TaintedString while true: if outp.readLine(line): result[0].string.add(line.string) result[0].string.add("\n") else: result[1] = peekExitCode(p) if result[1] != -1: break close(p)