diff options
-rw-r--r-- | compiler/extccomp.nim | 149 | ||||
-rw-r--r-- | compiler/main.nim | 4 | ||||
-rw-r--r-- | compiler/msgs.nim | 4 | ||||
-rw-r--r-- | compiler/semexprs.nim | 4 | ||||
-rw-r--r-- | compiler/vmdeps.nim | 13 | ||||
-rw-r--r-- | lib/pure/collections/deques.nim | 5 | ||||
-rw-r--r-- | lib/pure/collections/queues.nim | 5 | ||||
-rw-r--r-- | lib/pure/collections/sets.nim | 4 | ||||
-rw-r--r-- | lib/pure/collections/tables.nim | 4 | ||||
-rw-r--r-- | lib/pure/ioselectors.nim | 14 | ||||
-rw-r--r-- | lib/pure/ioselects/ioselectors_epoll.nim | 320 | ||||
-rw-r--r-- | lib/pure/osproc.nim | 14 | ||||
-rw-r--r-- | lib/pure/strscans.nim | 6 | ||||
-rw-r--r-- | lib/pure/times.nim | 29 | ||||
-rw-r--r-- | lib/system.nim | 10 | ||||
-rw-r--r-- | lib/system/mmdisp.nim | 2 | ||||
-rw-r--r-- | lib/system/sysio.nim | 4 | ||||
-rw-r--r-- | lib/upcoming/asyncdispatch.nim | 20 | ||||
-rw-r--r-- | tests/async/tioselectors.nim | 11 | ||||
-rw-r--r-- | tests/async/tupcoming_async.nim | 88 | ||||
-rw-r--r-- | tests/osproc/texitcode.nim | 5 |
21 files changed, 439 insertions, 276 deletions
diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 1f9af95a5..402c9592e 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -657,14 +657,62 @@ proc compileCFile(list: TLinkedList, script: var Rope, cmds: var TStringSeq, add(script, tnl) it = PStrEntry(it.next) +proc getLinkCmd(projectfile, objfiles: string): string = + if optGenStaticLib in gGlobalOptions: + var libname: string + if options.outFile.len > 0: + libname = options.outFile.expandTilde + if not libname.isAbsolute(): + libname = getCurrentDir() / libname + else: + libname = (libNameTmpl() % splitFile(gProjectName).name) + result = CC[cCompiler].buildLib % ["libfile", libname, + "objfiles", objfiles] + else: + var linkerExe = getConfigVar(cCompiler, ".linkerexe") + if len(linkerExe) == 0: linkerExe = cCompiler.getLinkerExe + if needsExeExt(): linkerExe = addFileExt(linkerExe, "exe") + if noAbsolutePaths(): result = quoteShell(linkerExe) + else: result = quoteShell(joinPath(ccompilerpath, linkerExe)) + let buildgui = if optGenGuiApp in gGlobalOptions: CC[cCompiler].buildGui + else: "" + var exefile, builddll: string + if optGenDynLib in gGlobalOptions: + exefile = platform.OS[targetOS].dllFrmt % splitFile(projectfile).name + builddll = CC[cCompiler].buildDll + else: + exefile = splitFile(projectfile).name & platform.OS[targetOS].exeExt + builddll = "" + if options.outFile.len > 0: + exefile = options.outFile.expandTilde + if not exefile.isAbsolute(): + exefile = getCurrentDir() / exefile + if not noAbsolutePaths(): + if not exefile.isAbsolute(): + exefile = joinPath(splitFile(projectfile).dir, exefile) + if optCDebug in gGlobalOptions: + writeDebugInfo(exefile.changeFileExt("ndb")) + exefile = quoteShell(exefile) + let linkOptions = getLinkOptions() & " " & + getConfigVar(cCompiler, ".options.linker") + result = quoteShell(result % ["builddll", builddll, + "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles, + "exefile", exefile, "nim", getPrefixDir(), "lib", libpath]) + result.add ' ' + addf(result, CC[cCompiler].linkTmpl, ["builddll", builddll, + "buildgui", buildgui, "options", linkOptions, + "objfiles", objfiles, "exefile", exefile, + "nim", quoteShell(getPrefixDir()), + "lib", quoteShell(libpath)]) + proc callCCompiler*(projectfile: string) = var - linkCmd, buildgui, builddll: string + linkCmd, buildgui: string if gGlobalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: return # speed up that call if only compiling and no script shall be # generated fileCounter = 0 - var c = cCompiler + #var c = cCompiler var script: Rope = nil var cmds: TStringSeq = @[] var prettyCmds: TStringSeq = @[] @@ -708,46 +756,7 @@ proc callCCompiler*(projectfile: string) = addFileExt(objFile, CC[cCompiler].objExt))) it = PStrEntry(it.next) - if optGenStaticLib in gGlobalOptions: - let name = splitFile(gProjectName).name - linkCmd = CC[c].buildLib % ["libfile", (libNameTmpl() % name), - "objfiles", objfiles] - else: - var linkerExe = getConfigVar(c, ".linkerexe") - if len(linkerExe) == 0: linkerExe = c.getLinkerExe - if needsExeExt(): linkerExe = addFileExt(linkerExe, "exe") - if noAbsolutePaths(): linkCmd = quoteShell(linkerExe) - else: linkCmd = quoteShell(joinPath(ccompilerpath, linkerExe)) - if optGenGuiApp in gGlobalOptions: buildgui = CC[c].buildGui - else: buildgui = "" - var exefile: string - if optGenDynLib in gGlobalOptions: - exefile = platform.OS[targetOS].dllFrmt % splitFile(projectfile).name - builddll = CC[c].buildDll - else: - exefile = splitFile(projectfile).name & platform.OS[targetOS].exeExt - builddll = "" - if options.outFile.len > 0: - exefile = options.outFile.expandTilde - if not exefile.isAbsolute(): - exefile = getCurrentDir() / exefile - if not noAbsolutePaths(): - if not exefile.isAbsolute(): - exefile = joinPath(splitFile(projectfile).dir, exefile) - if optCDebug in gGlobalOptions: - writeDebugInfo(exefile.changeFileExt("ndb")) - exefile = quoteShell(exefile) - let linkOptions = getLinkOptions() & " " & - getConfigVar(cCompiler, ".options.linker") - linkCmd = quoteShell(linkCmd % ["builddll", builddll, - "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles, - "exefile", exefile, "nim", getPrefixDir(), "lib", libpath]) - linkCmd.add ' ' - addf(linkCmd, CC[c].linkTmpl, ["builddll", builddll, - "buildgui", buildgui, "options", linkOptions, - "objfiles", objfiles, "exefile", exefile, - "nim", quoteShell(getPrefixDir()), - "lib", quoteShell(libpath)]) + linkCmd = getLinkCmd(projectfile, objfiles) if optCompileOnly notin gGlobalOptions: execExternalProgram(linkCmd, if optListCmd in gGlobalOptions or gVerbosity > 1: hintExecuting else: hintLinking) @@ -758,6 +767,62 @@ proc callCCompiler*(projectfile: string) = add(script, tnl) generateScript(projectfile, script) +from json import escapeJson + +proc writeJsonBuildInstructions*(projectfile: string) = + template lit(x: untyped) = f.write x + template str(x: untyped) = + buf.setLen 0 + escapeJson(x, buf) + f.write buf + + proc cfiles(f: File; buf: var string; list: TLinkedList, isExternal: bool) = + var it = PStrEntry(list.head) + while it != nil: + let compileCmd = getCompileCFileCmd(it.data, isExternal) + lit "[" + str it.data + lit ", " + str compileCmd + it = PStrEntry(it.next) + if it == nil: + lit "]\L" + else: + lit "],\L" + + proc linkfiles(f: File; buf, objfiles: var string; toLink: TLinkedList) = + var it = PStrEntry(toLink.head) + while it != nil: + let objfile = addFileExt(it.data, CC[cCompiler].objExt) + str objfile + add(objfiles, ' ') + add(objfiles, quoteShell(objfile)) + it = PStrEntry(it.next) + if it == nil: + lit "\L" + else: + lit ",\L" + + var buf = newStringOfCap(50) + + let file = projectfile.splitFile.name + let jsonFile = toGeneratedFile(file, "json") + + var f: File + if open(f, jsonFile, fmWrite): + lit "{\"compile\":[\L" + cfiles(f, buf, toCompile, false) + lit "],\L\"extcompile\":[\L" + cfiles(f, buf, externalToCompile, true) + lit "],\L\"link\":[\L" + var objfiles = "" + linkfiles(f, buf, objfiles, toLink) + + lit "],\L\"linkcmd\": " + str getLinkCmd(projectfile, objfiles) + lit "\L}\L" + close(f) + proc genMappingFiles(list: TLinkedList): Rope = var it = PStrEntry(list.head) while it != nil: diff --git a/compiler/main.nim b/compiler/main.nim index 2118078be..5896934ce 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -66,7 +66,9 @@ proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = compileProject(graph, cache) cgenWriteModules() if gCmd != cmdRun: - extccomp.callCCompiler(changeFileExt(gProjectFull, "")) + let proj = changeFileExt(gProjectFull, "") + extccomp.callCCompiler(proj) + extccomp.writeJsonBuildInstructions(proj) proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) = #incl(gGlobalOptions, optSafeCode) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 94b0bee00..e6a2b75a6 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -48,7 +48,7 @@ type errIndexOutOfBounds, errIndexTypesDoNotMatch, errBracketsInvalidForType, errValueOutOfSetBounds, errFieldInitTwice, errFieldNotInit, errExprXCannotBeCalled, errExprHasNoType, errExprXHasNoType, - errCastNotInSafeMode, errExprCannotBeCastedToX, errCommaOrParRiExpected, + errCastNotInSafeMode, errExprCannotBeCastToX, errCommaOrParRiExpected, errCurlyLeOrParLeExpected, errSectionExpected, errRangeExpected, errMagicOnlyInSystem, errPowerOfTwoExpected, errStringMayNotBeEmpty, errCallConvExpected, errProcOnlyOneCallConv, @@ -229,7 +229,7 @@ const errExprHasNoType: "expression has no type", errExprXHasNoType: "expression \'$1\' has no type (or is ambiguous)", errCastNotInSafeMode: "\'cast\' not allowed in safe mode", - errExprCannotBeCastedToX: "expression cannot be casted to $1", + errExprCannotBeCastToX: "expression cannot be cast to $1", errCommaOrParRiExpected: "',' or ')' expected", errCurlyLeOrParLeExpected: "\'{\' or \'(\' expected", errSectionExpected: "section (\'type\', \'proc\', etc.) expected", diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 723045fb0..c15a97c50 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -127,7 +127,7 @@ proc checkConvertible(c: PContext, castDest, src: PType): TConvStatus = discard proc isCastable(dst, src: PType): bool = - ## Checks whether the source type can be casted to the destination type. + ## Checks whether the source type can be cast to the destination type. ## Casting is very unrestrictive; casts are allowed as long as ## castDest.size >= src.size, and typeAllowed(dst, skParam) #const @@ -223,7 +223,7 @@ proc semCast(c: PContext, n: PNode): PNode = addSon(result, copyTree(n.sons[0])) addSon(result, semExprWithType(c, n.sons[1])) if not isCastable(result.typ, result.sons[1].typ): - localError(result.info, errExprCannotBeCastedToX, + localError(result.info, errExprCannotBeCastToX, typeToString(result.typ)) proc semLowHigh(c: PContext, n: PNode, m: TMagic): PNode = diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index bd6908722..7de30b7f0 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -73,6 +73,10 @@ proc atomicTypeX(name: string; m: TMagic; t: PType; info: TLineInfo): PNode = result = newSymNode(sym) result.typ = t +proc atomicTypeX(s: PSym; info: TLineInfo): PNode = + result = newSymNode(s) + result.info = info + proc mapTypeToAstX(t: PType; info: TLineInfo; inst=false; allowRecursionX=false): PNode @@ -103,6 +107,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; inst=false; allowRecursionX=false): PNode = var allowRecursion = allowRecursionX template atomicType(name, m): untyped = atomicTypeX(name, m, t, info) + template atomicType(s): untyped = atomicTypeX(s, info) template mapTypeToAst(t,info): untyped = mapTypeToAstX(t, info, inst) template mapTypeToAstR(t,info): untyped = mapTypeToAstX(t, info, inst, true) template mapTypeToAst(t,i,info): untyped = @@ -125,7 +130,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; if allowRecursion: # getTypeImpl behavior: turn off recursion allowRecursion = false else: # getTypeInst behavior: return symbol - return atomicType(t.sym.name.s, t.sym.magic) + return atomicType(t.sym) case t.kind of tyNone: result = atomicType("none", mNone) @@ -180,9 +185,9 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; if allowRecursion or t.sym == nil: result = mapTypeToBracket("distinct", mDistinct, t, info) else: - result = atomicType(t.sym.name.s, t.sym.magic) + result = atomicType(t.sym) of tyGenericParam, tyForward: - result = atomicType(t.sym.name.s, t.sym.magic) + result = atomicType(t.sym) of tyObject: if inst: result = newNodeX(nkObjectTy) @@ -206,7 +211,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add mapTypeToAst(t.sons[0], info) result.add copyTree(t.n) else: - result = atomicType(t.sym.name.s, t.sym.magic) + result = atomicType(t.sym) of tyEnum: result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t) result.add copyTree(t.n) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index c25429778..495d7896c 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -160,7 +160,10 @@ proc peekLast*[T](deq: Deque[T]): T {.inline.} = emptyCheck(deq) result = deq.data[(deq.tail - 1) and deq.mask] -proc default[T](t: typedesc[T]): T {.inline.} = discard +template default[T](t: typedesc[T]): T = + var v: T + v + proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = ## Remove and returns the first element of the `deq`. emptyCheck(deq) diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index e4d7eeef1..0490ae494 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -154,7 +154,10 @@ proc add*[T](q: var Queue[T], item: T) = q.data[q.wr] = item q.wr = (q.wr + 1) and q.mask -proc default[T](t: typedesc[T]): T {.inline.} = discard +template default[T](t: typedesc[T]): T = + var v: T + v + proc pop*[T](q: var Queue[T]): T {.inline, discardable.} = ## Remove and returns the first (oldest) element of the queue `q`. emptyCheck(q) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 552e41ef7..b2ffbe58d 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -261,7 +261,9 @@ template doWhile(a, b) = b if not a: break -proc default[T](t: typedesc[T]): T {.inline.} = discard +template default[T](t: typedesc[T]): T = + var v: T + v proc excl*[A](s: var HashSet[A], key: A) = ## Excludes `key` from the set `s`. diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index e6e72d9ed..57e98bf5c 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -954,7 +954,7 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) = inc(t.counter) proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] = - ## returns the largest (key,val)-pair. Efficiency: O(n) + ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) assert t.len > 0 var minIdx = 0 for h in 1..high(t.data): @@ -1080,7 +1080,7 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = t[].inc(key, val) proc smallest*[A](t: CountTableRef[A]): (A, int) = - ## returns the largest (key,val)-pair. Efficiency: O(n) + ## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n) t[].smallest proc largest*[A](t: CountTableRef[A]): (A, int) = diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index 744bdbaa1..5745d0b72 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -18,10 +18,12 @@ ## Supported features: files, sockets, pipes, timers, processes, signals ## and user events. ## -## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux. +## Fully supported OS: MacOSX, FreeBSD, OpenBSD, NetBSD, Linux (except +## for Android). ## ## Partially supported OS: Windows (only sockets and user events), ## Solaris (files, sockets, handles and user events). +## Android (files, sockets, handles and user events). ## ## TODO: ``/dev/poll``, ``event ports`` and filesystem events. @@ -29,9 +31,11 @@ import os const hasThreadSupport = compileOption("threads") and defined(threadsafe) -const supportedPlatform = defined(macosx) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(linux) +const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + (defined(linux) and not defined(android)) + ## This constant is used to determine whether the destination platform is + ## fully supported by ``ioselectors`` module. const bsdPlatform = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) @@ -244,7 +248,7 @@ else: skey.key.fd = pkeyfd skey.key.data = pdata - when supportedPlatform: + when ioselSupportedPlatform: template blockSignals(newmask: var Sigset, oldmask: var Sigset) = when hasThreadSupport: if posix.pthread_sigmask(SIG_BLOCK, newmask, oldmask) == -1: diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 4bc6e9d51..ceba670fb 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -14,27 +14,29 @@ import posix, times # Maximum number of events that can be returned const MAX_EPOLL_RESULT_EVENTS = 64 -type - SignalFdInfo* {.importc: "struct signalfd_siginfo", - header: "<sys/signalfd.h>", pure, final.} = object - ssi_signo*: uint32 - ssi_errno*: int32 - ssi_code*: int32 - ssi_pid*: uint32 - ssi_uid*: uint32 - ssi_fd*: int32 - ssi_tid*: uint32 - ssi_band*: uint32 - ssi_overrun*: uint32 - ssi_trapno*: uint32 - ssi_status*: int32 - ssi_int*: int32 - ssi_ptr*: uint64 - ssi_utime*: uint64 - ssi_stime*: uint64 - ssi_addr*: uint64 - pad* {.importc: "__pad".}: array[0..47, uint8] +when not defined(android): + type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "<sys/signalfd.h>", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] +type eventFdData {.importc: "eventfd_t", header: "<sys/eventfd.h>", pure, final.} = uint64 epoll_data {.importc: "union epoll_data", header: "<sys/epoll.h>", @@ -68,12 +70,22 @@ proc timerfd_create(clock_id: ClockId, flags: cint): cint proc timerfd_settime(ufd: cint, flags: cint, utmr: var Itimerspec, otmr: var Itimerspec): cint {.cdecl, importc: "timerfd_settime", header: "<sys/timerfd.h>".} -proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint - {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} proc eventfd(count: cuint, flags: cint): cint {.cdecl, importc: "eventfd", header: "<sys/eventfd.h>".} -proc ulimit(cmd: cint): clong - {.importc: "ulimit", header: "<ulimit.h>", varargs.} + +when not defined(android): + proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "<sys/signalfd.h>".} + +var RLIMIT_NOFILE {.importc: "RLIMIT_NOFILE", + header: "<sys/resource.h>".}: cint +type + rlimit {.importc: "struct rlimit", + header: "<sys/resource.h>", pure, final.} = object + rlim_cur: int + rlim_max: int +proc getrlimit(resource: cint, rlp: var rlimit): cint + {.importc: "getrlimit",header: "<sys/resource.h>".} when hasThreadSupport: type @@ -97,7 +109,10 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = - var maxFD = int(ulimit(4, 0)) + var a = rlimit() + if getrlimit(RLIMIT_NOFILE, a) != 0: + raiseOsError(osLastError()) + var maxFD = int(a.rlim_max) doAssert(maxFD > 0) var epollFD = epoll_create(MAX_EPOLL_RESULT_EVENTS) @@ -194,39 +209,53 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = doAssert(pkey.ident != 0) if pkey.events != {}: - if pkey.events * {Event.Read, Event.Write} != {}: - var epv = epoll_event() - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - dec(s.count) - elif Event.Timer in pkey.events: - var epv = epoll_event() - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - discard posix.close(fdi.cint) - dec(s.count) - elif Event.Signal in pkey.events: - var epv = epoll_event() - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - var nmask, omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, cint(s.fds[fdi].param)) - unblockSignals(nmask, omask) - discard posix.close(fdi.cint) - dec(s.count) - elif Event.Process in pkey.events: - var epv = epoll_event() - if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - var nmask, omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, SIGCHLD) - unblockSignals(nmask, omask) - discard posix.close(fdi.cint) - dec(s.count) + when not defined(android): + if pkey.events * {Event.Read, Event.Write} != {}: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Signal in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + elif Event.Process in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + discard posix.close(fdi.cint) + dec(s.count) + else: + if pkey.events * {Event.Read, Event.Write} != {}: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + var epv = epoll_event() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + discard posix.close(fdi.cint) + dec(s.count) + pkey.ident = 0 pkey.events = {} @@ -280,60 +309,61 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, inc(s.count) result = fdi -proc registerSignal*[T](s: Selector[T], signal: int, - data: T): int {.discardable.} = - var - nmask: Sigset - omask: Sigset +when not defined(android): + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, cint(signal)) - blockSignals(nmask, omask) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) - let fdi = signalfd(-1, nmask, 0).int - if fdi == -1: - raiseOSError(osLastError()) - setNonBlocking(fdi.cint) + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) - s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) - epv.data.u64 = fdi.uint - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - s.setKey(fdi, signal, {Event.Signal}, signal, data) - inc(s.count) - result = fdi + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, signal, {Event.Signal}, signal, data) + inc(s.count) + result = fdi -proc registerProcess*[T](s: Selector, pid: int, - data: T): int {.discardable.} = - var - nmask: Sigset - omask: Sigset + proc registerProcess*[T](s: Selector, pid: int, + data: T): int {.discardable.} = + var + nmask: Sigset + omask: Sigset - discard sigemptyset(nmask) - discard sigemptyset(omask) - discard sigaddset(nmask, posix.SIGCHLD) - blockSignals(nmask, omask) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) - let fdi = signalfd(-1, nmask, 0).int - if fdi == -1: - raiseOSError(osLastError()) - setNonBlocking(fdi.cint) + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseOSError(osLastError()) + setNonBlocking(fdi.cint) - s.checkFd(fdi) - doAssert(s.fds[fdi].ident == 0) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == 0) - var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) - epv.data.u64 = fdi.uint - epv.events = EPOLLIN or EPOLLRDHUP - if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: - raiseOSError(osLastError()) - s.setKey(fdi, pid, {Event.Process, Event.Oneshot}, pid, data) - inc(s.count) - result = fdi + var epv = epoll_event(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) == -1: + raiseOSError(osLastError()) + s.setKey(fdi, pid, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = let fdi = int(ev.efd) @@ -382,40 +412,60 @@ proc selectInto*[T](s: Selector[T], timeout: int, events.incl(Event.Error) if (pevents and EPOLLOUT) != 0: events.incl(Event.Write) - if (pevents and EPOLLIN) != 0: - if Event.Read in skey.events: - events.incl(Event.Read) - elif Event.Timer in skey.events: - var data: uint64 = 0 - if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): - raiseOSError(osLastError()) - events = {Event.Timer} - elif Event.Signal in skey.events: - var data = SignalFdInfo() - if posix.read(fdi.cint, addr data, - sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): - raiseOsError(osLastError()) - events = {Event.Signal} - elif Event.Process in skey.events: - var data = SignalFdInfo() - if posix.read(fdi.cint, addr data, - sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): - raiseOsError(osLastError()) - if cast[int](data.ssi_pid) == skey.param: - events = {Event.Process} - else: - inc(i) - continue - elif Event.User in skey.events: - var data: uint64 = 0 - if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): - let err = osLastError() - if err == OSErrorCode(EAGAIN): + when not defined(android): + if (pevents and EPOLLIN) != 0: + if Event.Read in skey.events: + events.incl(Event.Read) + elif Event.Timer in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + events = {Event.Timer} + elif Event.Signal in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + events = {Event.Signal} + elif Event.Process in skey.events: + var data = SignalFdInfo() + if posix.read(fdi.cint, addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseOsError(osLastError()) + if cast[int](data.ssi_pid) == skey.param: + events = {Event.Process} + else: inc(i) continue - else: - raiseOSError(err) - events = {Event.User} + elif Event.User in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseOSError(err) + events = {Event.User} + else: + if (pevents and EPOLLIN) != 0: + if Event.Read in skey.events: + events.incl(Event.Read) + elif Event.Timer in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + raiseOSError(osLastError()) + events = {Event.Timer} + elif Event.User in skey.events: + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseOSError(err) + events = {Event.User} skey.key.events = events results[k] = skey.key diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 76bd2dfe1..1d43bb321 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -965,10 +965,16 @@ elif not defined(useNimRtl): var ret : int var status : cint = 1 ret = waitpid(p.id, status, WNOHANG) - if WIFEXITED(status): - p.exitStatus = status - if ret == 0: return true # Can't establish status. Assume running. - result = ret == int(p.id) + if ret == int(p.id): + if WIFEXITED(status): + p.exitStatus = status + return false + else: + return true + elif ret == 0: + return true # Can't establish status. Assume running. + else: + return false proc terminate(p: Process) = if kill(p.id, SIGTERM) != 0'i32: diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 246f018c5..fc400173f 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -76,7 +76,7 @@ to a variable (that was passed to the ``scanf`` macro) while ``$[]`` merely optional tokens. -In this example, we define a helper proc ``skipSep`` that skips some separators +In this example, we define a helper proc ``someSep`` that skips some separators which we then use in our scanf pattern to help us in the matching process: .. code-block:: nim @@ -86,14 +86,14 @@ which we then use in our scanf pattern to help us in the matching process: result = 0 while input[start+result] in seps: inc result - if scanf(input, "$w${someSep}$w", key, value): + if scanf(input, "$w$[someSep]$w", key, value): ... It also possible to pass arguments to a user definable matcher: .. code-block:: nim - proc ndigits(input: string; start: int; intVal: var int; n: int): int = + proc ndigits(input: string; intVal: var int; start: int; n: int): int = # matches exactly ``n`` digits. Matchers need to return 0 if nothing # matched or otherwise the number of processed chars. var x = 0 diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 1767a37be..cf4e7dde6 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -436,6 +436,11 @@ when not defined(JS): TimeInfoPtr = ptr StructTM Clock {.importc: "clock_t".} = distinct int + when not defined(windows): + # This is not ANSI C, but common enough + proc timegm(t: StructTM): Time {. + importc: "timegm", header: "<time.h>", tags: [].} + proc localtime(timer: ptr Time): TimeInfoPtr {. importc: "localtime", header: "<time.h>", tags: [].} proc gmtime(timer: ptr Time): TimeInfoPtr {. @@ -515,20 +520,22 @@ when not defined(JS): # the conversion is not expensive proc timeInfoToTime(timeInfo: TimeInfo): Time = - var cTimeInfo = timeInfo # for C++ we have to make a copy, - # because the header of mktime is broken in my version of libc - result = mktime(timeInfoToTM(cTimeInfo)) - # mktime is defined to interpret the input as local time. As timeInfoToTM - # does ignore the timezone, we need to adjust this here. - result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) + toTime(timeInfo) proc toTime(timeInfo: TimeInfo): Time = - var cTimeInfo = timeInfo # for C++ we have to make a copy, + var cTimeInfo = timeInfo # for C++ we have to make a copy # because the header of mktime is broken in my version of libc - result = mktime(timeInfoToTM(cTimeInfo)) - # mktime is defined to interpret the input as local time. As timeInfoToTM - # does ignore the timezone, we need to adjust this here. - result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) + + when defined(windows): + # On Windows `mktime` is broken enough to make this work. + result = mktime(timeInfoToTM(cTimeInfo)) + # mktime is defined to interpret the input as local time. As timeInfoToTM + # does ignore the timezone, we need to adjust this here. + result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) + else: + result = timegm(timeInfoToTM(cTimeInfo)) + # As timeInfoToTM does ignore the timezone, we need to adjust this here. + result = Time(TimeImpl(result) + timeInfo.timezone) const epochDiff = 116444736000000000'i64 diff --git a/lib/system.nim b/lib/system.nim index 69d3db291..8209dbc23 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2595,6 +2595,14 @@ else: if x < 0: -x else: x {.pop.} +type + FileSeekPos* = enum ## Position relative to which seek should happen + # The values are ordered so that they match with stdio + # SEEK_SET, SEEK_CUR and SEEK_END respectively. + fspSet ## Seek to absolute value + fspCur ## Seek relative to current position + fspEnd ## Seek relative to end + when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} @@ -2858,7 +2866,7 @@ when not defined(JS): #and not defined(nimscript): ## file `f`. Returns the number of actual written bytes, which may be less ## than `len` in case of an error. - proc setFilePos*(f: File, pos: int64) {.benign.} + proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} ## sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 63af49e35..431f84bfd 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -20,7 +20,7 @@ const alwaysCycleGC = defined(smokeCycles) alwaysGC = defined(fulldebug) # collect after every memory # allocation (for debugging) - leakDetector = false + leakDetector = defined(leakDetector) overwriteFree = defined(nimBurnFree) # overwrite memory with 0xFF before free trackAllocationSource = leakDetector diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 5c10392f1..29c5777cc 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -336,8 +336,8 @@ proc open(f: var File, filehandle: FileHandle, mode: FileMode): bool = f = c_fdopen(filehandle, FormatOpen[mode]) result = f != nil -proc setFilePos(f: File, pos: int64) = - if c_fseek(f, clong(pos), 0) != 0: +proc setFilePos(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) = + if c_fseek(f, clong(pos), cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos(f: File): int64 = diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 9a35cf3c8..ca5c1f64c 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -1092,11 +1092,6 @@ else: import ioselectors from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK, MSG_NOSIGNAL - - const supportedPlatform = defined(linux) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(macosx) - type AsyncFD* = distinct cint Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} @@ -1191,7 +1186,7 @@ else: var keys: array[64, ReadyKey[AsyncData]] let p = getGlobalDispatcher() - when supportedPlatform: + when ioselSupportedPlatform: let customSet = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, Event.User} @@ -1225,7 +1220,7 @@ else: else: break - when supportedPlatform: + when ioselSupportedPlatform: if (customSet * events) != {}: for node in keys[i].data.readCBs[].nodes(): let cb = node.value @@ -1234,6 +1229,15 @@ else: if cb(fd.AsyncFD): keys[i].data.readCBs[].remove(node) p.selector.unregister(fd) + else: + if Event.User in events or events == {Event.Error}: + for node in keys[i].data.readCBs[].nodes(): + let cb = node.value + custom = true + if cb != nil: + if cb(fd.AsyncFD): + keys[i].data.readCBs[].remove(node) + p.selector.unregister(fd) # because state `data` can be modified in callback we need to update # descriptor events with currently registered callbacks. @@ -1496,7 +1500,7 @@ else: addRead(socket, cb) return retFuture - when supportedPlatform: + when ioselSupportedPlatform: proc addTimer*(timeout: int, oneshot: bool, cb: Callback) = ## Start watching for timeout expiration, and then call the diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index 71d901e69..e6b4196a3 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -12,11 +12,10 @@ template processTest(t, x: untyped) = if not x: echo(t & " FAILED\r\n") when not defined(windows): - import os, posix, osproc, nativesockets, times + import os, posix, nativesockets, times - const supportedPlatform = defined(macosx) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(linux) + when ioselSupportedPlatform: + import osproc proc socket_notification_test(): bool = proc create_test_socket(): SocketHandle = @@ -143,7 +142,7 @@ when not defined(windows): selector.close() result = true - when supportedPlatform: + when ioselSupportedPlatform: proc timer_notification_test(): bool = var selector = newSelector[int]() var timer = selector.registerTimer(100, false, 0) @@ -462,7 +461,7 @@ when not defined(windows): when hasThreadSupport: processTest("Multithreaded user event notification test...", mt_event_test()) - when supportedPlatform: + when ioselSupportedPlatform: processTest("Timer notification test...", timer_notification_test()) processTest("Process notification test...", process_notification_test()) processTest("Signal notification test...", signal_notification_test()) diff --git a/tests/async/tupcoming_async.nim b/tests/async/tupcoming_async.nim index 137794afd..7d255f213 100644 --- a/tests/async/tupcoming_async.nim +++ b/tests/async/tupcoming_async.nim @@ -8,11 +8,12 @@ OK """ when defined(upcoming): - import asyncdispatch, times, osproc, streams + import asyncdispatch, times, streams, posix + from ioselectors import ioselSupportedPlatform - const supportedPlatform = defined(linux) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or - defined(macosx) + proc delayedSet(ev: AsyncEvent, timeout: int): Future[void] {.async.} = + await sleepAsync(timeout) + ev.setEvent() proc waitEvent(ev: AsyncEvent, closeEvent = false): Future[void] = var retFuture = newFuture[void]("waitEvent") @@ -25,56 +26,55 @@ when defined(upcoming): addEvent(ev, cb) return retFuture - proc waitTimer(timeout: int): Future[void] = - var retFuture = newFuture[void]("waitTimer") - proc cb(fd: AsyncFD): bool = - retFuture.complete() - addTimer(timeout, true, cb) - return retFuture - - proc waitProcess(p: Process): Future[void] = - var retFuture = newFuture[void]("waitProcess") - proc cb(fd: AsyncFD): bool = - retFuture.complete() - addProcess(p.processID(), cb) - return retFuture - - proc delayedSet(ev: AsyncEvent, timeout: int): Future[void] {.async.} = - await waitTimer(timeout) - ev.setEvent() - - proc timerTest() = - waitFor(waitTimer(200)) - echo "OK" - proc eventTest() = var event = newAsyncEvent() var fut = waitEvent(event) asyncCheck(delayedSet(event, 500)) - waitFor(fut or waitTimer(1000)) + waitFor(fut or sleepAsync(1000)) if fut.finished: echo "OK" else: echo "eventTest: Timeout expired before event received!" - proc processTest() = - when defined(windows): - var process = startProcess("ping.exe", "", - ["127.0.0.1", "-n", "2", "-w", "100"], nil, - {poStdErrToStdOut, poUsePath, poInteractive, - poDemon}) - else: - var process = startProcess("/bin/sleep", "", ["1"], nil, - {poStdErrToStdOut, poUsePath}) - var fut = waitProcess(process) - waitFor(fut or waitTimer(2000)) - if fut.finished and process.peekExitCode() == 0: + when ioselSupportedPlatform or defined(windows): + + import osproc + + proc waitTimer(timeout: int): Future[void] = + var retFuture = newFuture[void]("waitTimer") + proc cb(fd: AsyncFD): bool = + retFuture.complete() + addTimer(timeout, true, cb) + return retFuture + + proc waitProcess(p: Process): Future[void] = + var retFuture = newFuture[void]("waitProcess") + proc cb(fd: AsyncFD): bool = + retFuture.complete() + addProcess(p.processID(), cb) + return retFuture + + proc timerTest() = + waitFor(waitTimer(200)) echo "OK" - else: - echo "processTest: Timeout expired before process exited!" - when supportedPlatform: - import posix + proc processTest() = + when defined(windows): + var process = startProcess("ping.exe", "", + ["127.0.0.1", "-n", "2", "-w", "100"], nil, + {poStdErrToStdOut, poUsePath, poInteractive, + poDemon}) + else: + var process = startProcess("/bin/sleep", "", ["1"], nil, + {poStdErrToStdOut, poUsePath}) + var fut = waitProcess(process) + waitFor(fut or waitTimer(2000)) + if fut.finished and process.peekExitCode() == 0: + echo "OK" + else: + echo "processTest: Timeout expired before process exited!" + + when ioselSupportedPlatform: proc waitSignal(signal: int): Future[void] = var retFuture = newFuture[void]("waitSignal") @@ -97,7 +97,7 @@ when defined(upcoming): else: echo "signalTest: Timeout expired before signal received!" - when supportedPlatform: + when ioselSupportedPlatform: timerTest() eventTest() processTest() diff --git a/tests/osproc/texitcode.nim b/tests/osproc/texitcode.nim index 1e83658c2..4eaab6da2 100644 --- a/tests/osproc/texitcode.nim +++ b/tests/osproc/texitcode.nim @@ -16,3 +16,8 @@ var running = true while running: running = running(p) doAssert(waitForExit(p) == QuitFailure) + +# make sure that first call to running() after process exit returns false +p = startProcess(filename, dir) +os.sleep(500) +doAssert(not running(p)) |