diff options
| author | Simon Hafner <hafnersimon@gmail.com> | 2014-03-31 15:49:04 -0500 |
|---|---|---|
| committer | Simon Hafner <hafnersimon@gmail.com> | 2014-03-31 15:49:04 -0500 |
| commit | ffb36db5a6caa147119aed1728c8042dfa68a3e8 (patch) | |
| tree | 0db6e9c9b39a21511cf6d5a1c2f489e35a10ed60 /lib | |
| parent | 565031f0cd4768962fb19ac4e17efb994dfb4735 (diff) | |
| parent | 44ee8aecfd70d1d381b5eed5ae52b01fae04452b (diff) | |
| download | Nim-ffb36db5a6caa147119aed1728c8042dfa68a3e8.tar.gz | |
Merge branch 'devel' of github.com:Araq/Nimrod into seq_toString
Diffstat (limited to 'lib')
41 files changed, 3798 insertions, 2682 deletions
diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 585ccf869..8ccad8fe3 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -48,17 +48,17 @@ type nnkYieldStmt, nnkTryStmt, nnkFinally, nnkRaiseStmt, nnkReturnStmt, nnkBreakStmt, nnkContinueStmt, nnkBlockStmt, nnkStaticStmt, nnkDiscardStmt, nnkStmtList, - nnkImportStmt, nnkImportExceptStmt, nnkExportStmt, nnkExportExceptStmt, nnkFromStmt, nnkIncludeStmt, - nnkBindStmt, nnkMixinStmt, nnkUsingStmt, nnkCommentStmt, nnkStmtListExpr, nnkBlockExpr, - nnkStmtListType, nnkBlockType, nnkTypeOfExpr, nnkObjectTy, + nnkStmtListType, nnkBlockType, + nnkWith, nnkWithout, + nnkTypeOfExpr, nnkObjectTy, nnkTupleTy, nnkTypeClassTy, nnkStaticTy, nnkRecList, nnkRecCase, nnkRecWhen, nnkRefTy, nnkPtrTy, nnkVarTy, @@ -88,7 +88,7 @@ type nskUnknown, nskConditional, nskDynLib, nskParam, nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet, nskConst, nskResult, - nskProc, nskMethod, nskIterator, + nskProc, nskMethod, nskIterator, nskClosureIterator, nskConverter, nskMacro, nskTemplate, nskField, nskEnumField, nskForVar, nskLabel, nskStub @@ -516,7 +516,7 @@ proc last*(node: PNimrodNode): PNimrodNode {.compileTime.} = node[node.high] const - RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda} + RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda, nnkIteratorDef} AtomicNodes* = {nnkNone..nnkNilLit} CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, nnkCallStrLit, nnkHiddenCallConv} diff --git a/lib/impure/db_mongo.nim b/lib/impure/db_mongo.nim deleted file mode 100644 index dc8a808f2..000000000 --- a/lib/impure/db_mongo.nim +++ /dev/null @@ -1,227 +0,0 @@ -# -# -# Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a higher level wrapper for `mongodb`:idx:. Example: -## -## .. code-block:: nimrod -## -## import mongo, db_mongo, oids, json -## -## var conn = db_mongo.open() -## -## # construct JSON data: -## var data = %{"a": %13, "b": %"my string value", -## "inner": %{"i": %71} } -## -## var id = insertID(conn, "test.test", data) -## -## for v in find(conn, "test.test", "this.a == 13"): -## print v -## -## delete(conn, "test.test", id) -## close(conn) - -import mongo, oids, json - -type - EDb* = object of EIO ## exception that is raised if a database error occurs - TDbConn* = TMongo ## a database connection; alias for ``TMongo`` - - FDb* = object of FIO ## effect that denotes a database operation - FReadDb* = object of FDB ## effect that denotes a read operation - FWriteDb* = object of FDB ## effect that denotes a write operation - -proc dbError*(db: TDbConn, msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - if db.errstr[0] != '\0': - e.msg = $db.errstr - else: - e.msg = $db.err & " " & msg - raise e - -proc close*(db: var TDbConn) {.tags: [FDB].} = - ## closes the database connection. - disconnect(db) - destroy(db) - -proc open*(host: string = defaultHost, port: int = defaultPort): TDbConn {. - tags: [FDB].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. - init(result) - - let x = client(result, host, port.cint) - if x != 0'i32: - dbError(result, "cannot open: " & host) - -proc jsonToBSon(b: var TBSon, key: string, j: PJsonNode) = - case j.kind - of JString: - add(b, key, j.str) - of JInt: - add(b, key, j.num) - of JFloat: - add(b, key, j.fnum) - of JBool: - addBool(b, key, ord(j.bval)) - of JNull: - addNull(b, key) - of JObject: - addStartObject(b, key) - for k, v in items(j.fields): - jsonToBSon(b, k, v) - addFinishObject(b) - of JArray: - addStartArray(b, key) - for i, e in pairs(j.elems): - jsonToBSon(b, $i, e) - addFinishArray(b) - -proc jsonToBSon*(j: PJsonNode, oid: TOid): TBSon = - ## converts a JSON value into the BSON format. The result must be - ## ``destroyed`` explicitely! - init(result) - assert j.kind == JObject - add(result, "_id", oid) - for key, val in items(j.fields): - jsonToBSon(result, key, val) - finish(result) - -proc `[]`*(obj: var TBSon, fieldname: cstring): TBSon = - ## retrieves the value belonging to `fieldname`. Raises `EInvalidKey` if - ## the attribute does not exist. - var it = initIter(obj) - let res = find(it, result, fieldname) - if res == bkEOO: - raise newException(EInvalidIndex, "key not in object") - -proc getId*(obj: var TBSon): TOid = - ## retrieves the ``_id`` attribute of `obj`. - var it = initIter(obj) - var b: TBSon - let res = find(it, b, "_id") - if res == bkOID: - result = oidVal(it)[] - else: - raise newException(EInvalidIndex, "_id not in object") - -proc insertId*(db: var TDbConn, namespace: string, data: PJsonNode): TOid {. - tags: [FWriteDb].} = - ## converts `data` to BSON format and inserts it in `namespace`. Returns - ## the generated OID for the ``_id`` field. - result = genOid() - var x = jsonToBSon(data, result) - insert(db, namespace, x, nil) - destroy(x) - -proc insert*(db: var TDbConn, namespace: string, data: PJsonNode) {. - tags: [FWriteDb].} = - ## converts `data` to BSON format and inserts it in `namespace`. - discard InsertID(db, namespace, data) - -proc update*(db: var TDbConn, namespace: string, obj: var TBSon) {. - tags: [FReadDB, FWriteDb].} = - ## updates `obj` in `namespace`. - var cond: TBson - init(cond) - cond.add("_id", getId(obj)) - finish(cond) - update(db, namespace, cond, obj, ord(UPDATE_UPSERT)) - destroy(cond) - -proc update*(db: var TDbConn, namespace: string, oid: TOid, obj: PJsonNode) {. - tags: [FReadDB, FWriteDb].} = - ## updates the data with `oid` to have the new data `obj`. - var a = jsonToBSon(obj, oid) - Update(db, namespace, a) - destroy(a) - -proc delete*(db: var TDbConn, namespace: string, oid: TOid) {. - tags: [FWriteDb].} = - ## Deletes the object belonging to `oid`. - var cond: TBson - init(cond) - cond.add("_id", oid) - finish(cond) - discard remove(db, namespace, cond) - destroy(cond) - -proc delete*(db: var TDbConn, namespace: string, obj: var TBSon) {. - tags: [FWriteDb].} = - ## Deletes the object `obj`. - delete(db, namespace, getId(obj)) - -iterator find*(db: var TDbConn, namespace: string): var TBSon {. - tags: [FReadDB].} = - ## iterates over any object in `namespace`. - var cursor: TCursor - init(cursor, db, namespace) - while next(cursor) == mongo.OK: - yield bson(cursor)[] - destroy(cursor) - -iterator find*(db: var TDbConn, namespace: string, - query, fields: var TBSon): var TBSon {.tags: [FReadDB].} = - ## yields the `fields` of any document that suffices `query`. - var cursor = find(db, namespace, query, fields, 0'i32, 0'i32, 0'i32) - if cursor != nil: - while next(cursor[]) == mongo.OK: - yield bson(cursor[])[] - destroy(cursor[]) - -proc setupFieldnames(fields: varargs[string]): TBSon = - init(result) - for x in fields: add(result, x, 1'i32) - finish(result) - -iterator find*(db: var TDbConn, namespace: string, - query: var TBSon, fields: varargs[string]): var TBSon {. - tags: [FReadDB].} = - ## yields the `fields` of any document that suffices `query`. If `fields` - ## is ``[]`` the whole document is yielded. - var f = setupFieldnames(fields) - var cursor = find(db, namespace, query, f, 0'i32, 0'i32, 0'i32) - if cursor != nil: - while next(cursor[]) == mongo.OK: - yield bson(cursor[])[] - destroy(cursor[]) - destroy(f) - -proc setupQuery(query: string): TBSon = - init(result) - add(result, "$where", query) - finish(result) - -iterator find*(db: var TDbConn, namespace: string, - query: string, fields: varargs[string]): var TBSon {. - tags: [FReadDB].} = - ## yields the `fields` of any document that suffices `query`. If `fields` - ## is ``[]`` the whole document is yielded. - var f = setupFieldnames(fields) - var q = setupQuery(query) - var cursor = find(db, namespace, q, f, 0'i32, 0'i32, 0'i32) - if cursor != nil: - while next(cursor[]) == mongo.OK: - yield bson(cursor[])[] - destroy(cursor[]) - destroy(q) - destroy(f) - -when false: - # this doesn't work this way; would require low level hacking - iterator fieldPairs*(obj: var TBSon): tuple[key: cstring, value: TBSon] = - ## iterates over `obj` and yields all (key, value)-Pairs. - var it = initIter(obj) - var v: TBSon - while next(it) != bkEOO: - let key = key(it) - discard init(v, value(it)) - yield (key, v) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 8cdccda01..32cda3e4d 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -87,7 +87,7 @@ proc newRow(L: int): TRow = proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = if row != nil: - while mysql.FetchRow(sqlres) != nil: nil + while mysql.FetchRow(sqlres) != nil: discard mysql.FreeResult(sqlres) iterator fastRows*(db: TDbConn, query: TSqlQuery, @@ -195,8 +195,14 @@ proc open*(connection, user, password, database: string): TDbConn {. ## be established. result = mysql.Init(nil) if result == nil: dbError("could not open database connection") - if mysql.RealConnect(result, "", user, password, database, - 0'i32, nil, 0) == nil: + let + colonPos = connection.find(':') + host = if colonPos < 0: connection + else: substr(connection, 0, colonPos-1) + port: int32 = if colonPos < 0: 0'i32 + else: substr(connection, colonPos+1).parseInt.int32 + if mysql.RealConnect(result, host, user, password, database, + port, nil, 0) == nil: var errmsg = $mysql.error(result) db_mysql.Close(result) dbError(errmsg) diff --git a/lib/nimbase.h b/lib/nimbase.h index 19d161adf..b16b27b2c 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -285,8 +285,8 @@ static N_INLINE(NI32, float32ToInt32)(float x) { typedef struct TStringDesc* string; -/* declared size of a sequence: */ -#if defined(__GNUC__) +/* declared size of a sequence/variable length array: */ +#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) # define SEQ_DECL_SIZE /* empty is correct! */ #else # define SEQ_DECL_SIZE 1000000 @@ -314,6 +314,9 @@ static unsigned long nimNaN[2]={0xffffffff, 0x7fffffff}; # define INF INFINITY # elif defined(HUGE_VAL) # define INF HUGE_VAL +# elif defined(_MSC_VER) +# include <float.h> +# define INF (DBL_MAX+DBL_MAX) # else # define INF (1.0 / 0.0) # endif @@ -373,5 +376,8 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } # define GC_GUARD #endif +/* Test to see if nimrod and the C compiler agrees on the size of a pointer. + On disagreement, your C compiler will say something like: + "error: 'assert_numbits' declared as an array with a negative size" */ typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1]; #endif diff --git a/lib/packages/docutils/docutils.babel b/lib/packages/docutils/docutils.babel new file mode 100644 index 000000000..1ed86ca05 --- /dev/null +++ b/lib/packages/docutils/docutils.babel @@ -0,0 +1,6 @@ +[Package] +name = "docutils" +version = "0.9.0" +author = "Andreas Rumpf" +description = "Nimrod's reStructuredText processor." +license = "MIT" diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 4ca0c79e0..c507f5e1c 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -61,9 +61,8 @@ proc getSourceLanguage*(name: string): TSourceLanguage = if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: return i result = langNone - -proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = - g.buf = cstring(buf) +proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: cstring) = + g.buf = buf g.kind = low(TTokenClass) g.start = 0 g.length = 0 @@ -71,6 +70,8 @@ proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = var pos = 0 # skip initial whitespace: while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) g.pos = pos +proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = + initGeneralTokenizer(g, cstring(buf)) proc deinitGeneralTokenizer*(g: var TGeneralTokenizer) = discard diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index bb018bc1e..30cc9026b 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1543,7 +1543,7 @@ proc dirRaw(p: var TRstParser): PRstNode = elif cmpIgnoreCase(result.sons[0].sons[0].text, "latex") == 0: dirRawAux(p, result, rnRawLatex, parseLiteralBlock) else: - rstMessage(p, meInvalidDirective, result.sons[0].text) + rstMessage(p, meInvalidDirective, result.sons[0].sons[0].text) else: dirRawAux(p, result, rnRaw, parseSectionWrapper) diff --git a/lib/posix/epoll.nim b/lib/posix/epoll.nim index d50394f60..57a2f001f 100644 --- a/lib/posix/epoll.nim +++ b/lib/posix/epoll.nim @@ -7,6 +7,8 @@ # distribution, for details about the copyright. # +from posix import TSocketHandle + const EPOLLIN* = 0x00000001 EPOLLPRI* = 0x00000002 @@ -33,8 +35,8 @@ const type epoll_data* {.importc: "union epoll_data", header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union. - thePtr* {.importc: "ptr".}: pointer # \ - #fd*: cint + #thePtr* {.importc: "ptr".}: pointer + fd* {.importc: "fd".}: cint # \ #u32*: uint32 #u64*: uint64 @@ -54,7 +56,7 @@ proc epoll_create1*(flags: cint): cint {.importc: "epoll_create1", ## Same as epoll_create but with an FLAGS parameter. The unused SIZE ## parameter has been dropped. -proc epoll_ctl*(epfd: cint; op: cint; fd: cint; event: ptr epoll_event): cint {. +proc epoll_ctl*(epfd: cint; op: cint; fd: cint | TSocketHandle; event: ptr epoll_event): cint {. importc: "epoll_ctl", header: "<sys/epoll.h>".} ## Manipulate an epoll instance "epfd". Returns 0 in case of success, ## -1 in case of error ( the "errno" variable will contain the diff --git a/lib/posix/linux.nim b/lib/posix/linux.nim new file mode 100644 index 000000000..1ed1af3b6 --- /dev/null +++ b/lib/posix/linux.nim @@ -0,0 +1,25 @@ +import posix + +const + CSIGNAL* = 0x000000FF + CLONE_VM* = 0x00000100 + CLONE_FS* = 0x00000200 + CLONE_FILES* = 0x00000400 + CLONE_SIGHAND* = 0x00000800 + CLONE_PTRACE* = 0x00002000 + CLONE_VFORK* = 0x00004000 + CLONE_PARENT* = 0x00008000 + CLONE_THREAD* = 0x00010000 + CLONE_NEWNS* = 0x00020000 + CLONE_SYSVSEM* = 0x00040000 + CLONE_SETTLS* = 0x00080000 + CLONE_PARENT_SETTID* = 0x00100000 + CLONE_CHILD_CLEARTID* = 0x00200000 + CLONE_DETACHED* = 0x00400000 + CLONE_UNTRACED* = 0x00800000 + CLONE_CHILD_SETTID* = 0x01000000 + CLONE_STOPPED* = 0x02000000 + +# fn should be of type proc (a2: pointer): void {.cdecl.} +proc clone*(fn: pointer; child_stack: pointer; flags: cint; + arg: pointer; ptid: ptr TPid; tls: pointer; ctid: ptr TPid): cint {.importc, header: "<sched.h>".} diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 41260b36f..131f23fdd 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -2066,6 +2066,7 @@ proc pthread_spin_unlock*(a1: ptr Tpthread_spinlock): cint {. proc pthread_testcancel*() {.importc, header: "<pthread.h>".} +proc exitnow*(code: int): void {.importc: "_exit", header: "<unistd.h>".} proc access*(a1: cstring, a2: cint): cint {.importc, header: "<unistd.h>".} proc alarm*(a1: cint): cint {.importc, header: "<unistd.h>".} proc chdir*(a1: cstring): cint {.importc, header: "<unistd.h>".} @@ -2265,6 +2266,7 @@ proc gmtime_r*(a1: var TTime, a2: var Ttm): ptr Ttm {.importc, header: "<time.h> proc localtime*(a1: var TTime): ptr Ttm {.importc, header: "<time.h>".} proc localtime_r*(a1: var TTime, a2: var Ttm): ptr Ttm {.importc, header: "<time.h>".} proc mktime*(a1: var Ttm): TTime {.importc, header: "<time.h>".} +proc timegm*(a1: var Ttm): TTime {.importc, header: "<time.h>".} proc nanosleep*(a1, a2: var Ttimespec): cint {.importc, header: "<time.h>".} proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Ttm): int {.importc, header: "<time.h>".} @@ -2356,7 +2358,7 @@ proc FD_ZERO*(a1: var TFdSet) {.importc, header: "<sys/select.h>".} proc pselect*(a1: cint, a2, a3, a4: ptr TFdSet, a5: ptr Ttimespec, a6: var Tsigset): cint {.importc, header: "<sys/select.h>".} -proc select*(a1: cint, a2, a3, a4: ptr TFdSet, a5: ptr Ttimeval): cint {. +proc select*(a1: cint | TSocketHandle, a2, a3, a4: ptr TFdSet, a5: ptr Ttimeval): cint {. importc, header: "<sys/select.h>".} when hasSpawnH: diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 921c659de..37fbc948c 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -55,6 +55,36 @@ proc smartBinarySearch*[T](a: openArray[T], key: T): int = const onlySafeCode = true +proc lowerBound*[T](a: openarray[T], key: T, cmp: proc(x,y: T): int {.closure.}): int = + ## same as binarySearch except that if key is not in `a` then this + ## returns the location where `key` would be if it were. In other + ## words if you have a sorted sequence and you call insert(thing, elm, lowerBound(thing, elm)) + ## the sequence will still be sorted + ## + ## `cmp` is the comparator function to use, the expected return values are the same as + ## that of system.cmp + ## + ## example:: + ## + ## var arr = @[1,2,3,5,6,7,8,9] + ## arr.insert(4, arr.lowerBound(4)) + ## `after running the above arr is `[1,2,3,4,5,6,7,8,9]` + result = a.low + var pos = result + var count, step: int + count = a.high - a.low + 1 + while count != 0: + pos = result + step = count div 2 + pos += step + if cmp(a[pos], key) < 0: + pos.inc + result = pos + count -= step + 1 + else: + count = step + +proc lowerBound*[T](a: openarray[T], key: T): int = lowerBound(a, key, cmp[T]) proc merge[T](a, b: var openArray[T], lo, m, hi: int, cmp: proc (x, y: T): int {.closure.}, order: TSortOrder) = template `<-` (a, b: expr) = diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim new file mode 100644 index 000000000..344652c9f --- /dev/null +++ b/lib/pure/asyncdispatch.nim @@ -0,0 +1,956 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import os, oids, tables, strutils, macros + +import rawsockets + +## AsyncDispatch +## -------- +## +## This module implements a brand new dispatcher based on Futures. +## On Windows IOCP is used and on other operating systems the selectors module +## is used instead. + +# -- Futures + +type + PFutureBase* = ref object of PObject + cb: proc () {.closure.} + finished: bool + + PFuture*[T] = ref object of PFutureBase + value: T + error*: ref EBase # TODO: This shouldn't be necessary, generics bug? + +proc newFuture*[T](): PFuture[T] = + ## Creates a new future. + new(result) + result.finished = false + +proc complete*[T](future: PFuture[T], val: T) = + ## Completes ``future`` with value ``val``. + assert(not future.finished, "Future already finished, cannot finish twice.") + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: PFuture[void]) = + ## Completes a void ``future``. + assert(not future.finished, "Future already finished, cannot finish twice.") + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc fail*[T](future: PFuture[T], error: ref EBase) = + ## Completes ``future`` with ``error``. + assert(not future.finished, "Future already finished, cannot finish twice.") + future.finished = true + future.error = error + if future.cb != nil: + future.cb() + +proc `callback=`*(future: PFutureBase, cb: proc () {.closure.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + future.cb() + +proc `callback=`*[T](future: PFuture[T], + cb: proc (future: PFuture[T]) {.closure.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc read*[T](future: PFuture[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``EInvalidValue`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished: + if future.error != nil: raise future.error + when T isnot void: + return future.value + else: + # TODO: Make a custom exception type for this? + raise newException(EInvalidValue, "Future still in progress.") + +proc readError*[T](future: PFuture[T]): ref EBase = + if future.error != nil: return future.error + else: + raise newException(EInvalidValue, "No error in future.") + +proc finished*[T](future: PFuture[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + future.finished + +proc failed*[T](future: PFuture[T]): bool = + ## Determines whether ``future`` completed with an error. + future.error != nil + +when defined(windows) or defined(nimdoc): + import winlean, sets, hashes + type + TCompletionKey = dword + + TCompletionData* = object + sock: TAsyncFD + cb: proc (sock: TAsyncFD, bytesTransferred: DWORD, + errcode: TOSErrorCode) {.closure.} + + PDispatcher* = ref object + ioPort: THandle + handles: TSet[TAsyncFD] + + TCustomOverlapped = object + Internal*: DWORD + InternalHigh*: DWORD + Offset*: DWORD + OffsetHigh*: DWORD + hEvent*: THANDLE + data*: TCompletionData + + PCustomOverlapped = ptr TCustomOverlapped + + TAsyncFD* = distinct int + + proc hash(x: TAsyncFD): THash {.borrow.} + proc `==`*(x: TAsyncFD, y: TAsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + new result + result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) + result.handles = initSet[TAsyncFD]() + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + ## Retrieves the global thread-local dispatcher. + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc register*(sock: TAsyncFD) = + ## Registers ``sock`` with the dispatcher. + let p = getGlobalDispatcher() + if CreateIOCompletionPort(sock.THandle, p.ioPort, + cast[TCompletionKey](sock), 1) == 0: + OSError(OSLastError()) + p.handles.incl(sock) + + proc verifyPresence(sock: TAsyncFD) = + ## Ensures that socket has been registered with the dispatcher. + let p = getGlobalDispatcher() + if sock notin p.handles: + raise newException(EInvalidValue, + "Operation performed on a socket which has not been registered with" & + " the dispatcher yet.") + + proc poll*(timeout = 500) = + ## Waits for completion events and processes them. + let p = getGlobalDispatcher() + if p.handles.len == 0: + raise newException(EInvalidValue, "No handles registered in dispatcher.") + + let llTimeout = + if timeout == -1: winlean.INFINITE + else: timeout.int32 + var lpNumberOfBytesTransferred: DWORD + var lpCompletionKey: ULONG + var lpOverlapped: POverlapped + let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, + addr lpCompletionKey, addr lpOverlapped, llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + var customOverlapped = cast[PCustomOverlapped](lpOverlapped) + if res: + # This is useful for ensuring the reliability of the overlapped struct. + assert customOverlapped.data.sock == lpCompletionKey.TAsyncFD + + customOverlapped.data.cb(customOverlapped.data.sock, + lpNumberOfBytesTransferred, TOSErrorCode(-1)) + dealloc(customOverlapped) + else: + let errCode = OSLastError() + if lpOverlapped != nil: + assert customOverlapped.data.sock == lpCompletionKey.TAsyncFD + customOverlapped.data.cb(customOverlapped.data.sock, + lpNumberOfBytesTransferred, errCode) + dealloc(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: OSError(errCode) + + var connectExPtr: pointer = nil + var acceptExPtr: pointer = nil + var getAcceptExSockAddrsPtr: pointer = nil + + proc initPointer(s: TSocketHandle, func: var pointer, guid: var TGUID): bool = + # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c + var bytesRet: DWord + func = nil + result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, + sizeof(TGUID).dword, addr func, sizeof(pointer).DWORD, + addr bytesRet, nil, nil) == 0 + + proc initAll() = + let dummySock = newRawSocket() + if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX): + OSError(OSLastError()) + if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX): + OSError(OSLastError()) + if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS): + OSError(OSLastError()) + + proc connectEx(s: TSocketHandle, name: ptr TSockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: dword, + lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool = + if connectExPtr.isNil: raise newException(EInvalidValue, "Need to initialise ConnectEx().") + let func = + cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, + lpSendBuffer: pointer, dwSendDataLength: dword, + lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr) + + result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent, + lpOverlapped) + + proc acceptEx(listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, + lpOverlapped: POverlapped): bool = + if acceptExPtr.isNil: raise newException(EInvalidValue, "Need to initialise AcceptEx().") + let func = + cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, + lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr) + result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, + lpOverlapped) + + proc getAcceptExSockaddrs(lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD, + LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint, + RemoteSockaddr: ptr ptr TSockAddr, RemoteSockaddrLength: lpint) = + if getAcceptExSockAddrsPtr.isNil: + raise newException(EInvalidValue, "Need to initialise getAcceptExSockAddrs().") + + let func = + cast[proc (lpOutputBuffer: pointer, + dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr, + LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr, + RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr) + + func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, + dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, + RemoteSockaddr, RemoteSockaddrLength) + + proc connect*(socket: TAsyncFD, address: string, port: TPort, + af = AF_INET): PFuture[void] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``PFuture`` which will complete when the connection succeeds + ## or an error occurs. + verifyPresence(socket) + var retFuture = newFuture[void]() + # Apparently ``ConnectEx`` expects the socket to be initially bound: + var saddr: Tsockaddr_in + saddr.sin_family = int16(toInt(af)) + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(socket.TSocketHandle, cast[ptr TSockAddr](addr(saddr)), + sizeof(saddr).TSockLen) < 0'i32: + OSError(OSLastError()) + + var aiList = getAddrInfo(address, port, af) + var success = false + var lastError: TOSErrorCode + var it = aiList + while it != nil: + # "the OVERLAPPED structure must remain valid until the I/O completes" + # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) = + if not retFuture.finished: + if errcode == TOSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + var ret = connectEx(socket.TSocketHandle, it.ai_addr, + sizeof(TSockAddrIn).cint, nil, 0, nil, + cast[POverlapped](ol)) + if ret: + # Request to connect completed immediately. + success = true + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + break + else: + lastError = OSLastError() + if lastError.int32 == ERROR_IO_PENDING: + # In this case ``ol`` will be deallocated in ``poll``. + success = true + break + else: + dealloc(ol) + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: TAsyncFD, size: int, + flags: int = 0): PFuture[string] = + ## Reads ``size`` bytes from ``socket``. Returned future will complete once + ## all of the requested data is read. If socket is disconnected during the + ## recv operation then the future may complete with only a part of the + ## requested data read. If socket is disconnected and no data is available + ## to be read then the future will complete with a value of ``""``. + verifyPresence(socket) + var retFuture = newFuture[string]() + + var dataBuf: TWSABuf + dataBuf.buf = newString(size) + dataBuf.len = size + + var bytesReceived: DWord + var flagsio = flags.dword + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) = + if not retFuture.finished: + if errcode == TOSErrorCode(-1): + if bytesCount == 0 and dataBuf.buf[0] == '\0': + retFuture.complete("") + else: + var data = newString(size) + copyMem(addr data[0], addr dataBuf.buf[0], size) + retFuture.complete($data) + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + let ret = WSARecv(socket.TSocketHandle, addr dataBuf, 1, addr bytesReceived, + addr flagsio, cast[POverlapped](ol), nil) + if ret == -1: + let err = OSLastError() + if err.int32 != ERROR_IO_PENDING: + retFuture.fail(newException(EOS, osErrorMsg(err))) + dealloc(ol) + elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': + # We have to ensure that the buffer is empty because WSARecv will tell + # us immediatelly when it was disconnected, even when there is still + # data in the buffer. + # We want to give the user as much data as we can. So we only return + # the empty string (which signals a disconnection) when there is + # nothing left to read. + retFuture.complete("") + # TODO: "For message-oriented sockets, where a zero byte message is often + # allowable, a failure with an error code of WSAEDISCON is used to + # indicate graceful closure." + # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx + else: + # Request to read completed immediately. + var data = newString(size) + copyMem(addr data[0], addr dataBuf.buf[0], size) + retFuture.complete($data) + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc send*(socket: TAsyncFD, data: string): PFuture[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + verifyPresence(socket) + var retFuture = newFuture[void]() + + var dataBuf: TWSABuf + dataBuf.buf = data + dataBuf.len = data.len + + var bytesReceived, flags: DWord + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) = + if not retFuture.finished: + if errcode == TOSErrorCode(-1): + retFuture.complete() + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket.TSocketHandle, addr dataBuf, 1, addr bytesReceived, + flags, cast[POverlapped](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + retFuture.fail(newException(EOS, osErrorMsg(err))) + dealloc(ol) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + + proc acceptAddr*(socket: TAsyncFD): + PFuture[tuple[address: string, client: TAsyncFD]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + ## + ## The resulting client socket is automatically registered to dispatcher. + verifyPresence(socket) + var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]() + + var clientSock = newRawSocket() + if clientSock == OSInvalidSocket: osError(osLastError()) + + const lpOutputLen = 1024 + var lpOutputBuf = newString(lpOutputLen) + var dwBytesReceived: DWORD + let dwReceiveDataLength = 0.DWORD # We don't want any data to be read. + let dwLocalAddressLength = DWORD(sizeof (TSockaddr_in) + 16) + let dwRemoteAddressLength = DWORD(sizeof(TSockaddr_in) + 16) + + template completeAccept(): stmt {.immediate, dirty.} = + var listenSock = socket + let setoptRet = setsockopt(clientSock, SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, + sizeof(listenSock).TSockLen) + if setoptRet != 0: osError(osLastError()) + + var LocalSockaddr, RemoteSockaddr: ptr TSockAddr + var localLen, remoteLen: int32 + getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, + dwLocalAddressLength, dwRemoteAddressLength, + addr LocalSockaddr, addr localLen, + addr RemoteSockaddr, addr remoteLen) + register(clientSock.TAsyncFD) + # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 + retFuture.complete( + (address: $inet_ntoa(cast[ptr Tsockaddr_in](remoteSockAddr).sin_addr), + client: clientSock.TAsyncFD) + ) + + var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) + ol.data = TCompletionData(sock: socket, cb: + proc (sock: TAsyncFD, bytesCount: DWord, errcode: TOSErrorCode) = + if not retFuture.finished: + if errcode == TOSErrorCode(-1): + completeAccept() + else: + retFuture.fail(newException(EOS, osErrorMsg(errcode))) + ) + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx + let ret = acceptEx(socket.TSocketHandle, clientSock, addr lpOutputBuf[0], + dwReceiveDataLength, + dwLocalAddressLength, + dwRemoteAddressLength, + addr dwBytesReceived, cast[POverlapped](ol)) + + if not ret: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + retFuture.fail(newException(EOS, osErrorMsg(err))) + dealloc(ol) + else: + completeAccept() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + + return retFuture + + proc newAsyncRawSocket*(domain: TDomain = AF_INET, + typ: TType = SOCK_STREAM, + protocol: TProtocol = IPPROTO_TCP): TAsyncFD = + ## Creates a new socket and registers it with the dispatcher implicitly. + result = newRawSocket(domain, typ, protocol).TAsyncFD + result.TSocketHandle.setBlocking(false) + register(result) + + proc close*(socket: TAsyncFD) = + ## Closes a socket and ensures that it is unregistered. + socket.TSocketHandle.close() + getGlobalDispatcher().handles.excl(socket) + + initAll() +else: + import selectors + from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK + type + TAsyncFD* = distinct cint + TCallback = proc (sock: TAsyncFD): bool {.closure.} + + PData* = ref object of PObject + sock: TAsyncFD + readCBs: seq[TCallback] + writeCBs: seq[TCallback] + + PDispatcher* = ref object + selector: PSelector + + proc `==`*(x, y: TAsyncFD): bool {.borrow.} + + proc newDispatcher*(): PDispatcher = + new result + result.selector = newSelector() + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + proc getGlobalDispatcher*(): PDispatcher = + if gDisp.isNil: gDisp = newDispatcher() + result = gDisp + + proc update(sock: TAsyncFD, events: set[TEvent]) = + let p = getGlobalDispatcher() + assert sock.TSocketHandle in p.selector + discard p.selector.update(sock.TSocketHandle, events) + + proc register(sock: TAsyncFD) = + let p = getGlobalDispatcher() + var data = PData(sock: sock, readCBs: @[], writeCBs: @[]) + p.selector.register(sock.TSocketHandle, {}, data.PObject) + + proc newAsyncRawSocket*(domain: TDomain = AF_INET, + typ: TType = SOCK_STREAM, + protocol: TProtocol = IPPROTO_TCP): TAsyncFD = + result = newRawSocket(domain, typ, protocol).TAsyncFD + result.TSocketHandle.setBlocking(false) + register(result) + + proc close*(sock: TAsyncFD) = + let disp = getGlobalDispatcher() + sock.TSocketHandle.close() + disp.selector.unregister(sock.TSocketHandle) + + proc addRead(sock: TAsyncFD, cb: TCallback) = + let p = getGlobalDispatcher() + if sock.TSocketHandle notin p.selector: + raise newException(EInvalidValue, "File descriptor not registered.") + p.selector[sock.TSocketHandle].data.PData.readCBs.add(cb) + update(sock, p.selector[sock.TSocketHandle].events + {EvRead}) + + proc addWrite(sock: TAsyncFD, cb: TCallback) = + let p = getGlobalDispatcher() + if sock.TSocketHandle notin p.selector: + raise newException(EInvalidValue, "File descriptor not registered.") + p.selector[sock.TSocketHandle].data.PData.writeCBs.add(cb) + update(sock, p.selector[sock.TSocketHandle].events + {EvWrite}) + + proc poll*(timeout = 500) = + let p = getGlobalDispatcher() + for info in p.selector.select(timeout): + let data = PData(info.key.data) + assert data.sock == info.key.fd.TAsyncFD + #echo("In poll ", data.sock.cint) + if EvRead in info.events: + # Callback may add items to ``data.readCBs`` which causes issues if + # we are iterating over ``data.readCBs`` at the same time. We therefore + # make a copy to iterate over. + let currentCBs = data.readCBs + data.readCBs = @[] + for cb in currentCBs: + if not cb(data.sock): + # Callback wants to be called again. + data.readCBs.add(cb) + + if EvWrite in info.events: + let currentCBs = data.writeCBs + data.writeCBs = @[] + for cb in currentCBs: + if not cb(data.sock): + # Callback wants to be called again. + data.writeCBs.add(cb) + + if info.key in p.selector: + var newEvents: set[TEvent] + if data.readCBs.len != 0: newEvents = {EvRead} + if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite} + if newEvents != info.key.events: + update(data.sock, newEvents) + else: + # FD no longer a part of the selector. Likely been closed + # (e.g. socket disconnected). + + proc connect*(socket: TAsyncFD, address: string, port: TPort, + af = AF_INET): PFuture[void] = + var retFuture = newFuture[void]() + + proc cb(sock: TAsyncFD): bool = + # We have connected. + retFuture.complete() + return true + + var aiList = getAddrInfo(address, port, af) + var success = false + var lastError: TOSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.TSocketHandle, it.ai_addr, it.ai_addrlen.TSocklen) + if ret == 0: + # Request to connect completed immediately. + success = true + retFuture.complete() + break + else: + lastError = osLastError() + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + addWrite(socket, cb) + break + else: + success = false + it = it.ai_next + + dealloc(aiList) + if not success: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) + return retFuture + + proc recv*(socket: TAsyncFD, size: int, + flags: int = 0): PFuture[string] = + var retFuture = newFuture[string]() + + var readBuffer = newString(size) + var sizeRead = 0 + + proc cb(sock: TAsyncFD): bool = + result = true + let netSize = size - sizeRead + let res = recv(sock.TSocketHandle, addr readBuffer[sizeRead], netSize, + flags.cint) + #echo("recv cb res: ", res) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + elif res == 0: + #echo("Disconnected recv: ", sizeRead) + # Disconnected + if sizeRead == 0: + retFuture.complete("") + else: + readBuffer.setLen(sizeRead) + retFuture.complete(readBuffer) + else: + sizeRead.inc(res) + if res != netSize: + result = false # We want to read all the data requested. + else: + retFuture.complete(readBuffer) + #echo("Recv cb result: ", result) + + addRead(socket, cb) + return retFuture + + proc send*(socket: TAsyncFD, data: string): PFuture[void] = + var retFuture = newFuture[void]() + + var written = 0 + + proc cb(sock: TAsyncFD): bool = + result = true + let netSize = data.len-written + var d = data.cstring + let res = send(sock.TSocketHandle, addr d[written], netSize, 0.cint) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + addWrite(socket, cb) + return retFuture + + proc acceptAddr*(socket: TAsyncFD): + PFuture[tuple[address: string, client: TAsyncFD]] = + var retFuture = newFuture[tuple[address: string, client: TAsyncFD]]() + proc cb(sock: TAsyncFD): bool = + result = true + var sockAddress: Tsockaddr_in + var addrLen = sizeof(sockAddress).TSocklen + var client = accept(sock.TSocketHandle, + cast[ptr TSockAddr](addr(sockAddress)), addr(addrLen)) + if client == osInvalidSocket: + let lastError = osLastError() + assert lastError.int32 notin {EWOULDBLOCK, EAGAIN} + if lastError.int32 == EINTR: + return false + else: + retFuture.fail(newException(EOS, osErrorMsg(lastError))) + else: + register(client.TAsyncFD) + retFuture.complete(($inet_ntoa(sockAddress.sin_addr), client.TAsyncFD)) + addRead(socket, cb) + return retFuture + +proc accept*(socket: TAsyncFD): PFuture[TAsyncFD] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[TAsyncFD]() + var fut = acceptAddr(socket) + fut.callback = + proc (future: PFuture[tuple[address: string, client: TAsyncFD]]) = + assert future.finished + if future.failed: + retFut.fail(future.error) + else: + retFut.complete(future.read.client) + return retFut + +# -- Await Macro + +template createCb*(cbName, varNameIterSym, retFutureSym: expr): stmt {.immediate, dirty.} = + proc cbName {.closure.} = + if not varNameIterSym.finished: + var next = varNameIterSym() + if next == nil: + assert retFutureSym.finished, "Async procedure's return Future was not finished." + else: + next.callback = cbName + +template createVar(futSymName: string, asyncProc: PNimrodNode, + valueReceiver: expr) {.immediate, dirty.} = + # TODO: Used template here due to bug #926 + result = newNimNode(nnkStmtList) + var futSym = genSym(nskVar, "future") + result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y + result.add newNimNode(nnkYieldStmt).add(futSym) # -> yield future<x> + valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read + +proc processBody(node, retFutureSym: PNimrodNode): PNimrodNode {.compileTime.} = + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList) + result.add newCall(newIdentNode("complete"), retFutureSym, + if node[0].kind == nnkEmpty: newIdentNode("result") else: node[0]) + result.add newNimNode(nnkYieldStmt).add(newNilLit()) + of nnkCommand: + if node[0].kind == nnkIdent and node[0].ident == !"await": + case node[1].kind + of nnkIdent: + # await x + result = newNimNode(nnkYieldStmt).add(node[1]) # -> yield x + of nnkCall: + # await foo(p, x) + var futureValue: PNimrodNode + createVar("future" & $node[1][0].toStrLit, node[1], futureValue) + result.add futureValue + else: + error("Invalid node kind in 'await', got: " & $node[1].kind) + elif node[1].kind == nnkCommand and node[1][0].kind == nnkIdent and + node[1][0].ident == !"await": + # foo await x + var newCommand = node + createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1]) + result.add newCommand + + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].ident == !"await": + # var x = await y + var newVarSection = node # TODO: Should this use copyNimNode? + createVar("future" & $node[0][0].ident, node[0][2][1], + newVarSection[0][2]) + result.add newVarSection + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + var newAsgn = node + createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1]) + result.add newAsgn + else: discard + of nnkDiscardStmt: + # discard await x + if node[0][0].kind == nnkIdent and node[0][0].ident == !"await": + var dummy = newNimNode(nnkStmtList) + createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], dummy) + else: discard + + for i in 0 .. <result.len: + result[i] = processBody(result[i], retFutureSym) + #echo(treeRepr(result)) + +proc getName(node: PNimrodNode): string {.compileTime.} = + case node.kind + of nnkPostfix: + return $node[1].ident + of nnkIdent: + return $node.ident + else: + assert false + +macro async*(prc: stmt): stmt {.immediate.} = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. + + expectKind(prc, nnkProcDef) + + hint("Processing " & prc[0].getName & " as an async proc.") + + let returnType = prc[3][0] + var subtypeName = "" + # Verify that the return type is a PFuture[T] + if returnType.kind == nnkIdent: + error("Expected return type of 'PFuture' got '" & $returnType & "'") + elif returnType.kind == nnkBracketExpr: + if $returnType[0] != "PFuture": + error("Expected return type of 'PFuture' got '" & $returnType[0] & "'") + subtypeName = $returnType[1].ident + elif returnType.kind == nnkEmpty: + subtypeName = "void" + + var outerProcBody = newNimNode(nnkStmtList) + + # -> var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr).add( + newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + newIdentNode(subtypeName))))) # Get type from return type of this proc + + # -> iterator nameIter(): PFutureBase {.closure.} = + # -> var result: T + # -> <proc_body> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") + var procBody = prc[6].processBody(retFutureSym) + if subtypeName != "void": + procBody.insert(0, newNimNode(nnkVarSection).add( + newIdentDefs(newIdentNode("result"), returnType[1]))) # -> var result: T + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + + var closureIterator = newProc(iteratorNameSym, [newIdentNode("PFutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) + + # -> var nameIterVar = nameIter + # -> var first = nameIterVar() + var varNameIterSym = genSym(nskVar, $prc[0].getName & "IterVar") + var varNameIter = newVarStmt(varNameIterSym, iteratorNameSym) + outerProcBody.add varNameIter + var varFirstSym = genSym(nskVar, "first") + var varFirst = newVarStmt(varFirstSym, newCall(varNameIterSym)) + outerProcBody.add varFirst + + # -> createCb(cb, nameIter, retFuture) + var cbName = newIdentNode("cb") + var procCb = newCall("createCb", cbName, varNameIterSym, retFutureSym) + outerProcBody.add procCb + + # -> first.callback = cb + outerProcBody.add newAssignment( + newDotExpr(varFirstSym, newIdentNode("callback")), + cbName) + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt).add(retFutureSym) + + result = prc + + # Remove the 'async' pragma. + for i in 0 .. <result[4].len: + if result[4][i].ident == !"async": + result[4].del(i) + if subtypeName == "void": + # Add discardable pragma. + result[4].add(newIdentNode("discardable")) + if returnType.kind == nnkEmpty: + # Add PFuture[void] + result[3][0] = parseExpr("PFuture[void]") + + result[6] = outerProcBody + + echo(toStrLit(result)) + +proc recvLine*(socket: TAsyncFD): PFuture[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + + result = "" + var c = "" + while true: + c = await recv(socket, 1) + if c.len == 0: + return "" + if c == "\r": + c = await recv(socket, 1, MSG_PEEK) + if c.len > 0 and c == "\L": + discard await recv(socket, 1) + addNLIfEmpty() + return + elif c == "\L": + addNLIfEmpty() + return + add(result, c) + +proc runForever*() = + ## Begins a never ending global dispatcher poll loop. + while true: + poll() diff --git a/lib/pure/asyncdispatch.nimrod.cfg b/lib/pure/asyncdispatch.nimrod.cfg new file mode 100644 index 000000000..e88f8eec3 --- /dev/null +++ b/lib/pure/asyncdispatch.nimrod.cfg @@ -0,0 +1,3 @@ +@if nimdoc: + --os:linux +@end diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index 96afc6f4f..ab09dc860 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -167,7 +167,7 @@ proc asyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, result = newAsyncSocket() result.socket = socket(domain, typ, protocol, buffered) result.proto = protocol - if result.socket == InvalidSocket: OSError(OSLastError()) + if result.socket == invalidSocket: osError(osLastError()) result.socket.setBlocking(false) proc toAsyncSocket*(sock: TSocket, state: TInfo = SockConnected): PAsyncSocket = @@ -357,7 +357,7 @@ proc acceptAddr*(server: PAsyncSocket, client: var PAsyncSocket, client.sslNeedAccept = false client.info = SockConnected - if c == InvalidSocket: SocketError(server.socket) + if c == invalidSocket: socketError(server.socket) c.setBlocking(false) # TODO: Needs to be tested. # deleg.open is set in ``toDelegate``. @@ -481,7 +481,7 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool {.deprecated.} = of RecvDisconnected: result = true of RecvFail: - s.SocketError(async = true) + s.socketError(async = true) result = false {.pop.} @@ -615,11 +615,11 @@ proc poll*(d: PDispatcher, timeout: int = 500): bool = if d.hasDataBuffered(d.deleVal): hasDataBufferedCount.inc() d.handleRead(d.deleVal) - if hasDataBufferedCount > 0: return True + if hasDataBufferedCount > 0: return true if readDg.len() == 0 and writeDg.len() == 0: ## TODO: Perhaps this shouldn't return if errorDg has something? - return False + return false if select(readDg, writeDg, errorDg, timeout) != 0: for i in 0..len(d.delegates)-1: diff --git a/lib/pure/asyncio2.nim b/lib/pure/asyncio2.nim deleted file mode 100644 index cdb4a6f49..000000000 --- a/lib/pure/asyncio2.nim +++ /dev/null @@ -1,485 +0,0 @@ -# -# -# Nimrod's Runtime Library -# (c) Copyright 2014 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import os, oids, tables, strutils - -import winlean - -import sockets2, net - -## Asyncio2 -## -------- -## -## This module implements a brand new asyncio module based on Futures. -## IOCP is used under the hood on Windows and the selectors module is used for -## other operating systems. - -# -- Futures - -type - PFutureVoid* = ref object of PObject - cbVoid: proc () {.closure.} - finished: bool - - PFuture*[T] = ref object of PFutureVoid - value: T - error: ref EBase - cb: proc (future: PFuture[T]) {.closure.} - -proc newFuture*[T](): PFuture[T] = - ## Creates a new future. - new(result) - result.finished = false - -proc complete*[T](future: PFuture[T], val: T) = - ## Completes ``future`` with value ``val``. - assert(not future.finished) - assert(future.error == nil) - future.value = val - future.finished = true - if future.cb != nil: - future.cb(future) - if future.cbVoid != nil: - future.cbVoid() - -proc fail*[T](future: PFuture[T], error: ref EBase) = - ## Completes ``future`` with ``error``. - assert(not future.finished) - future.finished = true - future.error = error - if future.cb != nil: - future.cb(future) - -proc `callback=`*[T](future: PFuture[T], - cb: proc (future: PFuture[T]) {.closure.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - future.cb = cb - if future.finished: - future.cb(future) - -proc `callbackVoid=`*(future: PFutureVoid, cb: proc () {.closure.}) = - ## Sets the **void** callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - ## - ## **Note**: This is used for the ``await`` functionality, you most likely - ## want to use ``callback``. - future.cbVoid = cb - if future.finished: - future.cbVoid() - -proc read*[T](future: PFuture[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``EInvalidValue`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished: - if future.error != nil: raise future.error - return future.value - else: - # TODO: Make a custom exception type for this? - raise newException(EInvalidValue, "Future still in progress.") - -proc finished*[T](future: PFuture[T]): bool = - ## Determines whether ``future`` has completed. - ## - ## ``True`` may indicate an error or a value. Use ``hasError`` to distinguish. - future.finished - -proc failed*[T](future: PFuture[T]): bool = - ## Determines whether ``future`` completed with an error. - future.error != nil - -when defined(windows): - type - TCompletionKey = dword - - TCompletionData* = object - sock: TSocketHandle - cb: proc (sock: TSocketHandle, errcode: TOSErrorCode) {.closure.} - - PDispatcher* = ref object - ioPort: THandle - - TCustomOverlapped = object - Internal*: DWORD - InternalHigh*: DWORD - Offset*: DWORD - OffsetHigh*: DWORD - hEvent*: THANDLE - data*: TCompletionData - - PCustomOverlapped = ptr TCustomOverlapped - - proc newDispatcher*(): PDispatcher = - ## Creates a new Dispatcher instance. - new result - result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) - - proc register*(p: PDispatcher, sock: TSocketHandle) = - ## Registers ``sock`` with the dispatcher ``p``. - if CreateIOCompletionPort(sock.THandle, p.ioPort, - cast[TCompletionKey](sock), 1) == 0: - OSError(OSLastError()) - - proc poll*(p: PDispatcher, timeout = 500) = - ## Waits for completion events and processes them. - let llTimeout = - if timeout == -1: winlean.INFINITE - else: timeout.int32 - var lpNumberOfBytesTransferred: DWORD - var lpCompletionKey: ULONG - var lpOverlapped: POverlapped - let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred, - addr lpCompletionKey, addr lpOverlapped, llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - var customOverlapped = cast[PCustomOverlapped](lpOverlapped) - if res: - assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle - - customOverlapped.data.cb(customOverlapped.data.sock, TOSErrorCode(-1)) - dealloc(customOverlapped) - else: - let errCode = OSLastError() - if lpOverlapped != nil: - assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle - dealloc(customOverlapped) - customOverlapped.data.cb(customOverlapped.data.sock, errCode) - else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: OSError(errCode) - - var connectExPtr: pointer = nil - var acceptExPtr: pointer = nil - var getAcceptExSockAddrsPtr: pointer = nil - - proc initPointer(s: TSocketHandle, func: var pointer, guid: var TGUID): bool = - # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c - var bytesRet: DWord - func = nil - result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid, - sizeof(TGUID).dword, addr func, sizeof(pointer).DWORD, - addr bytesRet, nil, nil) == 0 - - proc initAll() = - let dummySock = socket() - if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX): - OSError(OSLastError()) - if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX): - OSError(OSLastError()) - if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS): - OSError(OSLastError()) - - proc connectEx(s: TSocketHandle, name: ptr TSockAddr, namelen: cint, - lpSendBuffer: pointer, dwSendDataLength: dword, - lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool = - if connectExPtr.isNil: raise newException(EInvalidValue, "Need to initialise ConnectEx().") - let func = - cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, - lpSendBuffer: pointer, dwSendDataLength: dword, - lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr) - - result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent, - lpOverlapped) - - proc acceptEx(listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, - lpOverlapped: POverlapped): bool = - if acceptExPtr.isNil: raise newException(EInvalidValue, "Need to initialise AcceptEx().") - let func = - cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD, - lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr) - result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived, - lpOverlapped) - - proc getAcceptExSockaddrs(lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD, - LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint, - RemoteSockaddr: ptr ptr TSockAddr, RemoteSockaddrLength: lpint) = - if getAcceptExSockAddrsPtr.isNil: - raise newException(EInvalidValue, "Need to initialise getAcceptExSockAddrs().") - - let func = - cast[proc (lpOutputBuffer: pointer, - dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr, - LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr, - RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr) - - func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength, - dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength, - RemoteSockaddr, RemoteSockaddrLength) - - proc connect*(p: PDispatcher, socket: TSocketHandle, address: string, port: TPort, - af = AF_INET): PFuture[int] = - ## Connects ``socket`` to server at ``address:port``. - ## - ## Returns a ``PFuture`` which will complete when the connection succeeds - ## or an error occurs. - - var retFuture = newFuture[int]()# TODO: Change to void when that regression is fixed. - # Apparently ``ConnectEx`` expects the socket to be initially bound: - var saddr: Tsockaddr_in - saddr.sin_family = int16(toInt(af)) - saddr.sin_port = 0 - saddr.sin_addr.s_addr = INADDR_ANY - if bindAddr(socket, cast[ptr TSockAddr](addr(saddr)), - sizeof(saddr).TSockLen) < 0'i32: - OSError(OSLastError()) - - var aiList = getAddrInfo(address, port, af) - var success = false - var lastError: TOSErrorCode - var it = aiList - while it != nil: - # "the OVERLAPPED structure must remain valid until the I/O completes" - # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx - var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) - ol.data = TCompletionData(sock: socket, cb: - proc (sock: TSocketHandle, errcode: TOSErrorCode) = - if errcode == TOSErrorCode(-1): - retFuture.complete(0) - else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) - ) - - var ret = connectEx(socket, it.ai_addr, sizeof(TSockAddrIn).cint, - nil, 0, nil, cast[POverlapped](ol)) - if ret: - # Request to connect completed immediately. - success = true - retFuture.complete(0) - dealloc(ol) - break - else: - lastError = OSLastError() - if lastError.int32 == ERROR_IO_PENDING: - # In this case ``ol`` will be deallocated in ``poll``. - success = true - break - else: - dealloc(ol) - success = false - it = it.ai_next - - dealloc(aiList) - if not success: - retFuture.fail(newException(EOS, osErrorMsg(lastError))) - return retFuture - - proc recv*(p: PDispatcher, socket: TSocketHandle, size: int): PFuture[string] = - ## Reads ``size`` bytes from ``socket``. Returned future will complete once - ## all of the requested data is read. - - var retFuture = newFuture[string]() - - var dataBuf: TWSABuf - dataBuf.buf = newString(size) - dataBuf.len = size - - var bytesReceived, flags: DWord - var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) - ol.data = TCompletionData(sock: socket, cb: - proc (sock: TSocketHandle, errcode: TOSErrorCode) = - if errcode == TOSErrorCode(-1): - var data = newString(size) - copyMem(addr data[0], addr dataBuf.buf[0], size) - retFuture.complete($data) - else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) - ) - - let ret = WSARecv(socket, addr dataBuf, 1, addr bytesReceived, - addr flags, cast[POverlapped](ol), nil) - if ret == -1: - let err = OSLastError() - if err.int32 != ERROR_IO_PENDING: - retFuture.fail(newException(EOS, osErrorMsg(err))) - dealloc(ol) - else: - # Request to read completed immediately. - var data = newString(size) - copyMem(addr data[0], addr dataBuf.buf[0], size) - retFuture.complete($data) - dealloc(ol) - return retFuture - - proc send*(p: PDispatcher, socket: TSocketHandle, data: string): PFuture[int] = - ## Sends ``data`` to ``socket``. The returned future will complete once all - ## data has been sent. - var retFuture = newFuture[int]() - - var dataBuf: TWSABuf - dataBuf.buf = data - dataBuf.len = data.len - - var bytesReceived, flags: DWord - var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) - ol.data = TCompletionData(sock: socket, cb: - proc (sock: TSocketHandle, errcode: TOSErrorCode) = - if errcode == TOSErrorCode(-1): - retFuture.complete(0) - else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) - ) - - let ret = WSASend(socket, addr dataBuf, 1, addr bytesReceived, - flags, cast[POverlapped](ol), nil) - if ret == -1: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - retFuture.fail(newException(EOS, osErrorMsg(err))) - dealloc(ol) - else: - retFuture.complete(0) - dealloc(ol) - return retFuture - - proc acceptAddr*(p: PDispatcher, socket: TSocketHandle): - PFuture[tuple[address: string, client: TSocketHandle]] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection and the remote address of the client. - ## The future will complete when the connection is successfully accepted. - - var retFuture = newFuture[tuple[address: string, client: TSocketHandle]]() - - var clientSock = socket() - if clientSock == OSInvalidSocket: osError(osLastError()) - - const lpOutputLen = 1024 - var lpOutputBuf = newString(lpOutputLen) - var dwBytesReceived: DWORD - let dwReceiveDataLength = 0.DWORD # We don't want any data to be read. - let dwLocalAddressLength = DWORD(sizeof (TSockaddr_in) + 16) - let dwRemoteAddressLength = DWORD(sizeof(TSockaddr_in) + 16) - - template completeAccept(): stmt {.immediate, dirty.} = - var listenSock = socket - let setoptRet = setsockopt(clientSock, SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, - sizeof(listenSock).TSockLen) - if setoptRet != 0: osError(osLastError()) - - var LocalSockaddr, RemoteSockaddr: ptr TSockAddr - var localLen, remoteLen: int32 - getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength, - dwLocalAddressLength, dwRemoteAddressLength, - addr LocalSockaddr, addr localLen, - addr RemoteSockaddr, addr remoteLen) - # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186 - retFuture.complete( - (address: $inet_ntoa(cast[ptr Tsockaddr_in](remoteSockAddr).sin_addr), - client: clientSock) - ) - - var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped))) - ol.data = TCompletionData(sock: socket, cb: - proc (sock: TSocketHandle, errcode: TOSErrorCode) = - if errcode == TOSErrorCode(-1): - completeAccept() - else: - retFuture.fail(newException(EOS, osErrorMsg(errcode))) - ) - - # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx - let ret = acceptEx(socket, clientSock, addr lpOutputBuf[0], - dwReceiveDataLength, - dwLocalAddressLength, - dwRemoteAddressLength, - addr dwBytesReceived, cast[POverlapped](ol)) - - if not ret: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - retFuture.fail(newException(EOS, osErrorMsg(err))) - dealloc(ol) - else: - completeAccept() - dealloc(ol) - - return retFuture - - proc accept*(p: PDispatcher, socket: TSocketHandle): PFuture[TSocketHandle] = - ## Accepts a new connection. Returns a future containing the client socket - ## corresponding to that connection. - ## The future will complete when the connection is successfully accepted. - var retFut = newFuture[TSocketHandle]() - var fut = p.acceptAddr(socket) - fut.callback = - proc (future: PFuture[tuple[address: string, client: TSocketHandle]]) = - assert future.finished - if future.failed: - retFut.fail(future.error) - else: - retFut.complete(future.read.client) - return retFut - - initAll() -else: - # TODO: Selectors. - - -when isMainModule: - - var p = newDispatcher() - var sock = socket() - #sock.setBlocking false - p.register(sock) - - when true: - - var f = p.connect(sock, "irc.freenode.org", TPort(6667)) - f.callback = - proc (future: PFuture[int]) = - echo("Connected in future!") - echo(future.read) - for i in 0 .. 50: - var recvF = p.recv(sock, 10) - recvF.callback = - proc (future: PFuture[string]) = - echo("Read: ", future.read) - - else: - - sock.bindAddr(TPort(6667)) - sock.listen() - proc onAccept(future: PFuture[TSocketHandle]) = - echo "Accepted" - var t = p.send(future.read, "test\c\L") - t.callback = - proc (future: PFuture[int]) = - echo(future.read) - - var f = p.accept(sock) - f.callback = onAccept - - var f = p.accept(sock) - f.callback = onAccept - - while true: - p.poll() - echo "polled" - - - - - - - - \ No newline at end of file diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim new file mode 100644 index 000000000..24651b08c --- /dev/null +++ b/lib/pure/asyncnet.nim @@ -0,0 +1,195 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +import asyncdispatch +import rawsockets +import net + +when defined(ssl): + import openssl + +type + # TODO: I would prefer to just do: + # PAsyncSocket* {.borrow: `.`.} = distinct PSocket. But that doesn't work. + TAsyncSocket {.borrow: `.`.} = distinct TSocketImpl + PAsyncSocket* = ref TAsyncSocket + +# TODO: Save AF, domain etc info and reuse it in procs which need it like connect. + +proc newSocket(fd: TAsyncFD, isBuff: bool): PAsyncSocket = + assert fd != osInvalidSocket.TAsyncFD + new(result.PSocket) + result.fd = fd.TSocketHandle + result.isBuffered = isBuff + if isBuff: + result.currPos = 0 + +proc newAsyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, + protocol: TProtocol = IPPROTO_TCP, buffered = true): PAsyncSocket = + ## Creates a new asynchronous socket. + result = newSocket(newAsyncRawSocket(domain, typ, protocol), buffered) + +proc connect*(socket: PAsyncSocket, address: string, port: TPort, + af = AF_INET): PFuture[void] = + ## Connects ``socket`` to server at ``address:port``. + ## + ## Returns a ``PFuture`` which will complete when the connection succeeds + ## or an error occurs. + result = connect(socket.fd.TAsyncFD, address, port, af) + +proc recv*(socket: PAsyncSocket, size: int, + flags: int = 0): PFuture[string] = + ## Reads ``size`` bytes from ``socket``. Returned future will complete once + ## all of the requested data is read. If socket is disconnected during the + ## recv operation then the future may complete with only a part of the + ## requested data read. If socket is disconnected and no data is available + ## to be read then the future will complete with a value of ``""``. + result = recv(socket.fd.TAsyncFD, size, flags) + +proc send*(socket: PAsyncSocket, data: string): PFuture[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + result = send(socket.fd.TAsyncFD, data) + +proc acceptAddr*(socket: PAsyncSocket): + PFuture[tuple[address: string, client: PAsyncSocket]] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection and the remote address of the client. + ## The future will complete when the connection is successfully accepted. + var retFuture = newFuture[tuple[address: string, client: PAsyncSocket]]() + var fut = acceptAddr(socket.fd.TAsyncFD) + fut.callback = + proc (future: PFuture[tuple[address: string, client: TAsyncFD]]) = + assert future.finished + if future.failed: + retFuture.fail(future.readError) + else: + let resultTup = (future.read.address, + newSocket(future.read.client, socket.isBuffered)) + retFuture.complete(resultTup) + return retFuture + +proc accept*(socket: PAsyncSocket): PFuture[PAsyncSocket] = + ## Accepts a new connection. Returns a future containing the client socket + ## corresponding to that connection. + ## The future will complete when the connection is successfully accepted. + var retFut = newFuture[PAsyncSocket]() + var fut = acceptAddr(socket) + fut.callback = + proc (future: PFuture[tuple[address: string, client: PAsyncSocket]]) = + assert future.finished + if future.failed: + retFut.fail(future.readError) + else: + retFut.complete(future.read.client) + return retFut + +proc recvLine*(socket: PAsyncSocket): PFuture[string] {.async.} = + ## Reads a line of data from ``socket``. Returned future will complete once + ## a full line is read or an error occurs. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## If the socket is disconnected in the middle of a line (before ``\r\L`` + ## is read) then line will be set to ``""``. + ## The partial line **will be lost**. + + template addNLIfEmpty(): stmt = + if result.len == 0: + result.add("\c\L") + + result = "" + var c = "" + while true: + c = await recv(socket, 1) + if c.len == 0: + return "" + if c == "\r": + c = await recv(socket, 1, MSG_PEEK) + if c.len > 0 and c == "\L": + discard await recv(socket, 1) + addNLIfEmpty() + return + elif c == "\L": + addNLIfEmpty() + return + add(result.string, c) + +proc bindAddr*(socket: PAsyncSocket, port = TPort(0), address = "") = + ## Binds ``address``:``port`` to the socket. + ## + ## If ``address`` is "" then ADDR_ANY will be bound. + socket.PSocket.bindAddr(port, address) + +proc listen*(socket: PAsyncSocket, backlog = SOMAXCONN) = + ## Marks ``socket`` as accepting connections. + ## ``Backlog`` specifies the maximum length of the + ## queue of pending connections. + ## + ## Raises an EOS error upon failure. + socket.PSocket.listen(backlog) + +proc close*(socket: PAsyncSocket) = + ## Closes the socket. + socket.fd.TAsyncFD.close() + # TODO SSL + +when isMainModule: + type + TestCases = enum + HighClient, LowClient, LowServer + + const test = LowServer + + when test == HighClient: + proc main() {.async.} = + var sock = newAsyncSocket() + await sock.connect("irc.freenode.net", TPort(6667)) + while true: + let line = await sock.recvLine() + if line == "": + echo("Disconnected") + break + else: + echo("Got line: ", line) + main() + elif test == LowClient: + var sock = newAsyncSocket() + var f = connect(sock, "irc.freenode.net", TPort(6667)) + f.callback = + proc (future: PFuture[void]) = + echo("Connected in future!") + for i in 0 .. 50: + var recvF = recv(sock, 10) + recvF.callback = + proc (future: PFuture[string]) = + echo("Read ", future.read.len, ": ", future.read.repr) + elif test == LowServer: + var sock = newAsyncSocket() + sock.bindAddr(TPort(6667)) + sock.listen() + proc onAccept(future: PFuture[PAsyncSocket]) = + let client = future.read + echo "Accepted ", client.fd.cint + var t = send(client, "test\c\L") + t.callback = + proc (future: PFuture[void]) = + echo("Send") + client.close() + + var f = accept(sock) + f.callback = onAccept + + var f = accept(sock) + f.callback = onAccept + runForever() + diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim index 3ed00fdb2..54a553173 100644 --- a/lib/pure/dynlib.nim +++ b/lib/pure/dynlib.nim @@ -14,7 +14,7 @@ type TLibHandle* = pointer ## a handle to a dynamically loaded library -proc loadLib*(path: string): TLibHandle +proc loadLib*(path: string, global_symbols=false): TLibHandle ## loads a library from `path`. Returns nil if the library could not ## be loaded. @@ -53,6 +53,7 @@ when defined(posix): # var RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int + RTLD_GLOBAL {.importc: "RTLD_GLOBAL", header: "<dlfcn.h>".}: int proc dlclose(lib: TLibHandle) {.importc, header: "<dlfcn.h>".} proc dlopen(path: CString, mode: int): TLibHandle {. @@ -60,7 +61,10 @@ when defined(posix): proc dlsym(lib: TLibHandle, name: cstring): pointer {. importc, header: "<dlfcn.h>".} - proc loadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW) + proc loadLib(path: string, global_symbols=false): TLibHandle = + var flags = RTLD_NOW + if global_symbols: flags = flags or RTLD_GLOBAL + return dlopen(path, flags) proc loadLib(): TLibHandle = return dlopen(nil, RTLD_NOW) proc unloadLib(lib: TLibHandle) = dlclose(lib) proc symAddr(lib: TLibHandle, name: cstring): pointer = @@ -81,14 +85,14 @@ elif defined(windows) or defined(dos): proc getProcAddress(lib: THINSTANCE, name: cstring): pointer {. importc: "GetProcAddress", header: "<windows.h>", stdcall.} - proc loadLib(path: string): TLibHandle = + proc loadLib(path: string, global_symbols=false): TLibHandle = result = cast[TLibHandle](winLoadLibrary(path)) proc loadLib(): TLibHandle = result = cast[TLibHandle](winLoadLibrary(nil)) proc unloadLib(lib: TLibHandle) = FreeLibrary(cast[THINSTANCE](lib)) proc symAddr(lib: TLibHandle, name: cstring): pointer = - result = GetProcAddress(cast[THINSTANCE](lib), name) + result = getProcAddress(cast[THINSTANCE](lib), name) else: {.error: "no implementation for dynlib".} diff --git a/lib/pure/htmlgen.nim b/lib/pure/htmlgen.nim index 63737d583..b9d6aec7b 100644 --- a/lib/pure/htmlgen.nim +++ b/lib/pure/htmlgen.nim @@ -1,12 +1,17 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2014 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # +## **Warning**: This module uses ``immediate`` macros which are known to +## cause problems. Do yourself a favor and import the module +## as ``from htmlgen import nil`` and then fully qualify the macros. +## +## ## This module implements a simple `XML`:idx: and `HTML`:idx: code ## generator. Each commonly used HTML tag has a corresponding macro ## that generates a string with its HTML representation. @@ -15,11 +20,11 @@ ## ## .. code-block:: nimrod ## var nim = "Nimrod" -## echo h1(a(href="http://nimrod-code.org", nim)) +## echo h1(a(href="http://nimrod-lang.org", nim)) ## ## Writes the string:: ## -## <h1><a href="http://nimrod-code.org">Nimrod</a></h1> +## <h1><a href="http://nimrod-lang.org">Nimrod</a></h1> ## import diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index bb9835fe7..62d9bea7c 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -76,6 +76,8 @@ ## currently only basic authentication is supported. import sockets, strutils, parseurl, parseutils, strtabs, base64 +import asyncnet, asyncdispatch +import rawsockets type TResponse* = tuple[ @@ -286,16 +288,16 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "", add(headers, "\c\L") var s = socket() - var port = TPort(80) + var port = sockets.TPort(80) if r.scheme == "https": when defined(ssl): sslContext.wrapSocket(s) - port = TPort(443) + port = sockets.TPort(443) else: raise newException(EHttpRequestErr, "SSL support is not available. Cannot connect over SSL.") if r.port != "": - port = TPort(r.port.parseInt) + port = sockets.TPort(r.port.parseInt) if timeout == -1: s.connect(r.hostname, port) @@ -413,28 +415,217 @@ proc downloadFile*(url: string, outputFilename: string, else: fileError("Unable to open file") +proc generateHeaders(r: TURL, httpMethod: THttpMethod, + headers: PStringTable): string = + result = substr($httpMethod, len("http")) + # TODO: Proxies + result.add(" /" & r.path & r.query) + result.add(" HTTP/1.1\c\L") -when isMainModule: - #downloadFile("http://force7.de/nimrod/index.html", "nimrodindex.html") - #downloadFile("http://www.httpwatch.com/", "ChunkTest.html") - #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com", - # "validator.html") + add(result, "Host: " & r.hostname & "\c\L") + add(result, "Connection: Keep-Alive\c\L") + for key, val in headers: + add(result, key & ": " & val & "\c\L") + + add(result, "\c\L") + +type + PAsyncHttpClient = ref object + socket: PAsyncSocket + connected: bool + currentURL: TURL ## Where we are currently connected. + headers: PStringTable + userAgent: string + +proc newAsyncHttpClient*(): PAsyncHttpClient = + new result + result.socket = newAsyncSocket() + result.headers = newStringTable(modeCaseInsensitive) + result.userAgent = defUserAgent + +proc parseChunks(client: PAsyncHttpClient): PFuture[string] {.async.} = + result = "" + var ri = 0 + while true: + var chunkSize = 0 + var chunkSizeStr = await client.socket.recvLine() + var i = 0 + if chunkSizeStr == "": + httpError("Server terminated connection prematurely") + while true: + case chunkSizeStr[i] + of '0'..'9': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) + of 'a'..'f': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) + of 'A'..'F': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) + of '\0': + break + of ';': + # http://tools.ietf.org/html/rfc2616#section-3.6.1 + # We don't care about chunk-extensions. + break + else: + httpError("Invalid chunk size: " & chunkSizeStr) + inc(i) + if chunkSize <= 0: break + result.add await recv(client.socket, chunkSize) + discard await recv(client.socket, 2) # Skip \c\L + # Trailer headers will only be sent if the request specifies that we want + # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 + +proc parseBody(client: PAsyncHttpClient, + headers: PStringTable): PFuture[string] {.async.} = + result = "" + if headers["Transfer-Encoding"] == "chunked": + result = await parseChunks(client) + else: + # -REGION- Content-Length + # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3 + var contentLengthHeader = headers["Content-Length"] + if contentLengthHeader != "": + var length = contentLengthHeader.parseint() + result = await client.socket.recv(length) + if result == "": + httpError("Got disconnected while trying to recv body.") + else: + # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO + + # -REGION- Connection: Close + # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5 + if headers["Connection"] == "close": + var buf = "" + while True: + buf = await client.socket.recv(4000) + if buf == "": break + result.add(buf) + +proc parseResponse(client: PAsyncHttpClient, + getBody: bool): PFuture[TResponse] {.async.} = + var parsedStatus = false + var linei = 0 + var fullyRead = false + var line = "" + result.headers = newStringTable(modeCaseInsensitive) + while True: + linei = 0 + line = await client.socket.recvLine() + if line == "": break # We've been disconnected. + if line == "\c\L": + fullyRead = true + break + if not parsedStatus: + # Parse HTTP version info and status code. + var le = skipIgnoreCase(line, "HTTP/", linei) + if le <= 0: httpError("invalid http version") + inc(linei, le) + le = skipIgnoreCase(line, "1.1", linei) + if le > 0: result.version = "1.1" + else: + le = skipIgnoreCase(line, "1.0", linei) + if le <= 0: httpError("unsupported http version") + result.version = "1.0" + inc(linei, le) + # Status code + linei.inc skipWhitespace(line, linei) + result.status = line[linei .. -1] + parsedStatus = true + else: + # Parse headers + var name = "" + var le = parseUntil(line, name, ':', linei) + if le <= 0: httpError("invalid headers") + inc(linei, le) + if line[linei] != ':': httpError("invalid headers") + inc(linei) # Skip : + + result.headers[name] = line[linei.. -1].strip() + if not fullyRead: + httpError("Connection was closed before full request has been made") + if getBody: + result.body = await parseBody(client, result.headers) + else: + result.body = "" + +proc close*(client: PAsyncHttpClient) = + ## Closes any connections held by the HttpClient. + if client.connected: + client.socket.close() + client.connected = false + #client.socket = newAsyncSocket() + +proc newConnection(client: PAsyncHttpClient, url: TURL) {.async.} = + if not client.connected or client.currentURL.hostname != url.hostname or + client.currentURL.scheme != url.scheme: + if client.connected: client.close() + if url.scheme == "https": + assert false, "TODO SSL" - #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& - # charset=%28detect+automatically%29&doctype=Inline&group=0") + # TODO: I should be able to write 'net.TPort' here... + let port = + if url.port == "": rawsockets.TPort(80) + else: rawsockets.TPort(url.port.parseInt) + + await client.socket.connect(url.hostname, port) + client.currentURL = url + +proc request*(client: PAsyncHttpClient, url: string, httpMethod = httpGET, + body = ""): PFuture[TResponse] {.async.} = + let r = parseUrl(url) + await newConnection(client, r) + + if not client.headers.hasKey("user-agent") and client.userAgent != "": + client.headers["User-Agent"] = client.userAgent - var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L" - var body: string = "--xyz\c\L" - # soap 1.2 output - body.add("Content-Disposition: form-data; name=\"output\"\c\L") - body.add("\c\Lsoap12\c\L") + var headers = generateHeaders(r, httpMethod, client.headers) + + await client.socket.send(headers) + if body != "": + await client.socket.send(body) - # html - body.add("--xyz\c\L") - body.add("Content-Disposition: form-data; name=\"uploaded_file\";" & - " filename=\"test.html\"\c\L") - body.add("Content-Type: text/html\c\L") - body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L") - body.add("--xyz--") - - echo(postContent("http://validator.w3.org/check", headers, body)) + result = await parseResponse(client, httpMethod != httpHEAD) + +when isMainModule: + when true: + # Async + proc main() {.async.} = + var client = newAsyncHttpClient() + var resp = await client.request("http://picheta.me") + + echo("Got response: ", resp.status) + echo("Body:\n") + echo(resp.body) + + #var resp1 = await client.request("http://freenode.net") + #echo("Got response: ", resp1.status) + + var resp2 = await client.request("http://picheta.me/aasfasgf.html") + echo("Got response: ", resp2.status) + main() + runForever() + + else: + #downloadFile("http://force7.de/nimrod/index.html", "nimrodindex.html") + #downloadFile("http://www.httpwatch.com/", "ChunkTest.html") + #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com", + # "validator.html") + + #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& + # charset=%28detect+automatically%29&doctype=Inline&group=0") + + var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L" + var body: string = "--xyz\c\L" + # soap 1.2 output + body.add("Content-Disposition: form-data; name=\"output\"\c\L") + body.add("\c\Lsoap12\c\L") + + # html + body.add("--xyz\c\L") + body.add("Content-Disposition: form-data; name=\"uploaded_file\";" & + " filename=\"test.html\"\c\L") + body.add("Content-Type: text/html\c\L") + body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L") + body.add("--xyz--") + + echo(postContent("http://validator.w3.org/check", headers, body)) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 062cfae25..94570fc68 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -20,6 +20,8 @@ when defined(Posix) and not defined(haiku): {.passl: "-lm".} +import times + const PI* = 3.1415926535897932384626433 ## the circle constant PI (Ludolph's number) E* = 2.71828182845904523536028747 ## Euler's number @@ -201,7 +203,7 @@ when not defined(JS): result = drand48() * max proc randomize() = - randomize(gettime(nil)) + randomize(cast[int](epochTime())) proc randomize(seed: int) = srand(cint(seed)) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 8c404abe8..807f3da43 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Nimrod Contributors +# (c) Copyright 2014 Nimrod Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -34,6 +34,43 @@ type else: handle: cint + +proc mapMem*(m: var TMemFile, mode: TFileMode = fmRead, + mappedSize = -1, offset = 0): pointer = + var readonly = mode == fmRead + when defined(windows): + result = mapViewOfFileEx( + m.mapHandle, + if readonly: FILE_MAP_READ else: FILE_MAP_WRITE, + int32(offset shr 32), + int32(offset and 0xffffffff), + if mappedSize == -1: 0 else: mappedSize, + nil) + if result == nil: + osError(osLastError()) + else: + assert mappedSize > 0 + result = mmap( + nil, + mappedSize, + if readonly: PROT_READ else: PROT_READ or PROT_WRITE, + if readonly: MAP_PRIVATE else: MAP_SHARED, + m.handle, offset) + if result == cast[pointer](MAP_FAILED): + osError(osLastError()) + + +proc unmapMem*(f: var TMemFile, p: pointer, size: int) = + ## unmaps the memory region ``(p, <p+size)`` of the mapped file `f`. + ## All changes are written back to the file system, if `f` was opened + ## with write access. ``size`` must be of exactly the size that was requested + ## via ``mapMem``. + when defined(windows): + if unmapViewOfFile(p) == 0: osError(osLastError()) + else: + if munmap(p, size) != 0: osError(osLastError()) + + proc open*(filename: string, mode: TFileMode = fmRead, mappedSize = -1, offset = 0, newFileSize = -1): TMemFile = ## opens a memory mapped file. If this fails, ``EOS`` is raised. @@ -71,7 +108,7 @@ proc open*(filename: string, mode: TFileMode = fmRead, when useWinUnicode: result.fHandle = callCreateFile(createFileW, newWideCString(filename)) else: - result.fHandle = callCreateFile(CreateFileA, filename) + result.fHandle = callCreateFile(createFileA, filename) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") @@ -170,14 +207,14 @@ proc close*(f: var TMemFile) = when defined(windows): if f.fHandle != INVALID_HANDLE_VALUE: - lastErr = osLastError() error = unmapViewOfFile(f.mem) == 0 + lastErr = osLastError() error = (closeHandle(f.mapHandle) == 0) or error error = (closeHandle(f.fHandle) == 0) or error else: if f.handle != 0: - lastErr = osLastError() error = munmap(f.mem, f.size) != 0 + lastErr = osLastError() error = (close(f.handle) != 0) or error f.size = 0 diff --git a/lib/pure/net.nim b/lib/pure/net.nim index bdcae677e..4afb5c6ab 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -9,17 +9,519 @@ ## This module implements a high-level cross-platform sockets interface. -import sockets2, os +{.deadCodeElim: on.} +import rawsockets, os, strutils, unsigned, parseutils, times +export TPort +type + IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address + IPv6, ## IPv6 address + IPv4 ## IPv4 address + + TIpAddress* = object ## stores an arbitrary IP address + case family*: IpAddressFamily ## the type of the IP address (IPv4 or IPv6) + of IpAddressFamily.IPv6: + address_v6*: array[0..15, uint8] ## Contains the IP address in bytes in + ## case of IPv6 + of IpAddressFamily.IPv4: + address_v4*: array[0..3, uint8] ## Contains the IP address in bytes in + ## case of IPv4 + +proc IPv4_any*(): TIpAddress = + ## Returns the IPv4 any address, which can be used to listen on all available + ## network adapters + result = TIpAddress( + family: IpAddressFamily.IPv4, + address_v4: [0'u8, 0, 0, 0]) + +proc IPv4_loopback*(): TIpAddress = + ## Returns the IPv4 loopback address (127.0.0.1) + result = TIpAddress( + family: IpAddressFamily.IPv4, + address_v4: [127'u8, 0, 0, 1]) + +proc IPv4_broadcast*(): TIpAddress = + ## Returns the IPv4 broadcast address (255.255.255.255) + result = TIpAddress( + family: IpAddressFamily.IPv4, + address_v4: [255'u8, 255, 255, 255]) + +proc IPv6_any*(): TIpAddress = + ## Returns the IPv6 any address (::0), which can be used + ## to listen on all available network adapters + result = TIpAddress( + family: IpAddressFamily.IPv6, + address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + +proc IPv6_loopback*(): TIpAddress = + ## Returns the IPv6 loopback address (::1) + result = TIpAddress( + family: IpAddressFamily.IPv6, + address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) + +proc `==`*(lhs, rhs: TIpAddress): bool = + ## Compares two IpAddresses for Equality. Returns two if the addresses are equal + if lhs.family != rhs.family: return false + if lhs.family == IpAddressFamily.IPv4: + for i in low(lhs.address_v4) .. high(lhs.address_v4): + if lhs.address_v4[i] != rhs.address_v4[i]: return false + else: # IPv6 + for i in low(lhs.address_v6) .. high(lhs.address_v6): + if lhs.address_v6[i] != rhs.address_v6[i]: return false + return true + +proc `$`*(address: TIpAddress): string = + ## Converts an TIpAddress into the textual representation + result = "" + case address.family + of IpAddressFamily.IPv4: + for i in 0 .. 3: + if i != 0: + result.add('.') + result.add($address.address_v4[i]) + of IpAddressFamily.IPv6: + var + currentZeroStart = -1 + currentZeroCount = 0 + biggestZeroStart = -1 + biggestZeroCount = 0 + # Look for the largest block of zeros + for i in 0..7: + var isZero = address.address_v6[i*2] == 0 and address.address_v6[i*2+1] == 0 + if isZero: + if currentZeroStart == -1: + currentZeroStart = i + currentZeroCount = 1 + else: + currentZeroCount.inc() + if currentZeroCount > biggestZeroCount: + biggestZeroCount = currentZeroCount + biggestZeroStart = currentZeroStart + else: + currentZeroStart = -1 + + if biggestZeroCount == 8: # Special case ::0 + result.add("::") + else: # Print address + var printedLastGroup = false + for i in 0..7: + var word:uint16 = (cast[uint16](address.address_v6[i*2])) shl 8 + word = word or cast[uint16](address.address_v6[i*2+1]) + + if biggestZeroCount != 0 and # Check if group is in skip group + (i >= biggestZeroStart and i < (biggestZeroStart + biggestZeroCount)): + if i == biggestZeroStart: # skip start + result.add("::") + printedLastGroup = false + else: + if printedLastGroup: + result.add(':') + var + afterLeadingZeros = false + mask = 0xF000'u16 + for j in 0'u16..3'u16: + var val = (mask and word) shr (4'u16*(3'u16-j)) + if val != 0 or afterLeadingZeros: + if val < 0xA: + result.add(chr(uint16(ord('0'))+val)) + else: # val >= 0xA + result.add(chr(uint16(ord('a'))+val-0xA)) + afterLeadingZeros = true + mask = mask shr 4 + printedLastGroup = true + +proc parseIPv4Address(address_str: string): TIpAddress = + ## Parses IPv4 adresses + ## Raises EInvalidValue on errors + var + byteCount = 0 + currentByte:uint16 = 0 + seperatorValid = false + + result.family = IpAddressFamily.IPv4 + + for i in 0 .. high(address_str): + if address_str[i] in strutils.Digits: # Character is a number + currentByte = currentByte * 10 + + cast[uint16](ord(address_str[i]) - ord('0')) + if currentByte > 255'u16: + raise newException(EInvalidValue, + "Invalid IP Address. Value is out of range") + seperatorValid = true + elif address_str[i] == '.': # IPv4 address separator + if not seperatorValid or byteCount >= 3: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + result.address_v4[byteCount] = cast[uint8](currentByte) + currentByte = 0 + byteCount.inc + seperatorValid = false + else: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not seperatorValid: + raise newException(EInvalidValue, "Invalid IP Address") + result.address_v4[byteCount] = cast[uint8](currentByte) + +proc parseIPv6Address(address_str: string): TIpAddress = + ## Parses IPv6 adresses + ## Raises EInvalidValue on errors + result.family = IpAddressFamily.IPv6 + if address_str.len < 2: + raise newException(EInvalidValue, "Invalid IP Address") + + var + groupCount = 0 + currentGroupStart = 0 + currentShort:uint32 = 0 + seperatorValid = true + dualColonGroup = -1 + lastWasColon = false + v4StartPos = -1 + byteCount = 0 + + for i,c in address_str: + if c == ':': + if not seperatorValid: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid seperator") + if lastWasColon: + if dualColonGroup != -1: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains more than one \"::\" seperator") + dualColonGroup = groupCount + seperatorValid = false + elif i != 0 and i != high(address_str): + if groupCount >= 8: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + currentShort = 0 + groupCount.inc() + if dualColonGroup != -1: seperatorValid = false + elif i == 0: # only valid if address starts with :: + if address_str[1] != ':': + raise newException(EInvalidValue, + "Invalid IP Address. Address may not start with \":\"") + else: # i == high(address_str) - only valid if address ends with :: + if address_str[high(address_str)-1] != ':': + raise newException(EInvalidValue, + "Invalid IP Address. Address may not end with \":\"") + lastWasColon = true + currentGroupStart = i + 1 + elif c == '.': # Switch to parse IPv4 mode + if i < 3 or not seperatorValid or groupCount >= 7: + raise newException(EInvalidValue, "Invalid IP Address") + v4StartPos = currentGroupStart + currentShort = 0 + seperatorValid = false + break + elif c in strutils.HexDigits: + if c in strutils.Digits: # Normal digit + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) + elif c >= 'a' and c <= 'f': # Lower case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 + else: # Upper case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 + if currentShort > 65535'u32: + raise newException(EInvalidValue, + "Invalid IP Address. Value is out of range") + lastWasColon = false + seperatorValid = true + else: + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid character") + + + if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff + if seperatorValid: # Copy remaining data + if groupCount >= 8: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + groupCount.inc() + else: # Must parse IPv4 address + for i,c in address_str[v4StartPos..high(address_str)]: + if c in strutils.Digits: # Character is a number + currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) + if currentShort > 255'u32: + raise newException(EInvalidValue, + "Invalid IP Address. Value is out of range") + seperatorValid = true + elif c == '.': # IPv4 address separator + if not seperatorValid or byteCount >= 3: + raise newException(EInvalidValue, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + currentShort = 0 + byteCount.inc() + seperatorValid = false + else: # Invalid character + raise newException(EInvalidValue, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not seperatorValid: + raise newException(EInvalidValue, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + groupCount += 2 + + # Shift and fill zeros in case of :: + if groupCount > 8: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + elif groupCount < 8: # must fill + if dualColonGroup == -1: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too few groups") + var toFill = 8 - groupCount # The number of groups to fill + var toShift = groupCount - dualColonGroup # Nr of known groups after :: + for i in 0..2*toShift-1: # shift + result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] + for i in 0..2*toFill-1: # fill with 0s + result.address_v6[dualColonGroup*2+i] = 0 + elif dualColonGroup != -1: + raise newException(EInvalidValue, + "Invalid IP Address. The address consists of too many groups") + +proc parseIpAddress*(address_str: string): TIpAddress = + ## Parses an IP address + ## Raises EInvalidValue on error + if address_str == nil: + raise newException(EInvalidValue, "IP Address string is nil") + if address_str.contains(':'): + return parseIPv6Address(address_str) + else: + return parseIPv4Address(address_str) + +when defined(ssl): + import openssl + +# Note: The enumerations are mapped to Window's constants. + +when defined(ssl): + type + ESSL* = object of ESynch + + TSSLCVerifyMode* = enum + CVerifyNone, CVerifyPeer + + TSSLProtVersion* = enum + protSSLv2, protSSLv3, protTLSv1, protSSLv23 + + PSSLContext* = distinct PSSLCTX + + TSSLAcceptResult* = enum + AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess + +const + BufferSize*: int = 4000 ## size of a buffered socket's buffer type - TSocket* = TSocketHandle + TSocketImpl* = object ## socket type + fd*: TSocketHandle + case isBuffered*: bool # determines whether this socket is buffered. + of true: + buffer*: array[0..BufferSize, char] + currPos*: int # current index in buffer + bufLen*: int # current length of buffer + of false: nil + when defined(ssl): + case isSsl*: bool + of true: + sslHandle*: PSSL + sslContext*: PSSLContext + sslNoHandshake*: bool # True if needs handshake. + sslHasPeekChar*: bool + sslPeekChar*: char + of false: nil + + PSocket* = ref TSocketImpl -proc bindAddr*(socket: TSocket, port = TPort(0), address = "") {. - tags: [FReadIO].} = + TSOBool* = enum ## Boolean socket options. + OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive, + OptOOBInline, OptReuseAddr + + TReadLineResult* = enum ## result for readLineAsync + ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone + + ETimeout* = object of ESynch + +proc createSocket(fd: TSocketHandle, isBuff: bool): PSocket = + assert fd != osInvalidSocket + new(result) + result.fd = fd + result.isBuffered = isBuff + if isBuff: + result.currPos = 0 + +proc newSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, + protocol: TProtocol = IPPROTO_TCP, buffered = true): PSocket = + ## Creates a new socket. + ## + ## If an error occurs EOS will be raised. + let fd = newRawSocket(domain, typ, protocol) + if fd == osInvalidSocket: + osError(osLastError()) + result = createSocket(fd, buffered) + +when defined(ssl): + CRYPTO_malloc_init() + SslLibraryInit() + SslLoadErrorStrings() + ErrLoadBioStrings() + OpenSSL_add_all_algorithms() + + proc SSLError(s = "") = + if s != "": + raise newException(ESSL, s) + let err = ErrPeekLastError() + if err == 0: + raise newException(ESSL, "No error reported.") + if err == -1: + OSError(OSLastError()) + var errStr = ErrErrorString(err, nil) + raise newException(ESSL, $errStr) + + # http://simplestcodings.blogspot.co.uk/2010/08/secure-server-client-using-openssl-in-c.html + proc loadCertificates(ctx: PSSL_CTX, certFile, keyFile: string) = + if certFile != "" and not existsFile(certFile): + raise newException(system.EIO, "Certificate file could not be found: " & certFile) + if keyFile != "" and not existsFile(keyFile): + raise newException(system.EIO, "Key file could not be found: " & keyFile) + + if certFile != "": + var ret = SSLCTXUseCertificateChainFile(ctx, certFile) + if ret != 1: + SSLError() + + # TODO: Password? www.rtfm.com/openssl-examples/part1.pdf + if keyFile != "": + if SSL_CTX_use_PrivateKey_file(ctx, keyFile, + SSL_FILETYPE_PEM) != 1: + SSLError() + + if SSL_CTX_check_private_key(ctx) != 1: + SSLError("Verification of private key file failed.") - ## binds an address/port number to a socket. - ## Use address string in dotted decimal form like "a.b.c.d" - ## or leave "" for any address. + proc newContext*(protVersion = ProtSSLv23, verifyMode = CVerifyPeer, + certFile = "", keyFile = ""): PSSLContext = + ## Creates an SSL context. + ## + ## Protocol version specifies the protocol to use. SSLv2, SSLv3, TLSv1 are + ## are available with the addition of ``ProtSSLv23`` which allows for + ## compatibility with all of them. + ## + ## There are currently only two options for verify mode; + ## one is ``CVerifyNone`` and with it certificates will not be verified + ## the other is ``CVerifyPeer`` and certificates will be verified for + ## it, ``CVerifyPeer`` is the safest choice. + ## + ## The last two parameters specify the certificate file path and the key file + ## path, a server socket will most likely not work without these. + ## Certificates can be generated using the following command: + ## ``openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.pem -out mycert.pem``. + var newCTX: PSSL_CTX + case protVersion + of protSSLv23: + newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. + of protSSLv2: + when not defined(linux): + newCTX = SSL_CTX_new(SSLv2_method()) + else: + SSLError() + of protSSLv3: + newCTX = SSL_CTX_new(SSLv3_method()) + of protTLSv1: + newCTX = SSL_CTX_new(TLSv1_method()) + + if newCTX.SSLCTXSetCipherList("ALL") != 1: + SSLError() + case verifyMode + of CVerifyPeer: + newCTX.SSLCTXSetVerify(SSLVerifyPeer, nil) + of CVerifyNone: + newCTX.SSLCTXSetVerify(SSLVerifyNone, nil) + if newCTX == nil: + SSLError() + + discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY) + newCTX.loadCertificates(certFile, keyFile) + return PSSLContext(newCTX) + + proc wrapSocket*(ctx: PSSLContext, socket: PSocket) = + ## Wraps a socket in an SSL context. This function effectively turns + ## ``socket`` into an SSL socket. + ## + ## **Disclaimer**: This code is not well tested, may be very unsafe and + ## prone to security vulnerabilities. + + socket.isSSL = true + socket.sslContext = ctx + socket.sslHandle = SSLNew(PSSLCTX(socket.sslContext)) + socket.sslNoHandshake = false + socket.sslHasPeekChar = false + if socket.sslHandle == nil: + SSLError() + + if SSLSetFd(socket.sslHandle, socket.fd) != 1: + SSLError() + +proc socketError*(socket: PSocket, err: int = -1, async = false) = + ## Raises an EOS error based on the error code returned by ``SSLGetError`` + ## (for SSL sockets) and ``osLastError`` otherwise. + ## + ## If ``async`` is ``True`` no error will be thrown in the case when the + ## error was caused by no data being available to be read. + ## + ## If ``err`` is not lower than 0 no exception will be raised. + when defined(ssl): + if socket.isSSL: + if err <= 0: + var ret = SSLGetError(socket.sslHandle, err.cint) + case ret + of SSL_ERROR_ZERO_RETURN: + SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") + of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: + if async: + return + else: SSLError("Not enough data on socket.") + of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: + if async: + return + else: SSLError("Not enough data on socket.") + of SSL_ERROR_WANT_X509_LOOKUP: + SSLError("Function for x509 lookup has been called.") + of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + SSLError() + else: SSLError("Unknown Error") + + if err == -1 and not (when defined(ssl): socket.isSSL else: false): + let lastError = osLastError() + if async: + when defined(windows): + if lastError.int32 == WSAEWOULDBLOCK: + return + else: osError(lastError) + else: + if lastError.int32 == EAGAIN or lastError.int32 == EWOULDBLOCK: + return + else: osError(lastError) + else: osError(lastError) + +proc listen*(socket: PSocket, backlog = SOMAXCONN) {.tags: [FReadIO].} = + ## Marks ``socket`` as accepting connections. + ## ``Backlog`` specifies the maximum length of the + ## queue of pending connections. + ## + ## Raises an EOS error upon failure. + if listen(socket.fd, backlog) < 0'i32: osError(osLastError()) + +proc bindAddr*(socket: PSocket, port = TPort(0), address = "") {. + tags: [FReadIO].} = + ## Binds ``address``:``port`` to the socket. + ## + ## If ``address`` is "" then ADDR_ANY will be bound. if address == "": var name: TSockaddr_in @@ -29,12 +531,599 @@ proc bindAddr*(socket: TSocket, port = TPort(0), address = "") {. name.sin_family = toInt(AF_INET) name.sin_port = htons(int16(port)) name.sin_addr.s_addr = htonl(INADDR_ANY) - if bindAddr(socket, cast[ptr TSockAddr](addr(name)), + if bindAddr(socket.fd, cast[ptr TSockAddr](addr(name)), sizeof(name).TSocklen) < 0'i32: osError(osLastError()) else: var aiList = getAddrInfo(address, port, AF_INET) - if bindAddr(socket, aiList.ai_addr, aiList.ai_addrlen.TSocklen) < 0'i32: + if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.TSocklen) < 0'i32: dealloc(aiList) osError(osLastError()) - dealloc(aiList) \ No newline at end of file + dealloc(aiList) + +proc acceptAddr*(server: PSocket, client: var PSocket, address: var string) {. + tags: [FReadIO].} = + ## Blocks until a connection is being made from a client. When a connection + ## is made sets ``client`` to the client socket and ``address`` to the address + ## of the connecting client. + ## This function will raise EOS if an error occurs. + ## + ## The resulting client will inherit any properties of the server socket. For + ## example: whether the socket is buffered or not. + ## + ## **Note**: ``client`` must be initialised (with ``new``), this function + ## makes no effort to initialise the ``client`` variable. + assert(client != nil) + var sockAddress: Tsockaddr_in + var addrLen = sizeof(sockAddress).TSocklen + var sock = accept(server.fd, cast[ptr TSockAddr](addr(sockAddress)), + addr(addrLen)) + + if sock == osInvalidSocket: + let err = osLastError() + osError(err) + else: + client.fd = sock + client.isBuffered = server.isBuffered + + # Handle SSL. + when defined(ssl): + if server.isSSL: + # We must wrap the client sock in a ssl context. + + server.sslContext.wrapSocket(client) + let ret = SSLAccept(client.sslHandle) + socketError(client, ret, false) + + # Client socket is set above. + address = $inet_ntoa(sockAddress.sin_addr) + +when false: #defined(ssl): + proc acceptAddrSSL*(server: PSocket, client: var PSocket, + address: var string): TSSLAcceptResult {. + tags: [FReadIO].} = + ## This procedure should only be used for non-blocking **SSL** sockets. + ## It will immediately return with one of the following values: + ## + ## ``AcceptSuccess`` will be returned when a client has been successfully + ## accepted and the handshake has been successfully performed between + ## ``server`` and the newly connected client. + ## + ## ``AcceptNoHandshake`` will be returned when a client has been accepted + ## but no handshake could be performed. This can happen when the client + ## connects but does not yet initiate a handshake. In this case + ## ``acceptAddrSSL`` should be called again with the same parameters. + ## + ## ``AcceptNoClient`` will be returned when no client is currently attempting + ## to connect. + template doHandshake(): stmt = + when defined(ssl): + if server.isSSL: + client.setBlocking(false) + # We must wrap the client sock in a ssl context. + + if not client.isSSL or client.sslHandle == nil: + server.sslContext.wrapSocket(client) + let ret = SSLAccept(client.sslHandle) + while ret <= 0: + let err = SSLGetError(client.sslHandle, ret) + if err != SSL_ERROR_WANT_ACCEPT: + case err + of SSL_ERROR_ZERO_RETURN: + SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") + of SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE, + SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: + client.sslNoHandshake = true + return AcceptNoHandshake + of SSL_ERROR_WANT_X509_LOOKUP: + SSLError("Function for x509 lookup has been called.") + of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + SSLError() + else: + SSLError("Unknown error") + client.sslNoHandshake = false + + if client.isSSL and client.sslNoHandshake: + doHandshake() + return AcceptSuccess + else: + acceptAddrPlain(AcceptNoClient, AcceptSuccess): + doHandshake() + +proc accept*(server: PSocket, client: var PSocket) {.tags: [FReadIO].} = + ## Equivalent to ``acceptAddr`` but doesn't return the address, only the + ## socket. + ## + ## **Note**: ``client`` must be initialised (with ``new``), this function + ## makes no effort to initialise the ``client`` variable. + + var addrDummy = "" + acceptAddr(server, client, addrDummy) + +proc close*(socket: PSocket) = + ## Closes a socket. + socket.fd.close() + when defined(ssl): + if socket.isSSL: + let res = SSLShutdown(socket.sslHandle) + if res == 0: + if SSLShutdown(socket.sslHandle) != 1: + socketError(socket) + elif res != 1: + socketError(socket) + +proc toCInt(opt: TSOBool): cint = + case opt + of OptAcceptConn: SO_ACCEPTCONN + of OptBroadcast: SO_BROADCAST + of OptDebug: SO_DEBUG + of OptDontRoute: SO_DONTROUTE + of OptKeepAlive: SO_KEEPALIVE + of OptOOBInline: SO_OOBINLINE + of OptReuseAddr: SO_REUSEADDR + +proc getSockOpt*(socket: PSocket, opt: TSOBool, level = SOL_SOCKET): bool {. + tags: [FReadIO].} = + ## Retrieves option ``opt`` as a boolean value. + var res = getsockoptint(socket.fd, cint(level), toCInt(opt)) + result = res != 0 + +proc setSockOpt*(socket: PSocket, opt: TSOBool, value: bool, level = SOL_SOCKET) {. + tags: [FWriteIO].} = + ## Sets option ``opt`` to a boolean value specified by ``value``. + var valuei = cint(if value: 1 else: 0) + setsockoptint(socket.fd, cint(level), toCInt(opt), valuei) + +proc connect*(socket: PSocket, address: string, port = TPort(0), + af: TDomain = AF_INET) {.tags: [FReadIO].} = + ## Connects socket to ``address``:``port``. ``Address`` can be an IP address or a + ## host name. If ``address`` is a host name, this function will try each IP + ## of that host name. ``htons`` is already performed on ``port`` so you must + ## not do it. + ## + ## If ``socket`` is an SSL socket a handshake will be automatically performed. + var aiList = getAddrInfo(address, port, af) + # try all possibilities: + var success = false + var lastError: TOSErrorCode + var it = aiList + while it != nil: + if connect(socket.fd, it.ai_addr, it.ai_addrlen.TSocklen) == 0'i32: + success = true + break + else: lastError = osLastError() + it = it.ai_next + + dealloc(aiList) + if not success: osError(lastError) + + when defined(ssl): + if socket.isSSL: + let ret = SSLConnect(socket.sslHandle) + socketError(socket, ret) + +when defined(ssl): + proc handshake*(socket: PSocket): bool {.tags: [FReadIO, FWriteIO].} = + ## This proc needs to be called on a socket after it connects. This is + ## only applicable when using ``connectAsync``. + ## This proc performs the SSL handshake. + ## + ## Returns ``False`` whenever the socket is not yet ready for a handshake, + ## ``True`` whenever handshake completed successfully. + ## + ## A ESSL error is raised on any other errors. + result = true + if socket.isSSL: + var ret = SSLConnect(socket.sslHandle) + if ret <= 0: + var errret = SSLGetError(socket.sslHandle, ret) + case errret + of SSL_ERROR_ZERO_RETURN: + SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") + of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT, + SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE: + return false + of SSL_ERROR_WANT_X509_LOOKUP: + SSLError("Function for x509 lookup has been called.") + of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + SSLError() + else: + SSLError("Unknown Error") + socket.sslNoHandshake = false + else: + SSLError("Socket is not an SSL socket.") + + proc gotHandshake*(socket: PSocket): bool = + ## Determines whether a handshake has occurred between a client (``socket``) + ## and the server that ``socket`` is connected to. + ## + ## Throws ESSL if ``socket`` is not an SSL socket. + if socket.isSSL: + return not socket.sslNoHandshake + else: + SSLError("Socket is not an SSL socket.") + +proc hasDataBuffered*(s: PSocket): bool = + ## Determines whether a socket has data buffered. + result = false + if s.isBuffered: + result = s.bufLen > 0 and s.currPos != s.bufLen + + when defined(ssl): + if s.isSSL and not result: + result = s.sslHasPeekChar + +proc select(readfd: PSocket, timeout = 500): int = + ## Used for socket operation timeouts. + if readfd.hasDataBuffered: + return 1 + + var fds = @[readFd.fd] + result = select(fds, timeout) + +proc readIntoBuf(socket: PSocket, flags: int32): int = + result = 0 + when defined(ssl): + if socket.isSSL: + result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) + else: + result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) + else: + result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags) + if result <= 0: + socket.bufLen = 0 + socket.currPos = 0 + return result + socket.bufLen = result + socket.currPos = 0 + +template retRead(flags, readBytes: int) {.dirty.} = + let res = socket.readIntoBuf(flags.int32) + if res <= 0: + if readBytes > 0: + return readBytes + else: + return res + +proc recv*(socket: PSocket, data: pointer, size: int): int {.tags: [FReadIO].} = + ## Receives data from a socket. + ## + ## **Note**: This is a low-level function, you may be interested in the higher + ## level versions of this function which are also named ``recv``. + if size == 0: return + if socket.isBuffered: + if socket.bufLen == 0: + retRead(0'i32, 0) + + var read = 0 + while read < size: + if socket.currPos >= socket.bufLen: + retRead(0'i32, read) + + let chunk = min(socket.bufLen-socket.currPos, size-read) + var d = cast[cstring](data) + copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk) + read.inc(chunk) + socket.currPos.inc(chunk) + + result = read + else: + when defined(ssl): + if socket.isSSL: + if socket.sslHasPeekChar: + copyMem(data, addr(socket.sslPeekChar), 1) + socket.sslHasPeekChar = false + if size-1 > 0: + var d = cast[cstring](data) + result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 + else: + result = 1 + else: + result = SSLRead(socket.sslHandle, data, size) + else: + result = recv(socket.fd, data, size.cint, 0'i32) + else: + result = recv(socket.fd, data, size.cint, 0'i32) + +proc waitFor(socket: PSocket, waited: var float, timeout, size: int, + funcName: string): int {.tags: [FTime].} = + ## determines the amount of characters that can be read. Result will never + ## be larger than ``size``. For unbuffered sockets this will be ``1``. + ## For buffered sockets it can be as big as ``BufferSize``. + ## + ## If this function does not determine that there is data on the socket + ## within ``timeout`` ms, an ETimeout error will be raised. + result = 1 + if size <= 0: assert false + if timeout == -1: return size + if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos: + result = socket.bufLen - socket.currPos + result = min(result, size) + else: + if timeout - int(waited * 1000.0) < 1: + raise newException(ETimeout, "Call to '" & funcName & "' timed out.") + + when defined(ssl): + if socket.isSSL: + if socket.hasDataBuffered: + # sslPeekChar is present. + return 1 + let sslPending = SSLPending(socket.sslHandle) + if sslPending != 0: + return sslPending + + var startTime = epochTime() + let selRet = select(socket, timeout - int(waited * 1000.0)) + if selRet < 0: osError(osLastError()) + if selRet != 1: + raise newException(ETimeout, "Call to '" & funcName & "' timed out.") + waited += (epochTime() - startTime) + +proc recv*(socket: PSocket, data: pointer, size: int, timeout: int): int {. + tags: [FReadIO, FTime].} = + ## overload with a ``timeout`` parameter in miliseconds. + var waited = 0.0 # number of seconds already waited + + var read = 0 + while read < size: + let avail = waitFor(socket, waited, timeout, size-read, "recv") + var d = cast[cstring](data) + result = recv(socket, addr(d[read]), avail) + if result == 0: break + if result < 0: + return result + inc(read, result) + + result = read + +proc recv*(socket: PSocket, data: var string, size: int, timeout = -1): int = + ## Higher-level version of ``recv``. + ## + ## When 0 is returned the socket's connection has been closed. + ## + ## This function will throw an EOS exception when an error occurs. A value + ## lower than 0 is never returned. + ## + ## A timeout may be specified in miliseconds, if enough data is not received + ## within the time specified an ETimeout exception will be raised. + ## + ## **Note**: ``data`` must be initialised. + data.setLen(size) + result = recv(socket, cstring(data), size, timeout) + if result < 0: + data.setLen(0) + socket.socketError(result) + data.setLen(result) + +proc peekChar(socket: PSocket, c: var char): int {.tags: [FReadIO].} = + if socket.isBuffered: + result = 1 + if socket.bufLen == 0 or socket.currPos > socket.bufLen-1: + var res = socket.readIntoBuf(0'i32) + if res <= 0: + result = res + + c = socket.buffer[socket.currPos] + else: + when defined(ssl): + if socket.isSSL: + if not socket.sslHasPeekChar: + result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) + socket.sslHasPeekChar = true + + c = socket.sslPeekChar + return + result = recv(socket.fd, addr(c), 1, MSG_PEEK) + +proc readLine*(socket: PSocket, line: var TaintedString, timeout = -1) {. + tags: [FReadIO, FTime].} = + ## Reads a line of data from ``socket``. + ## + ## If a full line is read ``\r\L`` is not + ## added to ``line``, however if solely ``\r\L`` is read then ``line`` + ## will be set to it. + ## + ## If the socket is disconnected, ``line`` will be set to ``""``. + ## + ## An EOS exception will be raised in the case of a socket error. + ## + ## A timeout can be specified in miliseconds, if data is not received within + ## the specified time an ETimeout exception will be raised. + + template addNLIfEmpty(): stmt = + if line.len == 0: + line.add("\c\L") + + var waited = 0.0 + + setLen(line.string, 0) + while true: + var c: char + discard waitFor(socket, waited, timeout, 1, "readLine") + var n = recv(socket, addr(c), 1) + if n < 0: socket.socketError() + elif n == 0: return + if c == '\r': + discard waitFor(socket, waited, timeout, 1, "readLine") + n = peekChar(socket, c) + if n > 0 and c == '\L': + discard recv(socket, addr(c), 1) + elif n <= 0: socket.socketError() + addNLIfEmpty() + return + elif c == '\L': + addNLIfEmpty() + return + add(line.string, c) + +proc recvFrom*(socket: PSocket, data: var string, length: int, + address: var string, port: var TPort, flags = 0'i32): int {. + tags: [FReadIO].} = + ## Receives data from ``socket``. This function should normally be used with + ## connection-less sockets (UDP sockets). + ## + ## If an error occurs an EOS exception will be raised. Otherwise the return + ## value will be the length of data received. + ## + ## **Warning:** This function does not yet have a buffered implementation, + ## so when ``socket`` is buffered the non-buffered implementation will be + ## used. Therefore if ``socket`` contains something in its buffer this + ## function will make no effort to return it. + + # TODO: Buffered sockets + data.setLen(length) + var sockAddress: Tsockaddr_in + var addrLen = sizeof(sockAddress).TSocklen + result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, + cast[ptr TSockAddr](addr(sockAddress)), addr(addrLen)) + + if result != -1: + data.setLen(result) + address = $inet_ntoa(sockAddress.sin_addr) + port = ntohs(sockAddress.sin_port).TPort + else: + osError(osLastError()) + +proc skip*(socket: PSocket, size: int, timeout = -1) = + ## Skips ``size`` amount of bytes. + ## + ## An optional timeout can be specified in miliseconds, if skipping the + ## bytes takes longer than specified an ETimeout exception will be raised. + ## + ## Returns the number of skipped bytes. + var waited = 0.0 + var dummy = alloc(size) + var bytesSkipped = 0 + while bytesSkipped != size: + let avail = waitFor(socket, waited, timeout, size-bytesSkipped, "skip") + bytesSkipped += recv(socket, dummy, avail) + dealloc(dummy) + +proc send*(socket: PSocket, data: pointer, size: int): int {. + tags: [FWriteIO].} = + ## Sends data to a socket. + ## + ## **Note**: This is a low-level version of ``send``. You likely should use + ## the version below. + when defined(ssl): + if socket.isSSL: + return SSLWrite(socket.sslHandle, cast[cstring](data), size) + + when defined(windows) or defined(macosx): + result = send(socket.fd, data, size.cint, 0'i32) + else: + when defined(solaris): + const MSG_NOSIGNAL = 0 + result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) + +proc send*(socket: PSocket, data: string) {.tags: [FWriteIO].} = + ## sends data to a socket. + let sent = send(socket, cstring(data), data.len) + if sent < 0: + socketError(socket) + + if sent != data.len: + raise newException(EOS, "Could not send all data.") + +proc trySend*(socket: PSocket, data: string): bool {.tags: [FWriteIO].} = + ## Safe alternative to ``send``. Does not raise an EOS when an error occurs, + ## and instead returns ``false`` on failure. + result = send(socket, cstring(data), data.len) == data.len + +proc sendTo*(socket: PSocket, address: string, port: TPort, data: pointer, + size: int, af: TDomain = AF_INET, flags = 0'i32): int {. + tags: [FWriteIO].} = + ## This proc sends ``data`` to the specified ``address``, + ## which may be an IP address or a hostname, if a hostname is specified + ## this function will try each IP of that hostname. + ## + ## + ## **Note:** You may wish to use the high-level version of this function + ## which is defined below. + ## + ## **Note:** This proc is not available for SSL sockets. + var aiList = getAddrInfo(address, port, af) + + # try all possibilities: + var success = false + var it = aiList + while it != nil: + result = sendto(socket.fd, data, size.cint, flags.cint, it.ai_addr, + it.ai_addrlen.TSocklen) + if result != -1'i32: + success = true + break + it = it.ai_next + + dealloc(aiList) + +proc sendTo*(socket: PSocket, address: string, port: TPort, + data: string): int {.tags: [FWriteIO].} = + ## This proc sends ``data`` to the specified ``address``, + ## which may be an IP address or a hostname, if a hostname is specified + ## this function will try each IP of that hostname. + ## + ## This is the high-level version of the above ``sendTo`` function. + result = socket.sendTo(address, port, cstring(data), data.len) + +proc connectAsync(socket: PSocket, name: string, port = TPort(0), + af: TDomain = AF_INET) {.tags: [FReadIO].} = + ## A variant of ``connect`` for non-blocking sockets. + ## + ## This procedure will immediatelly return, it will not block until a connection + ## is made. It is up to the caller to make sure the connection has been established + ## by checking (using ``select``) whether the socket is writeable. + ## + ## **Note**: For SSL sockets, the ``handshake`` procedure must be called + ## whenever the socket successfully connects to a server. + var aiList = getAddrInfo(name, port, af) + # try all possibilities: + var success = false + var lastError: TOSErrorCode + var it = aiList + while it != nil: + var ret = connect(socket.fd, it.ai_addr, it.ai_addrlen.TSocklen) + if ret == 0'i32: + success = true + break + else: + lastError = osLastError() + when defined(windows): + # Windows EINTR doesn't behave same as POSIX. + if lastError.int32 == WSAEWOULDBLOCK: + success = true + break + else: + if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS: + success = true + break + + it = it.ai_next + + dealloc(aiList) + if not success: osError(lastError) + +proc connect*(socket: PSocket, address: string, port = TPort(0), timeout: int, + af: TDomain = AF_INET) {.tags: [FReadIO, FWriteIO].} = + ## Connects to server as specified by ``address`` on port specified by ``port``. + ## + ## The ``timeout`` paremeter specifies the time in miliseconds to allow for + ## the connection to the server to be made. + socket.fd.setBlocking(false) + + socket.connectAsync(address, port, af) + var s = @[socket.fd] + if selectWrite(s, timeout) != 1: + raise newException(ETimeout, "Call to 'connect' timed out.") + else: + when defined(ssl): + if socket.isSSL: + socket.fd.setBlocking(true) + doAssert socket.handshake() + socket.fd.setBlocking(true) + +proc isSSL*(socket: PSocket): bool = return socket.isSSL + ## Determines whether ``socket`` is a SSL socket. + +proc getFD*(socket: PSocket): TSocketHandle = return socket.fd + ## Returns the socket's file descriptor diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 02f0366cd..3d0cc2154 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -67,7 +67,7 @@ when withThreads: proc hookAux(st: TStackTrace, costs: int) = # this is quite performance sensitive! - when withThreads: Acquire profilingLock + when withThreads: acquire profilingLock inc totalCalls var last = high(st) while last > 0 and isNil(st[last]): dec last @@ -106,7 +106,7 @@ proc hookAux(st: TStackTrace, costs: int) = h = ((5 * h) + 1) and high(profileData) inc chain maxChainLen = max(maxChainLen, chain) - when withThreads: Release profilingLock + when withThreads: release profilingLock when defined(memProfiler): const diff --git a/lib/pure/os.nim b/lib/pure/os.nim index bb70f28b6..faca17e98 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -260,11 +260,12 @@ proc osError*(errorCode: TOSErrorCode) = ## ## If the error code is ``0`` or an error message could not be retrieved, ## the message ``unknown OS error`` will be used. - let msg = osErrorMsg(errorCode) - if msg == "": - raise newException(EOS, "unknown OS error") - else: - raise newException(EOS, msg) + var e: ref EOS; new(e) + e.errorCode = errorCode.int32 + e.msg = osErrorMsg(errorCode) + if e.msg == "": + e.msg = "unknown OS error" + raise e {.push stackTrace:off.} proc osLastError*(): TOSErrorCode = @@ -1037,7 +1038,10 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1", ## the process has finished. To execute a program without having a ## shell involved, use the `execProcess` proc of the `osproc` ## module. - result = c_system(command) shr 8 + when defined(linux): + result = c_system(command) shr 8 + else: + result = c_system(command) # Environment handling cannot be put into RTL, because the ``envPairs`` # iterator depends on ``environment``. @@ -1189,7 +1193,8 @@ iterator walkFiles*(pattern: string): string {.tags: [FReadDir].} = res = findFirstFile(pattern, f) if res != -1: while true: - if not skipFindData(f): + if not skipFindData(f) and + (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32: yield splitFile(pattern).dir / extractFilename(getFilename(f)) if findNextFile(res, f) == 0'i32: break findClose(res) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6df85bbc6..5d6848565 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -13,13 +13,16 @@ include "system/inclrtl" import - strutils, os, strtabs, streams, sequtils + strutils, os, strtabs, streams when defined(windows): import winlean else: import posix +when defined(linux): + import linux + type TProcess = object of TObject when defined(windows): @@ -44,7 +47,7 @@ type poStdErrToStdOut, ## merge stdout and stderr to the stdout stream poParentStreams ## use the parent's streams -template poUseShell*: TProcessOption {.deprecated.} = poUsePath +const poUseShell* {.deprecated.} = poUsePath ## Deprecated alias for poUsePath. proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = @@ -165,6 +168,9 @@ proc processID*(p: PProcess): int {.rtl, extern: "nosp$1".} = proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl, extern: "nosp$1", tags: [].} ## waits for the process to finish and returns `p`'s error code. + ## + ## **Warning**: Be careful when using waitForExit for processes created without + ## poParentStreams because they may fill output buffers, causing deadlock. proc peekExitCode*(p: PProcess): int {.tags: [].} ## return -1 if the process is still running. Otherwise the process' exit code @@ -590,6 +596,24 @@ elif not defined(useNimRtl): copyMem(result[i], addr(x[0]), x.len+1) inc(i) + type TStartProcessData = object + sysCommand: cstring + sysArgs: cstringArray + sysEnv: cstringArray + workingDir: cstring + pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint] + optionPoUsePath: bool + optionPoParentStreams: bool + optionPoStdErrToStdOut: bool + + when not defined(useFork): + proc startProcessAuxSpawn(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].} + proc startProcessAuxFork(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].} + {.push stacktrace: off, profiler: off.} + proc startProcessAfterFork(data: ptr TStartProcessData) {. + tags: [FExecIO, FReadEnv], cdecl.} + {.pop.} + proc startProcess(command: string, workingDir: string = "", args: openArray[string] = [], @@ -604,23 +628,76 @@ elif not defined(useNimRtl): pipe(pStderr) != 0'i32: osError(osLastError()) - var sys_command: string - var sys_args_raw: seq[string] + var sysCommand: string + var sysArgsRaw: seq[string] if poEvalCommand in options: - sys_command = "/bin/sh" - sys_args_raw = @[sys_command, "-c", command] + sysCommand = "/bin/sh" + sysArgsRaw = @[sysCommand, "-c", command] assert args.len == 0 else: - sys_command = command - sys_args_raw = @[command] + sysCommand = command + sysArgsRaw = @[command] for arg in args.items: - sys_args_raw.add arg - - var sys_args = allocCStringArray(sys_args_raw) - finally: deallocCStringArray(sys_args) + sysArgsRaw.add arg var pid: TPid - when defined(posix_spawn) and not defined(useFork): + + var sysArgs = allocCStringArray(sysArgsRaw) + finally: deallocCStringArray(sysArgs) + + var sysEnv = if env == nil: + envToCStringArray() + else: + envToCStringArray(env) + + finally: deallocCStringArray(sysEnv) + + var data: TStartProcessData + data.sysCommand = sysCommand + data.sysArgs = sysArgs + data.sysEnv = sysEnv + data.pStdin = pStdin + data.pStdout = pStdout + data.pStderr = pStderr + data.optionPoParentStreams = poParentStreams in options + data.optionPoUsePath = poUsePath in options + data.optionPoStdErrToStdOut = poStdErrToStdOut in options + data.workingDir = workingDir + + + when defined(posix_spawn) and not defined(useFork) and + not defined(useClone) and not defined(linux): + pid = startProcessAuxSpawn(data) + else: + pid = startProcessAuxFork(data) + + # Parent process. Copy process information. + if poEchoCmd in options: + echo(command, " ", join(args, " ")) + result.id = pid + + if poParentStreams in options: + # does not make much sense, but better than nothing: + result.inHandle = 0 + result.outHandle = 1 + if poStdErrToStdOut in options: + result.errHandle = result.outHandle + else: + result.errHandle = 2 + else: + result.inHandle = pStdin[writeIdx] + result.outHandle = pStdout[readIdx] + if poStdErrToStdOut in options: + result.errHandle = result.outHandle + discard close(pStderr[readIdx]) + else: + result.errHandle = pStderr[readIdx] + discard close(pStderr[writeIdx]) + discard close(pStdin[readIdx]) + discard close(pStdout[writeIdx]) + + when not defined(useFork): + proc startProcessAuxSpawn(data: TStartProcessData): TPid = var attr: Tposix_spawnattr var fops: Tposix_spawn_file_actions @@ -639,89 +716,117 @@ elif not defined(useNimRtl): POSIX_SPAWN_SETSIGMASK or POSIX_SPAWN_SETPGROUP) - if poParentStreams notin options: - chck posix_spawn_file_actions_addclose(fops, pStdin[writeIdx]) - chck posix_spawn_file_actions_adddup2(fops, pStdin[readIdx], readIdx) - chck posix_spawn_file_actions_addclose(fops, pStdout[readIdx]) - chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], writeIdx) - chck posix_spawn_file_actions_addclose(fops, pStderr[readIdx]) - if poStdErrToStdOut in options: - chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], 2) + if not data.optionPoParentStreams: + chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx]) + chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx) + chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx]) + chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx) + chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx]) + if data.optionPoStdErrToStdOut: + chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2) else: - chck posix_spawn_file_actions_adddup2(fops, p_stderr[writeIdx], 2) + chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) - var sys_env = if env == nil: envToCStringArray() else: envToCStringArray(env) var res: cint - # This is incorrect! - if workingDir.len > 0: os.setCurrentDir(workingDir) - if poUsePath in options: - res = posix_spawnp(pid, sys_command, fops, attr, sys_args, sys_env) + # FIXME: chdir is global to process + if data.workingDir.len > 0: + setCurrentDir($data.workingDir) + var pid: TPid + + if data.optionPoUsePath: + res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) else: - res = posix_spawn(pid, sys_command, fops, attr, sys_args, sys_env) - deallocCStringArray(sys_env) + res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) chck res + return pid + + proc startProcessAuxFork(data: TStartProcessData): TPid = + if pipe(data.pErrorPipe) != 0: + osError(osLastError()) + finally: + discard close(data.pErrorPipe[readIdx]) + + var pid: TPid + var dataCopy = data + + when defined(useClone): + const stackSize = 65536 + let stackEnd = cast[clong](alloc(stackSize)) + let stack = cast[pointer](stackEnd + stackSize) + let fn: pointer = startProcessAfterFork + pid = clone(fn, stack, + cint(CLONE_VM or CLONE_VFORK or SIGCHLD), + pointer(addr dataCopy), nil, nil, nil) + discard close(data.pErrorPipe[writeIdx]) + dealloc(stack) else: pid = fork() - if pid < 0: osError(osLastError()) if pid == 0: - ## child process: - - if poParentStreams notin options: - discard close(p_stdin[writeIdx]) - if dup2(p_stdin[readIdx], readIdx) < 0: osError(osLastError()) - discard close(p_stdout[readIdx]) - if dup2(p_stdout[writeIdx], writeIdx) < 0: osError(osLastError()) - discard close(p_stderr[readIdx]) - if poStdErrToStdOut in options: - if dup2(p_stdout[writeIdx], 2) < 0: osError(osLastError()) - else: - if dup2(p_stderr[writeIdx], 2) < 0: osError(osLastError()) - - # Create a new process group - if setpgid(0, 0) == -1: quit("setpgid call failed: " & $strerror(errno)) - - if workingDir.len > 0: os.setCurrentDir(workingDir) - - if env == nil: - if poUsePath in options: - discard execvp(sys_command, sys_args) - else: - discard execv(sys_command, sys_args) - else: - var c_env = envToCStringArray(env) - if poUsePath in options: - discard execvpe(sys_command, sys_args, c_env) - else: - discard execve(sys_command, sys_args, c_env) - # too risky to raise an exception here: - quit("execve call failed: " & $strerror(errno)) - # Parent process. Copy process information. - if poEchoCmd in options: - echo(command, " ", join(args, " ")) - result.id = pid - - if poParentStreams in options: - # does not make much sense, but better than nothing: - result.inHandle = 0 - result.outHandle = 1 - if poStdErrToStdOut in options: - result.errHandle = result.outHandle + startProcessAfterFork(addr(dataCopy)) + exitnow(1) + + discard close(data.pErrorPipe[writeIdx]) + if pid < 0: osError(osLastError()) + + var error: cint + let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) + if sizeRead == sizeof(error): + osError($strerror(error)) + + return pid + + {.push stacktrace: off, profiler: off.} + proc startProcessFail(data: ptr TStartProcessData) = + var error: cint = errno + discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) + exitnow(1) + + when defined(macosx): + var environ {.importc.}: cstringArray + + proc startProcessAfterFork(data: ptr TStartProcessData) = + # Warning: no GC here! + # Or anythink that touches global structures - all called nimrod procs + # must be marked with noStackFrame. Inspect C code after making changes. + if not data.optionPoParentStreams: + discard close(data.pStdin[writeIdx]) + if dup2(data.pStdin[readIdx], readIdx) < 0: + startProcessFail(data) + discard close(data.pStdout[readIdx]) + if dup2(data.pStdout[writeIdx], writeIdx) < 0: + startProcessFail(data) + discard close(data.pStderr[readIdx]) + if data.optionPoStdErrToStdOut: + if dup2(data.pStdout[writeIdx], 2) < 0: + startProcessFail(data) else: - result.errHandle = 2 - else: - result.inHandle = pStdin[writeIdx] - result.outHandle = pStdout[readIdx] - if poStdErrToStdOut in options: - result.errHandle = result.outHandle - discard close(pStderr[readIdx]) + if dup2(data.pStderr[writeIdx], 2) < 0: + startProcessFail(data) + + if data.workingDir.len > 0: + if chdir(data.workingDir) < 0: + startProcessFail(data) + + discard close(data.pErrorPipe[readIdx]) + discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) + + if data.optionPoUsePath: + when defined(macosx): + # MacOSX doesn't have execvpe, so we need workaround. + # On MacOSX we can arrive here only from fork, so this is safe: + environ = data.sysEnv + discard execvp(data.sysCommand, data.sysArgs) else: - result.errHandle = pStderr[readIdx] - discard close(pStderr[writeIdx]) - discard close(pStdin[readIdx]) - discard close(pStdout[writeIdx]) + discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv) + else: + discard execve(data.sysCommand, data.sysArgs, data.sysEnv) + + startProcessFail(data) + {.pop} proc close(p: PProcess) = if p.inStream != nil: close(p.inStream) @@ -791,7 +896,10 @@ elif not defined(useNimRtl): proc csystem(cmd: cstring): cint {.nodecl, importc: "system".} proc execCmd(command: string): int = - result = csystem(command) shr 8 + when defined(linux): + result = csystem(command) shr 8 + else: + result = csystem(command) proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) = FD_ZERO(fd) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 3f9686e1e..bd8836f7c 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -267,7 +267,7 @@ proc getSymbol(c: var TSqlLexer, tok: var TToken) = while true: add(tok.literal, buf[pos]) Inc(pos) - if not (buf[pos] in {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}): + if buf[pos] notin {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}: break c.bufpos = pos tok.kind = tkIdentifier diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 70b617393..68b1ab223 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -836,9 +836,11 @@ iterator findAll*(s: string, pattern: TPeg, start = 0): string = while i < s.len: c.ml = 0 var L = rawMatch(s, pattern, i, c) - if L < 0: break - yield substr(s, i, i+L-1) - inc(i, L) + if L < 0: + inc(i, 1) + else: + yield substr(s, i, i+L-1) + inc(i, L) proc findAll*(s: string, pattern: TPeg, start = 0): seq[string] {. nosideEffect, rtl, extern: "npegs$1".} = diff --git a/lib/pure/rawsockets.nim b/lib/pure/rawsockets.nim new file mode 100644 index 000000000..aeaa7f3b5 --- /dev/null +++ b/lib/pure/rawsockets.nim @@ -0,0 +1,421 @@ +# +# +# Nimrod's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements a low-level cross-platform sockets interface. Look +## at the ``net`` module for the higher-level version. + +# TODO: Clean up the exports a bit and everything else in general. + +import unsigned, os + +when hostos == "solaris": + {.passl: "-lsocket -lnsl".} + +when defined(Windows): + import winlean + export WSAEWOULDBLOCK +else: + import posix + export fcntl, F_GETFL, O_NONBLOCK, F_SETFL, EAGAIN, EWOULDBLOCK, MSG_NOSIGNAL, + EINTR, EINPROGRESS + +export TSocketHandle, TSockaddr_in, TAddrinfo, INADDR_ANY, TSockAddr, TSockLen, + inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto + +export + SO_ERROR, + SOL_SOCKET, + SOMAXCONN, + SO_ACCEPTCONN, SO_BROADCAST, SO_DEBUG, SO_DONTROUTE, + SO_KEEPALIVE, SO_OOBINLINE, SO_REUSEADDR, + MSG_PEEK + +type + + TPort* = distinct uint16 ## port type + + TDomain* = enum ## domain, which specifies the protocol family of the + ## created socket. Other domains than those that are listed + ## here are unsupported. + AF_UNIX, ## for local socket (using a file). Unsupported on Windows. + AF_INET = 2, ## for network protocol IPv4 or + AF_INET6 = 23 ## for network protocol IPv6. + + TType* = enum ## second argument to `socket` proc + SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets + SOCK_DGRAM = 2, ## datagram service or Datagram Sockets + SOCK_RAW = 3, ## raw protocols atop the network layer. + SOCK_SEQPACKET = 5 ## reliable sequenced packet service + + TProtocol* = enum ## third argument to `socket` proc + IPPROTO_TCP = 6, ## Transmission control protocol. + IPPROTO_UDP = 17, ## User datagram protocol. + IPPROTO_IP, ## Internet protocol. Unsupported on Windows. + IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. + IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. + IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. + + TServent* {.pure, final.} = object ## information about a service + name*: string + aliases*: seq[string] + port*: TPort + proto*: string + + Thostent* {.pure, final.} = object ## information about a given host + name*: string + aliases*: seq[string] + addrtype*: TDomain + length*: int + addrList*: seq[string] + +when defined(windows): + let + osInvalidSocket* = winlean.INVALID_SOCKET + + const + IOCPARM_MASK* = 127 + IOC_IN* = int(-2147483648) + FIONBIO* = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or + (102 shl 8) or 126 + + proc ioctlsocket*(s: TSocketHandle, cmd: clong, + argptr: ptr clong): cint {. + stdcall, importc: "ioctlsocket", dynlib: "ws2_32.dll".} +else: + let + osInvalidSocket* = posix.INVALID_SOCKET + +proc `==`*(a, b: TPort): bool {.borrow.} + ## ``==`` for ports. + +proc `$`*(p: TPort): string {.borrow.} + ## returns the port number as a string + +proc toInt*(domain: TDomain): cint + ## Converts the TDomain enum to a platform-dependent ``cint``. + +proc toInt*(typ: TType): cint + ## Converts the TType enum to a platform-dependent ``cint``. + +proc toInt*(p: TProtocol): cint + ## Converts the TProtocol enum to a platform-dependent ``cint``. + +when defined(posix): + proc toInt(domain: TDomain): cint = + case domain + of AF_UNIX: result = posix.AF_UNIX + of AF_INET: result = posix.AF_INET + of AF_INET6: result = posix.AF_INET6 + else: discard + + proc toInt(typ: TType): cint = + case typ + of SOCK_STREAM: result = posix.SOCK_STREAM + of SOCK_DGRAM: result = posix.SOCK_DGRAM + of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET + of SOCK_RAW: result = posix.SOCK_RAW + else: discard + + proc toInt(p: TProtocol): cint = + case p + of IPPROTO_TCP: result = posix.IPPROTO_TCP + of IPPROTO_UDP: result = posix.IPPROTO_UDP + of IPPROTO_IP: result = posix.IPPROTO_IP + of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 + of IPPROTO_RAW: result = posix.IPPROTO_RAW + of IPPROTO_ICMP: result = posix.IPPROTO_ICMP + else: discard + +else: + proc toInt(domain: TDomain): cint = + result = toU16(ord(domain)) + + proc toInt(typ: TType): cint = + result = cint(ord(typ)) + + proc toInt(p: TProtocol): cint = + result = cint(ord(p)) + + +proc newRawSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, + protocol: TProtocol = IPPROTO_TCP): TSocketHandle = + ## Creates a new socket; returns `InvalidSocket` if an error occurs. + socket(toInt(domain), toInt(typ), toInt(protocol)) + +proc close*(socket: TSocketHandle) = + ## closes a socket. + when defined(windows): + discard winlean.closeSocket(socket) + else: + discard posix.close(socket) + # TODO: These values should not be discarded. An EOS should be raised. + # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times + +proc bindAddr*(socket: TSocketHandle, name: ptr TSockAddr, namelen: TSockLen): cint = + result = bindSocket(socket, name, namelen) + +proc listen*(socket: TSocketHandle, backlog = SOMAXCONN): cint {.tags: [FReadIO].} = + ## Marks ``socket`` as accepting connections. + ## ``Backlog`` specifies the maximum length of the + ## queue of pending connections. + when defined(windows): + result = winlean.listen(socket, cint(backlog)) + else: + result = posix.listen(socket, cint(backlog)) + +proc getAddrInfo*(address: string, port: TPort, af: TDomain = AF_INET, typ: TType = SOCK_STREAM, + prot: TProtocol = IPPROTO_TCP): ptr TAddrInfo = + ## + ## + ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``! + var hints: TAddrInfo + result = nil + hints.ai_family = toInt(af) + hints.ai_socktype = toInt(typ) + hints.ai_protocol = toInt(prot) + var gaiResult = getAddrInfo(address, $port, addr(hints), result) + if gaiResult != 0'i32: + when defined(windows): + OSError(OSLastError()) + else: + raise newException(EOS, $gai_strerror(gaiResult)) + +proc dealloc*(ai: ptr TAddrInfo) = + freeaddrinfo(ai) + +proc ntohl*(x: int32): int32 = + ## Converts 32-bit integers from network to host byte order. + ## On machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + when cpuEndian == bigEndian: result = x + else: result = (x shr 24'i32) or + (x shr 8'i32 and 0xff00'i32) or + (x shl 8'i32 and 0xff0000'i32) or + (x shl 24'i32) + +proc ntohs*(x: int16): int16 = + ## Converts 16-bit integers from network to host byte order. On machines + ## where the host byte order is the same as network byte order, this is + ## a no-op; otherwise, it performs a 2-byte swap operation. + when cpuEndian == bigEndian: result = x + else: result = (x shr 8'i16) or (x shl 8'i16) + +proc htonl*(x: int32): int32 = + ## Converts 32-bit integers from host to network byte order. On machines + ## where the host byte order is the same as network byte order, this is + ## a no-op; otherwise, it performs a 4-byte swap operation. + result = rawsockets.ntohl(x) + +proc htons*(x: int16): int16 = + ## Converts 16-bit positive integers from host to network byte order. + ## On machines where the host byte order is the same as network byte + ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. + result = rawsockets.ntohs(x) + +proc getServByName*(name, proto: string): TServent {.tags: [FReadIO].} = + ## Searches the database from the beginning and finds the first entry for + ## which the service name specified by ``name`` matches the s_name member + ## and the protocol name specified by ``proto`` matches the s_proto member. + ## + ## On posix this will search through the ``/etc/services`` file. + when defined(Windows): + var s = winlean.getservbyname(name, proto) + else: + var s = posix.getservbyname(name, proto) + if s == nil: raise newException(EOS, "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = TPort(s.s_port) + result.proto = $s.s_proto + +proc getServByPort*(port: TPort, proto: string): TServent {.tags: [FReadIO].} = + ## Searches the database from the beginning and finds the first entry for + ## which the port specified by ``port`` matches the s_port member and the + ## protocol name specified by ``proto`` matches the s_proto member. + ## + ## On posix this will search through the ``/etc/services`` file. + when defined(Windows): + var s = winlean.getservbyport(ze(int16(port)).cint, proto) + else: + var s = posix.getservbyport(ze(int16(port)).cint, proto) + if s == nil: raise newException(EOS, "Service not found.") + result.name = $s.s_name + result.aliases = cstringArrayToSeq(s.s_aliases) + result.port = TPort(s.s_port) + result.proto = $s.s_proto + +proc getHostByAddr*(ip: string): Thostent {.tags: [FReadIO].} = + ## This function will lookup the hostname of an IP Address. + var myaddr: TInAddr + myaddr.s_addr = inet_addr(ip) + + when defined(windows): + var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, + cint(rawsockets.AF_INET)) + if s == nil: osError(osLastError()) + else: + var s = posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).TSocklen, + cint(posix.AF_INET)) + if s == nil: + raise newException(EOS, $hstrerror(h_errno)) + + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when defined(windows): + result.addrtype = TDomain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raise newException(EOS, "unknown h_addrtype") + result.addrList = cstringArrayToSeq(s.h_addr_list) + result.length = int(s.h_length) + +proc getHostByName*(name: string): Thostent {.tags: [FReadIO].} = + ## This function will lookup the IP address of a hostname. + when defined(Windows): + var s = winlean.gethostbyname(name) + else: + var s = posix.gethostbyname(name) + if s == nil: osError(osLastError()) + result.name = $s.h_name + result.aliases = cstringArrayToSeq(s.h_aliases) + when defined(windows): + result.addrtype = TDomain(s.h_addrtype) + else: + if s.h_addrtype == posix.AF_INET: + result.addrtype = AF_INET + elif s.h_addrtype == posix.AF_INET6: + result.addrtype = AF_INET6 + else: + raise newException(EOS, "unknown h_addrtype") + result.addrList = cstringArrayToSeq(s.h_addr_list) + result.length = int(s.h_length) + +proc getSockName*(socket: TSocketHandle): TPort = + ## returns the socket's associated port number. + var name: Tsockaddr_in + when defined(Windows): + name.sin_family = int16(ord(AF_INET)) + else: + name.sin_family = posix.AF_INET + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) + var namelen = sizeof(name).TSocklen + if getsockname(socket, cast[ptr TSockAddr](addr(name)), + addr(namelen)) == -1'i32: + osError(osLastError()) + result = TPort(rawsockets.ntohs(name.sin_port)) + +proc getSockOptInt*(socket: TSocketHandle, level, optname: int): int {. + tags: [FReadIO].} = + ## getsockopt for integer options. + var res: cint + var size = sizeof(res).TSocklen + if getsockopt(socket, cint(level), cint(optname), + addr(res), addr(size)) < 0'i32: + osError(osLastError()) + result = int(res) + +proc setSockOptInt*(socket: TSocketHandle, level, optname, optval: int) {. + tags: [FWriteIO].} = + ## setsockopt for integer options. + var value = cint(optval) + if setsockopt(socket, cint(level), cint(optname), addr(value), + sizeof(value).TSocklen) < 0'i32: + osError(osLastError()) + +proc setBlocking*(s: TSocketHandle, blocking: bool) = + ## Sets blocking mode on socket. + ## + ## Raises EOS on error. + when defined(Windows): + var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking + if ioctlsocket(s, FIONBIO, addr(mode)) == -1: + osError(osLastError()) + else: # BSD sockets + var x: int = fcntl(s, F_GETFL, 0) + if x == -1: + osError(osLastError()) + else: + var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK + if fcntl(s, F_SETFL, mode) == -1: + osError(osLastError()) + +proc timeValFromMilliseconds(timeout = 500): Ttimeval = + if timeout != -1: + var seconds = timeout div 1000 + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + +proc createFdSet(fd: var TFdSet, s: seq[TSocketHandle], m: var int) = + FD_ZERO(fd) + for i in items(s): + m = max(m, int(i)) + FD_SET(i, fd) + +proc pruneSocketSet(s: var seq[TSocketHandle], fd: var TFdSet) = + var i = 0 + var L = s.len + while i < L: + if FD_ISSET(s[i], fd) == 0'i32: + s[i] = s[L-1] + dec(L) + else: + inc(i) + setLen(s, L) + +proc select*(readfds: var seq[TSocketHandle], timeout = 500): int = + ## Traditional select function. This function will return the number of + ## sockets that are ready to be read from, written to, or which have errors. + ## If there are none; 0 is returned. + ## ``Timeout`` is in miliseconds and -1 can be specified for no timeout. + ## + ## A socket is removed from the specific ``seq`` when it has data waiting to + ## be read/written to or has errors (``exceptfds``). + var tv {.noInit.}: Ttimeval = timeValFromMilliseconds(timeout) + + var rd: TFdSet + var m = 0 + createFdSet((rd), readfds, m) + + if timeout != -1: + result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) + else: + result = int(select(cint(m+1), addr(rd), nil, nil, nil)) + + pruneSocketSet(readfds, (rd)) + +proc selectWrite*(writefds: var seq[TSocketHandle], + timeout = 500): int {.tags: [FReadIO].} = + ## When a socket in ``writefds`` is ready to be written to then a non-zero + ## value will be returned specifying the count of the sockets which can be + ## written to. The sockets which can be written to will also be removed + ## from ``writefds``. + ## + ## ``timeout`` is specified in miliseconds and ``-1`` can be specified for + ## an unlimited time. + var tv {.noInit.}: Ttimeval = timeValFromMilliseconds(timeout) + + var wr: TFdSet + var m = 0 + createFdSet((wr), writefds, m) + + if timeout != -1: + result = int(select(cint(m+1), nil, addr(wr), nil, addr(tv))) + else: + result = int(select(cint(m+1), nil, addr(wr), nil, nil)) + + pruneSocketSet(writefds, (wr)) + +when defined(Windows): + var wsa: TWSADATA + if WSAStartup(0x0101'i16, addr wsa) != 0: OSError(OSLastError()) diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 83c158da1..085344e3e 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2013 Dominik Picheta +# (c) Copyright 2014 Dominik Picheta # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -9,212 +9,230 @@ # TODO: Docs. -import tables, os, unsigned -when defined(windows): - import winlean -else: - import posix +import tables, os, unsigned, hashes + +when defined(linux): import posix, epoll +elif defined(windows): import winlean + +proc hash*(x: TSocketHandle): THash {.borrow.} +proc `$`*(x: TSocketHandle): string {.borrow.} type TEvent* = enum EvRead, EvWrite - TSelectorKey* = object - fd: cint - events: set[TEvent] - data: PObject - - TReadyInfo* = tuple[key: TSelectorKey, events: set[TEvent]] - - PSelector* = ref object of PObject ## Selector interface. - fds*: TTable[cint, TSelectorKey] - registerImpl*: proc (s: PSelector, fd: cint, events: set[TEvent], - data: PObject): TSelectorKey {.nimcall, tags: [FWriteIO].} - unregisterImpl*: proc (s: PSelector, fd: cint): TSelectorKey {.nimcall, tags: [FWriteIO].} - selectImpl*: proc (s: PSelector, timeout: int): seq[TReadyInfo] {.nimcall, tags: [FReadIO].} - closeImpl*: proc (s: PSelector) {.nimcall.} - -template initSelector(r: expr) = - new r - r.fds = initTable[cint, TSelectorKey]() - -proc register*(s: PSelector, fd: cint, events: set[TEvent], data: PObject): - TSelectorKey = - if not s.registerImpl.isNil: result = s.registerImpl(s, fd, events, data) + PSelectorKey* = ref object + fd*: TSocketHandle + events*: set[TEvent] ## The events which ``fd`` listens for. + data*: PObject ## User object. -proc unregister*(s: PSelector, fd: cint): TSelectorKey = - ## - ## **Note:** For the ``epoll`` implementation the resulting ``TSelectorKey`` - ## will only have the ``fd`` field set. This is an optimisation and may - ## change in the future if a viable use case is presented. - if not s.unregisterImpl.isNil: result = s.unregisterImpl(s, fd) + TReadyInfo* = tuple[key: PSelectorKey, events: set[TEvent]] -proc select*(s: PSelector, timeout = 500): seq[TReadyInfo] = - ## - ## The ``events`` field of the returned ``key`` contains the original events - ## for which the ``fd`` was bound. This is contrary to the ``events`` field - ## of the ``TReadyInfo`` tuple which determines which events are ready - ## on the ``fd``. - - if not s.selectImpl.isNil: result = s.selectImpl(s, timeout) - -proc close*(s: PSelector) = - if not s.closeImpl.isNil: s.closeImpl(s) - -# ---- Select() ---------------------------------------------------------------- - -type - PSelectSelector* = ref object of PSelector ## Implementation of select() - -proc ssRegister(s: PSelector, fd: cint, events: set[TEvent], - data: PObject): TSelectorKey = - if s.fds.hasKey(fd): - raise newException(EInvalidValue, "FD already exists in selector.") - var sk = TSelectorKey(fd: fd, events: events, data: data) - s.fds[fd] = sk - result = sk - -proc ssUnregister(s: PSelector, fd: cint): TSelectorKey = - result = s.fds[fd] - s.fds.del(fd) - -proc ssClose(s: PSelector) = nil - -proc timeValFromMilliseconds(timeout: int): TTimeVal = - if timeout != -1: - var seconds = timeout div 1000 - result.tv_sec = seconds.int32 - result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 - -proc createFdSet(rd, wr: var TFdSet, fds: TTable[cint, TSelectorKey], - m: var int) = - FD_ZERO(rd); FD_ZERO(wr) - for k, v in pairs(fds): - if EvRead in v.events: - m = max(m, int(k)) - FD_SET(k, rd) - if EvWrite in v.events: - m = max(m, int(k)) - FD_SET(k, wr) - -proc getReadyFDs(rd, wr: var TFdSet, fds: TTable[cint, TSelectorKey]): - seq[TReadyInfo] = - result = @[] - for k, v in pairs(fds): - var events: set[TEvent] = {} - if FD_ISSET(k, rd) != 0'i32: - events = events + {EvRead} - if FD_ISSET(k, wr) != 0'i32: - events = events + {EvWrite} - result.add((v, events)) - -proc select(fds: TTable[cint, TSelectorKey], timeout = 500): - seq[TReadyInfo] = - var tv {.noInit.}: TTimeVal = timeValFromMilliseconds(timeout) - - var rd, wr: TFdSet - var m = 0 - createFdSet(rd, wr, fds, m) - - var retCode = 0 - if timeout != -1: - retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv))) - else: - retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, nil)) - - if retCode < 0: - OSError(OSLastError()) - elif retCode == 0: - return @[] - else: - return getReadyFDs(rd, wr, fds) - -proc ssSelect(s: PSelector, timeout: int): seq[TReadyInfo] = - result = select(s.fds, timeout) - -proc newSelectSelector*(): PSelectSelector = - initSelector(result) - result.registerImpl = ssRegister - result.unregisterImpl = ssUnregister - result.selectImpl = ssSelect - result.closeImpl = ssClose - -# ---- Epoll ------------------------------------------------------------------- - -when defined(linux): - import epoll +when defined(linux) or defined(nimdoc): type - PEpollSelector* = ref object of PSelector + PSelector* = ref object epollFD: cint - events: array[64, ptr epoll_event] - - TDataWrapper = object - fd: cint - boundEvents: set[TEvent] ## The events which ``fd`` listens for. - data: PObject ## User object. + events: array[64, epoll_event] + fds: TTable[TSocketHandle, PSelectorKey] - proc esRegister(s: PSelector, fd: cint, events: set[TEvent], - data: PObject): TSelectorKey = - var es = PEpollSelector(s) - var event: epoll_event + proc createEventStruct(events: set[TEvent], fd: TSocketHandle): epoll_event = if EvRead in events: - event.events = EPOLLIN + result.events = EPOLLIN if EvWrite in events: - event.events = event.events or EPOLLOUT - - var dw = cast[ptr TDataWrapper](alloc0(sizeof(TDataWrapper))) # TODO: This needs to be dealloc'd - dw.fd = fd - dw.boundEvents = events - dw.data = data - event.data.thePtr = dw - - if epoll_ctl(es.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: - OSError(OSLastError()) - - result = TSelectorKey(fd: fd, events: events, data: data) + result.events = result.events or EPOLLOUT + result.events = result.events or EPOLLRDHUP + result.data.fd = fd.cint - proc esUnregister(s: PSelector, fd: cint): TSelectorKey = - # We cannot find out the information about this ``fd`` from the epoll - # context. As such I will simply return an almost empty TSelectorKey. - var es = PEpollSelector(s) - if epoll_ctl(es.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: + proc register*(s: PSelector, fd: TSocketHandle, events: set[TEvent], + data: PObject): PSelectorKey {.discardable.} = + ## Registers file descriptor ``fd`` to selector ``s`` with a set of TEvent + ## ``events``. + var event = createEventStruct(events, fd) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0: OSError(OSLastError()) - # We could fill in the ``fds`` TTable to get the info, but that wouldn't - # be nice for our memory. - result = TSelectorKey(fd: fd, events: {}, data: nil) + + var key = PSelectorKey(fd: fd, events: events, data: data) + + s.fds[fd] = key + result = key + + proc update*(s: PSelector, fd: TSocketHandle, + events: set[TEvent]): PSelectorKey {.discardable.} = + ## Updates the events which ``fd`` wants notifications for. + if s.fds[fd].events != events: + var event = createEventStruct(events, fd) + + s.fds[fd].events = events + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: + OSError(OSLastError()) + result = s.fds[fd] + + proc unregister*(s: PSelector, fd: TSocketHandle): PSelectorKey {.discardable.} = + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0: + let err = OSLastError() + if err.cint notin {ENOENT, EBADF}: # TODO: Why do we sometimes get an EBADF? Is this normal? + OSError(err) + result = s.fds[fd] + s.fds.del(fd) - proc esClose(s: PSelector) = - var es = PEpollSelector(s) - if es.epollFD.close() != 0: OSError(OSLastError()) - dealloc(addr es.events) # TODO: Test this + proc close*(s: PSelector) = + if s.epollFD.close() != 0: OSError(OSLastError()) + dealloc(addr s.events) # TODO: Test this + + proc epollHasFd(s: PSelector, fd: TSocketHandle): bool = + result = true + var event = createEventStruct(s.fds[fd].events, fd) + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0: + let err = osLastError() + if err.cint in {ENOENT, EBADF}: + return false + OSError(OSLastError()) - proc esSelect(s: PSelector, timeout: int): seq[TReadyInfo] = + proc select*(s: PSelector, timeout: int): seq[TReadyInfo] = + ## + ## The ``events`` field of the returned ``key`` contains the original events + ## for which the ``fd`` was bound. This is contrary to the ``events`` field + ## of the ``TReadyInfo`` tuple which determines which events are ready + ## on the ``fd``. result = @[] - var es = PEpollSelector(s) - - let evNum = epoll_wait(es.epollFD, es.events[0], 64.cint, timeout.cint) + let evNum = epoll_wait(s.epollFD, addr s.events[0], 64.cint, timeout.cint) if evNum < 0: OSError(OSLastError()) if evNum == 0: return @[] for i in 0 .. <evNum: + let fd = s.events[i].data.fd.TSocketHandle + var evSet: set[TEvent] = {} - if (es.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead} - if (es.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite} - let dw = cast[ptr TDataWrapper](es.events[i].data.thePtr) - - let selectorKey = TSelectorKey(fd: dw.fd, events: dw.boundEvents, - data: dw.data) + if (s.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead} + if (s.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite} + let selectorKey = s.fds[fd] + assert selectorKey != nil result.add((selectorKey, evSet)) + + #echo("Epoll: ", result[i].key.fd, " ", result[i].events, " ", result[i].key.events) - proc newEpollSelector*(): PEpollSelector = + proc newSelector*(): PSelector = new result result.epollFD = epoll_create(64) - result.events = cast[array[64, ptr epoll_event]](alloc0(sizeof(epoll_event)*64)) + result.events = cast[array[64, epoll_event]](alloc0(sizeof(epoll_event)*64)) + result.fds = initTable[TSocketHandle, PSelectorKey]() if result.epollFD < 0: OSError(OSLastError()) - result.registerImpl = esRegister - result.unregisterImpl = esUnregister - result.closeImpl = esClose - result.selectImpl = esSelect + + proc contains*(s: PSelector, fd: TSocketHandle): bool = + ## Determines whether selector contains a file descriptor. + if s.fds.hasKey(fd): + # Ensure the underlying epoll instance still contains this fd. + result = epollHasFd(s, fd) + else: + return false + + proc contains*(s: PSelector, key: PSelectorKey): bool = + ## Determines whether selector contains this selector key. More accurate + ## than checking if the file descriptor is in the selector because it + ## ensures that the keys are equal. File descriptors may not always be + ## unique especially when an fd is closed and then a new one is opened, + ## the new one may have the same value. + return key.fd in s and s.fds[key.fd] == key + + proc `[]`*(s: PSelector, fd: TSocketHandle): PSelectorKey = + ## Retrieves the selector key for ``fd``. + return s.fds[fd] + +elif defined(windows): + type + PSelector* = ref object + fds: TTable[TSocketHandle, PSelectorKey] + + proc register*(s: PSelector, fd: TSocketHandle, events: set[TEvent], + data: PObject): PSelectorKey {.discardable.} = + if s.fds.hasKey(fd): + raise newException(EInvalidValue, "File descriptor already exists.") + var sk = PSelectorKey(fd: fd, events: events, data: data) + s.fds[fd] = sk + result = sk + + proc update*(s: PSelector, fd: TSocketHandle, + events: set[TEvent]): PSelectorKey {.discardable.} = + ## Updates the events which ``fd`` wants notifications for. + if not s.fds.hasKey(fd): + raise newException(EInvalidValue, "File descriptor not found.") + + s.fds[fd].events = events + result = s.fds[fd] + + proc unregister*(s: PSelector, fd: TSocketHandle): PSelectorKey {.discardable.} = + result = s.fds[fd] + s.fds.del(fd) + + proc close*(s: PSelector) = nil + + proc timeValFromMilliseconds(timeout: int): TTimeVal = + if timeout != -1: + var seconds = timeout div 1000 + result.tv_sec = seconds.int32 + result.tv_usec = ((timeout - seconds * 1000) * 1000).int32 + + proc createFdSet(rd, wr: var TFdSet, fds: TTable[TSocketHandle, PSelectorKey], + m: var int) = + FD_ZERO(rd); FD_ZERO(wr) + for k, v in pairs(fds): + if EvRead in v.events: + m = max(m, int(k)) + FD_SET(k, rd) + if EvWrite in v.events: + m = max(m, int(k)) + FD_SET(k, wr) + + proc getReadyFDs(rd, wr: var TFdSet, fds: TTable[TSocketHandle, PSelectorKey]): + seq[TReadyInfo] = + result = @[] + for k, v in pairs(fds): + var events: set[TEvent] = {} + if FD_ISSET(k, rd) != 0'i32: + events = events + {EvRead} + if FD_ISSET(k, wr) != 0'i32: + events = events + {EvWrite} + result.add((v, events)) + + proc select(fds: TTable[TSocketHandle, PSelectorKey], timeout = 500): + seq[TReadyInfo] = + var tv {.noInit.}: TTimeVal = timeValFromMilliseconds(timeout) + + var rd, wr: TFdSet + var m = 0 + createFdSet(rd, wr, fds, m) + + var retCode = 0 + if timeout != -1: + retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, addr(tv))) + else: + retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, nil)) + + if retCode < 0: + OSError(OSLastError()) + elif retCode == 0: + return @[] + else: + return getReadyFDs(rd, wr, fds) + + proc select*(s: PSelector, timeout: int): seq[TReadyInfo] = + result = select(s.fds, timeout) + + proc newSelector*(): PSelector = + new result + result.fds = initTable[TSocketHandle, PSelectorKey]() + + proc contains*(s: PSelector, fd: TSocketHandle): bool = + return s.fds.hasKey(fd) + + proc `[]`*(s: PSelector, fd: TSocketHandle): PSelectorKey = + return s.fds[fd] + +elif defined(bsd) or defined(macosx): + # TODO: kqueue + {.error: "Sorry your platform is not supported yet.".} +else: + {.error: "Sorry your platform is not supported.".} when isMainModule: # Select() @@ -224,11 +242,12 @@ when isMainModule: sock: TSocket var sock = socket() + sock.setBlocking(false) sock.connect("irc.freenode.net", TPort(6667)) - var selector = newEpollSelector() + var selector = newSelector() var data = PSockWrapper(sock: sock) - let key = selector.register(sock.getFD.cint, {EvRead}, data) + let key = selector.register(sock.getFD, {EvWrite}, data) var i = 0 while true: let ready = selector.select(1000) @@ -236,6 +255,7 @@ when isMainModule: if ready.len > 0: echo ready[0].events i.inc if i == 6: + assert selector.unregister(sock.getFD).fd == sock.getFD selector.close() break @@ -246,4 +266,4 @@ when isMainModule: - \ No newline at end of file + diff --git a/lib/pure/sockets2.nim b/lib/pure/sockets2.nim deleted file mode 100644 index 22624bbad..000000000 --- a/lib/pure/sockets2.nim +++ /dev/null @@ -1,202 +0,0 @@ -# -# -# Nimrod's Runtime Library -# (c) Copyright 2014 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a low-level cross-platform sockets interface. Look -## at the ``net`` module for the higher-level version. - -import unsigned, os - -when hostos == "solaris": - {.passl: "-lsocket -lnsl".} - -when defined(Windows): - import winlean -else: - import posix - -export TSocketHandle, TSockaddr_in, TAddrinfo, INADDR_ANY, TSockAddr, TSockLen, - inet_ntoa - -type - - TPort* = distinct uint16 ## port type - - TDomain* = enum ## domain, which specifies the protocol family of the - ## created socket. Other domains than those that are listed - ## here are unsupported. - AF_UNIX, ## for local socket (using a file). Unsupported on Windows. - AF_INET = 2, ## for network protocol IPv4 or - AF_INET6 = 23 ## for network protocol IPv6. - - TType* = enum ## second argument to `socket` proc - SOCK_STREAM = 1, ## reliable stream-oriented service or Stream Sockets - SOCK_DGRAM = 2, ## datagram service or Datagram Sockets - SOCK_RAW = 3, ## raw protocols atop the network layer. - SOCK_SEQPACKET = 5 ## reliable sequenced packet service - - TProtocol* = enum ## third argument to `socket` proc - IPPROTO_TCP = 6, ## Transmission control protocol. - IPPROTO_UDP = 17, ## User datagram protocol. - IPPROTO_IP, ## Internet protocol. Unsupported on Windows. - IPPROTO_IPV6, ## Internet Protocol Version 6. Unsupported on Windows. - IPPROTO_RAW, ## Raw IP Packets Protocol. Unsupported on Windows. - IPPROTO_ICMP ## Control message protocol. Unsupported on Windows. - - TServent* {.pure, final.} = object ## information about a service - name*: string - aliases*: seq[string] - port*: TPort - proto*: string - - Thostent* {.pure, final.} = object ## information about a given host - name*: string - aliases*: seq[string] - addrtype*: TDomain - length*: int - addrList*: seq[string] - -when defined(windows): - let - OSInvalidSocket* = winlean.INVALID_SOCKET -else: - let - OSInvalidSocket* = posix.INVALID_SOCKET - -proc `==`*(a, b: TPort): bool {.borrow.} - ## ``==`` for ports. - -proc `$`*(p: TPort): string {.borrow.} - ## returns the port number as a string - -proc toInt*(domain: TDomain): cint - ## Converts the TDomain enum to a platform-dependent ``cint``. - -proc toInt*(typ: TType): cint - ## Converts the TType enum to a platform-dependent ``cint``. - -proc toInt*(p: TProtocol): cint - ## Converts the TProtocol enum to a platform-dependent ``cint``. - -when defined(posix): - proc toInt(domain: TDomain): cint = - case domain - of AF_UNIX: result = posix.AF_UNIX - of AF_INET: result = posix.AF_INET - of AF_INET6: result = posix.AF_INET6 - else: nil - - proc toInt(typ: TType): cint = - case typ - of SOCK_STREAM: result = posix.SOCK_STREAM - of SOCK_DGRAM: result = posix.SOCK_DGRAM - of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET - of SOCK_RAW: result = posix.SOCK_RAW - else: nil - - proc toInt(p: TProtocol): cint = - case p - of IPPROTO_TCP: result = posix.IPPROTO_TCP - of IPPROTO_UDP: result = posix.IPPROTO_UDP - of IPPROTO_IP: result = posix.IPPROTO_IP - of IPPROTO_IPV6: result = posix.IPPROTO_IPV6 - of IPPROTO_RAW: result = posix.IPPROTO_RAW - of IPPROTO_ICMP: result = posix.IPPROTO_ICMP - else: nil - -else: - proc toInt(domain: TDomain): cint = - result = toU16(ord(domain)) - - proc toInt(typ: TType): cint = - result = cint(ord(typ)) - - proc toInt(p: TProtocol): cint = - result = cint(ord(p)) - - -proc socket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, - protocol: TProtocol = IPPROTO_TCP): TSocketHandle = - ## Creates a new socket; returns `InvalidSocket` if an error occurs. - - # TODO: The function which will use this will raise EOS. - socket(toInt(domain), toInt(typ), toInt(protocol)) - -proc close*(socket: TSocketHandle) = - ## closes a socket. - when defined(windows): - discard winlean.closeSocket(socket) - else: - discard posix.close(socket) - # TODO: These values should not be discarded. An EOS should be raised. - # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times - -proc bindAddr*(socket: TSocketHandle, name: ptr TSockAddr, namelen: TSockLen): cint = - result = bindSocket(socket, name, namelen) - -proc listen*(socket: TSocketHandle, backlog = SOMAXCONN) {.tags: [FReadIO].} = - ## Marks ``socket`` as accepting connections. - ## ``Backlog`` specifies the maximum length of the - ## queue of pending connections. - when defined(windows): - if winlean.listen(socket, cint(backlog)) < 0'i32: osError(osLastError()) - else: - if posix.listen(socket, cint(backlog)) < 0'i32: osError(osLastError()) - -proc getAddrInfo*(address: string, port: TPort, af: TDomain = AF_INET, typ: TType = SOCK_STREAM, - prot: TProtocol = IPPROTO_TCP): ptr TAddrInfo = - ## - ## - ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``! - var hints: TAddrInfo - result = nil - hints.ai_family = toInt(af) - hints.ai_socktype = toInt(typ) - hints.ai_protocol = toInt(prot) - var gaiResult = getAddrInfo(address, $port, addr(hints), result) - if gaiResult != 0'i32: - when defined(windows): - OSError(OSLastError()) - else: - raise newException(EOS, $gai_strerror(gaiResult)) - -proc dealloc*(ai: ptr TAddrInfo) = - freeaddrinfo(ai) - -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. - ## On machines where the host byte order is the same as network byte order, - ## this is a no-op; otherwise, it performs a 4-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) - -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. - when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) - -proc htonl*(x: int32): int32 = - ## Converts 32-bit integers from host to network byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 4-byte swap operation. - result = sockets2.ntohl(x) - -proc htons*(x: int16): int16 = - ## Converts 16-bit positive integers from host to network byte order. - ## On machines where the host byte order is the same as network byte - ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. - result = sockets2.ntohs(x) - -when defined(Windows): - var wsa: TWSADATA - if WSAStartup(0x0101'i16, addr wsa) != 0: OSError(OSLastError()) \ No newline at end of file diff --git a/lib/pure/times.nim b/lib/pure/times.nim index de6c4e4fa..2fce235e2 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -557,6 +557,119 @@ proc `$`*(m: TMonth): string = "November", "December"] return lookup[m] +proc format_token(info: TTimeInfo, token: string, buf: var string) = + ## Helper of the format proc to parse individual tokens. + ## + ## Pass the found token in the user input string, and the buffer where the + ## final string is being built. This has to be a var value because certain + ## formatting tokens require modifying the previous characters. + case token + of "d": + buf.add($info.monthday) + of "dd": + if info.monthday < 10: + buf.add("0") + buf.add($info.monthday) + of "ddd": + buf.add(($info.weekday)[0 .. 2]) + of "dddd": + buf.add($info.weekday) + of "h": + buf.add($(if info.hour > 12: info.hour - 12 else: info.hour)) + of "hh": + let amerHour = if info.hour > 12: info.hour - 12 else: info.hour + if amerHour < 10: + buf.add('0') + buf.add($amerHour) + of "H": + buf.add($info.hour) + of "HH": + if info.hour < 10: + buf.add('0') + buf.add($info.hour) + of "m": + buf.add($info.minute) + of "mm": + if info.minute < 10: + buf.add('0') + buf.add($info.minute) + of "M": + buf.add($(int(info.month)+1)) + of "MM": + if info.month < mOct: + buf.add('0') + buf.add($(int(info.month)+1)) + of "MMM": + buf.add(($info.month)[0..2]) + of "MMMM": + buf.add($info.month) + of "s": + buf.add($info.second) + of "ss": + if info.second < 10: + buf.add('0') + buf.add($info.second) + of "t": + if info.hour >= 12: + buf.add('P') + else: buf.add('A') + of "tt": + if info.hour >= 12: + buf.add("PM") + else: buf.add("AM") + of "y": + var fr = ($info.year).len()-1 + if fr < 0: fr = 0 + buf.add(($info.year)[fr .. ($info.year).len()-1]) + of "yy": + var fr = ($info.year).len()-2 + if fr < 0: fr = 0 + var fyear = ($info.year)[fr .. ($info.year).len()-1] + if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear + buf.add(fyear) + of "yyy": + var fr = ($info.year).len()-3 + if fr < 0: fr = 0 + var fyear = ($info.year)[fr .. ($info.year).len()-1] + if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear + buf.add(fyear) + of "yyyy": + var fr = ($info.year).len()-4 + if fr < 0: fr = 0 + var fyear = ($info.year)[fr .. ($info.year).len()-1] + if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear + buf.add(fyear) + of "yyyyy": + var fr = ($info.year).len()-5 + if fr < 0: fr = 0 + var fyear = ($info.year)[fr .. ($info.year).len()-1] + if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear + buf.add(fyear) + of "z": + let hrs = (info.timezone div 60) div 60 + buf.add($hrs) + of "zz": + let hrs = (info.timezone div 60) div 60 + + buf.add($hrs) + if hrs.abs < 10: + var atIndex = buf.len-(($hrs).len-(if hrs < 0: 1 else: 0)) + buf.insert("0", atIndex) + of "zzz": + let hrs = (info.timezone div 60) div 60 + + buf.add($hrs & ":00") + if hrs.abs < 10: + var atIndex = buf.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0)) + buf.insert("0", atIndex) + of "ZZZ": + buf.add(info.tzname) + of "": + discard + else: + raise newException(EInvalidValue, "Invalid format string: " & token) + + proc format*(info: TTimeInfo, f: string): string = ## This function formats `info` as specified by `f`. The following format ## specifiers are available: @@ -591,8 +704,11 @@ proc format*(info: TTimeInfo, f: string): string = ## ZZZ Displays the name of the timezone. ``GMT -> GMT``, ``EST -> EST`` ## ========== ================================================================================= ================================================ ## - ## Other strings can be inserted by putting them in ``''``. For example ``hh'->'mm`` will give ``01->56``. - ## The following characters can be inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` ``,`` + ## Other strings can be inserted by putting them in ``''``. For example + ## ``hh'->'mm`` will give ``01->56``. The following characters can be + ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` + ## ``,``. However you don't need to necessarily separate format specifiers, a + ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too. result = "" var i = 0 @@ -600,112 +716,8 @@ proc format*(info: TTimeInfo, f: string): string = while true: case f[i] of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': - case currentF - of "d": - result.add($info.monthday) - of "dd": - if info.monthday < 10: - result.add("0") - result.add($info.monthday) - of "ddd": - result.add(($info.weekday)[0 .. 2]) - of "dddd": - result.add($info.weekday) - of "h": - result.add($(if info.hour > 12: info.hour - 12 else: info.hour)) - of "hh": - let amerHour = if info.hour > 12: info.hour - 12 else: info.hour - if amerHour < 10: - result.add('0') - result.add($amerHour) - of "H": - result.add($info.hour) - of "HH": - if info.hour < 10: - result.add('0') - result.add($info.hour) - of "m": - result.add($info.minute) - of "mm": - if info.minute < 10: - result.add('0') - result.add($info.minute) - of "M": - result.add($(int(info.month)+1)) - of "MM": - if info.month < mOct: - result.add('0') - result.add($(int(info.month)+1)) - of "MMM": - result.add(($info.month)[0..2]) - of "MMMM": - result.add($info.month) - of "s": - result.add($info.second) - of "ss": - if info.second < 10: - result.add('0') - result.add($info.second) - of "t": - if info.hour >= 12: - result.add('P') - else: result.add('A') - of "tt": - if info.hour >= 12: - result.add("PM") - else: result.add("AM") - of "y": - var fr = ($info.year).len()-1 - if fr < 0: fr = 0 - result.add(($info.year)[fr .. ($info.year).len()-1]) - of "yy": - var fr = ($info.year).len()-2 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear - result.add(fyear) - of "yyy": - var fr = ($info.year).len()-3 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear - result.add(fyear) - of "yyyy": - var fr = ($info.year).len()-4 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear - result.add(fyear) - of "yyyyy": - var fr = ($info.year).len()-5 - if fr < 0: fr = 0 - var fyear = ($info.year)[fr .. ($info.year).len()-1] - if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear - result.add(fyear) - of "z": - let hrs = (info.timezone div 60) div 60 - result.add($hrs) - of "zz": - let hrs = (info.timezone div 60) div 60 - - result.add($hrs) - if hrs.abs < 10: - var atIndex = result.len-(($hrs).len-(if hrs < 0: 1 else: 0)) - result.insert("0", atIndex) - of "zzz": - let hrs = (info.timezone div 60) div 60 - - result.add($hrs & ":00") - if hrs.abs < 10: - var atIndex = result.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0)) - result.insert("0", atIndex) - of "ZZZ": - result.add(info.tzname) - of "": - discard - else: - raise newException(EInvalidValue, "Invalid format string: " & currentF) - + format_token(info, currentF, result) + currentF = "" if f[i] == '\0': break @@ -716,7 +728,15 @@ proc format*(info: TTimeInfo, f: string): string = inc(i) else: result.add(f[i]) - else: currentF.add(f[i]) + else: + # Check if the letter being added matches previous accumulated buffer. + if currentF.len < 1 or currentF[high(currentF)] == f[i]: + currentF.add(f[i]) + else: + format_token(info, currentF, result) + dec(i) # Move position back to re-process the character separately. + currentF = "" + inc(i) {.pop.} @@ -727,11 +747,15 @@ when isMainModule: var t = getGMTime(fromSeconds(2147483647)) echo t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") + echo t.format("ddd ddMMMhhmmssZZZyyyy") assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" + assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC" + + assert t.format("yyyyMMddhhmmss") == "20380119031407" var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & diff --git a/lib/stdlib.babel b/lib/stdlib.babel new file mode 100644 index 000000000..f22598aba --- /dev/null +++ b/lib/stdlib.babel @@ -0,0 +1,6 @@ +[Package] +name = "stdlib" +version = "0.9.0" +author = "Dominik Picheta" +description = "Nimrod's standard library." +license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index fde24a9d0..38839387d 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -185,6 +185,8 @@ proc `..`*[T](b: T): TSlice[T] {.noSideEffect, inline.} = when not defined(niminheritable): {.pragma: inheritable.} +when not defined(nimunion): + {.pragma: unchecked.} const NoFakeVars* = defined(NimrodVM) ## true if the backend doesn't support \ ## "fake variables" like 'var EBADF {.importc.}: cint'. @@ -194,9 +196,10 @@ when not defined(JS): TGenericSeq {.compilerproc, pure, inheritable.} = object len, reserved: int PGenericSeq {.exportc.} = ptr TGenericSeq + UncheckedCharArray {.unchecked.} = array[0..100_000_000, char] # len and space without counting the terminating zero: NimStringDesc {.compilerproc, final.} = object of TGenericSeq - data: array[0..100_000_000, char] + data: UncheckedCharArray NimString = ptr NimStringDesc when not defined(JS) and not defined(NimrodVM): @@ -257,6 +260,7 @@ type ## system raises. EIO* = object of ESystem ## raised if an IO error occured. EOS* = object of ESystem ## raised if an operating system service failed. + errorCode*: int32 ## OS-defined error code describing this error. EInvalidLibrary* = object of EOS ## raised if a dynamic library ## could not be loaded. EResourceExhausted* = object of ESystem ## raised if a resource request @@ -1161,6 +1165,14 @@ when not defined(nimrodVM): ## from it before writing to it is undefined behaviour! ## The allocated memory belongs to its allocating thread! ## Use `allocShared` to allocate from a shared heap. + proc createU*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + ## allocates a new memory block with at least ``T.sizeof * size`` + ## bytes. The block has to be freed with ``resize(block, 0)`` or + ## ``free(block)``. The block is not initialized, so reading + ## from it before writing to it is undefined behaviour! + ## The allocated memory belongs to its allocating thread! + ## Use `createSharedU` to allocate from a shared heap. + cast[ptr T](alloc(T.sizeof * size)) proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].} ## allocates a new memory block with at least ``size`` bytes. The ## block has to be freed with ``realloc(block, 0)`` or @@ -1168,14 +1180,31 @@ when not defined(nimrodVM): ## containing zero, so it is somewhat safer than ``alloc``. ## The allocated memory belongs to its allocating thread! ## Use `allocShared0` to allocate from a shared heap. - proc realloc*(p: pointer, newsize: int): pointer {.noconv, rtl, tags: [].} + proc create*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + ## allocates a new memory block with at least ``T.sizeof * size`` + ## bytes. The block has to be freed with ``resize(block, 0)`` or + ## ``free(block)``. The block is initialized with all bytes + ## containing zero, so it is somewhat safer than ``createU``. + ## The allocated memory belongs to its allocating thread! + ## Use `createShared` to allocate from a shared heap. + cast[ptr T](alloc0(T.sizeof * size)) + proc realloc*(p: pointer, newSize: int): pointer {.noconv, rtl, tags: [].} ## grows or shrinks a given memory block. If p is **nil** then a new ## memory block is returned. In either way the block has at least - ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil** + ## ``newSize`` bytes. If ``newSize == 0`` and p is not **nil** ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to ## be freed with ``dealloc``. ## The allocated memory belongs to its allocating thread! ## Use `reallocShared` to reallocate from a shared heap. + proc resize*[T](p: ptr T, newSize: Natural): ptr T {.inline.} = + ## grows or shrinks a given memory block. If p is **nil** then a new + ## memory block is returned. In either way the block has at least + ## ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is not + ## **nil** ``resize`` calls ``free(p)``. In other cases the block + ## has to be freed with ``free``. The allocated memory belongs to + ## its allocating thread! + ## Use `resizeShared` to reallocate from a shared heap. + cast[ptr T](realloc(p, T.sizeof * newSize)) proc dealloc*(p: pointer) {.noconv, rtl, tags: [].} ## frees the memory allocated with ``alloc``, ``alloc0`` or ## ``realloc``. This procedure is dangerous! If one forgets to @@ -1184,31 +1213,60 @@ when not defined(nimrodVM): ## or other memory may be corrupted. ## The freed memory must belong to its allocating thread! ## Use `deallocShared` to deallocate from a shared heap. - + proc free*[T](p: ptr T) {.inline.} = + dealloc(p) proc allocShared*(size: int): pointer {.noconv, rtl.} ## allocates a new memory block on the shared heap with at ## least ``size`` bytes. The block has to be freed with ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block ## is not initialized, so reading from it before writing to it is ## undefined behaviour! + proc createSharedU*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + ## allocates a new memory block on the shared heap with at + ## least ``T.sizeof * size`` bytes. The block has to be freed with + ## ``resizeShared(block, 0)`` or ``freeShared(block)``. The block + ## is not initialized, so reading from it before writing to it is + ## undefined behaviour! + cast[ptr T](allocShared(T.sizeof * size)) proc allocShared0*(size: int): pointer {.noconv, rtl.} ## allocates a new memory block on the shared heap with at ## least ``size`` bytes. The block has to be freed with ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. ## The block is initialized with all bytes ## containing zero, so it is somewhat safer than ``allocShared``. - proc reallocShared*(p: pointer, newsize: int): pointer {.noconv, rtl.} + proc createShared*(T: typedesc, size = 1.Positive): ptr T {.inline.} = + ## allocates a new memory block on the shared heap with at + ## least ``T.sizeof * size`` bytes. The block has to be freed with + ## ``resizeShared(block, 0)`` or ``freeShared(block)``. + ## The block is initialized with all bytes + ## containing zero, so it is somewhat safer than ``createSharedU``. + cast[ptr T](allocShared0(T.sizeof * size)) + proc reallocShared*(p: pointer, newSize: int): pointer {.noconv, rtl.} ## grows or shrinks a given memory block on the heap. If p is **nil** - ## then a new memory block is returned. In either way the block has at least - ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil** + ## then a new memory block is returned. In either way the block has at + ## least ``newSize`` bytes. If ``newSize == 0`` and p is not **nil** ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the ## block has to be freed with ``deallocShared``. + proc resizeShared*[T](p: ptr T, newSize: Natural): ptr T {.inline.} = + ## grows or shrinks a given memory block on the heap. If p is **nil** + ## then a new memory block is returned. In either way the block has at + ## least ``T.sizeof * newSize`` bytes. If ``newSize == 0`` and p is + ## not **nil** ``resizeShared`` calls ``freeShared(p)``. In other + ## cases the block has to be freed with ``freeShared``. + cast[ptr T](reallocShared(p, T.sizeof * newSize)) proc deallocShared*(p: pointer) {.noconv, rtl.} ## frees the memory allocated with ``allocShared``, ``allocShared0`` or ## ``reallocShared``. This procedure is dangerous! If one forgets to ## free the memory a leak occurs; if one tries to access freed ## memory (or just freeing it twice!) a core dump may happen ## or other memory may be corrupted. + proc freeShared*[T](p: ptr T) {.inline.} = + ## frees the memory allocated with ``createShared``, ``createSharedU`` or + ## ``resizeShared``. This procedure is dangerous! If one forgets to + ## free the memory a leak occurs; if one tries to access freed + ## memory (or just freeing it twice!) a core dump may happen + ## or other memory may be corrupted. + deallocShared(p) proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} ## swaps the values `a` and `b`. This is often more efficient than @@ -1399,20 +1457,6 @@ iterator items*[IX, T](a: array[IX, T]): T {.inline.} = if i >= high(IX): break inc(i) -iterator items*[T](a: seq[T]): T {.inline.} = - ## iterates over each item of `a`. - var i = 0 - while i < len(a): - yield a[i] - inc(i) - -iterator items*(a: string): char {.inline.} = - ## iterates over each item of `a`. - var i = 0 - while i < len(a): - yield a[i] - inc(i) - iterator items*[T](a: set[T]): T {.inline.} = ## iterates over each element of `a`. `items` iterates only over the ## elements that are really in the set (and not over the ones the set is @@ -1514,7 +1558,7 @@ when not defined(NimrodVM): proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = result = cast[pointer](x) else: - proc seqToPtr[T](x: seq[T]): pointer {.noStackFrame, nosideeffect.} = + proc seqToPtr[T](x: seq[T]): pointer {.asmNoStackFrame, nosideeffect.} = asm """return `x`""" proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} = @@ -1802,7 +1846,7 @@ type len*: int ## length of the inspectable slots when defined(JS): - proc add*(x: var string, y: cstring) {.noStackFrame.} = + proc add*(x: var string, y: cstring) {.asmNoStackFrame.} = asm """ var len = `x`[0].length-1; for (var i = 0; i < `y`.length; ++i) { @@ -2019,8 +2063,10 @@ when not defined(JS): #and not defined(NimrodVM): ## Flushes `f`'s buffer. proc readAll*(file: TFile): TaintedString {.tags: [FReadIO].} - ## Reads all data from the stream `file`. Raises an IO exception - ## in case of an error + ## Reads all data from the stream `file`. + ## + ## Raises an IO exception in case of an error. It is an error if the + ## current file position is not at the beginning of the file. proc readFile*(filename: string): TaintedString {.tags: [FReadIO].} ## Opens a file named `filename` for reading. Then calls `readAll` @@ -2309,12 +2355,32 @@ when not defined(JS): #and not defined(NimrodVM): when not defined(NimrodVM): proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} - ## can be used to mark a condition to be likely. This is a hint for the - ## optimizer. + ## Hints the optimizer that `val` is likely going to be true. + ## + ## You can use this proc to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## + ## .. code-block:: nimrod + ## for value in inputValues: + ## if likely(value <= 100): + ## process(value) + ## else: + ## echo "Value too big!" proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} - ## can be used to mark a condition to be unlikely. This is a hint for the - ## optimizer. + ## Hints the optimizer that `val` is likely going to be false. + ## + ## You can use this proc to decorate a branch condition. On certain + ## platforms this can help the processor predict better which branch is + ## going to be run. Example: + ## + ## .. code-block:: nimrod + ## for value in inputValues: + ## if unlikely(value > 100): + ## echo "Value too big!" + ## else: + ## process(value) proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = ## retrieves the raw proc pointer of the closure `x`. This is @@ -2612,6 +2678,24 @@ template doAssert*(cond: bool, msg = "") = if not cond: raiseAssert(astToStr(cond) & ' ' & msg) +iterator items*[T](a: seq[T]): T {.inline.} = + ## iterates over each item of `a`. + var i = 0 + let L = len(a) + while i < L: + yield a[i] + inc(i) + assert(len(a) == L, "seq modified while iterating over it") + +iterator items*(a: string): char {.inline.} = + ## iterates over each item of `a`. + var i = 0 + let L = len(a) + while i < L: + yield a[i] + inc(i) + assert(len(a) == L, "string modified while iterating over it") + when not defined(nimhygiene): {.pragma: inject.} diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim index d764a6672..d9b3aebac 100644 --- a/lib/system/arithm.nim +++ b/lib/system/arithm.nim @@ -111,7 +111,7 @@ const when asmVersion and not defined(gcc) and not defined(llvm_gcc): # assembler optimized versions for compilers that # have an intel syntax assembler: - proc addInt(a, b: int): int {.compilerProc, noStackFrame.} = + proc addInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = # a in eax, and b in edx asm """ mov eax, `a` @@ -121,7 +121,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc): theEnd: """ - proc subInt(a, b: int): int {.compilerProc, noStackFrame.} = + proc subInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = asm """ mov eax, `a` sub eax, `b` @@ -130,7 +130,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc): theEnd: """ - proc negInt(a: int): int {.compilerProc, noStackFrame.} = + proc negInt(a: int): int {.compilerProc, asmNoStackFrame.} = asm """ mov eax, `a` neg eax @@ -139,7 +139,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc): theEnd: """ - proc divInt(a, b: int): int {.compilerProc, noStackFrame.} = + proc divInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = asm """ mov eax, `a` mov ecx, `b` @@ -150,7 +150,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc): theEnd: """ - proc modInt(a, b: int): int {.compilerProc, noStackFrame.} = + proc modInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = asm """ mov eax, `a` mov ecx, `b` @@ -162,7 +162,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc): mov eax, edx """ - proc mulInt(a, b: int): int {.compilerProc, noStackFrame.} = + proc mulInt(a, b: int): int {.compilerProc, asmNoStackFrame.} = asm """ mov eax, `a` mov ecx, `b` diff --git a/lib/system/endb.nim b/lib/system/endb.nim index 2d6a25824..74d3c37ac 100644 --- a/lib/system/endb.nim +++ b/lib/system/endb.nim @@ -179,7 +179,7 @@ proc scanAndAppendWord(src: cstring, a: var TStaticStr, start: int): int = while True: case src[result] of 'a'..'z', '0'..'9': add(a, src[result]) - of '_': nil # just skip it + of '_': discard # just skip it of 'A'..'Z': add(a, chr(ord(src[result]) - ord('A') + ord('a'))) else: break inc(result) @@ -203,7 +203,7 @@ proc scanNumber(src: cstring, a: var int, start: int): int = while true: case src[result] of '0'..'9': a = a * 10 + ord(src[result]) - ord('0') - of '_': nil # skip underscores (nice for long line numbers) + of '_': discard # skip underscores (nice for long line numbers) else: break inc(result) @@ -524,7 +524,7 @@ proc lineHookImpl() {.nimcall.} = of dbBreakpoints: # debugger is only interested in breakpoints checkForBreakpoint() - else: nil + else: discard proc watchpointHookImpl(name: cstring) {.nimcall.} = dbgWriteStackTrace(framePtr) diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 4fc5f479b..1720804c4 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -23,9 +23,9 @@ type PCallFrame = ptr TCallFrame TCallFrame {.importc, nodecl, final.} = object prev: PCallFrame - procname: CString + procname: cstring line: int # current line number - filename: CString + filename: cstring var framePtr {.importc, nodecl, volatile.}: PCallFrame @@ -48,7 +48,7 @@ proc getCurrentExceptionMsg*(): string = proc auxWriteStackTrace(f: PCallFrame): string = type - TTempFrame = tuple[procname: CString, line: int] + TTempFrame = tuple[procname: cstring, line: int] var it = f i = 0 @@ -84,7 +84,7 @@ proc rawWriteStackTrace(): string = framePtr = nil proc raiseException(e: ref E_Base, ename: cstring) {. - compilerproc, noStackFrame.} = + compilerproc, asmNoStackFrame.} = e.name = ename if excHandler != nil: excHandler.exc = e @@ -104,7 +104,7 @@ proc raiseException(e: ref E_Base, ename: cstring) {. alert(buf) asm """throw `e`;""" -proc reraiseException() {.compilerproc, noStackFrame.} = +proc reraiseException() {.compilerproc, asmNoStackFrame.} = if excHandler == nil: raise newException(ENoExceptionToReraise, "no exception to reraise") else: @@ -125,7 +125,7 @@ proc raiseIndexError() {.compilerproc, noreturn.} = proc raiseFieldError(f: string) {.compilerproc, noreturn.} = raise newException(EInvalidField, f & " is not accessible") -proc SetConstr() {.varargs, noStackFrame, compilerproc.} = +proc SetConstr() {.varargs, asmNoStackFrame, compilerproc.} = asm """ var result = {}; for (var i = 0; i < arguments.length; ++i) { @@ -141,7 +141,7 @@ proc SetConstr() {.varargs, noStackFrame, compilerproc.} = return result; """ -proc cstrToNimstr(c: cstring): string {.noStackFrame, compilerproc.} = +proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} = asm """ var result = []; for (var i = 0; i < `c`.length; ++i) { @@ -151,7 +151,7 @@ proc cstrToNimstr(c: cstring): string {.noStackFrame, compilerproc.} = return result; """ -proc toJSStr(s: string): cstring {.noStackFrame, compilerproc.} = +proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} = asm """ var len = `s`.length-1; var result = new Array(len); @@ -162,7 +162,7 @@ proc toJSStr(s: string): cstring {.noStackFrame, compilerproc.} = return result.join(""); """ -proc mnewString(len: int): string {.noStackFrame, compilerproc.} = +proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} = asm """ var result = new Array(`len`+1); result[0] = 0; @@ -170,7 +170,7 @@ proc mnewString(len: int): string {.noStackFrame, compilerproc.} = return result; """ -proc SetCard(a: int): int {.compilerproc, noStackFrame.} = +proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} = # argument type is a fake asm """ var result = 0; @@ -178,14 +178,14 @@ proc SetCard(a: int): int {.compilerproc, noStackFrame.} = return result; """ -proc SetEq(a, b: int): bool {.compilerproc, noStackFrame.} = +proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} = asm """ for (var elem in `a`) { if (!`b`[elem]) return false; } for (var elem in `b`) { if (!`a`[elem]) return false; } return true; """ -proc SetLe(a, b: int): bool {.compilerproc, noStackFrame.} = +proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} = asm """ for (var elem in `a`) { if (!`b`[elem]) return false; } return true; @@ -194,7 +194,7 @@ proc SetLe(a, b: int): bool {.compilerproc, noStackFrame.} = proc SetLt(a, b: int): bool {.compilerproc.} = result = SetLe(a, b) and not SetEq(a, b) -proc SetMul(a, b: int): int {.compilerproc, noStackFrame.} = +proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} = asm """ var result = {}; for (var elem in `a`) { @@ -203,7 +203,7 @@ proc SetMul(a, b: int): int {.compilerproc, noStackFrame.} = return result; """ -proc SetPlus(a, b: int): int {.compilerproc, noStackFrame.} = +proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} = asm """ var result = {}; for (var elem in `a`) { result[elem] = true; } @@ -211,7 +211,7 @@ proc SetPlus(a, b: int): int {.compilerproc, noStackFrame.} = return result; """ -proc SetMinus(a, b: int): int {.compilerproc, noStackFrame.} = +proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} = asm """ var result = {}; for (var elem in `a`) { @@ -220,7 +220,7 @@ proc SetMinus(a, b: int): int {.compilerproc, noStackFrame.} = return result; """ -proc cmpStrings(a, b: string): int {.noStackFrame, compilerProc.} = +proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerProc.} = asm """ if (`a` == `b`) return 0; if (!`a`) return -1; @@ -234,7 +234,7 @@ proc cmpStrings(a, b: string): int {.noStackFrame, compilerProc.} = proc cmp(x, y: string): int = return cmpStrings(x, y) -proc eqStrings(a, b: string): bool {.noStackFrame, compilerProc.} = +proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerProc.} = asm """ if (`a` == `b`) return true; if ((!`a`) || (!`b`)) return false; @@ -300,7 +300,7 @@ type setAttributeNode*: proc (attr: ref TNode) {.nimcall.} when defined(kwin): - proc rawEcho {.compilerproc, nostackframe.} = + proc rawEcho {.compilerproc, asmNoStackFrame.} = asm """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { @@ -312,7 +312,7 @@ when defined(kwin): elif defined(nodejs): proc ewriteln(x: cstring) = log(x) - proc rawEcho {.compilerproc, nostackframe.} = + proc rawEcho {.compilerproc, asmNoStackFrame.} = asm """ var buf = ""; for (var i = 0; i < arguments.length; ++i) { @@ -345,42 +345,42 @@ else: node.appendChild(document.createElement("br")) # Arithmetic: -proc addInt(a, b: int): int {.noStackFrame, compilerproc.} = +proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ var result = `a` + `b`; if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); return result; """ -proc subInt(a, b: int): int {.noStackFrame, compilerproc.} = +proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ var result = `a` - `b`; if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); return result; """ -proc mulInt(a, b: int): int {.noStackFrame, compilerproc.} = +proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ var result = `a` * `b`; if (result > 2147483647 || result < -2147483648) `raiseOverflow`(); return result; """ -proc divInt(a, b: int): int {.noStackFrame, compilerproc.} = +proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); return Math.floor(`a` / `b`); """ -proc modInt(a, b: int): int {.noStackFrame, compilerproc.} = +proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 2147483647) `raiseOverflow`(); return Math.floor(`a` % `b`); """ -proc addInt64(a, b: int): int {.noStackFrame, compilerproc.} = +proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ var result = `a` + `b`; if (result > 9223372036854775807 @@ -388,7 +388,7 @@ proc addInt64(a, b: int): int {.noStackFrame, compilerproc.} = return result; """ -proc subInt64(a, b: int): int {.noStackFrame, compilerproc.} = +proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ var result = `a` - `b`; if (result > 9223372036854775807 @@ -396,7 +396,7 @@ proc subInt64(a, b: int): int {.noStackFrame, compilerproc.} = return result; """ -proc mulInt64(a, b: int): int {.noStackFrame, compilerproc.} = +proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ var result = `a` * `b`; if (result > 9223372036854775807 @@ -404,90 +404,89 @@ proc mulInt64(a, b: int): int {.noStackFrame, compilerproc.} = return result; """ -proc divInt64(a, b: int): int {.noStackFrame, compilerproc.} = +proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); return Math.floor(`a` / `b`); """ -proc modInt64(a, b: int): int {.noStackFrame, compilerproc.} = +proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} = asm """ if (`b` == 0) `raiseDivByZero`(); if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`(); return Math.floor(`a` % `b`); """ -proc NegInt(a: int): int {.compilerproc.} = +proc negInt(a: int): int {.compilerproc.} = result = a*(-1) -proc NegInt64(a: int64): int64 {.compilerproc.} = +proc negInt64(a: int64): int64 {.compilerproc.} = result = a*(-1) -proc AbsInt(a: int): int {.compilerproc.} = +proc absInt(a: int): int {.compilerproc.} = result = if a < 0: a*(-1) else: a -proc AbsInt64(a: int64): int64 {.compilerproc.} = +proc absInt64(a: int64): int64 {.compilerproc.} = result = if a < 0: a*(-1) else: a -proc LeU(a, b: int): bool {.compilerproc.} = +proc leU(a, b: int): bool {.compilerproc.} = result = abs(a) <= abs(b) -proc LtU(a, b: int): bool {.compilerproc.} = +proc ltU(a, b: int): bool {.compilerproc.} = result = abs(a) < abs(b) -proc LeU64(a, b: int64): bool {.compilerproc.} = +proc leU64(a, b: int64): bool {.compilerproc.} = result = abs(a) <= abs(b) - -proc LtU64(a, b: int64): bool {.compilerproc.} = +proc ltU64(a, b: int64): bool {.compilerproc.} = result = abs(a) < abs(b) -proc AddU(a, b: int): int {.compilerproc.} = +proc addU(a, b: int): int {.compilerproc.} = result = abs(a) + abs(b) -proc AddU64(a, b: int64): int64 {.compilerproc.} = +proc addU64(a, b: int64): int64 {.compilerproc.} = result = abs(a) + abs(b) -proc SubU(a, b: int): int {.compilerproc.} = +proc subU(a, b: int): int {.compilerproc.} = result = abs(a) - abs(b) -proc SubU64(a, b: int64): int64 {.compilerproc.} = +proc subU64(a, b: int64): int64 {.compilerproc.} = result = abs(a) - abs(b) -proc MulU(a, b: int): int {.compilerproc.} = +proc mulU(a, b: int): int {.compilerproc.} = result = abs(a) * abs(b) -proc MulU64(a, b: int64): int64 {.compilerproc.} = +proc mulU64(a, b: int64): int64 {.compilerproc.} = result = abs(a) * abs(b) -proc DivU(a, b: int): int {.compilerproc.} = +proc divU(a, b: int): int {.compilerproc.} = result = abs(a) div abs(b) -proc DivU64(a, b: int64): int64 {.compilerproc.} = +proc divU64(a, b: int64): int64 {.compilerproc.} = result = abs(a) div abs(b) -proc ModU(a, b: int): int {.compilerproc.} = +proc modU(a, b: int): int {.compilerproc.} = result = abs(a) mod abs(b) -proc ModU64(a, b: int64): int64 {.compilerproc.} = +proc modU64(a, b: int64): int64 {.compilerproc.} = result = abs(a) mod abs(b) -proc Ze(a: int): int {.compilerproc.} = +proc ze*(a: int): int {.compilerproc.} = result = a -proc Ze64(a: int64): int64 {.compilerproc.} = + +proc ze64*(a: int64): int64 {.compilerproc.} = result = a -proc ToU8(a: int): int8 {.noStackFrame, compilerproc.} = +proc ToU8(a: int): int8 {.asmNoStackFrame, compilerproc.} = asm """ return `a`; """ -proc ToU16(a: int): int16 {.noStackFrame, compilerproc.} = +proc ToU16(a: int): int16 {.asmNoStackFrame, compilerproc.} = asm """ return `a`; """ -proc ToU32(a: int): int32 {.noStackFrame, compilerproc.} = +proc ToU32(a: int): int32 {.asmNoStackFrame, compilerproc.} = asm """ return `a`; """ - proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b @@ -500,9 +499,9 @@ proc isFatPointer(ti: PNimType): bool = tyArray, tyArrayConstr, tyTuple, tyOpenArray, tySet, tyVar, tyRef, tyPtr} -proc NimCopy(x: pointer, ti: PNimType): pointer {.compilerproc.} +proc nimCopy(x: pointer, ti: PNimType): pointer {.compilerproc.} -proc NimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} = +proc nimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} = case n.kind of nkNone: sysAssert(false, "NimCopyAux") of nkSlot: @@ -518,7 +517,7 @@ proc NimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} = } """ -proc NimCopy(x: pointer, ti: PNimType): pointer = +proc nimCopy(x: pointer, ti: PNimType): pointer = case ti.kind of tyPtr, tyRef, tyVar, tyNil: if not isFatPointer(ti): @@ -586,7 +585,7 @@ proc genericReset(x: Pointer, ti: PNimType): pointer {.compilerproc.} = result = nil proc ArrayConstr(len: int, value: pointer, typ: PNimType): pointer {. - noStackFrame, compilerproc.} = + asmNoStackFrame, compilerproc.} = # types are fake asm """ var result = new Array(`len`); @@ -620,7 +619,7 @@ proc isObj(obj, subclass: PNimType): bool {.compilerproc.} = x = x.base return true -proc addChar(x: string, c: char) {.compilerproc, noStackFrame.} = +proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} = asm """ `x`[`x`.length-1] = `c`; `x`.push(0); """ diff --git a/lib/system/repr.nim b/lib/system/repr.nim index cd3f7c3f4..7c1a68bc7 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -59,7 +59,11 @@ proc reprChar(x: char): string {.compilerRtl.} = proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = # we read an 'int' but this may have been too large, so mask the other bits: - let e = e and (1 shl (typ.size*8)-1) + let e = if typ.size == 1: e and 0xff + elif typ.size == 2: e and 0xffff + else: e + # XXX we need a proper narrowing based on signedness here + #e and ((1 shl (typ.size*8)) - 1) if ntfEnumHole notin typ.flags: if e <% typ.node.len: return $typ.node.sons[e].name diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 6c8fa4882..4d87cf4b2 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -199,14 +199,14 @@ else: importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall.} proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall.} - proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. + proc createDirectoryA*(pathName: cstring, security: Pointer=nil): int32 {. importc: "CreateDirectoryA", dynlib: "kernel32", stdcall.} proc removeDirectoryA*(lpPathName: cstring): int32 {. importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall.} proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA".} - proc getModuleFileNameA*(handle: THandle, buf: cstring, size: int32): int32 {. + proc getModuleFileNameA*(handle: THandle, buf: CString, size: int32): int32 {. importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} when useWinUnicode: @@ -304,7 +304,7 @@ else: dwFileAttributes: int32): WINBOOL {. stdcall, dynlib: "kernel32", importc: "SetFileAttributesA".} - proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, + proc copyFileA*(lpExistingFileName, lpNewFileName: CString, bFailIfExists: cint): cint {. importc: "CopyFileA", stdcall, dynlib: "kernel32".} @@ -456,6 +456,7 @@ var SO_DONTLINGER* {.importc, header: "Winsock2.h".}: cint SO_EXCLUSIVEADDRUSE* {.importc, header: "Winsock2.h".}: cint # disallow local address reuse + SO_ERROR* {.importc, header: "Winsock2.h".}: cint proc `==`*(x, y: TSocketHandle): bool {.borrow.} diff --git a/lib/wrappers/mongo.nim b/lib/wrappers/mongo.nim deleted file mode 100644 index 098b4f4d3..000000000 --- a/lib/wrappers/mongo.nim +++ /dev/null @@ -1,1204 +0,0 @@ -# -# -# Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module is a wrapper for the `mongodb`:idx: client C library. -## It allows you to connect to a mongo-server instance, send commands and -## receive replies. - -# -# Copyright 2009-2011 10gen Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import oids, times - -{.deadCodeElim: on.} - -when defined(windows): - const - mongodll* = "mongoc.dll" - bsondll* = "bson.dll" -elif defined(macosx): - const - mongodll* = "libmongoc.dylib" - bsondll* = "libbson.dylib" -else: - const - mongodll* = "libmongoc.so" - bsondll* = "libbson.so" - -# -# This package supports both compile-time and run-time determination of CPU -# byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be -# compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is -# defined as non-zero, the code will be compiled to run only on big-endian -# CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to -# run on either big- or little-endian CPUs, but will run slightly less -# efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. -# - -type - Tmd5_state*{.pure, final.} = object - count*: array[0..2 - 1, int32] # message length in bits, lsw first - abcd*: array[0..4 - 1, int32] # digest buffer - buf*: array[0..64 - 1, byte] # accumulate block - - -proc sock_init*(): cint{.stdcall, importc: "mongo_sock_init", dynlib: mongodll.} -const - OK* = 0 - ERROR* = - 1 - SIZE_OVERFLOW* = 1 - defaultHost* = "127.0.0.1" - defaultPort* = 27017 - -type - TValidity* = enum ## validity - VALID = 0, ## BSON is valid and UTF-8 compliant. - NOT_UTF8 = (1 shl 1), ## A key or a string is not valid UTF-8. - FIELD_HAS_DOT = (1 shl 2), ## Warning: key contains '.' character. - FIELD_INIT_DOLLAR = (1 shl 3), ## Warning: key starts with '$' character. - ALREADY_FINISHED = (1 shl 4) ## Trying to modify a finished BSON object. - TBinarySubtype* = enum - BIN_BINARY = 0, BIN_FUNC = 1, BIN_BINARY_OLD = 2, BIN_UUID = 3, BIN_MD5 = 5, - BIN_USER = 128 - TBsonKind* {.size: sizeof(cint).} = enum - bkEOO = 0, - bkDOUBLE = 1, - bkSTRING = 2, - bkOBJECT = 3, - bkARRAY = 4, - bkBINDATA = 5, - bkUNDEFINED = 6, - bkOID = 7, - bkBOOL = 8, - bkDATE = 9, - bkNULL = 10, - bkREGEX = 11, - bkDBREF = 12, #*< Deprecated. - bkCODE = 13, - bkSYMBOL = 14, - bkCODEWSCOPE = 15, - bkINT = 16, - bkTIMESTAMP = 17, - bkLONG = 18 - TBsonBool* = cint - TIter* {.pure, final.} = object - cur*: cstring - first*: TBsonBool - - TBson* {.pure, final.} = object - data*: cstring - cur*: cstring - dataSize*: cint - finished*: TBsonBool - ownsData*: TBsonBool - err*: cint - stackSize*: cint - stackPos*: cint - stackPtr*: ptr csize - stack*: array[0..32 - 1, csize] - - TDate* = int64 - -# milliseconds since epoch UTC - -type - TTimestamp*{.pure, final.} = object ## a timestamp - i*: cint # increment - t*: cint # time in seconds - -proc create*(): ptr TBson{.stdcall, importc: "bson_create", dynlib: bsondll.} -proc dispose*(b: ptr TBson){.stdcall, importc: "bson_dispose", dynlib: bsondll.} - -proc size*(b: var TBson): cint {.stdcall, importc: "bson_size", dynlib: bsondll.} - ## Size of a BSON object. - -proc bufferSize*(b: var TBson): cint{.stdcall, importc: "bson_buffer_size", - dynlib: bsondll.} - ## Buffer size of a BSON object. - -proc print*(b: var TBson){.stdcall, importc: "bson_print", dynlib: bsondll.} - ## Print a string representation of a BSON object. - -proc print*(TBson: cstring, depth: cint) {.stdcall, - importc: "bson_print_raw", dynlib: bsondll.} - ## Print a string representation of a BSON object up to `depth`. - - -proc data*(b: var TBson): cstring{.stdcall, importc: "bson_data", - dynlib: bsondll.} - ## Return a pointer to the raw buffer stored by this bson object. - -proc find*(it: var TIter, obj: var TBson, name: cstring): TBsonKind {.stdcall, - importc: "bson_find", dynlib: bsondll.} - ## Advance `it` to the named field. `obj` is the BSON object to use. - ## `name` is the name of the field to find. Returns the type of the found - ## object or ``bkEOO`` if it is not found. - -proc createIter*(): ptr TIter{.stdcall, importc: "bson_iterator_create", - dynlib: bsondll.} -proc dispose*(a2: ptr TIter){.stdcall, importc: "bson_iterator_dispose", - dynlib: bsondll.} - -proc initIter*(b: var TBson): TIter = - ## Initialize a bson iterator from the value `b`. - proc iterator_init(i: var TIter, b: var TBson){.stdcall, - importc: "bson_iterator_init", dynlib: bsondll.} - - iterator_init(result, b) - -proc fromBuffer*(i: var TIter, buffer: cstring) {.stdcall, - importc: "bson_iterator_from_buffer", dynlib: bsondll.} - ## Initialize a bson iterator from a cstring buffer. Note - ## that this is mostly used internally. - -proc more*(i: var TIter): bool = - ## Check to see if the bson_iterator has more data. - proc iterator_more(i: var TIter): TBsonBool{.stdcall, - importc: "bson_iterator_more", dynlib: bsondll.} - result = iterator_more(i) != 0'i32 - -proc next*(i: var TIter): TBsonKind {.stdcall, - importc: "bson_iterator_next", dynlib: bsondll.} - ## Point the iterator at the next BSON object. - -proc kind*(i: var TIter): TBsonKind{.stdcall, - importc: "bson_iterator_type", dynlib: bsondll.} - ## Get the type of the BSON object currently pointed to by the iterator. - -proc key*(i: var TIter): cstring{.stdcall, - importc: "bson_iterator_key", dynlib: bsondll.} - ## Get the key of the BSON object currently pointed to by the iterator. - -proc value*(i: var TIter): cstring{.stdcall, - importc: "bson_iterator_value", dynlib: bsondll.} - ## Get the value of the BSON object currently pointed to by the iterator. - -proc floatVal*(i: var TIter): float {.stdcall, - importc: "bson_iterator_double", dynlib: bsondll.} - ## Get the double value of the BSON object currently pointed to by the - ## iterator. - -proc intVal*(i: var TIter): cint{.stdcall, importc: "bson_iterator_int", - dynlib: bsondll.} - ## Get the int value of the BSON object currently pointed to by the iterator. - -proc int64Val*(i: var TIter): int64{.stdcall, - importc: "bson_iterator_long", dynlib: bsondll.} - ## Get the long value of the BSON object currently pointed to by the iterator. - -proc timestamp*(i: var TIter): Ttimestamp {.stdcall, - importc: "bson_iterator_timestamp", dynlib: bsondll.} - # return the bson timestamp as a whole or in parts - -proc timestampTime*(i: var TIter): cint {.stdcall, - importc: "bson_iterator_timestamp_time", dynlib: bsondll.} - # return the bson timestamp as a whole or in parts -proc timestampIncrement*(i: var TIter): cint{.stdcall, - importc: "bson_iterator_timestamp_increment", dynlib: bsondll.} - # return the bson timestamp as a whole or in parts - -proc boolVal*(i: var TIter): TBsonBool{.stdcall, - importc: "bson_iterator_bool", dynlib: bsondll.} - ## Get the boolean value of the BSON object currently pointed to by - ## the iterator. - ## - ## | false: boolean false, 0 in any type, or null - ## | true: anything else (even empty strings and objects) - -proc floatRaw*(i: var TIter): cdouble{.stdcall, - importc: "bson_iterator_double_raw", dynlib: bsondll.} - ## Get the double value of the BSON object currently pointed to by the - ## iterator. Assumes the correct type is used. - -proc intRaw*(i: var TIter): cint{.stdcall, - importc: "bson_iterator_int_raw", dynlib: bsondll.} - ## Get the int value of the BSON object currently pointed to by the - ## iterator. Assumes the correct type is used. - -proc int64Raw*(i: var TIter): int64{.stdcall, - importc: "bson_iterator_long_raw", dynlib: bsondll.} - ## Get the long value of the BSON object currently pointed to by the - ## iterator. Assumes the correct type is used. - -proc boolRaw*(i: var TIter): TBsonBool{.stdcall, - importc: "bson_iterator_bool_raw", dynlib: bsondll.} - ## Get the bson_bool_t value of the BSON object currently pointed to by the - ## iterator. Assumes the correct type is used. - -proc oidVal*(i: var TIter): ptr TOid {.stdcall, - importc: "bson_iterator_oid", dynlib: bsondll.} - ## Get the bson_oid_t value of the BSON object currently pointed to by the - ## iterator. - -proc strVal*(i: var TIter): cstring {.stdcall, - importc: "bson_iterator_string", dynlib: bsondll.} - ## Get the string value of the BSON object currently pointed to by the - ## iterator. - -proc strLen*(i: var TIter): cint {.stdcall, - importc: "bson_iterator_string_len", dynlib: bsondll.} - ## Get the string length of the BSON object currently pointed to by the - ## iterator. - -proc code*(i: var TIter): cstring {.stdcall, - importc: "bson_iterator_code", dynlib: bsondll.} - ## Get the code value of the BSON object currently pointed to by the - ## iterator. Works with bson_code, bson_codewscope, and BSON_STRING - ## returns ``nil`` for everything else. - -proc codeScope*(i: var TIter, scope: var TBson) {.stdcall, - importc: "bson_iterator_code_scope", dynlib: bsondll.} - ## Calls bson_empty on scope if not a bson_codewscope - -proc date*(i: var TIter): Tdate {.stdcall, - importc: "bson_iterator_date", dynlib: bsondll.} - ## Get the date value of the BSON object currently pointed to by the - ## iterator. - -proc time*(i: var TIter): TTime {.stdcall, - importc: "bson_iterator_time_t", dynlib: bsondll.} - ## Get the time value of the BSON object currently pointed to by the - ## iterator. - -proc binLen*(i: var TIter): cint {.stdcall, - importc: "bson_iterator_bin_len", dynlib: bsondll.} - ## Get the length of the BSON binary object currently pointed to by the - ## iterator. - -proc binType*(i: var TIter): char {.stdcall, - importc: "bson_iterator_bin_type", dynlib: bsondll.} - ## Get the type of the BSON binary object currently pointed to by the - ## iterator. - -proc binData*(i: var TIter): cstring {.stdcall, - importc: "bson_iterator_bin_data", dynlib: bsondll.} - ## Get the value of the BSON binary object currently pointed to by the - ## iterator. - -proc regex*(i: var TIter): cstring {.stdcall, - importc: "bson_iterator_regex", dynlib: bsondll.} - ## Get the value of the BSON regex object currently pointed to by the - ## iterator. - -proc regexOpts*(i: var TIter): cstring {.stdcall, - importc: "bson_iterator_regex_opts", dynlib: bsondll.} - ## Get the options of the BSON regex object currently pointed to by the - ## iterator. - -proc subobject*(i: var TIter, sub: var TBson) {.stdcall, - importc: "bson_iterator_subobject", dynlib: bsondll.} - ## Get the BSON subobject currently pointed to by the - ## iterator. - -proc subiterator*(i: var TIter, sub: var TIter) {.stdcall, - importc: "bson_iterator_subiterator", dynlib: bsondll.} - ## Get a bson_iterator that on the BSON subobject. - - -# ---------------------------- -# BUILDING -# ---------------------------- - -proc init*(b: var TBson) {.stdcall, importc: "bson_init", dynlib: bsondll.} - ## Initialize a new bson object. If not created - ## with bson_new, you must initialize each new bson - ## object using this function. - ## - ## When finished, you must pass the bson object to bson_destroy(). - -proc init*(b: var TBson, data: cstring): cint {.stdcall, - importc: "bson_init_data", dynlib: bsondll.} - ## Initialize a BSON object, and point its data - ## pointer to the provided `data`. - ## Returns OK or ERROR. - -proc initFinished*(b: var TBson, data: cstring): cint {.stdcall, - importc: "bson_init_finished_data", dynlib: bsondll.} - -proc initSize*(b: var TBson, size: cint) {.stdcall, importc: "bson_init_size", - dynlib: bsondll.} - ## Initialize a BSON object, and set its buffer to the given size. - ## Returns OK or ERROR. - -proc ensureSpace*(b: var TBson, bytesNeeded: cint): cint {.stdcall, - importc: "bson_ensure_space", dynlib: bsondll.} - ## Grow a bson object. `bytesNeeded` is the additional number of bytes needed. - -proc finish*(b: var TBson): cint{.stdcall, importc: "bson_finish", - dynlib: bsondll, discardable.} - ## Finalize a bson object. Returns the standard error code. - ## To deallocate memory, call destroy on the bson object. - -proc destroy*(b: var TBson){.stdcall, importc: "bson_destroy", dynlib: bsondll.} - ## Destroy a bson object. - -proc empty*(obj: var TBson) {.stdcall, importc: "bson_empty", - dynlib: bsondll.} - ## Sets a pointer to a static empty BSON object. - ## `obj` is the BSON object to initialize. - -proc copy*(outp, inp: var TBson): cint{.stdcall, importc: "bson_copy", - dynlib: bsondll.} - ## Make a complete copy of the a BSON object. - ## The source bson object must be in a finished - ## state; otherwise, the copy will fail. - -proc add*(b: var TBson, name: cstring, oid: TOid) = - ## adds an OID to `b`. - proc appendOid(b: var TBson, name: cstring, oid: ptr TOid): cint {.stdcall, - importc: "bson_append_oid", dynlib: bsondll.} - - var oid = oid - discard appendOid(b, name, addr(oid)) - -proc add*(b: var TBson, name: cstring, i: cint): cint{.stdcall, - importc: "bson_append_int", dynlib: bsondll, discardable.} - ## Append an int to a bson. - -proc add*(b: var TBson, name: cstring, i: int64): cint{.stdcall, - importc: "bson_append_long", dynlib: bsondll, discardable.} - ## Append an long to a bson. - -proc add*(b: var TBson, name: cstring, d: float): cint{.stdcall, - importc: "bson_append_double", dynlib: bsondll, discardable.} - ## Append an double to a bson. - -proc add*(b: var TBson, name: cstring, str: cstring): cint {.stdcall, - importc: "bson_append_string", dynlib: bsondll, discardable.} - ## Append a string to a bson. - -proc add*(b: var TBson, name: cstring, str: cstring, len: cint): cint{. - stdcall, importc: "bson_append_string_n", dynlib: bsondll, discardable.} - ## Append len bytes of a string to a bson. - -proc add*(b: var TBson, name: cstring, str: string) = - ## Append a Nimrod string `str` to a bson. - discard add(b, name, str, str.len.cint) - -proc addSymbol*(b: var TBson, name: cstring, str: cstring): cint{.stdcall, - importc: "bson_append_symbol", dynlib: bsondll, discardable.} - ## Append a symbol to a bson. - -proc addSymbol*(b: var TBson, name: cstring, str: cstring, len: cint): cint{. - stdcall, importc: "bson_append_symbol_n", dynlib: bsondll, discardable.} - ## Append len bytes of a symbol to a bson. - -proc addCode*(b: var TBson, name: cstring, str: cstring): cint{.stdcall, - importc: "bson_append_code", dynlib: bsondll, discardable.} - ## Append code to a bson. - -proc addCode*(b: var TBson, name: cstring, str: cstring, len: cint): cint{. - stdcall, importc: "bson_append_code_n", dynlib: bsondll, discardable.} - ## Append len bytes of code to a bson. - -proc addCode*(b: var TBson, name: cstring, code: cstring, - scope: var TBson): cint{.stdcall, - importc: "bson_append_code_w_scope", dynlib: bsondll, discardable.} - ## Append code to a bson with scope. - -proc addCode*(b: var TBson, name: cstring, code: cstring, - size: cint, scope: var TBson): cint{.stdcall, - importc: "bson_append_code_w_scope_n", dynlib: bsondll, discardable.} - ## Append len bytes of code to a bson with scope. - -proc addBinary*(b: var TBson, name: cstring, typ: char, str: cstring, - len: cint): cint{.stdcall, importc: "bson_append_binary", - dynlib: bsondll, discardable.} - ## Append binary data to a bson. - -proc addBinary*(b: var TBson, name: cstring, data: string) = - ## Append binary data to a bson. - addBinary(b, name, '\5', data, data.len.cint) - -proc addBool*(b: var TBson, name: cstring, v: TBsonBool): cint{.stdcall, - importc: "bson_append_bool", dynlib: bsondll, discardable.} - ## Append a bson_bool_t to a bson. - -proc addNull*(b: var TBson, name: cstring): cint {.stdcall, - importc: "bson_append_null", dynlib: bsondll, discardable.} - ## Append a null value to a bson. - -proc addUndefined*(b: var TBson, name: cstring): cint{.stdcall, - importc: "bson_append_undefined", dynlib: bsondll, discardable.} - ## Append an undefined value to a bson. - -proc addRegex*(b: var TBson, name: cstring, pattern: cstring, opts: cstring): cint{. - stdcall, importc: "bson_append_regex", dynlib: bsondll, discardable.} - ## Append a regex value to a bson. - -proc add*(b: var TBson, name: cstring, TBson: var TBson): cint {.stdcall, - importc: "bson_append_bson", dynlib: bsondll, discardable.} - ## Append bson data to a bson. - -proc addElement*(b: var TBson, name_or_null: cstring, elem: var TIter): cint{. - stdcall, importc: "bson_append_element", dynlib: bsondll, discardable.} - ## Append a BSON element to a bson from the current point of an iterator. - -proc addTimestamp*(b: var TBson, name: cstring, ts: var TTimestamp): cint{. - stdcall, importc: "bson_append_timestamp", dynlib: bsondll, discardable.} - ## Append a bson_timestamp_t value to a bson. - -proc addTimestamp2*(b: var TBson, name: cstring, time: cint, increment: cint): cint{. - stdcall, importc: "bson_append_timestamp2", dynlib: bsondll, discardable.} -proc addDate*(b: var TBson, name: cstring, millis: TDate): cint{.stdcall, - importc: "bson_append_date", dynlib: bsondll, discardable.} - ## Append a bson_date_t value to a bson. - -proc addTime*(b: var TBson, name: cstring, secs: TTime): cint{.stdcall, - importc: "bson_append_time_t", dynlib: bsondll, discardable.} - ## Append a time_t value to a bson. - -proc addStartObject*(b: var TBson, name: cstring): cint {.stdcall, - importc: "bson_append_start_object", dynlib: bsondll, discardable.} - ## Start appending a new object to a bson. - -proc addStartArray*(b: var TBson, name: cstring): cint {.stdcall, - importc: "bson_append_start_array", dynlib: bsondll, discardable.} - ## Start appending a new array to a bson. - -proc addFinishObject*(b: var TBson): cint {.stdcall, - importc: "bson_append_finish_object", dynlib: bsondll, discardable.} - ## Finish appending a new object or array to a bson. - -proc addFinishArray*(b: var TBson): cint {.stdcall, - importc: "bson_append_finish_array", dynlib: bsondll, discardable.} - ## Finish appending a new object or array to a bson. This - ## is simply an alias for bson_append_finish_object. - -proc numstr*(str: cstring, i: cint){.stdcall, importc: "bson_numstr", - dynlib: bsondll.} -proc incnumstr*(str: cstring){.stdcall, importc: "bson_incnumstr", - dynlib: bsondll.} - -type - TErrHandler* = proc (errmsg: cstring){. - stdcall.} ## an error handler. Error handlers shouldn't return! - -proc setBsonErrHandler*(func: TErrHandler): TErrHandler {.stdcall, - importc: "set_bson_err_handler", dynlib: bsondll.} - ## Set a function for error handling. - ## Returns the old error handling function, or nil. - -proc fatal*(ok: cint){.stdcall, importc: "bson_fatal", dynlib: bsondll.} - ## does nothing if ok != 0. Exit fatally. - -proc fatal*(ok: cint, msg: cstring){.stdcall, importc: "bson_fatal_msg", - dynlib: bsondll.} - ## Exit fatally with an error message. - -proc builderError*(b: var TBson){.stdcall, importc: "bson_builder_error", - dynlib: bsondll.} - ## Invoke the error handler, but do not exit. - -proc int64ToDouble*(i64: int64): cdouble {.stdcall, - importc: "bson_int64_to_double", dynlib: bsondll.} - ## Cast an int64_t to double. This is necessary for embedding in - ## certain environments. - -const - MAJOR* = 0 - MINOR* = 4 - PATCH* = 0 - -type - TError*{.size: sizeof(cint).} = enum ## connection errors - CONN_SUCCESS = 0, ## Connection success! - CONN_NO_SOCKET, ## Could not create a socket. - CONN_FAIL, ## An error occured while calling connect(). - CONN_ADDR_FAIL, ## An error occured while calling getaddrinfo(). - CONN_NOT_MASTER, ## Warning: connected to a non-master node (read-only). - CONN_BAD_SET_NAME, ## Given rs name doesn't match this replica set. - CONN_NO_PRIMARY, ## Can't find primary in replica set. Connection closed. - IO_ERROR, ## An error occurred while reading or writing on the socket. - READ_SIZE_ERROR, ## The response is not the expected length. - COMMAND_FAILED, ## The command returned with 'ok' value of 0. - BSON_INVALID, ## BSON not valid for the specified op. - BSON_NOT_FINISHED ## BSON object has not been finished. - TCursorError*{.size: sizeof(cint).} = enum ## cursor error - CURSOR_EXHAUSTED, ## The cursor has no more results. - CURSOR_INVALID, ## The cursor has timed out or is not recognized. - CURSOR_PENDING, ## Tailable cursor still alive but no data. - CURSOR_QUERY_FAIL, ## The server returned an '$err' object, indicating query failure. - ## See conn.lasterrcode and conn.lasterrstr for details. - CURSOR_BSON_ERROR ## Something is wrong with the BSON provided. See conn.err - ## for details. - TCursorFlags* = enum ## cursor flags - CURSOR_MUST_FREE = 1, ## mongo_cursor_destroy should free cursor. - CURSOR_QUERY_SENT = (1 shl 1) ## Initial query has been sent. - TindexOpts* = enum - INDEX_UNIQUE = (1 shl 0), INDEX_DROP_DUPS = (1 shl 2), - INDEX_BACKGROUND = (1 shl 3), INDEX_SPARSE = (1 shl 4) - TupdateOpts* = enum - UPDATE_UPSERT = 0x00000001, - UPDATE_MULTI = 0x00000002, - UPDATE_BASIC = 0x00000004 - TCursorOpts* = enum - TAILABLE = (1 shl 1), ## Create a tailable cursor. - SLAVE_OK = (1 shl 2), ## Allow queries on a non-primary node. - NO_CURSOR_TIMEOUT = (1 shl 4), ## Disable cursor timeouts. - AWAIT_DATA = (1 shl 5), ## Momentarily block for more data. - EXHAUST = (1 shl 6), ## Stream in multiple 'more' packages. - PARTIAL = (1 shl 7) ## Allow reads even if a shard is down. - Toperations* = enum - OP_MSG = 1000, OP_UPDATE = 2001, OP_INSERT = 2002, OP_QUERY = 2004, - OP_GET_MORE = 2005, OP_DELETE = 2006, OP_KILL_CURSORS = 2007 - THeader* {.pure, final.} = object - len*: cint - id*: cint - responseTo*: cint - op*: cint - - TMessage* {.pure, final.} = object - head*: Theader - data*: char - - TReplyFields*{.pure, final.} = object - flag*: cint # FIX THIS COMMENT non-zero on failure - cursorID*: int64 - start*: cint - num*: cint - - TReply*{.pure, final.} = object - head*: Theader - fields*: Treply_fields - objs*: char - - THostPort*{.pure, final.} = object - host*: array[0..255 - 1, char] - port*: cint - next*: ptr THostPort - - TReplset*{.pure, final.} = object ## replset - seeds*: ptr THostPort ## List of seeds provided by the user. - hosts*: ptr THostPort ## List of host/ports given by the replica set - name*: cstring ## Name of the replica set. - primary_connected*: TBsonBool ## Primary node connection status. - - TWriteConcern*{.pure, final.} = object ## mongo_write_concern - w*: cint - wtimeout*: cint - j*: cint - fsync*: cint - mode*: cstring - cmd*: TBSon - - TMongo*{.pure, final.} = object ## mongo - primary*: ptr THostPort ## Primary connection info. - replset*: ptr TReplSet ## replset object if connected to a replica set. - sock*: cint ## Socket file descriptor. - flags*: cint ## Flags on this connection object. - conn_timeout_ms*: cint ## Connection timeout in milliseconds. - op_timeout_ms*: cint ## Read and write timeout in milliseconds. - max_bson_size*: cint ## Largest BSON object allowed on this connection. - connected*: TBsonBool ## Connection status. - write_concern*: TWriteConcern ## The default write concern. - err*: TError ## Most recent driver error code. - errcode*: cint ## Most recent errno or WSAGetLastError(). - errstr*: array[0..128 - 1, char] ## String version of most recent driver error code. - lasterrcode*: cint ## getlasterror code given by the server on error. - lasterrstr*: array[0..128 - 1, char] ## getlasterror string generated by server. - - TCursor*{.pure, final.} = object ## cursor - reply*: ptr TReply ## reply is owned by cursor - conn*: ptr TMongo ## connection is *not* owned by cursor - ns*: cstring ## owned by cursor - flags*: cint ## Flags used internally by this drivers. - seen*: cint ## Number returned so far. - current*: TBson ## This cursor's current bson object. - err*: TCursorError ## Errors on this cursor. - query*: ptr TBson ## Bitfield containing cursor options. - fields*: ptr TBson ## Bitfield containing cursor options. - options*: cint ## Bitfield containing cursor options. - limit*: cint ## Bitfield containing cursor options. - skip*: cint ## Bitfield containing cursor options. - - -# Connection API - -proc createMongo*(): ptr TMongo{.stdcall, importc: "mongo_create", dynlib: mongodll.} -proc dispose*(conn: ptr TMongo){.stdcall, importc: "mongo_dispose", - dynlib: mongodll.} -proc getErr*(conn: var TMongo): cint{.stdcall, importc: "mongo_get_err", - dynlib: mongodll.} -proc isConnected*(conn: var TMongo): cint{.stdcall, - importc: "mongo_is_connected", dynlib: mongodll.} -proc getOpTimeout*(conn: var TMongo): cint{.stdcall, - importc: "mongo_get_op_timeout", dynlib: mongodll.} -proc getPrimary*(conn: var TMongo): cstring{.stdcall, - importc: "mongo_get_primary", dynlib: mongodll.} -proc getSocket*(conn: var TMongo): cint {.stdcall, importc: "mongo_get_socket", - dynlib: mongodll.} -proc getHostCount*(conn: var TMongo): cint{.stdcall, - importc: "mongo_get_host_count", dynlib: mongodll.} -proc getHost*(conn: var TMongo, i: cint): cstring {.stdcall, - importc: "mongo_get_host", dynlib: mongodll.} -proc createCursor*(): ptr TCursor{.stdcall, importc: "mongo_cursor_create", - dynlib: mongodll.} -proc dispose*(cursor: ptr TCursor){.stdcall, - importc: "mongo_cursor_dispose", dynlib: mongodll.} -proc getServerErr*(conn: var TMongo): cint{.stdcall, - importc: "mongo_get_server_err", dynlib: mongodll.} -proc getServerErrString*(conn: var TMongo): cstring{.stdcall, - importc: "mongo_get_server_err_string", dynlib: mongodll.} - -proc init*(conn: var TMongo){.stdcall, importc: "mongo_init", dynlib: mongodll.} - ## Initialize a new mongo connection object. You must initialize each mongo - ## object using this function. - ## When finished, you must pass this object to ``destroy``. - -proc connect*(conn: var TMongo, host: cstring = defaultHost, - port: cint = defaultPort): cint {.stdcall, - importc: "mongo_connect", dynlib: mongodll, deprecated.} - ## Connect to a single MongoDB server. -proc client*(conn: var TMongo, host: cstring = defaultHost, - port: cint = defaultPort): cint {.stdcall, - importc: "mongo_client", dynlib: mongodll.} - ## Connect to a single MongoDB server. - -proc replsetInit*(conn: var TMongo, name: cstring){.stdcall, - importc: "mongo_replset_init", dynlib: mongodll.} - ## Set up this connection object for connecting to a replica set. - ## To connect, pass the object to replsetConnect. - ## `name` is the name of the replica set to connect to. - -proc replsetAddSeed*(conn: var TMongo, host: cstring = defaultHost, - port: cint = defaultPort){.stdcall, - importc: "mongo_replset_add_seed", dynlib: mongodll.} - ## Add a seed node to the replica set connection object. - ## You must specify at least one seed node before connecting - ## to a replica set. - -proc parseHost*(hostString: cstring, hostPort: var ThostPort){.stdcall, - importc: "mongo_parse_host", dynlib: mongodll.} - ## Utility function for converting a host-port string to a mongo_host_port. - ## `hostString` is a string containing either a host or a host and port - ## separated by a colon. - ## `hostPort` is the mongo_host_port object to write the result to. - -proc replsetConnect*(conn: var TMongo): cint{.stdcall, - importc: "mongo_replset_connect", dynlib: mongodll.} - ## Connect to a replica set. - ## Before passing a connection object to this function, you must already - ## have called setReplset and replsetAddSeed. - -proc setOpTimeout*(conn: var TMongo, millis: cint): cint{.stdcall, - importc: "mongo_set_op_timeout", dynlib: mongodll.} - ## Set a timeout for operations on this connection. This - ## is a platform-specific feature, and only work on Unix-like - ## systems. You must also compile for linux to support this. - -proc checkConnection*(conn: var TMongo): cint {.stdcall, - importc: "mongo_check_connection", dynlib: mongodll.} - ## Ensure that this connection is healthy by performing - ## a round-trip to the server. - ## Returns OK if connected; otherwise ERROR. - -proc reconnect*(conn: var TMongo): cint {.stdcall, importc: "mongo_reconnect", - dynlib: mongodll.} - ## Try reconnecting to the server using the existing connection settings. - ## This function will disconnect the current socket. If you've authenticated, - ## you'll need to re-authenticate after calling this function. - -proc disconnect*(conn: var TMongo){.stdcall, importc: "mongo_disconnect", - dynlib: mongodll.} - ## Close the current connection to the server. After calling - ## this function, you may call reconnect with the same - ## connection object. - -proc destroy*(conn: var TMongo){.stdcall, importc: "mongo_destroy", - dynlib: mongodll.} - ## Close any existing connection to the server and free all allocated - ## memory associated with the conn object. - ## You must always call this function when finished with the connection - ## object. - -proc insert*(conn: var TMongo, ns: cstring, data: var TBson, - custom_write_concern: ptr TWriteConcern): cint{.stdcall, - importc: "mongo_insert", dynlib: mongodll, discardable.} - ## Insert a BSON document into a MongoDB server. This function - ## will fail if the supplied BSON struct is not UTF-8 or if - ## the keys are invalid for insert (contain '.' or start with '$'). - -proc insertBatch*(conn: var TMongo, ns: cstring, - data: ptr ptr TBson, num: cint): cint{. - stdcall, importc: "mongo_insert_batch", dynlib: mongodll, discardable.} - ## Insert a batch of BSON documents into a MongoDB server. This function - ## will fail if any of the documents to be inserted is invalid. - ## `num` is the number of documents in data. - -proc update*(conn: var TMongo, ns: cstring, cond, op: var TBson, - flags: cint): cint{.stdcall, importc: "mongo_update", - dynlib: mongodll, discardable.} - ## Update a document in a MongoDB server. - ## - ## | conn a mongo object. - ## | ns the namespace. - ## | cond the bson update query. - ## | op the bson update data. - ## | flags flags for the update. - ## | returns OK or ERROR with error stored in conn object. - -proc remove*(conn: var TMongo, namespace: cstring, cond: var TBson): cint{.stdcall, - importc: "mongo_remove", dynlib: mongodll.} - ## Remove a document from a MongoDB server. - ## - ## | conn a mongo object. - ## | ns the namespace. - ## | cond the bson query. - ## | returns OK or ERROR with error stored in conn object. - -proc find*(conn: var TMongo, namespace: cstring, query, fields: var TBson, - limit, skip: cint, options: cint): ptr TCursor{.stdcall, - importc: "mongo_find", dynlib: mongodll.} - ## Find documents in a MongoDB server. - ## - ## | conn a mongo object. - ## | ns the namespace. - ## | query the bson query. - ## | fields a bson document of fields to be returned. - ## | limit the maximum number of documents to return. - ## | skip the number of documents to skip. - ## | options A bitfield containing cursor options. - ## | returns A cursor object allocated on the heap or nil if - ## an error has occurred. For finer-grained error checking, - ## use the cursor builder API instead. - -proc init*(cursor: var TCursor, conn: var TMongo, namespace: cstring){.stdcall, - importc: "mongo_cursor_init", dynlib: mongodll.} - ## Initalize a new cursor object. - ## - ## The namespace is represented as the database - ## name and collection name separated by a dot. e.g., "test.users". - -proc setQuery*(cursor: var TCursor, query: var TBson) {.stdcall, - importc: "mongo_cursor_set_query", dynlib: mongodll.} - ## Set the bson object specifying this cursor's query spec. If - ## your query is the empty bson object "{}", then you need not - ## set this value. - ## - ## `query` is a bson object representing the query spec. This may - ## be either a simple query spec or a complex spec storing values for - ## $query, $orderby, $hint, and/or $explain. See - ## http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol for details. - -proc setFields*(cursor: var TCursor, fields: var TBson){.stdcall, - importc: "mongo_cursor_set_fields", dynlib: mongodll.} - ## Set the fields to return for this cursor. If you want to return - ## all fields, you need not set this value. - ## `fields` is a bson object representing the fields to return. - ## See http://www.mongodb.org/display/DOCS/Retrieving+a+Subset+of+Fields. - -proc setSkip*(cursor: var TCursor, skip: cint){.stdcall, - importc: "mongo_cursor_set_skip", dynlib: mongodll.} - ## Set the number of documents to skip. - -proc setLimit*(cursor: var TCursor, limit: cint){.stdcall, - importc: "mongo_cursor_set_limit", dynlib: mongodll.} - ## Set the number of documents to return. - -proc setOptions*(cursor: var TCursor, options: cint){.stdcall, - importc: "mongo_cursor_set_options", dynlib: mongodll.} - ## Set any of the available query options (e.g., TAILABLE). - ## See `TCursorOpts` for available constants. - -proc data*(cursor: var TCursor): cstring {.stdcall, - importc: "mongo_cursor_data", dynlib: mongodll.} - ## Return the current BSON object data as a ``cstring``. This is useful - ## for creating bson iterators. - -proc bson*(cursor: var TCursor): ptr TBson{.stdcall, - importc: "mongo_cursor_bson", dynlib: mongodll.} - ## Return the current BSON object. - -proc next*(cursor: var TCursor): cint {.stdcall, - importc: "mongo_cursor_next", dynlib: mongodll.} - ## Iterate the cursor, returning the next item. When successful, - ## the returned object will be stored in cursor.current; - -proc destroy*(cursor: var TCursor): cint {.stdcall, - importc: "mongo_cursor_destroy", dynlib: mongodll, discardable.} - ## Destroy a cursor object. When finished with a cursor, you - ## must pass it to this function. - -proc findOne*(conn: var TMongo, namespace: cstring, query: var TBson, - fields: var TBson, outp: var TBson): cint{.stdcall, - importc: "mongo_find_one", dynlib: mongodll.} - ## Find a single document in a MongoDB server. - ## - ## | conn a mongo object. - ## | ns the namespace. - ## | query the bson query. - ## | fields a bson document of the fields to be returned. - ## | outp a bson document in which to put the query result. - ## outp can be nil if you don't care about results. Useful for commands. - -proc count*(conn: var TMongo, db: cstring, coll: cstring, query: var TBson): cdouble{. - stdcall, importc: "mongo_count", dynlib: mongodll.} - ## Count the number of documents in a collection matching a query. - ## - ## | conn a mongo object. - ## | db the db name. - ## | coll the collection name. - ## | query the BSON query. - ## | returns the number of matching documents. If the command fails, - ## ERROR is returned. - -proc createIndex*(conn: var TMongo, namespace: cstring, key: var TBson, - options: cint, outp: var TBson): cint {.stdcall, - importc: "mongo_create_index", dynlib: mongodll.} - ## Create a compouned index. - ## - ## | conn a mongo object. - ## | ns the namespace. - ## | data the bson index data. - ## | options a bitfield for setting index options. Possibilities include - ## INDEX_UNIQUE, INDEX_DROP_DUPS, INDEX_BACKGROUND, - ## and INDEX_SPARSE. - ## | out a bson document containing errors, if any. - ## | returns MONGO_OK if index is created successfully; otherwise, MONGO_ERROR. - -proc createSimpleIndex*(conn: var TMongo, namespace, field: cstring, - options: cint, outp: var TBson): TBsonBool {.stdcall, - importc: "mongo_create_simple_index", dynlib: mongodll.} - ## Create an index with a single key. - ## - ## | conn a mongo object. - ## | ns the namespace. - ## | field the index key. - ## | options index options. - ## | out a BSON document containing errors, if any. - ## | returns true if the index was created. - - -# ---------------------------- -# COMMANDS -# ---------------------------- - - -proc runCommand*(conn: var TMongo, db: cstring, command: var TBson, - outp: var TBson): cint{.stdcall, importc: "mongo_run_command", - dynlib: mongodll.} - ## Run a command on a MongoDB server. - ## - ## | conn a mongo object. - ## | db the name of the database. - ## | command the BSON command to run. - ## | out the BSON result of the command. - ## | returns OK if the command ran without error. - -proc simpleIntCommand*(conn: var TMongo, db: cstring, cmd: cstring, arg: cint, - outp: var TBson): cint{.stdcall, - importc: "mongo_simple_int_command", dynlib: mongodll.} - ## Run a command that accepts a simple string key and integer value. - ## - ## | conn a mongo object. - ## | db the name of the database. - ## | cmd the command to run. - ## | arg the integer argument to the command. - ## | out the BSON result of the command. - ## | returns OK or an error code. - -proc simpleStrCommand*(conn: var TMongo, db: cstring, cmd: cstring, - arg: cstring, outp: var TBson): cint{.stdcall, - importc: "mongo_simple_str_command", dynlib: mongodll.} - ## Run a command that accepts a simple string key and value. - ## - ## | conn a mongo object. - ## | db the name of the database. - ## | cmd the command to run. - ## | arg the string argument to the command. - ## | out the BSON result of the command. - ## | returns true if the command ran without error. - -proc cmdDropDb*(conn: var TMongo, db: cstring): cint{.stdcall, - importc: "mongo_cmd_drop_db", dynlib: mongodll.} - ## Drop a database. - ## - ## | conn a mongo object. - ## | db the name of the database to drop. - ## | returns OK or an error code. - -proc cmdDropCollection*(conn: var TMongo, db: cstring, collection: cstring, - outp: var TBson): cint{.stdcall, - importc: "mongo_cmd_drop_collection", dynlib: mongodll.} - ## Drop a collection. - ## - ## | conn a mongo object. - ## | db the name of the database. - ## | collection the name of the collection to drop. - ## | out a BSON document containing the result of the command. - ## | returns true if the collection drop was successful. - -proc cmdAddUser*(conn: var TMongo, db: cstring, user: cstring, pass: cstring): cint{. - stdcall, importc: "mongo_cmd_add_user", dynlib: mongodll.} - ## Add a database user. - ## - ## | conn a mongo object. - ## | db the database in which to add the user. - ## | user the user name - ## | pass the user password - ## | returns OK or ERROR. - -proc cmdAuthenticate*(conn: var TMongo, db: cstring, user: cstring, - pass: cstring): cint{.stdcall, - importc: "mongo_cmd_authenticate", dynlib: mongodll.} - ## Authenticate a user. - ## - ## | conn a mongo object. - ## | db the database to authenticate against. - ## | user the user name to authenticate. - ## | pass the user's password. - ## | returns OK on sucess and ERROR on failure. - -proc cmdIsMaster*(conn: var TMongo, outp: var TBson): TBsonBool {.stdcall, - importc: "mongo_cmd_ismaster", dynlib: mongodll.} - ## Check if the current server is a master. - ## - ## | conn a mongo object. - ## | outp a BSON result of the command. - ## | returns true if the server is a master. - -proc cmdGetLastError*(conn: var TMongo, db: cstring, outp: var TBson): cint{. - stdcall, importc: "mongo_cmd_get_last_error", dynlib: mongodll.} - ## Get the error for the last command with the current connection. - ## - ## | conn a mongo object. - ## | db the name of the database. - ## | outp a BSON object containing the error details. - ## | returns OK or ERROR - -proc cmdGetPrevError*(conn: var TMongo, db: cstring, outp: var TBson): cint{. - stdcall, importc: "mongo_cmd_get_prev_error", dynlib: mongodll.} - ## Get the most recent error with the current connection. - ## - ## | conn a mongo object. - ## | db the name of the database. - ## | outp a BSON object containing the error details. - ## | returns OK or ERROR. - -proc cmdResetError*(conn: var TMongo, db: cstring){.stdcall, - importc: "mongo_cmd_reset_error", dynlib: mongodll.} - ## Reset the error state for the connection. `db` is the name of the database. - -# gridfs.h - -const - DEFAULT_CHUNK_SIZE* = 262144 - -type - TOffset* = int64 - -# A GridFS represents a single collection of GridFS files in the database. - -type - TGridfs*{.pure, final.} = object - client*: ptr TMongo ## The client to db-connection. - dbname*: cstring ## The root database name - prefix*: cstring ## The prefix of the GridFS's collections, - ## default is nil - files_ns*: cstring ## The namespace where the file's metadata - ## is stored - chunks_ns*: cstring ## The namespace where the files's data is - ## stored in chunks - -# A GridFile is a single GridFS file. - -type - TGridFile*{.pure, final.} = object - gfs*: ptr TGridfs ## GridFS where the GridFile is located - meta*: ptr TBson ## GridFile's bson object where all - ## its metadata is located - pos*: TOffset ## position is the offset in the file - id*: TOid ## files_id of the gridfile - remote_name*: cstring ## name of the gridfile as a string - content_type*: cstring ## gridfile's content type - length*: TOffset ## length of this gridfile - chunk_num*: cint ## number of the current chunk being written to - pending_data*: cstring ## buffer storing data still to be - ## written to chunks - pending_len*: cint ## length of pending_data buffer - - -proc createGridfs*(): ptr TGridfs{.stdcall, importc: "gridfs_create", dynlib: mongodll.} -proc dispose*(gfs: ptr TGridfs){.stdcall, importc: "gridfs_dispose", - dynlib: mongodll.} -proc createGridfile*(): ptr TGridFile{.stdcall, importc: "gridfile_create", - dynlib: mongodll.} -proc dispose*(gf: ptr TGridFile){.stdcall, importc: "gridfile_dispose", - dynlib: mongodll.} -proc getDescriptor*(gf: var TGridFile, outp: var TBson){.stdcall, - importc: "gridfile_get_descriptor", dynlib: mongodll.} - - -proc init*(client: var TMongo, dbname: cstring, prefix: cstring, - gfs: var TGridfs): cint{.stdcall, importc: "gridfs_init", - dynlib: mongodll.} - ## Initializes a GridFS object - ## - ## | client - db connection - ## | dbname - database name - ## | prefix - collection prefix, default is fs if NULL or empty - ## | gfs - the GridFS object to initialize - ## | returns - OK or ERROR. - -proc destroy*(gfs: var TGridfs){.stdcall, importc: "gridfs_destroy", - dynlib: mongodll.} - ## Destroys a GridFS object. Call this when finished with the object. - -proc writerInit*(gfile: var TGridFile, gfs: var TGridfs, remote_name: cstring, - content_type: cstring){.stdcall, - importc: "gridfile_writer_init", dynlib: mongodll.} - ## Initializes a gridfile for writing incrementally with ``writeBuffer``. - ## Once initialized, you can write any number of buffers with ``writeBuffer``. - ## When done, you must call ``writerDone`` to save the file metadata. - -proc writeBuffer*(gfile: var TGridFile, data: cstring, length: TOffset){. - stdcall, importc: "gridfile_write_buffer", dynlib: mongodll.} - ## Write to a GridFS file incrementally. You can call this function any number - ## of times with a new buffer each time. This allows you to effectively - ## stream to a GridFS file. When finished, be sure to call ``writerDone``. - -proc writerDone*(gfile: var TGridFile): cint{.stdcall, - importc: "gridfile_writer_done", dynlib: mongodll.} - ## Signal that writing of this gridfile is complete by - ## writing any buffered chunks along with the entry in the - ## files collection. Returns OK or ERROR. - -proc storeBuffer*(gfs: var TGridfs, data: cstring, length: TOffset, - remotename: cstring, contenttype: cstring): cint{.stdcall, - importc: "gridfs_store_buffer", dynlib: mongodll.} - ## Store a buffer as a GridFS file. - ## - ## | gfs - the working GridFS - ## | data - pointer to buffer to store in GridFS - ## | length - length of the buffer - ## | remotename - filename for use in the database - ## | contenttype - optional MIME type for this object - ## | returns - MONGO_OK or MONGO_ERROR. - -proc storeFile*(gfs: var TGridfs, filename: cstring, remotename: cstring, - contenttype: cstring): cint{.stdcall, - importc: "gridfs_store_file", dynlib: mongodll.} - ## Open the file referenced by filename and store it as a GridFS file. - ## - ## | gfs - the working GridFS - ## | filename - local filename relative to the process - ## | remotename - optional filename for use in the database - ## | contenttype - optional MIME type for this object - ## | returns - OK or ERROR. - -proc removeFilename*(gfs: var TGridfs, filename: cstring){.stdcall, - importc: "gridfs_remove_filename", dynlib: mongodll.} - ## Removes the files referenced by filename from the db. - -proc findQuery*(gfs: var TGridfs, query: var TBson, gfile: var TGridFile): cint{. - stdcall, importc: "gridfs_find_query", dynlib: mongodll.} - ## Find the first file matching the provided query within the - ## GridFS files collection, and return the file as a GridFile. - ## Returns OK if successful, ERROR otherwise. - -proc findFilename*(gfs: var TGridfs, filename: cstring, gfile: var TGridFile): cint{. - stdcall, importc: "gridfs_find_filename", dynlib: mongodll.} - ## Find the first file referenced by filename within the GridFS - ## and return it as a GridFile. Returns OK or ERROR. - -proc init*(gfs: var TGridfs, meta: var TBson, gfile: var TGridFile): cint{. - stdcall, importc: "gridfile_init", dynlib: mongodll.} - ## Initializes a GridFile containing the GridFS and file bson. - -proc destroy*(gfile: var TGridFile){.stdcall, importc: "gridfile_destroy", - dynlib: mongodll.} - ## Destroys the GridFile. - -proc exists*(gfile: var TGridFile): TBsonBool{.stdcall, - importc: "gridfile_exists", dynlib: mongodll.} - ## Returns whether or not the GridFile exists. - -proc getFilename*(gfile: var TGridFile): cstring{.stdcall, - importc: "gridfile_get_filename", dynlib: mongodll.} - ## Returns the filename of GridFile. - -proc getChunksize*(gfile: var TGridFile): cint{.stdcall, - importc: "gridfile_get_chunksize", dynlib: mongodll.} - ## Returns the size of the chunks of the GridFile. - -proc getContentlength*(gfile: var TGridFile): TOffset{.stdcall, - importc: "gridfile_get_contentlength", dynlib: mongodll.} - ## Returns the length of GridFile's data. - -proc getContenttype*(gfile: var TGridFile): cstring{.stdcall, - importc: "gridfile_get_contenttype", dynlib: mongodll.} - ## Returns the MIME type of the GridFile (nil if no type specified). - -proc getUploaddate*(gfile: var TGridFile): Tdate{.stdcall, - importc: "gridfile_get_uploaddate", dynlib: mongodll.} - ## Returns the upload date of GridFile. - -proc getMd5*(gfile: var TGridFile): cstring {.stdcall, - importc: "gridfile_get_md5", dynlib: mongodll.} - ## Returns the MD5 of GridFile. - -proc getField*(gfile: var TGridFile, name: cstring): cstring{.stdcall, - importc: "gridfile_get_field", dynlib: mongodll.} - ## Returns the field in GridFile specified by name. Returns the data of the - ## field specified (nil if none exists). - -proc getBoolean*(gfile: var TGridFile, name: cstring): TBsonBool{.stdcall, - importc: "gridfile_get_boolean", dynlib: mongodll.} - ## Returns a boolean field in GridFile specified by name. - -proc getMetadata*(gfile: var TGridFile, outp: var TBson){.stdcall, - importc: "gridfile_get_metadata", dynlib: mongodll.} - ## Returns the metadata of GridFile (an empty bson is returned if none - ## exists). - -proc getNumchunks*(gfile: var TGridFile): cint{.stdcall, - importc: "gridfile_get_numchunks", dynlib: mongodll.} - ## Returns the number of chunks in the GridFile. - -proc getChunk*(gfile: var TGridFile, n: cint, outp: var TBson){.stdcall, - importc: "gridfile_get_chunk", dynlib: mongodll.} - ## Returns chunk `n` of GridFile. - -proc getChunks*(gfile: var TGridFile, start: cint, size: cint): ptr TCursor{. - stdcall, importc: "gridfile_get_chunks", dynlib: mongodll.} - ## Returns a mongo_cursor of `size` chunks starting with chunk `start`. - ## The cursor must be destroyed after use. - -proc writeFile*(gfile: ptr TGridFile, stream: TFile): TOffset{.stdcall, - importc: "gridfile_write_file", dynlib: mongodll.} - ## Writes the GridFile to a stream. - -proc read*(gfile: var TGridFile, size: TOffset, buf: cstring): TOffset{.stdcall, - importc: "gridfile_read", dynlib: mongodll.} - ## Reads length bytes from the GridFile to a buffer - ## and updates the position in the file. - ## (assumes the buffer is large enough) - ## (if size is greater than EOF gridfile_read reads until EOF). - ## Returns the number of bytes read. - -proc seek*(gfile: var TGridFile, offset: TOffset): TOffset{.stdcall, - importc: "gridfile_seek", dynlib: mongodll.} - ## Updates the position in the file - ## (If the offset goes beyond the contentlength, - ## the position is updated to the end of the file.) - ## Returns the offset location diff --git a/lib/wrappers/zip/zlib.nim b/lib/wrappers/zip/zlib.nim index c4c6ac071..f505b95a7 100644 --- a/lib/wrappers/zip/zlib.nim +++ b/lib/wrappers/zip/zlib.nim @@ -7,7 +7,7 @@ when defined(windows): elif defined(macosx): const libz = "libz.dylib" else: - const libz = "libz.so" + const libz = "libz.so.1" type Uint* = int32 @@ -134,6 +134,7 @@ proc gzerror*(thefile: gzFile, errnum: var int32): pbytef{.cdecl, dynlib: libz, importc: "gzerror".} proc adler32*(adler: uLong, buf: pbytef, length: uInt): uLong{.cdecl, dynlib: libz, importc: "adler32".} + ## **Warning**: Adler-32 requires at least a few hundred bytes to get rolling. proc crc32*(crc: uLong, buf: pbytef, length: uInt): uLong{.cdecl, dynlib: libz, importc: "crc32".} proc deflateInitu*(strm: var TZStream, level: int32, version: cstring, diff --git a/lib/wrappers/zmq.nim b/lib/wrappers/zmq.nim index 4e658028e..9826ab813 100644 --- a/lib/wrappers/zmq.nim +++ b/lib/wrappers/zmq.nim @@ -299,12 +299,12 @@ proc open*(address: string, server: bool, mode: TConnectionMode = conDEALER, else: if connect(result.s, address) != 0'i32: zmqError() -proc close*(c: var TConnection) = +proc close*(c: TConnection) = ## closes the connection. if close(c.s) != 0'i32: zmqError() if term(c.c) != 0'i32: zmqError() -proc send*(c: var TConnection, msg: string) = +proc send*(c: TConnection, msg: string) = ## sends a message over the connection. var m: TMsg if msg_init(m, msg.len) != 0'i32: zmqError() @@ -312,7 +312,7 @@ proc send*(c: var TConnection, msg: string) = if send(c.s, m, 0'i32) != 0'i32: zmqError() discard msg_close(m) -proc receive*(c: var TConnection): string = +proc receive*(c: TConnection): string = ## receives a message from a connection. var m: TMsg if msg_init(m) != 0'i32: zmqError() @@ -320,4 +320,3 @@ proc receive*(c: var TConnection): string = result = newString(msg_size(m)) copyMem(addr(result[0]), msg_data(m), result.len) discard msg_close(m) - |